nedo-vision-worker 1.3.1__py3-none-any.whl → 1.3.5__py3-none-any.whl

nedo_vision_worker/__init__.py

@@ -6,5 +6,5 @@ A library for running worker agents in the Nedo Vision platform.
 
 from .worker_service import WorkerService
 
-__version__ = "1.3.1"
+__version__ = "1.3.5"
 __all__ = ["WorkerService"] 

nedo_vision_worker/worker/DataSenderWorker.py

@@ -153,7 +153,7 @@ class DataSenderWorker:
     def _calculate_sleep_interval(self, ppe_pending: int, violation_pending: int) -> float:
         """
         Calculate optimal sleep interval based on pending data.
-        Conservative approach to prevent server overload.
+        Conservative approach to prevent server overload and storage rate limits.
         
         Args:
             ppe_pending: Number of pending PPE detections
@@ -169,11 +169,13 @@ class DataSenderWorker:
         elif total_pending < 50:
             return self.send_interval
         elif total_pending < 200:
-            return 3.0
+            return 4.0
         elif total_pending < 500:
-            return 2.0
+            return 3.0
+        elif total_pending < 1000:
+            return 2.5
         else:
-            return 1.5
+            return 2.0
 
     def _run_worker_source_updater(self):
         """Dedicated loop for updating worker sources at a different interval."""

nedo_vision_worker/worker/PPEDetectionManager.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import threading
 import time
 from ..services.PPEDetectionClient import PPEDetectionClient
@@ -50,41 +51,91 @@ class PPEDetectionManager:
 
         logger.info("📡 [APP] PPE detection monitoring started.")
 
-    def _calculate_batch_size(self, pending_count: int) -> int:
+    def _calculate_fetch_size(self, pending_count: int) -> int:
         """
-        Calculates optimal batch size based on pending items.
-        Limited to 50 to stay within 50MB gRPC message size limit.
+        Dynamically calculates how many records to fetch based on pending count.
+        Reduced to prevent storage rate limits (each detection = 2 images = 2 dir checks).
         
         Args:
             pending_count (int): Number of pending detections
             
         Returns:
-            int: Optimal batch size (max 50)
+            int: Number of records to fetch
         """
-        if pending_count < 20:
-            return 10
-        elif pending_count < 100:
-            return 20
-        elif pending_count < 500:
-            return 30
+        if pending_count <= 50:
+            return min(pending_count, 5)
+        elif pending_count <= 200:
+            return min(pending_count, 10)
+        elif pending_count <= 1000:
+            return min(pending_count, 15)
         else:
-            return 50
+            return min(pending_count, 20)
+
+    def _calculate_batch_by_size(self, all_detections: list, max_size_mb: int = 40) -> list:
+        """
+        Calculates batch based on actual image file sizes to stay within gRPC limit.
+        
+        Args:
+            all_detections (list): All pending detections
+            max_size_mb (int): Maximum batch size in MB (default 40MB for 50MB limit with margin)
+            
+        Returns:
+            list: Detections that fit within size limit
+        """
+        import os
+        
+        batch = []
+        total_size = 0
+        max_size_bytes = max_size_mb * 1024 * 1024
+        
+        for detection in all_detections:
+            try:
+                image_size = os.path.getsize(detection['image']) if os.path.exists(detection['image']) else 0
+                tile_size = os.path.getsize(detection['image_tile']) if os.path.exists(detection['image_tile']) else 0
+                detection_size = image_size + tile_size
+                
+                if total_size + detection_size > max_size_bytes:
+                    if batch:
+                        break
+                    else:
+                        logger.warning(f"⚠️ Single detection exceeds {max_size_mb}MB, skipping")
+                        continue
+                
+                batch.append(detection)
+                total_size += detection_size
+                
+            except Exception as e:
+                logger.error(f"❌ Error checking file size: {e}")
+                continue
+        
+        return batch
 
     def send_ppe_detection_batch(self):
-        """Sends a batch of collected PPE detection data to the server with dynamic batch sizing."""
+        """Sends a batch of collected PPE detection data to the server with dynamic size-based batching."""
         try:
             pending_count = self.ppe_detection_repo.get_total_pending_count()
             
             if pending_count == 0:
                 return
             
-            batch_size = self._calculate_batch_size(pending_count)
-            self.ppe_detection_data = self.ppe_detection_repo.get_latest_detections(batch_size)
+            fetch_size = self._calculate_fetch_size(pending_count)
+            all_detections = self.ppe_detection_repo.get_latest_detections(fetch_size)
+            
+            if not all_detections:
+                return
+
+            self.ppe_detection_data = self._calculate_batch_by_size(all_detections)
             
             if not self.ppe_detection_data:
+                logger.warning("⚠️ [APP] No valid detections within size limit")
                 return
 
-            logger.info(f"📤 [APP] Sending {len(self.ppe_detection_data)} PPE detections ({pending_count} pending)")
+            batch_size_mb = sum(
+                os.path.getsize(d['image']) + os.path.getsize(d['image_tile']) 
+                for d in self.ppe_detection_data if os.path.exists(d['image']) and os.path.exists(d['image_tile'])
+            ) / (1024 * 1024)
+            
+            logger.info(f"📤 [APP] Sending {len(self.ppe_detection_data)} PPE detections (~{batch_size_mb:.1f}MB, {pending_count} pending)")
 
             response = self.ppe_detection_client.send_upsert_batch(
                 worker_id=self.worker_id,

nedo_vision_worker/worker/RestrictedAreaManager.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import threading
 import time
 
@@ -50,41 +51,89 @@ class RestrictedAreaManager:
 
         logger.info("📡 [APP] Restricted area violation monitoring started.")
 
-    def _calculate_batch_size(self, pending_count: int) -> int:
+    def _calculate_fetch_size(self, pending_count: int) -> int:
         """
-        Calculates optimal batch size based on pending items.
-        Limited to 50 to stay within 50MB gRPC message size limit.
+        Dynamically calculates how many records to fetch based on pending count.
+        Reduced to prevent storage rate limits (each violation = 2 images = 2 dir checks).
         
         Args:
             pending_count (int): Number of pending violations
             
         Returns:
-            int: Optimal batch size (max 50)
+            int: Number of records to fetch
         """
-        if pending_count < 20:
-            return 10
-        elif pending_count < 100:
-            return 20
-        elif pending_count < 500:
-            return 30
+        if pending_count <= 50:
+            return min(pending_count, 5)
+        elif pending_count <= 200:
+            return min(pending_count, 10)
+        elif pending_count <= 1000:
+            return min(pending_count, 15)
         else:
-            return 50
+            return min(pending_count, 20)
+
+    def _calculate_batch_by_size(self, all_violations: list, max_size_mb: int = 40) -> list:
+        """
+        Calculates batch based on actual image file sizes to stay within gRPC limit.
+        
+        Args:
+            all_violations (list): All pending violations
+            max_size_mb (int): Maximum batch size in MB (default 40MB for 50MB limit with margin)
+            
+        Returns:
+            list: Violations that fit within size limit
+        """
+        batch = []
+        total_size = 0
+        max_size_bytes = max_size_mb * 1024 * 1024
+        
+        for violation in all_violations:
+            try:
+                image_size = os.path.getsize(violation['image']) if os.path.exists(violation['image']) else 0
+                tile_size = os.path.getsize(violation['image_tile']) if os.path.exists(violation['image_tile']) else 0
+                violation_size = image_size + tile_size
+                
+                if total_size + violation_size > max_size_bytes:
+                    if batch:
+                        break
+                    else:
+                        logger.warning(f"⚠️ Single violation exceeds {max_size_mb}MB, skipping")
+                        continue
+                
+                batch.append(violation)
+                total_size += violation_size
+                
+            except Exception as e:
+                logger.error(f"❌ Error checking file size: {e}")
+                continue
+        
+        return batch
 
     def send_violation_batch(self):
-        """Sends a batch of collected violation data to the server with dynamic batch sizing."""
+        """Sends a batch of collected violation data to the server with dynamic size-based batching."""
         try:
             pending_count = self.repo.get_total_pending_count()
             
             if pending_count == 0:
                 return
             
-            batch_size = self._calculate_batch_size(pending_count)
-            self.violations_data = self.repo.get_latest_violations(batch_size)
+            fetch_size = self._calculate_fetch_size(pending_count)
+            all_violations = self.repo.get_latest_violations(fetch_size)
+            
+            if not all_violations:
+                return
+
+            self.violations_data = self._calculate_batch_by_size(all_violations)
             
             if not self.violations_data:
+                logger.warning("⚠️ [APP] No valid violations within size limit")
                 return
 
-            logger.info(f"📤 [APP] Sending {len(self.violations_data)} violations ({pending_count} pending)")
+            batch_size_mb = sum(
+                os.path.getsize(v['image']) + os.path.getsize(v['image_tile']) 
+                for v in self.violations_data if os.path.exists(v['image']) and os.path.exists(v['image_tile'])
+            ) / (1024 * 1024)
+            
+            logger.info(f"📤 [APP] Sending {len(self.violations_data)} violations (~{batch_size_mb:.1f}MB, {pending_count} pending)")
 
             response = self.client.send_upsert_batch(
                 worker_id=self.worker_id,

{nedo_vision_worker-1.3.1.dist-info → nedo_vision_worker-1.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker
-Version: 1.3.1
+Version: 1.3.5
 Summary: Nedo Vision Worker Service Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>

{nedo_vision_worker-1.3.1.dist-info → nedo_vision_worker-1.3.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-nedo_vision_worker/__init__.py,sha256=ymUax1kADKc7nS9WyGBL5U3KmJaYaJsSNFXbaxeB5iY,203
+nedo_vision_worker/__init__.py,sha256=5XGhmiZwTkqHlXOZd-dVjqCEOx21tOtxk24cH0N_kqM,203
 nedo_vision_worker/cli.py,sha256=ddWspJmSgVkcUYvRdkvTtMNuMTDvNCqLLuMVU9KE3Ik,7457
 nedo_vision_worker/doctor.py,sha256=wNkpe8gLVd76Y_ViyK2h1ZFdqeSl37MnzZN5frWKu30,48410
 nedo_vision_worker/worker_service.py,sha256=9zz8hKwDwqwpfS0KPQfftGJtRci0uj_wiwcr_TGf-E0,11039
@@ -80,22 +80,22 @@ nedo_vision_worker/util/SystemMonitor.py,sha256=2kkqj9mOlywAS2fHdN1TaIXSXvCApcIH
 nedo_vision_worker/util/VideoProbeUtil.py,sha256=cF-vJ7hIDlXfEJby2a0s9tqwkPGVz_6B3Vv4D5pMmIw,12876
 nedo_vision_worker/util/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
 nedo_vision_worker/worker/CoreActionWorker.py,sha256=lb7zPY3yui6I3F4rX4Ii7JwpWZahLEO72rh3iWOgFmg,5441
-nedo_vision_worker/worker/DataSenderWorker.py,sha256=BNA5mL2guBVp0O5MKULEWd78dY32YjiyWZX7ECFjVFc,9089
+nedo_vision_worker/worker/DataSenderWorker.py,sha256=57bhNhpbherg_gCuUNnKt_1JwSQk1ukbx8mFWNsmrJ4,9171
 nedo_vision_worker/worker/DataSyncWorker.py,sha256=LmDPt2J1frmXwuR46L6b0MjlFOHfgG-4_0MGQa78zF4,6288
 nedo_vision_worker/worker/DatasetFrameSender.py,sha256=1SFYj8LJFNi-anBTapsbq8U_NGMM7mnoMKg9NeFAHys,8087
 nedo_vision_worker/worker/DatasetFrameWorker.py,sha256=Hh_wZuMjwovxsEKFqXSuTRin9eYRBZCbcFKm3CKLMbE,19335
-nedo_vision_worker/worker/PPEDetectionManager.py,sha256=FZbPuxY5dc4e919DJHXQSJHPF3btZMrC1xwD0jYYypU,4588
+nedo_vision_worker/worker/PPEDetectionManager.py,sha256=lb46NfhrZ-g_pxhurZl2uozPnWneZzptYOaIdzoFQxM,6775
 nedo_vision_worker/worker/PipelineActionWorker.py,sha256=xgvryjKtEsMj4BKqWzDIaK_lFny-DfMCj5Y2DxHnWww,5651
 nedo_vision_worker/worker/PipelineImageWorker.py,sha256=J8VBUG0cwcH3qOJp2zTl30B-XhmPFyvJLjxitKJYq0E,5642
 nedo_vision_worker/worker/PipelinePreviewWorker.py,sha256=owFiBbktcOZkdImQeykZSeBIR2-mpt6HNkmYIkLRKzE,6397
 nedo_vision_worker/worker/RabbitMQListener.py,sha256=9gR49MDplgpyb-D5HOH0K77-DJQFvhS2E7biL92SjSU,6950
-nedo_vision_worker/worker/RestrictedAreaManager.py,sha256=IWuCqpE-IPSf5Mdo5NUnbRUnY9pSH2471EB2eS0XRWs,4482
+nedo_vision_worker/worker/RestrictedAreaManager.py,sha256=LAIrOhzGlR0gQNuTJZx6zvUmcg5WwGKehthONCMjbCQ,6639
 nedo_vision_worker/worker/SystemUsageManager.py,sha256=mkh4sT-HkIEY1CJHMEG6LP9ATu39YXvLRLyf995OkoQ,5315
 nedo_vision_worker/worker/VideoStreamWorker.py,sha256=5n6v1PNO7IB-jj_McALLkUP-cBjJoIEw4UiSAs3vTb0,7606
 nedo_vision_worker/worker/WorkerManager.py,sha256=2bxXi19fp3p1qjYBStYRdVVgko8dnevXx1_M_sqH5og,5521
 nedo_vision_worker/worker/__init__.py,sha256=Nqnn8clbgv-5l0PgxcTOldg8mkMKrFn4TvPL-rYUUGg,1
-nedo_vision_worker-1.3.1.dist-info/METADATA,sha256=eZ2JS5SEBuZrrvpkJDNxc4svJHCVLzf4beEcTdRO4iA,14728
-nedo_vision_worker-1.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nedo_vision_worker-1.3.1.dist-info/entry_points.txt,sha256=LrglS-8nCi8C_PL_pa6uxdgCe879hBETHDVXAckvs-8,60
-nedo_vision_worker-1.3.1.dist-info/top_level.txt,sha256=vgilhlkyD34YsEKkaBabmhIpcKSvF3XpzD2By68L-XI,19
-nedo_vision_worker-1.3.1.dist-info/RECORD,,
+nedo_vision_worker-1.3.5.dist-info/METADATA,sha256=S6qk8Fmb9pN2zpeAZ15Eqa8uxv85fSUIL6tWrV7S7OA,14728
+nedo_vision_worker-1.3.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nedo_vision_worker-1.3.5.dist-info/entry_points.txt,sha256=LrglS-8nCi8C_PL_pa6uxdgCe879hBETHDVXAckvs-8,60
+nedo_vision_worker-1.3.5.dist-info/top_level.txt,sha256=vgilhlkyD34YsEKkaBabmhIpcKSvF3XpzD2By68L-XI,19
+nedo_vision_worker-1.3.5.dist-info/RECORD,,