nedo-vision-worker 1.2.6__tar.gz → 1.2.8__tar.gz

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker
-Version: 1.2.6
+Version: 1.2.8
 Summary: Nedo Vision Worker Service Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
@@ -40,9 +40,10 @@ Requires-Dist: protobuf>=3.20.0
 Requires-Dist: psutil>=5.9.0
 Requires-Dist: requests>=2.28.0
 Requires-Dist: SQLAlchemy>=1.4.0
-Requires-Dist: opencv-python>=4.6.0; platform_machine not in "aarch64 armv7l"
-Requires-Dist: opencv-python-headless>=4.6.0; platform_machine in "aarch64 armv7l"
 Requires-Dist: pynvml>=11.4.1; platform_system != "Darwin" or platform_machine != "arm64"
+Provides-Extra: opencv
+Requires-Dist: opencv-python>=4.6.0; platform_machine not in "aarch64 armv7l" and extra == "opencv"
+Requires-Dist: opencv-python-headless>=4.6.0; platform_machine in "aarch64 armv7l" and extra == "opencv"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: black>=22.0.0; extra == "dev"

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/__init__.py

@@ -6,5 +6,5 @@ A library for running worker agents in the Nedo Vision platform.
 
 from .worker_service import WorkerService
 
-__version__ = "1.2.6"
+__version__ = "1.2.8"
 __all__ = ["WorkerService"] 

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/database/DatabaseManager.py

@@ -104,7 +104,23 @@ class DatabaseManager:
         # Initialize engines and session factories for each database
         for name, path in DB_PATHS.items():
             path.parent.mkdir(parents=True, exist_ok=True)  # Ensure directory exists
-            engine = create_engine(f"sqlite:///{path.as_posix()}")
+            
+            # Configure connection pool for multi-threaded usage
+            # pool_size: Max connections to keep open
+            # max_overflow: Additional connections that can be created temporarily
+            # pool_pre_ping: Test connections before using (prevents stale connections)
+            # pool_recycle: Recycle connections after N seconds (prevents long-lived stale connections)
+            engine = create_engine(
+                f"sqlite:///{path.as_posix()}",
+                pool_size=20,          # Base pool size for persistent connections
+                max_overflow=30,       # Allow up to 30 additional temporary connections
+                pool_pre_ping=True,    # Verify connection health before use
+                pool_recycle=3600,     # Recycle connections after 1 hour
+                connect_args={
+                    "check_same_thread": False,  # Required for SQLite with multiple threads
+                    "timeout": 30.0              # Connection timeout
+                }
+            )
             ENGINES[name] = engine
             SESSION_FACTORIES[name] = scoped_session(sessionmaker(bind=engine))  # Use scoped sessions
             DatabaseManager.synchronize(name)

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/initializer/AppInitializer.py

@@ -76,3 +76,63 @@ class AppInitializer:
             logging.error(f"Grpc Error: {ge}")
         except Exception as e:
             logging.error(f"Unexpected error during initialization: {e}")
+
+    @staticmethod
+    def update_connection_info(server_host: str, server_port: int, token: str):
+        """
+        Fetch and update connection information (RabbitMQ credentials) from the server.
+        This should be called on startup to ensure credentials are up-to-date.
+        
+        Args:
+            server_host: The server hostname or IP address
+            server_port: The gRPC server port
+            token: Authentication token for the worker
+            
+        Returns:
+            bool: True if update was successful, False otherwise
+        """
+        try:
+            # Validate server host
+            AppInitializer.validate_server_host(server_host)
+
+            # Get connection info using the ConnectionInfoClient
+            connection_client = ConnectionInfoClient(server_host, server_port, token)
+            connection_result = connection_client.get_connection_info()
+
+            if not connection_result["success"]:
+                logging.error(f"Failed to fetch connection info: {connection_result['message']}")
+                return False
+
+            # Check if any RabbitMQ credentials have changed
+            current_config = ConfigurationManager.get_all_configs()
+            config_updated = False
+            
+            rabbitmq_fields = {
+                'rabbitmq_host': connection_result['rabbitmq_host'],
+                'rabbitmq_port': str(connection_result['rabbitmq_port']),
+                'rabbitmq_username': connection_result['rabbitmq_username'],
+                'rabbitmq_password': connection_result['rabbitmq_password']
+            }
+            
+            for field, new_value in rabbitmq_fields.items():
+                if current_config.get(field) != new_value:
+                    ConfigurationManager.set_config(field, new_value)
+                    config_updated = True
+                    logging.info(f"✅ [APP] Updated {field}")
+            
+            if config_updated:
+                logging.info("✅ [APP] RabbitMQ connection info updated successfully")
+            else:
+                logging.info("✅ [APP] RabbitMQ connection info is up-to-date")
+            
+            return True
+        
+        except ValueError as ve:
+            logging.error(f"Validation error: {ve}")
+            return False
+        except grpc.RpcError as ge:
+            logging.error(f"gRPC Error: {ge}")
+            return False
+        except Exception as e:
+            logging.error(f"Unexpected error updating connection info: {e}")
+            return False

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/models/worker_source_pipeline.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, String
+from sqlalchemy import Column, String, DateTime
 from sqlalchemy.orm import relationship
 from ..database.DatabaseManager import Base
 
@@ -13,6 +13,7 @@ class WorkerSourcePipelineEntity(Base):
     ai_model_id = Column(String, nullable=True)
     pipeline_status_code = Column(String, nullable=False)
     location_name = Column(String, nullable=True)
+    last_preview_request_at = Column(DateTime, nullable=True)
 
     worker_source_pipeline_configs = relationship(
         "WorkerSourcePipelineConfigEntity",

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/services/DirectDeviceToRTMPStreamer.py

@@ -8,6 +8,7 @@ import cv2
 import os
 from .VideoSharingDaemon import VideoSharingClient
 from ..database.DatabaseManager import get_storage_path
+from ..util.EncoderSelector import EncoderSelector
 
 
 class DirectDeviceToRTMPStreamer:
@@ -68,6 +69,10 @@ class DirectDeviceToRTMPStreamer:
 
     def _start_ffmpeg_stream(self):
         """Starts an FFmpeg process to stream frames to the RTMP server silently."""
+        # Get optimal encoder for hardware
+        encoder_args, encoder_name = EncoderSelector.get_encoder_args()
+        logging.info(f"🎬 [APP] Using encoder: {encoder_name}")
+        
         ffmpeg_command = [
             "ffmpeg",
             "-y",
@@ -79,16 +84,17 @@ class DirectDeviceToRTMPStreamer:
             "-video_size", f"{self.width}x{self.height}",
             "-framerate", str(self.fps),
             "-i", "-",
-            "-c:v", "libx264",
-            "-preset", "ultrafast",
-            "-tune", "zerolatency",
+            
+            # Video encoding with optimal encoder
+            *encoder_args,
             "-b:v", self.bitrate,
-            # Disable Audio (Avoid unnecessary encoding overhead)
-            "-an",
             "-maxrate", "2500k",
             "-bufsize", "5000k",
+            
+            # Disable Audio
+            "-an",
+            
             "-f", "flv",
-            # Remove duration limit - let application control duration
             self.rtmp_url,
         ]
 

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/services/FileToRTMPServer.py

@@ -1,6 +1,7 @@
 import subprocess
 import logging
 import os
+from ..util.EncoderSelector import EncoderSelector
 
 class FileToRTMPStreamer:
     def __init__(self, video_path, rtmp_url, stream_key, fps=30, resolution="1280x720", loop=False):
@@ -31,6 +32,10 @@ class FileToRTMPStreamer:
 
         logging.info(f"📼 [APP] Starting file stream: {self.video_path} → {self.rtmp_url}")
 
+        # Get optimal encoder for hardware
+        encoder_args, encoder_name = EncoderSelector.get_encoder_args()
+        logging.info(f"🎬 [APP] Using encoder: {encoder_name}")
+
         # FFmpeg command
         ffmpeg_command = [
             "ffmpeg",
@@ -38,9 +43,8 @@ class FileToRTMPStreamer:
             "-stream_loop", "-1" if self.loop else "0",  # Loop if needed
             "-i", self.video_path,
 
-            "-c:v", "libx264",
-            "-preset", "ultrafast",
-            "-tune", "zerolatency",
+            # Video encoding with optimal encoder
+            *encoder_args,
             "-r", str(self.fps),
             "-b:v", "1500k",
             "-maxrate", "2000k",

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/services/GrpcClientBase.py

@@ -3,6 +3,7 @@ import logging
 import time
 from grpc import StatusCode
 from typing import Callable, Optional, Any, Dict
+from .GrpcConnection import GrpcConnection
 
 logger = logging.getLogger(__name__)
 
@@ -42,42 +43,21 @@ class GrpcClientBase:
     def __init__(self, server_host: str, server_port: int = 50051, max_retries: int = 3):
         self.server_address = f"{server_host}:{server_port}"
         self.channel: Optional[grpc.Channel] = None
-        self.stub: Optional[Any] = None
         self.connected = False
         self.max_retries = max_retries
 
+        self.connection = GrpcConnection(server_host, server_port)
+
     def connect(self, stub_class, retry_interval: int = 2) -> bool:
-        attempts = 0
-        while attempts < self.max_retries and not self.connected:
-            try:
-                if self.channel:
-                    self._close_channel()
-                
-                self.channel = grpc.insecure_channel(self.server_address)
-                
-                future = grpc.channel_ready_future(self.channel)
-                future.result(timeout=30)
-
-                self.stub = stub_class(self.channel)
-                self.connected = True
-                logger.info(f"🚀 Connected to gRPC server at {self.server_address}")
-                return True
-
-            except (grpc.RpcError, grpc.FutureTimeoutError, Exception) as e:
-                attempts += 1
-                self.connected = False
-                error_msg = str(e)
-                
-                logger.error(f"⚠️ Connection failed ({attempts}/{self.max_retries}): {error_msg}", exc_info=True)
-
-                if attempts < self.max_retries:
-                    sleep_time = retry_interval * (2 ** (attempts - 1))
-                    logger.info(f"⏳ Retrying in {sleep_time}s...")
-                    time.sleep(sleep_time)
-                else:
-                    logger.critical("❌ Max retries reached. Connection failed.")
-
-        return False
+        conn = self.connection.get_connection()
+        if conn is None:
+            return False
+        requested_stub = stub_class(conn)
+        
+        self.stub = requested_stub
+        self.connected = True
+
+        return True
 
     def _close_channel(self) -> None:
         try:
@@ -89,6 +69,7 @@ class GrpcClientBase:
             self.channel = None
             self.stub = None
 
+    # MARK:
     def close(self) -> None:
         self._close_channel()
         self.connected = False
@@ -104,6 +85,11 @@ class GrpcClientBase:
         except grpc.RpcError as e:
             return self._handle_grpc_error(e, rpc_call, *args, **kwargs)
         except Exception as e:
+            print(e)
+            print(str(e) == "Cannot invoke RPC on closed channel!")
+            if str(e) == "Cannot invoke RPC on closed channel!":
+                self.connect(type(self.stub))
+
             logger.error(f"💥 Unexpected RPC error: {e}")
             return None
 
@@ -127,8 +113,11 @@ class GrpcClientBase:
 
         return None
 
+    # MARK:
+    # Should request for reconnection two times. Notify grpc connection to do reconnect
     def _handle_unavailable(self, rpc_call: Callable, *args, **kwargs) -> Optional[Any]:
-        self.connected = False
+        # self.connected = False
+        self.connection.try_reconnect()
         
         if self.stub:
             stub_class = type(self.stub)
@@ -154,7 +143,7 @@ class GrpcClientBase:
         return error_message.split("debug_error_string")[0].strip()
 
     def is_connected(self) -> bool:
-        return self.connected and self.channel and self.stub
+        return self.connected and self.connection.get_connection() is not None and self.stub is not None
 
     def get_connection_info(self) -> Dict[str, Any]:
         return {

nedo_vision_worker-1.2.8/nedo_vision_worker/services/GrpcConnection.py

@@ -0,0 +1,147 @@
+import grpc
+import logging
+import time
+import threading
+from grpc import StatusCode
+from typing import Optional, Any, Dict
+
+logger = logging.getLogger(__name__)
+
+
+class GrpcConnection:
+    """
+    Grpc connection management. Responsible for initiating connection with gRPC server that
+    will be used across gRPC clients in the project.
+    Expected to be initiated as a singleton.
+    """
+    
+    _instance = None
+    _init_done = False
+    _lock = threading.Lock()
+    _reconnectLock = threading.Lock()
+    _reconnecting = False
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self, server_host: str, server_port: int = 50051, max_retries: int = 3):
+        if self.__class__._init_done:
+            return  # prevent re-initialization
+        self.__class__._init_done = True
+
+        self.server_address = f"{server_host}:{server_port}"
+        self.channel: Optional[grpc.Channel] = None
+        self.connected = False
+        self.max_retries = max_retries
+        self.connect()
+
+    def connect(self, retry_interval: int = 2) -> bool:
+        attempts = 0
+        while attempts < self.max_retries and not self.connected:
+            try:
+                if self.channel:
+                    self._close_channel()
+                
+                self.channel = grpc.insecure_channel(self.server_address)
+                
+                future = grpc.channel_ready_future(self.channel)
+                future.result(timeout=30)
+
+                self.connected = True
+                logger.info(f"🚀 Connected to gRPC server at {self.server_address}")
+                return True
+
+            except (grpc.RpcError, grpc.FutureTimeoutError, Exception) as e:
+                attempts += 1
+                self.connected = False
+                error_msg = str(e)
+                
+                logger.error(f"⚠️ Connection failed ({attempts}/{self.max_retries}): {error_msg}")
+
+                if attempts < self.max_retries:
+                    sleep_time = retry_interval * (2 ** (attempts - 1))
+                    logger.info(f"⏳ Retrying in {sleep_time}s...")
+                    time.sleep(sleep_time)
+                else:
+                    logger.critical("❌ Max retries reached. Connection failed.")
+
+        return False
+    
+    def get_connection(self):
+        if self._reconnecting or not self.connected:
+            return None
+        
+        return self.channel
+    
+    def _reconnect(self):
+        logger.info(f"⏳ Reconnecting...")
+        attempts = 0
+
+        while not self.connected:
+            try:
+                if self.channel:
+                    self._close_channel()
+                
+                self.channel = grpc.insecure_channel(self.server_address)
+                
+                future = grpc.channel_ready_future(self.channel)
+                future.result(timeout=30)
+
+                self.connected = True
+                self._reconnecting = False
+                logger.info(f"🚀 Connected to gRPC server at {self.server_address}")
+                self._reconnectLock.release_lock()
+            except (grpc.RpcError, grpc.FutureTimeoutError, Exception) as e:
+                attempts += 1
+                self.connected = False
+                error_msg = str(e)
+                
+                logger.error(f"⚠️ Connection failed ({attempts}/{self.max_retries}): {error_msg}")
+
+                sleep_time = 2 * (2 ** (attempts - 1))
+                logger.info(f"⏳ Retrying in {sleep_time}s...")
+                time.sleep(sleep_time)
+    
+    def try_reconnect(self):
+        if self._reconnecting:
+            return
+        
+        if self._reconnectLock.acquire_lock(blocking=False):
+            self._reconnecting = True
+            self.connected = False
+            self._reconnect()
+
+    def _close_channel(self) -> None:
+        try:
+            if self.channel:
+                self.channel.close()
+            logger.info("🔌 gRPC connection closed")
+        except Exception as e:
+            logger.warning(f"⚠️ Error closing channel: {e}")
+        finally:
+            self.channel = None
+            self.connected = False
+
+    def close(self) -> None:
+        if self.channel:
+            self.channel.close()
+        self.connected = False
+        logger.info("🔌 gRPC connection closed")
+
+    def is_connected(self) -> bool:
+        return self.connected and self.channel is not None
+
+    def get_connection_info(self) -> Dict[str, Any]:
+        return {
+            "server_address": self.server_address,
+            "connected": self.connected,
+            "max_retries": self.max_retries,
+            "has_channel": self.channel is not None,
+        }
+
+    def __enter__(self):
+        return self

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/services/RTSPtoRTMPStreamer.py

@@ -3,6 +3,7 @@ import logging
 import time
 import os
 from urllib.parse import urlparse
+from ..util.EncoderSelector import EncoderSelector
 
 class RTSPtoRTMPStreamer:
     def __init__(self, rtsp_url, rtmp_url, stream_key, fps=30, resolution="1280x720", duration=120):
@@ -38,6 +39,10 @@ class RTSPtoRTMPStreamer:
 
         logging.info(f"📡 [APP] Starting RTSP to RTMP stream: {self.rtsp_url} → {self.rtmp_url} for {self.duration} seconds")
 
+        # Get optimal encoder for hardware
+        encoder_args, encoder_name = EncoderSelector.get_encoder_args()
+        logging.info(f"🎬 [APP] Using encoder: {encoder_name}")
+
         # FFmpeg command
         ffmpeg_command = [
             "ffmpeg",
@@ -47,17 +52,14 @@ class RTSPtoRTMPStreamer:
             "-strict", "experimental",
             "-i", self.rtsp_url,
 
-            # ✅ Video Encoding (Fastest possible)
-            "-c:v", "libx264",
-            "-preset", "ultrafast",  # 🚀 Reduce CPU usage
-            "-tune", "zerolatency",  # 🚀 Optimize for real-time streaming
-            "-x264-params", "keyint=40:min-keyint=40",  # 🚀 Keyframe optimization
-            "-r", "25",  # ⏳ Limit FPS to 20 (prevents excessive encoding load)
-            "-b:v", "1500k",  # ✅ Lower bitrate to improve performance
+            # Video encoding with optimal encoder
+            *encoder_args,
+            "-r", "25",  # ⏳ Limit FPS to 25
+            "-b:v", "1500k",  # ✅ Bitrate
             "-maxrate", "2000k",  # ✅ Set max bitrate
             "-bufsize", "4000k",  # ✅ Reduce buffer latency
             "-g", "25",  # ✅ Reduce GOP size for faster keyframes
-            "-vf", "scale='min(1024,iw)':-2",  # ✅ Resize width to max 800px
+            "-vf", "scale='min(1024,iw)':-2",  # ✅ Resize width to max 1024px
 
             # ❌ Disable Audio (Avoid unnecessary encoding overhead)
             "-an",

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/services/SystemUsageClient.py

@@ -21,7 +21,7 @@ class SystemUsageClient(GrpcClientBase):
             logging.error(f"Failed to connect to gRPC server: {e}")
             self.stub = None
 
-    def send_system_usage(self, device_id: str, cpu_usage: float, ram_usage: dict, gpu_usage: list, latency: float, token: str) -> dict:
+    def send_system_usage(self, device_id: str, cpu_usage: float, ram_usage: dict, gpu_usage: list, latency: float, cpu_temperature: float, token: str) -> dict:
         """
         Send system usage data to the server using token authentication.
 
@@ -44,6 +44,7 @@ class SystemUsageClient(GrpcClientBase):
             request = SystemUsageRequest(
                 device_id=device_id,
                 cpu_usage=cpu_usage,
+                cpu_temperature=cpu_temperature,
                 ram_usage_percent=ram_usage.get("percent", 0.0),
                 ram_total=ram_usage.get("total", 0),
                 ram_used=ram_usage.get("used", 0),

{nedo_vision_worker-1.2.6 → nedo_vision_worker-1.2.8}/nedo_vision_worker/services/WorkerSourcePipelineClient.py

@@ -377,7 +377,15 @@ class WorkerSourcePipelineClient(GrpcClientBase):
             debug_entries = self.debug_repo.get_debug_entries_with_data()
 
             for debug_entry in debug_entries:
-                image_binary = self.read_image_as_binary(debug_entry.image_path)
+                try:
+                    image_binary = self.read_image_as_binary(debug_entry.image_path)
+                except FileNotFoundError:
+                    logging.warning(f"Image file not found: {debug_entry.image_path}, deleting entry {debug_entry.id}")
+                    self.debug_repo.delete_entry_by_id(debug_entry.id)
+                    continue
+                except Exception as e:
+                    logging.error(f"Error reading image {debug_entry.image_path}: {e}")
+                    continue
 
                 request = SendPipelineDebugRequest(
                     worker_source_pipeline_id=debug_entry.worker_source_pipeline_id,
@@ -391,12 +399,13 @@ class WorkerSourcePipelineClient(GrpcClientBase):
                 if response and response.success:
                     self.debug_repo.delete_entry_by_id(debug_entry.id)
                 else:
-                    return {"success": False, "message": response.message if response else "Unknown error"}
+                    logging.warning(f"Failed to sync debug entry {debug_entry.id}: {response.message if response else 'Unknown error'}")
 
             return {"success": True, "message": "Successfully synced debug entries"}
 
         except Exception as e:
-            logging.error(f"Error syncing pipeline debug: {e}")
+            logging.error(f"Error syncing pipeline debug: {e}", exc_info=True)
+            return {"success": False, "message": f"Exception: {str(e)}"}
 
     def sync_pipeline_detection(self, token: str):
         if not self.stub:
@@ -406,7 +415,15 @@ class WorkerSourcePipelineClient(GrpcClientBase):
             entries = self.detection_repo.get_entries()
 
             for entry in entries:
-                image_binary = self.read_image_as_binary(entry.image_path)
+                try:
+                    image_binary = self.read_image_as_binary(entry.image_path)
+                except FileNotFoundError:
+                    logging.warning(f"Image file not found: {entry.image_path}, deleting entry {entry.id}")
+                    self.detection_repo.delete_entry_by_id(entry.id)
+                    continue
+                except Exception as e:
+                    logging.error(f"Error reading image {entry.image_path}: {e}")
+                    continue
 
                 request = SendPipelineDetectionDataRequest(
                     worker_source_pipeline_id=entry.worker_source_pipeline_id,
@@ -420,9 +437,10 @@ class WorkerSourcePipelineClient(GrpcClientBase):
                 if response and response.success:
                     self.detection_repo.delete_entry_by_id(entry.id)
                 else:
-                    return {"success": False, "message": response.message if response else "Unknown error"}
+                    logging.warning(f"Failed to sync detection entry {entry.id}: {response.message if response else 'Unknown error'}")
 
-            return {"success": True, "message": "Successfully synced debug entries"}
+            return {"success": True, "message": "Successfully synced detection entries"}
 
         except Exception as e:
-            logging.error(f"Error syncing pipeline debug: {e}")
+            logging.error(f"Error syncing pipeline detection: {e}", exc_info=True)
+            return {"success": False, "message": f"Exception: {str(e)}"}