nedo-vision-worker 1.3.7__tar.gz → 1.3.9__tar.gz

{nedo_vision_worker-1.3.7 → nedo_vision_worker-1.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker
-Version: 1.3.7
+Version: 1.3.9
 Summary: Nedo Vision Worker Service Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
@@ -40,6 +40,7 @@ Requires-Dist: protobuf>=3.20.0
 Requires-Dist: psutil>=5.9.0
 Requires-Dist: requests>=2.28.0
 Requires-Dist: SQLAlchemy>=1.4.0
+Requires-Dist: Pillow>=12.1.0
 Requires-Dist: pynvml>=11.4.1; platform_system != "Darwin" or platform_machine != "arm64"
 Provides-Extra: opencv
 Requires-Dist: opencv-python>=4.6.0; platform_machine not in "aarch64 armv7l" and extra == "opencv"

{nedo_vision_worker-1.3.7 → nedo_vision_worker-1.3.9}/nedo_vision_worker/__init__.py

@@ -6,5 +6,5 @@ A library for running worker agents in the Nedo Vision platform.
 
 from .worker_service import WorkerService
 
-__version__ = "1.3.7"
+__version__ = "1.3.9"
 __all__ = ["WorkerService"] 

nedo_vision_worker-1.3.9/nedo_vision_worker/bootstrap.py

@@ -0,0 +1,51 @@
+import logging
+from nedo_vision_worker.config.ConfigurationManager import ConfigurationManager
+from nedo_vision_worker.database.DatabaseManager import set_storage_path, DatabaseManager
+from nedo_vision_worker.worker_service import WorkerService
+
+
+def start_worker(
+    *,
+    server_host: str,
+    server_port: int,
+    token: str,
+    system_usage_interval: int,
+    rtmp_server: str,
+    storage_path: str,
+    log_level: str = "INFO",
+) -> WorkerService:
+    # Logging (only once, force allowed)
+    logging.basicConfig(
+        level=getattr(logging, log_level),
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        force=True,
+    )
+
+    logger = logging.getLogger("nedo-worker")
+
+    # Storage & DB
+    set_storage_path(storage_path)
+    DatabaseManager.init_databases()
+
+    # Configuration
+    configuration_manager = ConfigurationManager(
+        worker_token=token,
+        server_host=server_host,
+        server_port=server_port,
+        rtmp_server_url=rtmp_server,
+        logger=logging.getLogger("configuration_manager"),
+    )
+
+    service = WorkerService(
+        configuration_manager=configuration_manager,
+        server_host=server_host,
+        token=token,
+        system_usage_interval=system_usage_interval,
+        rtmp_server=rtmp_server,
+        storage_path=storage_path,
+    )
+
+    logger.info("🚀 Starting Nedo Vision Worker Service")
+    service.run()
+
+    return service

nedo_vision_worker-1.3.9/nedo_vision_worker/cli.py

@@ -0,0 +1,75 @@
+import argparse
+import signal
+import sys
+import logging
+from typing import NoReturn
+
+from nedo_vision_worker.bootstrap import start_worker
+from nedo_vision_worker import __version__
+
+class NedoWorkerCLI:
+    def __init__(self):
+        self.logger = logging.getLogger(__name__)
+        self._setup_signal_handlers()
+        self._service = None
+
+    def _setup_signal_handlers(self) -> None:
+        signal.signal(signal.SIGINT, self._signal_handler)
+        signal.signal(signal.SIGTERM, self._signal_handler)
+
+    def _signal_handler(self, signum: int, frame) -> NoReturn:
+        self.logger.info("🛑 Shutdown signal received")
+        if self._service and hasattr(self._service, "stop"):
+            self._service.stop()
+        sys.exit(0)
+
+    def create_parser(self) -> argparse.ArgumentParser:
+        parser = argparse.ArgumentParser(description="Nedo Vision Worker Service")
+
+        parser.add_argument("--version", action="version", version=f"nedo-vision-worker {__version__}")
+
+        subparsers = parser.add_subparsers(dest="command", required=True)
+
+        subparsers.add_parser("doctor", help="Check system dependencies")
+
+        run = subparsers.add_parser("run", help="Start worker")
+
+        run.add_argument("--token", required=True)
+        run.add_argument("--server-host", default="be.vision.sindika.co.id")
+        run.add_argument("--server-port", type=int, default=50051)
+        run.add_argument("--rtmp-server", default="rtmp://live.vision.sindika.co.id:1935/live")
+        run.add_argument("--storage-path", default="data")
+        run.add_argument("--system-usage-interval", type=int, default=30)
+        run.add_argument("--log-level", default="INFO")
+
+        return parser
+
+    def run(self) -> int:
+        parser = self.create_parser()
+        args = parser.parse_args()
+
+        if args.command == "doctor":
+            from nedo_vision_worker.doctor import main as doctor_main
+            return doctor_main()
+
+        if args.command == "run":
+            self._service = start_worker(
+                server_host=args.server_host,
+                server_port=args.server_port,
+                token=args.token,
+                system_usage_interval=args.system_usage_interval,
+                rtmp_server=args.rtmp_server,
+                storage_path=args.storage_path,
+                log_level=args.log_level,
+            )
+            signal.pause()  # wait for signal
+
+        return 0
+
+
+def main() -> int:
+    return NedoWorkerCLI().run()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{nedo_vision_worker-1.3.7 → nedo_vision_worker-1.3.9}/nedo_vision_worker/config/ConfigurationManager.py

@@ -1,84 +1,138 @@
 import logging
+import grpc
+from typing import Optional
 from ..models.config import ConfigEntity  # ORM model for server_config
 from ..database.DatabaseManager import DatabaseManager  # DatabaseManager for managing sessions
 
+from ..util.HardwareID import HardwareID
+from ..services.ConnectionInfoClient import ConnectionInfoClient
+from .ConfigurationManagerInterface import ConfigurationManagerInterface
 
-class ConfigurationManager:
+class ConfigurationManager(ConfigurationManagerInterface):
     """
-    A class to manage server configuration stored in the 'config' database using SQLAlchemy.
+    A class to manage local and remote configuration stored in the 'config' database.
     """
 
-    @staticmethod
-    def init_database():
-        """
-        Initialize the 'config' database and create the `server_config` table if it doesn't exist.
-        """
+    def __init__(self, worker_token: str, server_host: str, server_port: int, rtmp_server_url: str, logger: logging.Logger):
         try:
-            DatabaseManager.init_databases()
+            self._logger = logger
+            self._worker_token = worker_token
+            self._server_host = server_host
+            self._server_port = server_port
+            self._rtmp_server_url = rtmp_server_url
+
+            self._initialize_remote_configuration()
+
             logging.info("✅ [APP] Configuration database initialized successfully.")
         except Exception as e:
             logging.exception("❌ [APP] Failed to initialize the configuration database.")
             raise RuntimeError("Database initialization failed.") from e
-
-    @staticmethod
-    def is_config_initialized() -> bool:
+        
+    def get_config(self, key: str) -> str:
         """
-        Check if the configuration is already initialized.
+        Retrieve the value of a specific configuration key from the 'config' database.
+
+        Args:
+            key (str): The configuration key.
 
         Returns:
-            bool: True if configuration exists, False otherwise.
+            str: The configuration value, or None if the key does not exist.
         """
+        if not key or not isinstance(key, str):
+            raise ValueError("⚠️ The 'key' must be a non-empty string.")
+
         session = None
         try:
             session = DatabaseManager.get_session("config")
-            config_count = session.query(ConfigEntity).count()
-            return config_count > 0  # True if at least one config exists
+            logging.info(f"🔍 [APP] Retrieving configuration key: {key}")
+            config = session.query(ConfigEntity).filter_by(key=key).first()
+            if config:
+                logging.info(f"✅ [APP] Configuration key '{key}' retrieved successfully.")
+                return config.value
+            else:
+                logging.warning(f"⚠️ [APP] Configuration key '{key}' not found.")
+                return ""
         except Exception as e:
-            logging.exception("❌ [APP] Failed to check if configuration is initialized.")
-            return False
+            logging.exception(f"❌ [APP] Failed to retrieve configuration key '{key}': {e}")
+            raise RuntimeError(f"Failed to retrieve configuration key '{key}'") from e
         finally:
             if session:
                 session.close()
-
-    @staticmethod
-    def set_config(key: str, value: str):
+    
+    def get_all_configs(self) -> Optional[dict]:
         """
-        Set or update a configuration key-value pair in the 'config' database.
+        Retrieve all configuration key-value pairs from the 'config' database.
 
-        Args:
-            key (str): The configuration key.
-            value (str): The configuration value.
+        Returns:
+            dict: A dictionary of all configuration key-value pairs.
         """
-        if not key or not isinstance(key, str):
-            raise ValueError("⚠️ [APP] The 'key' must be a non-empty string.")
-        if not isinstance(value, str):
-            raise ValueError("⚠️ [APP] The 'value' must be a string.")
-
         session = None
         try:
             session = DatabaseManager.get_session("config")
-            logging.info(f"🔧 Attempting to set configuration: {key} = {value}")
-            existing_config = session.query(ConfigEntity).filter_by(key=key).first()
-            if existing_config:
-                logging.info(f"🔄 [APP] Updating configuration key: {key}")
-                existing_config.value = value
+            logging.info("🔍 [APP] Retrieving all configuration keys.")
+            configs = session.query(ConfigEntity).all()
+            if configs:
+                logging.info("✅ [APP] All configuration keys retrieved successfully.")
+                return {config.key: config.value for config in configs}
             else:
-                logging.info(f"➕ [APP] Adding new configuration key: {key}")
-                new_config = ConfigEntity(key=key, value=value)
-                session.add(new_config)
-            session.commit()
-            logging.info(f"✅ [APP] Configuration key '{key}' set successfully.")
+                logging.info("⚠️ [APP] No configuration keys found.")
+                return None
         except Exception as e:
-            if session:
-                session.rollback()
-            logging.exception(f"❌ [APP] Failed to set configuration key '{key}': {e}")
-            raise RuntimeError(f"Failed to set configuration key '{key}'") from e
+            logging.exception("❌ [APP] Failed to retrieve all configuration keys.")
+            raise RuntimeError("Failed to retrieve all configuration keys.") from e
         finally:
             if session:
                 session.close()
 
-    @staticmethod
-    def set_config_batch(configs: dict):
+    def _initialize_remote_configuration(self):
+        """
+        Initialize the application configuration using the provided token
+        and saving configuration data locally.
+        """
+        try:
+            # Get hardware ID
+            hardware_id = HardwareID.get_unique_id()
+
+            self._logger.info(f"🖥️ [APP] Detected Hardware ID: {hardware_id}")
+            self._logger.info(f"🌐 [APP] Using Server Host: {self._server_host}")
+
+            # Check if token is provided
+            if not self._worker_token:
+                raise ValueError("Token is required for worker initialization. Please provide a token obtained from the frontend.")
+
+            # Get connection info using the ConnectionInfoClient
+            connection_client = ConnectionInfoClient(self._server_host, self._server_port, self._worker_token)
+            connection_result = connection_client.get_connection_info()
+            
+            if not connection_result["success"]:
+                logging.error(f"Device connection info failed: {connection_result['message']}")
+                raise ValueError(f"Initializing remote config failed, reason: {connection_result['message']}")
+
+            worker_id = connection_result.get('id')
+            if not worker_id:
+                raise ValueError("No worker_id returned from connection info!")
+
+            self._set_config_batch({
+                "worker_id": worker_id,
+                "server_host": self._server_host,
+                "rtmp_server": self._rtmp_server_url,
+                "server_port": str(self._server_port),
+                "token": self._worker_token,
+                "rabbitmq_host": connection_result['rabbitmq_host'],
+                "rabbitmq_port": str(connection_result['rabbitmq_port']),
+                "rabbitmq_username": connection_result['rabbitmq_username'],
+                "rabbitmq_password": connection_result['rabbitmq_password']
+            })
+            self._print_config()
+        
+        except ValueError as ve:
+            logging.error(f"Validation error: {ve}")
+        except grpc.RpcError as ge:
+            logging.error(f"Grpc Error: {ge}")
+        except Exception as e:
+            logging.error(f"Unexpected error during initialization: {e}")
+
+    def _set_config_batch(self, configs: dict):
         """
         Set or update multiple configuration key-value pairs in the 'config' database in a batch operation.
 
@@ -116,71 +170,12 @@ class ConfigurationManager:
             if session:
                 session.close()
 
-    @staticmethod
-    def get_config(key: str) -> str:
-        """
-        Retrieve the value of a specific configuration key from the 'config' database.
-
-        Args:
-            key (str): The configuration key.
-
-        Returns:
-            str: The configuration value, or None if the key does not exist.
-        """
-        if not key or not isinstance(key, str):
-            raise ValueError("⚠️ The 'key' must be a non-empty string.")
-
-        session = None
-        try:
-            session = DatabaseManager.get_session("config")
-            logging.info(f"🔍 [APP] Retrieving configuration key: {key}")
-            config = session.query(ConfigEntity).filter_by(key=key).first()
-            if config:
-                logging.info(f"✅ [APP] Configuration key '{key}' retrieved successfully.")
-                return config.value
-            else:
-                logging.warning(f"⚠️ [APP] Configuration key '{key}' not found.")
-                return None
-        except Exception as e:
-            logging.exception(f"❌ [APP] Failed to retrieve configuration key '{key}': {e}")
-            raise RuntimeError(f"Failed to retrieve configuration key '{key}'") from e
-        finally:
-            if session:
-                session.close()
-
-    @staticmethod
-    def get_all_configs() -> dict:
-        """
-        Retrieve all configuration key-value pairs from the 'config' database.
-
-        Returns:
-            dict: A dictionary of all configuration key-value pairs.
-        """
-        session = None
-        try:
-            session = DatabaseManager.get_session("config")
-            logging.info("🔍 [APP] Retrieving all configuration keys.")
-            configs = session.query(ConfigEntity).all()
-            if configs:
-                logging.info("✅ [APP] All configuration keys retrieved successfully.")
-                return {config.key: config.value for config in configs}
-            else:
-                logging.info("⚠️ [APP] No configuration keys found.")
-                return {}
-        except Exception as e:
-            logging.exception("❌ [APP] Failed to retrieve all configuration keys.")
-            raise RuntimeError("Failed to retrieve all configuration keys.") from e
-        finally:
-            if session:
-                session.close()
-
-    @staticmethod
-    def print_config():
+    def _print_config(self):
         """
         Print all configuration key-value pairs to the console.
         """
         try:
-            configs = ConfigurationManager.get_all_configs()
+            configs = self.get_all_configs()
             if configs:
                 print("📄 Current Configuration:")
                 for key, value in configs.items():

nedo_vision_worker-1.3.9/nedo_vision_worker/config/ConfigurationManagerInterface.py

@@ -0,0 +1,12 @@
+from typing import Optional
+from abc import ABC, abstractmethod
+
+
+class ConfigurationManagerInterface(ABC):
+    @abstractmethod
+    def get_config(self, key: str) -> str:
+        pass
+    
+    @abstractmethod
+    def get_all_configs(self) -> Optional[dict]:
+        pass

nedo_vision_worker-1.3.9/nedo_vision_worker/config/DummyConfigurationManager.py

@@ -0,0 +1,52 @@
+import logging
+from typing import Optional
+
+from .ConfigurationManagerInterface import ConfigurationManagerInterface
+
+class DummyConfigurationManager(ConfigurationManagerInterface):
+    """
+    A class to manage local and remote configuration stored in the 'config' database.
+    """
+
+    def __init__(
+        self,
+        worker_id: str,
+        worker_token: str,
+        server_host: str,
+        server_port: str,
+        rtmp_server_url: str,
+        rabbitmq_host: str,
+        rabbitmq_port: str,
+        rabbitmq_username: str,
+        rabbitmq_password: str,
+        logger: logging.Logger
+    ):
+        self._logger = logger
+        self._worker_id = worker_id
+        self._worker_token = worker_token
+        self._server_host = server_host
+        self._server_port = server_port
+        self._rtmp_server_url = rtmp_server_url
+        self._rabbitmq_host = rabbitmq_host
+        self._rabbitmq_port = rabbitmq_port
+        self._rabbitmq_username = rabbitmq_username
+        self._rabbitmq_password = rabbitmq_password
+
+        self._config = {
+            "worker_id": self._worker_id,
+            "token": self._worker_token,
+            "server_host": self._server_host,
+            "server_port": str(self._server_port),
+            "rtmp_server": self._rtmp_server_url,
+            "rabbitmq_host": self._rabbitmq_host,
+            "rabbitmq_port": self._rabbitmq_port,
+            "rabbitmq_username": self._rabbitmq_username,
+            "rabbitmq_password": self._rabbitmq_password,
+        }
+
+        
+    def get_config(self, key: str) -> str:
+        return self._config[key]
+    
+    def get_all_configs(self) -> Optional[dict]:
+        return self._config

{nedo_vision_worker-1.3.7 → nedo_vision_worker-1.3.9}/nedo_vision_worker/services/ConnectionInfoClient.py

@@ -31,7 +31,7 @@ class ConnectionInfoClient(GrpcClientBase):
         try:
             if not self.stub:
                 raise Exception("Not connected to manager")
-
+            
             # Prepare the request
             request = GetWorkerConnectionInfoRequest(token=self.token)
 

{nedo_vision_worker-1.3.7 → nedo_vision_worker-1.3.9}/nedo_vision_worker/services/GrpcClientBase.py

@@ -41,6 +41,7 @@ class GrpcClientBase:
     }
 
     def __init__(self, server_host: str, server_port: int = 50051, max_retries: int = 3):
+        self.stub = None
         self.server_address = f"{server_host}:{server_port}"
         self.channel: Optional[grpc.Channel] = None
         self.connected = False

nedo_vision_worker-1.3.9/nedo_vision_worker/util/CorruptedImageValidator.py

@@ -0,0 +1,55 @@
+from PIL import Image
+import io
+import numpy as np
+
+# Tunables (adjust once, then forget)
+MIN_IMAGE_KB = 10
+GRAY_RATIO_THRESHOLD = 0.55
+GRAY_MIN = 90
+GRAY_MAX = 210
+GRAY_DELTA = 4
+
+def validate_image_gray_area(image_bytes: bytes) -> bool:
+    print(len(image_bytes))
+    # ---- Stage 1: size check (O(1)) ----
+    if len(image_bytes) < (MIN_IMAGE_KB * 1024):
+        return False
+
+    # ---- Stage 2: structural integrity (cheap) ----
+    try:
+        with Image.open(io.BytesIO(image_bytes)) as img:
+            img.verify()
+    except Exception:
+        return False
+
+    # ---- Stage 3: partial pixel decode (cropped, fast) ----
+    try:
+        with Image.open(io.BytesIO(image_bytes)) as img:
+            img = img.convert("RGB")
+
+            w, h = img.size
+            if w == 0 or h == 0:
+                return False
+
+            # Sample bottom 35% where corruption usually appears
+            crop_y = int(h * 0.65)
+            img_crop = img.crop((0, crop_y, w, h))
+
+            # Downscale aggressively (huge speedup)
+            img_small = img_crop.resize((w // 8, h // 8))
+
+            arr = np.asarray(img_small, dtype=np.uint8)
+
+            r, g, b = arr[..., 0], arr[..., 1], arr[..., 2]
+
+            gray_mask = (
+                (np.abs(r - g) < GRAY_DELTA) &
+                (np.abs(r - b) < GRAY_DELTA) &
+                (r > GRAY_MIN) & (r < GRAY_MAX)
+            )
+
+            gray_ratio = gray_mask.mean()
+            return gray_ratio < GRAY_RATIO_THRESHOLD
+
+    except Exception:
+        return False

{nedo_vision_worker-1.3.7 → nedo_vision_worker-1.3.9}/nedo_vision_worker/worker/DatasetFrameWorker.py

@@ -7,6 +7,7 @@ from typing import Dict
 from ..services.WorkerSourcePipelineClient import WorkerSourcePipelineClient
 from ..services.GrpcClientManager import GrpcClientManager
 from ..repositories.DatasetSourceRepository import DatasetSourceRepository
+from ..util.CorruptedImageValidator import validate_image_gray_area
 
 logger = logging.getLogger(__name__)
 
@@ -96,6 +97,9 @@ class DatasetSourceThread:
         try:
             # Get frame from source
             frame_bytes = self._get_frame_from_source(self.dataset_source.worker_source_url)
+            if not validate_image_gray_area(frame_bytes):
+                logger.warning(f"⚠️ [APP] Detected gray area corruption in image while processing {self.dataset_source.dataset_name} (ID: {self.dataset_source.id})")
+                return True
             
             if frame_bytes:
                 # Generate unique filename

{nedo_vision_worker-1.3.7 → nedo_vision_worker-1.3.9}/nedo_vision_worker/worker/RabbitMQListener.py

@@ -88,10 +88,6 @@ class RabbitMQListener:
         self.exchange_name = exchange_name
         self.queue_name = queue_name
 
-        if self.listener_thread and self.listener_thread.is_alive():
-            logger.warning("⚠️ [APP] RabbitMQ listener is already running.")
-            return
-            
         # Clean up any dead thread reference
         if self.listener_thread and not self.listener_thread.is_alive():
             self.listener_thread = None