nedo-vision-worker-core 0.3.4__tar.gz → 0.3.5__tar.gz

{nedo_vision_worker_core-0.3.4 → nedo_vision_worker_core-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker-core
-Version: 0.3.4
+Version: 0.3.5
 Summary: Nedo Vision Worker Core Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>

{nedo_vision_worker_core-0.3.4 → nedo_vision_worker_core-0.3.5}/nedo_vision_worker_core/__init__.py

@@ -7,7 +7,7 @@ A library for running AI vision processing and detection in the Nedo Vision plat
 from .core_service import CoreService
 from .callbacks import DetectionType, CallbackTrigger, DetectionData, IntervalMetadata
 
-__version__ = "0.3.4"
+__version__ = "0.3.5"
 __all__ = [
     "CoreService", 
     "DetectionType", 

{nedo_vision_worker_core-0.3.4 → nedo_vision_worker_core-0.3.5}/nedo_vision_worker_core/pipeline/PipelineManager.py

@@ -39,6 +39,11 @@ class PipelineManager:
 
         logging.info(f"🚀 Starting Pipeline processing for pipeline: {pipeline_id} | Source: {worker_source_id} ({pipeline.name})")
 
+        # Acquire the video stream (starts it if not already running)
+        if not self.video_manager.acquire_stream(worker_source_id, pipeline_id):
+            logging.error(f"❌ Failed to acquire stream {worker_source_id} for pipeline {pipeline_id}")
+            return
+
         processor = PipelineProcessor(pipeline, detector, False)
         processor.frame_drawer.location_name = pipeline.location_name
         self.processors[pipeline_id] = processor  # Store processor instance
@@ -79,6 +84,10 @@ class PipelineManager:
             self._stopping_pipelines.add(pipeline_id)
 
         try:
+            # Get worker_source_id before removing metadata
+            pipeline = self.pipeline_metadata.get(pipeline_id)
+            worker_source_id = pipeline.worker_source_id if pipeline else None
+
             # Stop AI processing
             processor = self.processors.pop(pipeline_id, None)
             if processor:
@@ -92,6 +101,10 @@ class PipelineManager:
             # Remove metadata
             self.pipeline_metadata.pop(pipeline_id, None)
 
+            # Release the video stream (stops it if no more pipelines use it)
+            if worker_source_id:
+                self.video_manager.release_stream(worker_source_id, pipeline_id)
+
             logging.info(f"✅ Pipeline {pipeline_id} stopped successfully.")
 
         except Exception as e:

{nedo_vision_worker_core-0.3.4 → nedo_vision_worker_core-0.3.5}/nedo_vision_worker_core/pipeline/PipelineProcessor.py

@@ -459,9 +459,10 @@ class PipelineProcessor:
                 logging.error(f"   Cannot get stream URL for {worker_source_id}")
                 return False
             
-            video_manager.remove_stream(worker_source_id)
+            # Use internal methods to restart the stream without affecting reference counting
+            video_manager._stop_stream(worker_source_id)
             time.sleep(1.0)
-            video_manager.add_stream(worker_source_id, stream_url)
+            video_manager._start_stream(worker_source_id, stream_url)
             time.sleep(2.0)
             
             if not video_manager.has_stream(worker_source_id):

{nedo_vision_worker_core-0.3.4 → nedo_vision_worker_core-0.3.5}/nedo_vision_worker_core/pipeline/PipelineSyncThread.py

@@ -103,8 +103,22 @@ class PipelineSyncThread(threading.Thread):
 
         for pid in pipeline_ids:
             db_pipeline = db_pipelines[pid]
+            
+            # Check if pipeline should be stopped (status changed to stop/restart in DB)
+            if db_pipeline.pipeline_status_code in ['stop', 'restart']:
+                if self.pipeline_manager.is_running(pid):
+                    logging.info(f"⏹️ Stopping pipeline due to status change: {pid}")
+                    self.pipeline_manager.stop_pipeline(pid)
+                continue
+            
             processor = self.pipeline_manager.processors.get(pid)
             if not processor:
+                # Pipeline exists in both sets but processor doesn't exist - shouldn't happen
+                # but if it does, try to start it if status is 'run'
+                if db_pipeline.pipeline_status_code == 'run':
+                    logging.warning(f"⚠️ Pipeline {pid} exists locally but has no processor. Restarting...")
+                    detector = self.model_manager.get_detector(db_pipeline.ai_model_id)
+                    self.pipeline_manager.start_pipeline(db_pipeline, detector)
                 continue
 
             local_detector = processor.detector
@@ -114,29 +128,12 @@ class PipelineSyncThread(threading.Thread):
                 processor.enable_debug()
 
     def update_pipeline(self, pid: str, db_pipeline: object, local_detector: object) -> None:
-        """Updates a single pipeline if necessary."""
+        """Updates a single pipeline if necessary (only called for running pipelines)."""
         processor = self.pipeline_manager.processors.get(pid)
         if not processor:
             return
 
-        # Stop/start based on status change
-        if db_pipeline.pipeline_status_code != processor._pipeline.pipeline_status_code:
-            if db_pipeline.pipeline_status_code == 'run':
-                logging.info(f"▶️ Resuming pipeline: {pid}")
-                self.pipeline_manager.start_pipeline(db_pipeline, self.model_manager.get_detector(db_pipeline.ai_model_id))
-            elif db_pipeline.pipeline_status_code in ['stop', 'restart']:
-                logging.info(f"⏹️ Stopping pipeline: {pid}")
-                self.pipeline_manager.stop_pipeline(pid)
-                if db_pipeline.pipeline_status_code == 'restart':
-                    # This will be picked up by the 'add_new_pipelines' logic in the next cycle
-                    return 
-            else:
-                 processor.update_config(db_pipeline) # Update config for non-running pipelines
-            return
-        elif db_pipeline.pipeline_status_code != 'run':
-            processor.update_config(db_pipeline)
-            return
-
+        # At this point, we know db_pipeline.pipeline_status_code == 'run' (checked in caller)
         # Check for significant changes that require a restart
         db_detector = self.model_manager.get_detector(db_pipeline.ai_model_id)
 
@@ -146,7 +143,7 @@ class PipelineSyncThread(threading.Thread):
             local_detector != db_detector
         ])
 
-        if requires_restart and db_pipeline.pipeline_status_code == 'run':
+        if requires_restart:
             logging.info(f"🔄 Restarting pipeline due to significant changes: {pid}")
             self.pipeline_manager.stop_pipeline(pid)
             self.pipeline_manager.start_pipeline(db_pipeline, db_detector)

nedo_vision_worker_core-0.3.5/nedo_vision_worker_core/streams/StreamSyncThread.py

@@ -0,0 +1,107 @@
+import logging
+import os
+import time
+import threading
+from ..database.DatabaseManager import DatabaseManager
+from ..repositories.WorkerSourceRepository import WorkerSourceRepository
+from .VideoStreamManager import VideoStreamManager
+
+class StreamSyncThread(threading.Thread):
+    """Thread responsible for synchronizing video streams from the database in real-time."""
+
+    def __init__(self, manager: VideoStreamManager, polling_interval=5):
+        super().__init__()  # Set as a daemon so it stops with the main process
+
+        self.source_file_path = DatabaseManager.STORAGE_PATHS["files"] / "source_files"
+
+        self.manager = manager
+        self.polling_interval = polling_interval
+        self.worker_source_repo = WorkerSourceRepository()
+        self.running = True
+
+    def _get_source_file_path(self, file):
+        """Returns the file path for a given source file."""
+        return self.source_file_path / os.path.basename(file)
+
+    def run(self):
+        """Continuously updates the VideoStreamManager with database changes."""
+        while self.running:
+            try:
+                sources = self.worker_source_repo.get_worker_sources()
+                db_sources = {
+                    source.id: 
+                        source.url if source.type_code == "live" 
+                        else source.url if source.type_code == "direct"
+                        else self._get_source_file_path(source.file_path)
+                     for source in sources
+                }  # Store latest sources
+                
+                # Get both active streams and pending streams
+                active_stream_ids = set(self.manager.get_active_stream_ids())
+                with self.manager._lock:
+                    pending_stream_ids = set(self.manager.pending_streams.keys())
+                registered_stream_ids = active_stream_ids | pending_stream_ids
+
+                # **1️⃣ Register new streams (lazy loading - don't start them yet)**
+                for source_id, url in db_sources.items():
+                    if source_id not in registered_stream_ids:
+                        logging.info(f"🟢 Registering new stream: {source_id} ({url})")
+                        self.manager.register_stream(source_id, url)
+
+                # **2️⃣ Unregister deleted or disconnected streams**
+                for stream_id in registered_stream_ids:
+                    if stream_id not in db_sources:
+                        logging.info(f"🔴 Unregistering stream: {stream_id}")
+                        self.manager.unregister_stream(stream_id)
+
+                # Refresh registered streams
+                with self.manager._lock:
+                    pending_stream_ids = set(self.manager.pending_streams.keys())
+                registered_stream_ids = active_stream_ids | pending_stream_ids
+
+                # **3️⃣ Update streams if URL has changed**
+                for source_id, url in db_sources.items():
+                    if source_id in registered_stream_ids:
+                        # Check if it's an active stream or pending stream
+                        with self.manager._lock:
+                            is_pending = source_id in self.manager.pending_streams
+                            if is_pending:
+                                existing_url = self.manager.pending_streams.get(source_id)
+                            else:
+                                existing_url = None
+                        
+                        if existing_url is None:
+                            # It's an active stream, get URL from stream manager
+                            existing_url = self.manager.get_stream_url(source_id)
+                        
+                        # Only update if URL actually changed
+                        if existing_url != url:
+                            if is_pending:
+                                # It's pending, just update the URL
+                                with self.manager._lock:
+                                    self.manager.pending_streams[source_id] = url
+                                    logging.info(f"🟡 Updated pending stream {source_id} URL")
+                            else:
+                                # It's active, need to restart it
+                                logging.info(f"🟡 Updating active stream {source_id}: New URL {url}")
+                                # Unregister and re-register with new URL
+                                self.manager.unregister_stream(source_id)
+                                # Add a small delay for device cleanup
+                                if self._is_direct_device(url) or self._is_direct_device(existing_url):
+                                    time.sleep(0.5)  # Allow device to be properly released
+                                self.manager.register_stream(source_id, url)
+
+            except Exception as e:
+                logging.error(f"⚠️ Error syncing streams from database: {e}")
+
+            time.sleep(self.polling_interval)  # Poll every X seconds
+
+    def _is_direct_device(self, url) -> bool:
+        """Check if URL represents a direct video device."""
+        if isinstance(url, str):
+            return url.isdigit() or url.startswith('/dev/video')
+        return isinstance(url, int)
+
+    def stop(self):
+        """Stops the synchronization thread."""
+        self.running = False

{nedo_vision_worker_core-0.3.4 → nedo_vision_worker_core-0.3.5}/nedo_vision_worker_core/streams/VideoStreamManager.py

@@ -1,7 +1,6 @@
 import logging
 import time
 import threading
-import cv2
 from typing import Any, Dict, Optional
 
 from .VideoStream import VideoStream
@@ -18,6 +17,11 @@ class VideoStreamManager:
         self.direct_device_streams: Dict[Any, Dict[str, Any]] = {}
         # Per-direct-device locks: {worker_source_id: threading.Lock}
         self.direct_device_locks: Dict[Any, threading.Lock] = {}
+        
+        # Reference counting for lazy loading: {worker_source_id: set of pipeline_ids}
+        self.stream_references: Dict[Any, set] = {}
+        # Store URLs for streams that aren't started yet: {worker_source_id: url}
+        self.pending_streams: Dict[Any, str] = {}
 
         self.shared_device_manager = SharedVideoDeviceManager()
 
@@ -38,16 +42,71 @@ class VideoStreamManager:
     # -----------------------
     # Public API
     # -----------------------
-    def add_stream(self, worker_source_id, url):
-        """Adds and starts a stream (regular file/RTSP or a shared direct device) if not already present."""
+    def register_stream(self, worker_source_id, url):
+        """Register a stream URL without starting it (lazy loading)."""
         with self._lock:
+            if worker_source_id not in self.pending_streams:
+                self.pending_streams[worker_source_id] = url
+                logging.debug(f"📝 Registered stream {worker_source_id} for lazy loading")
+    
+    def unregister_stream(self, worker_source_id):
+        """Unregister a stream that's no longer available in the database."""
+        with self._lock:
+            # If it's pending, remove it
+            if worker_source_id in self.pending_streams:
+                del self.pending_streams[worker_source_id]
+                logging.debug(f"🗑️ Unregistered pending stream {worker_source_id}")
+            
+            # If it's active and has no references, remove it
+            if worker_source_id in self.stream_references:
+                if len(self.stream_references[worker_source_id]) == 0:
+                    self._stop_stream(worker_source_id)
+    
+    def acquire_stream(self, worker_source_id, pipeline_id):
+        """Request access to a stream for a pipeline. Starts the stream if not already running."""
+        with self._lock:
+            # Initialize reference set if needed
+            if worker_source_id not in self.stream_references:
+                self.stream_references[worker_source_id] = set()
+            
+            # Add pipeline reference
+            self.stream_references[worker_source_id].add(pipeline_id)
+            logging.info(f"🔗 Pipeline {pipeline_id} acquired stream {worker_source_id} (refs: {len(self.stream_references[worker_source_id])})")
+            
+            # If stream is already running, we're done
             if worker_source_id in self.streams or worker_source_id in self.direct_device_streams:
-                logging.warning("⚠️ Stream %s is already active.", worker_source_id)
+                return True
+            
+            # Get URL from pending streams
+            url = self.pending_streams.get(worker_source_id)
+            if not url:
+                logging.error(f"❌ Cannot acquire stream {worker_source_id}: URL not registered")
+                return False
+        
+        # Start the stream (outside lock to avoid blocking)
+        return self._start_stream(worker_source_id, url)
+    
+    def release_stream(self, worker_source_id, pipeline_id):
+        """Release a stream reference from a pipeline. Stops the stream if no more references."""
+        with self._lock:
+            if worker_source_id not in self.stream_references:
                 return
-
+            
+            # Remove pipeline reference
+            self.stream_references[worker_source_id].discard(pipeline_id)
+            ref_count = len(self.stream_references[worker_source_id])
+            logging.info(f"🔓 Pipeline {pipeline_id} released stream {worker_source_id} (refs: {ref_count})")
+            
+            # If no more references, stop the stream
+            if ref_count == 0:
+                logging.info(f"💤 Stream {worker_source_id} has no more references, stopping...")
+                self._stop_stream(worker_source_id)
+    
+    def _start_stream(self, worker_source_id, url):
+        """Internal method to actually start a stream."""
         if self._is_direct_device(url):
             self._add_direct_device_stream(worker_source_id, url)
-            return
+            return True
 
         # Regular stream
         stream = VideoStream(url)
@@ -55,18 +114,19 @@ class VideoStreamManager:
             stream.start()  # start thread
             with self._lock:
                 self.streams[worker_source_id] = stream
-            logging.info("✅ Added and started video stream: %s", worker_source_id)
+            logging.info("✅ Started video stream: %s", worker_source_id)
+            return True
         except Exception as e:
             logging.error("❌ Failed to start regular stream %s: %s", worker_source_id, e)
-
-    def remove_stream(self, worker_source_id):
-        """Stops and removes a stream (regular or direct device)."""
-        if not worker_source_id:
-            return
-
+            return False
+    
+    def _stop_stream(self, worker_source_id):
+        """Internal method to stop a stream."""
         # Direct device?
         with self._lock:
             is_direct = worker_source_id in self.direct_device_streams
+            # Clean up references
+            self.stream_references.pop(worker_source_id, None)
 
         if is_direct:
             self._remove_direct_device_stream(worker_source_id)
@@ -77,20 +137,16 @@ class VideoStreamManager:
             stream = self.streams.pop(worker_source_id, None)
 
         if stream is None:
-            logging.warning("⚠️ Stream %s not found in manager.", worker_source_id)
             return
 
-        logging.info("🛑 Removing video stream: %s", worker_source_id)
+        logging.info("🛑 Stopping video stream: %s", worker_source_id)
         try:
-            # Expectation: VideoStream.stop() should signal and join internally.
             stream.stop()
         except Exception as e:
             logging.error("❌ Error stopping stream %s: %s", worker_source_id, e)
         finally:
             stream = None
 
-        logging.info("✅ Stream %s removed successfully.", worker_source_id)
-
     def start_all(self):
         """Starts all regular streams that are not alive. (Direct devices are publisher-driven.)"""
         logging.info("🔄 Starting all video streams...")
@@ -114,13 +170,13 @@ class VideoStreamManager:
 
         for wid in regular_ids:
             try:
-                self.remove_stream(wid)
+                self._stop_stream(wid)
             except Exception as e:
                 logging.error("Error stopping regular stream %s: %s", wid, e)
 
         for wid in direct_ids:
             try:
-                self.remove_stream(wid)
+                self._stop_stream(wid)
             except Exception as e:
                 logging.error("Error stopping direct device stream %s: %s", wid, e)
 

{nedo_vision_worker_core-0.3.4 → nedo_vision_worker_core-0.3.5}/nedo_vision_worker_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedo-vision-worker-core
-Version: 0.3.4
+Version: 0.3.5
 Summary: Nedo Vision Worker Core Library for AI Vision Processing
 Author-email: Willy Achmat Fauzi <willy.achmat@gmail.com>
 Maintainer-email: Willy Achmat Fauzi <willy.achmat@gmail.com>

nedo_vision_worker_core-0.3.4/nedo_vision_worker_core/streams/StreamSyncThread.py

@@ -1,80 +0,0 @@
-import logging
-import os
-import time
-import threading
-from ..database.DatabaseManager import DatabaseManager
-from ..repositories.WorkerSourceRepository import WorkerSourceRepository
-from .VideoStreamManager import VideoStreamManager
-
-class StreamSyncThread(threading.Thread):
-    """Thread responsible for synchronizing video streams from the database in real-time."""
-
-    def __init__(self, manager: VideoStreamManager, polling_interval=5):
-        super().__init__()  # Set as a daemon so it stops with the main process
-
-        self.source_file_path = DatabaseManager.STORAGE_PATHS["files"] / "source_files"
-
-        self.manager = manager
-        self.polling_interval = polling_interval
-        self.worker_source_repo = WorkerSourceRepository()
-        self.running = True
-
-    def _get_source_file_path(self, file):
-        """Returns the file path for a given source file."""
-        return self.source_file_path / os.path.basename(file)
-
-    def run(self):
-        """Continuously updates the VideoStreamManager with database changes."""
-        while self.running:
-            try:
-                sources = self.worker_source_repo.get_worker_sources()
-                db_sources = {
-                    source.id: (
-                        source.url if source.type_code == "live" 
-                        else source.url if source.type_code == "direct"
-                        else self._get_source_file_path(source.file_path), 
-                        source.status_code
-                    ) for source in sources
-                }  # Store latest sources
-                active_stream_ids = set(self.manager.get_active_stream_ids())
-
-                # **1️⃣ Add new streams**
-                for source_id, (url, status_code) in db_sources.items():
-                    if source_id not in active_stream_ids and status_code == "connected":
-                        logging.info(f"🟢 Adding new stream: {source_id} ({url})")
-                        self.manager.add_stream(source_id, url)
-
-                # **2️⃣ Remove deleted streams**
-                for stream_id in active_stream_ids:
-                    if stream_id not in db_sources or db_sources[stream_id][1] != "connected":
-                        logging.info(f"🔴 Removing deleted stream: {stream_id}")
-                        self.manager.remove_stream(stream_id)
-
-                active_stream_ids = set(self.manager.get_active_stream_ids())
-
-                # **3️⃣ Update streams if URL has changed**
-                for source_id, (url, status_code) in db_sources.items():
-                    if source_id in active_stream_ids:
-                        existing_url = self.manager.get_stream_url(source_id)
-                        if existing_url != url:
-                            logging.info(f"🟡 Updating stream {source_id}: New URL {url}")
-                            self.manager.remove_stream(source_id)
-                            # Add a small delay for device cleanup
-                            if self._is_direct_device(url) or self._is_direct_device(existing_url):
-                                time.sleep(0.5)  # Allow device to be properly released
-                            self.manager.add_stream(source_id, url)
-
-            except Exception as e:
-                logging.error(f"⚠️ Error syncing streams from database: {e}")
-
-            time.sleep(self.polling_interval)  # Poll every X seconds
-
-    def _is_direct_device(self, url) -> bool:
-        """Check if URL represents a direct video device."""
-        if isinstance(url, str):
-            return url.isdigit() or url.startswith('/dev/video')
-        return isinstance(url, int)
-
-    def stop(self):
-        """Stops the synchronization thread."""
-        self.running = False