nedo-vision-worker 1.0.0__py3-none-any.whl

nedo_vision_worker/repositories/DatasetSourceRepository.py

@@ -0,0 +1,150 @@
+import logging
+from typing import List, Optional
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from ..models.dataset_source import DatasetSourceEntity
+from ..database.DatabaseManager import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+class DatasetSourceRepository:
+    """Repository for managing dataset source data using SQLAlchemy."""
+    
+    def __init__(self):
+        """Initialize the repository."""
+        self.db_manager = DatabaseManager()
+        self.session: Session = self.db_manager.get_session("default")
+    
+    def sync_dataset_sources(self, dataset_sources_data, update_callback=None):
+        """
+        Synchronize dataset sources from server data with intelligent change detection.
+        
+        Args:
+            dataset_sources_data: List of dataset source data from server
+            update_callback: Optional callback function for status updates
+        """
+        try:
+            # Get existing dataset sources from local database
+            local_dataset_sources = {ds.id: ds for ds in self.session.query(DatasetSourceEntity).all()}
+            
+            new_dataset_sources = []
+            updated_dataset_sources = []
+            changed_dataset_sources = []
+            server_dataset_source_ids = set()
+
+            for dataset_source_data in dataset_sources_data:
+                server_dataset_source_ids.add(dataset_source_data.id)
+                existing_dataset_source = local_dataset_sources.get(dataset_source_data.id)
+                changes = []
+
+                if existing_dataset_source:
+                    # Check for changes in each field
+                    if existing_dataset_source.dataset_id != dataset_source_data.dataset_id:
+                        changes.append(f"dataset_id: {existing_dataset_source.dataset_id} → {dataset_source_data.dataset_id}")
+                    if existing_dataset_source.worker_source_id != dataset_source_data.worker_source_id:
+                        changes.append(f"worker_source_id: {existing_dataset_source.worker_source_id} → {dataset_source_data.worker_source_id}")
+                    if existing_dataset_source.sampling_interval != dataset_source_data.sampling_interval:
+                        changes.append(f"sampling_interval: {existing_dataset_source.sampling_interval} → {dataset_source_data.sampling_interval}")
+                    if existing_dataset_source.dataset_name != dataset_source_data.dataset_name:
+                        changes.append(f"dataset_name: '{existing_dataset_source.dataset_name}' → '{dataset_source_data.dataset_name}'")
+                    if existing_dataset_source.worker_source_name != dataset_source_data.worker_source_name:
+                        changes.append(f"worker_source_name: '{existing_dataset_source.worker_source_name}' → '{dataset_source_data.worker_source_name}'")
+                    if existing_dataset_source.worker_source_url != dataset_source_data.worker_source_url:
+                        changes.append(f"worker_source_url: '{existing_dataset_source.worker_source_url}' → '{dataset_source_data.worker_source_url}'")
+
+                    if changes:
+                        # Update existing record
+                        existing_dataset_source.dataset_id = dataset_source_data.dataset_id
+                        existing_dataset_source.worker_source_id = dataset_source_data.worker_source_id
+                        existing_dataset_source.sampling_interval = dataset_source_data.sampling_interval
+                        existing_dataset_source.dataset_name = dataset_source_data.dataset_name
+                        existing_dataset_source.worker_source_name = dataset_source_data.worker_source_name
+                        existing_dataset_source.worker_source_url = dataset_source_data.worker_source_url
+                        updated_dataset_sources.append(existing_dataset_source)
+                        changed_dataset_sources.append(f"🔄 [APP] [UPDATE] Dataset Source ID {dataset_source_data.id}: " + ", ".join(changes))
+                else:
+                    # Create new record
+                    new_dataset_source = DatasetSourceEntity(
+                        id=dataset_source_data.id,
+                        dataset_id=dataset_source_data.dataset_id,
+                        worker_source_id=dataset_source_data.worker_source_id,
+                        sampling_interval=dataset_source_data.sampling_interval,
+                        dataset_name=dataset_source_data.dataset_name,
+                        worker_source_name=dataset_source_data.worker_source_name,
+                        worker_source_url=dataset_source_data.worker_source_url
+                    )
+                    new_dataset_sources.append(new_dataset_source)
+                    logger.info(f"🆕 [APP] [INSERT] Added Dataset Source ID {dataset_source_data.id} - {dataset_source_data.dataset_name}")
+
+            # Identify and delete dataset sources not in the server response
+            records_to_delete = [
+                dataset_source for dataset_source_id, dataset_source in local_dataset_sources.items()
+                if dataset_source_id not in server_dataset_source_ids
+            ]
+
+            # Perform batch operations in a single transaction
+            if new_dataset_sources:
+                self.session.bulk_save_objects(new_dataset_sources)  # Bulk insert
+
+            if updated_dataset_sources:
+                self.session.bulk_save_objects(updated_dataset_sources)  # Bulk update
+
+            if records_to_delete:
+                for record in records_to_delete:
+                    self.session.delete(record)  # Mark for deletion
+                    logger.info(f"❌ [APP] [DELETE] Dataset Source ID {record.id} - {record.dataset_name}")
+
+            self.session.commit()  # Commit once (reducing DB round trips)
+
+            # Log all changes
+            for change in changed_dataset_sources:
+                logger.info(change)
+                
+            if new_dataset_sources or updated_dataset_sources or records_to_delete:
+                logger.info(f"✅ [APP] Synced {len(dataset_sources_data)} dataset sources (Added: {len(new_dataset_sources)}, Updated: {len(updated_dataset_sources)}, Deleted: {len(records_to_delete)})")
+            
+        except SQLAlchemyError as e:
+            self.session.rollback()
+            logger.error(f"🚨 [APP] Database error while syncing dataset sources: {e}", exc_info=True)
+        except Exception as e:
+            self.session.rollback()
+            logger.error(f"🚨 [APP] Error syncing dataset sources: {e}")
+    
+    def get_all_dataset_sources(self) -> List[DatasetSourceEntity]:
+        """Get all dataset sources from local database."""
+        try:
+            dataset_sources = self.session.query(DatasetSourceEntity).all()
+            return dataset_sources
+            
+        except SQLAlchemyError as e:
+            logger.error(f"🚨 [APP] Database error while fetching dataset sources: {e}", exc_info=True)
+            return []
+        except Exception as e:
+            logger.error(f"🚨 [APP] Error fetching dataset sources: {e}")
+            return []
+    
+    def get_dataset_source_by_id(self, dataset_source_id: str) -> Optional[DatasetSourceEntity]:
+        """Get a specific dataset source by ID."""
+        try:
+            dataset_source = self.session.query(DatasetSourceEntity).filter_by(id=dataset_source_id).first()
+            return dataset_source
+            
+        except SQLAlchemyError as e:
+            logger.error(f"🚨 [APP] Database error while fetching dataset source by ID {dataset_source_id}: {e}", exc_info=True)
+            return None
+        except Exception as e:
+            logger.error(f"🚨 [APP] Error fetching dataset source by ID {dataset_source_id}: {e}")
+            return None
+    
+    def get_dataset_sources_by_worker_source_id(self, worker_source_id: str) -> List[DatasetSourceEntity]:
+        """Get all dataset sources for a specific worker source."""
+        try:
+            dataset_sources = self.session.query(DatasetSourceEntity).filter_by(worker_source_id=worker_source_id).all()
+            return dataset_sources
+            
+        except SQLAlchemyError as e:
+            logger.error(f"🚨 [APP] Database error while fetching dataset sources by worker source ID {worker_source_id}: {e}", exc_info=True)
+            return []
+        except Exception as e:
+            logger.error(f"🚨 [APP] Error fetching dataset sources by worker source ID {worker_source_id}: {e}")
+            return [] 

nedo_vision_worker/repositories/PPEDetectionRepository.py

@@ -0,0 +1,112 @@
+import os
+import logging
+from pathlib import Path
+from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import asc
+from sqlalchemy.exc import SQLAlchemyError
+from ..database.DatabaseManager import DatabaseManager, get_storage_path
+from ..models.ppe_detection import PPEDetectionEntity
+from ..models.ppe_detection_label import PPEDetectionLabelEntity
+
+class PPEDetectionRepository:
+    """Handles storage of PPE detections into SQLite using SQLAlchemy."""
+
+    def __init__(self):
+        
+        self.storage_dir = get_storage_path("files")
+        self.db_manager = DatabaseManager()
+        self.session: Session = self.db_manager.get_session("default")
+        os.makedirs(self.storage_dir, exist_ok=True) 
+
+    
+    def get_latest_5_detections(self) -> list:
+        """
+        Retrieves the latest 5 PPE detections ordered by the 'created_at' timestamp.
+
+        Returns:
+            list: A list of PPEDetectionEntity objects.
+        """
+        try:
+            latest_detections = (
+                self.session.query(PPEDetectionEntity)
+                .options(joinedload(PPEDetectionEntity.ppe_labels)) 
+                .order_by(asc(PPEDetectionEntity.created_at))
+                .limit(5)
+                .all()
+            )
+            
+            # Prepare the list of UpsertPPEDetectionRequest messages
+            ppe_detection_requests = []
+            for detection in latest_detections:
+                ppe_detection_labels = [
+                    {
+                        'code': label.code,
+                        'confidence_score': label.confidence_score,
+                        'b_box_x1': label.b_box_x1,
+                        'b_box_y1': label.b_box_y2,
+                        'b_box_x2': label.b_box_x2,
+                        'b_box_y2': label.b_box_y2,
+                    }
+                    for label in detection.ppe_labels
+                ]
+
+                worker_timestamp = detection.created_at.strftime('%Y-%m-%dT%H:%M:%SZ')  # Remove microseconds and add Z for UTC
+
+                # Create request object
+                request = {
+                    'person_id': detection.person_id,
+                    'image': detection.image_path,
+                    'image_tile': detection.image_tile_path,
+                    'worker_source_id': detection.worker_source_id,
+                    "worker_timestamp": worker_timestamp,
+                    'ppe_detection_labels': ppe_detection_labels
+                }
+                ppe_detection_requests.append(request)
+            
+            return ppe_detection_requests
+
+        except SQLAlchemyError as e:
+            self.session.rollback()
+            logging.error(f"❌ Database error while retrieving latest 5 detections: {e}")
+            return []
+
+    def delete_records_from_db(self, detection_data: list):
+        """
+        Deletes PPE detection records from the database based on detection data.
+
+        Args:
+            detection_data (list): List of dictionaries containing the detection data.
+        """
+        try:
+            # Extract person_id from detection data to delete the corresponding records
+            person_ids_to_delete = [data['person_id'] for data in detection_data]
+
+            # Delete corresponding PPEDetectionEntity records
+            detections_to_delete = (
+                self.session.query(PPEDetectionEntity)
+                .filter(PPEDetectionEntity.person_id.in_(person_ids_to_delete))
+                .all()
+            )
+
+            for detection in detections_to_delete:
+                # Also delete related PPE detection labels from PPEDetectionLabelEntity
+                self.session.query(PPEDetectionLabelEntity).filter(PPEDetectionLabelEntity.detection_id == detection.id).delete()
+
+                # Delete the image file associated with the detection if it exists
+                image_path = detection.image_path
+                if os.path.exists(image_path):
+                    os.remove(image_path)
+                    logging.info(f"Deleted image file: {image_path}")
+                else:
+                    logging.warning(f"Image file not found for detection {detection.id}: {image_path}")
+
+                # Delete the detection record
+                self.session.delete(detection)
+
+            # Commit the transaction
+            self.session.commit()
+            logging.info(f"Successfully deleted {len(detections_to_delete)} PPE detection records.")
+
+        except SQLAlchemyError as e:
+            self.session.rollback()
+            logging.error(f"❌ Error occurred while deleting records from DB: {e}")

nedo_vision_worker/repositories/RestrictedAreaRepository.py

@@ -0,0 +1,88 @@
+import os
+import logging
+from pathlib import Path
+from sqlalchemy.orm import Session
+from sqlalchemy import asc
+from sqlalchemy.exc import SQLAlchemyError
+from ..database.DatabaseManager import DatabaseManager, get_storage_path
+from ..models.restricted_area_violation import RestrictedAreaViolationEntity
+
+class RestrictedAreaRepository:
+    """Handles storage and retrieval of restricted area violations in SQLite using SQLAlchemy."""
+
+    def __init__(self):
+        self.storage_dir = get_storage_path("restricted_violations")
+        self.db_manager = DatabaseManager()
+        self.session: Session = self.db_manager.get_session("default")
+        os.makedirs(self.storage_dir, exist_ok=True)
+
+    def get_latest_5_violations(self) -> list:
+        """
+        Retrieves the latest 5 restricted area violations ordered by the 'created_at' timestamp.
+
+        Returns:
+            list: A list of dictionaries representing restricted area violations.
+        """
+        try:
+            latest_violations = (
+                self.session.query(RestrictedAreaViolationEntity)
+                .order_by(asc(RestrictedAreaViolationEntity.created_at))
+                .limit(5)
+                .all()
+            )
+
+            result = []
+            for violation in latest_violations:
+                timestamp = violation.created_at.strftime('%Y-%m-%dT%H:%M:%SZ')
+                result.append({
+                    'person_id': violation.person_id,
+                    'image': violation.image_path,
+                    'image_tile': violation.image_tile_path,
+                    'worker_source_id': violation.worker_source_id,
+                    'worker_timestamp': timestamp,
+                    'confidence_score': violation.confidence_score,
+                    'b_box_x1': violation.b_box_x1,
+                    'b_box_y1': violation.b_box_y1,
+                    'b_box_x2': violation.b_box_x2,
+                    'b_box_y2': violation.b_box_y2,
+                })
+
+            return result
+
+        except SQLAlchemyError as e:
+            self.session.rollback()
+            logging.error(f"❌ Database error while retrieving latest 5 violations: {e}")
+            return []
+
+    def delete_records_from_db(self, violation_data: list):
+        """
+        Deletes restricted area violation records from the database based on the provided data.
+
+        Args:
+            violation_data (list): List of dictionaries containing the violation data.
+        """
+        try:
+            person_ids_to_delete = [data['person_id'] for data in violation_data]
+
+            violations_to_delete = (
+                self.session.query(RestrictedAreaViolationEntity)
+                .filter(RestrictedAreaViolationEntity.person_id.in_(person_ids_to_delete))
+                .all()
+            )
+
+            for violation in violations_to_delete:
+                image_path = violation.image_path
+                if os.path.exists(image_path):
+                    os.remove(image_path)
+                    logging.info(f"Deleted image file: {image_path}")
+                else:
+                    logging.warning(f"Image file not found for violation {violation.id}: {image_path}")
+
+                self.session.delete(violation)
+
+            self.session.commit()
+            logging.info(f"Successfully deleted {len(violations_to_delete)} violation records.")
+
+        except SQLAlchemyError as e:
+            self.session.rollback()
+            logging.error(f"❌ Error occurred while deleting violation records: {e}")

nedo_vision_worker/repositories/WorkerSourcePipelineDebugRepository.py

@@ -0,0 +1,90 @@
+from datetime import datetime, timedelta, timezone
+import os
+from sqlalchemy.orm import Session
+from ..database.DatabaseManager import DatabaseManager
+from ..models.worker_source_pipeline_debug import WorkerSourcePipelineDebugEntity
+
+
+class WorkerSourcePipelineDebugRepository:
+    def __init__(self):
+        self.db_manager = DatabaseManager()
+        self.session: Session = self.db_manager.get_session("default")
+
+    def create_debug_entry(self, uuid: str, worker_source_pipeline_id: str) -> WorkerSourcePipelineDebugEntity:
+        """
+        Create a new debug entry for a worker source pipeline.
+        
+        Args:
+            uuid (str): The requester ID
+            worker_source_pipeline_id (str): The ID of the worker source pipeline
+            
+        Returns:
+            WorkerSourcePipelineDebugEntity: The created debug entry
+        """
+        debug_entry = WorkerSourcePipelineDebugEntity(
+            uuid=uuid,
+            worker_source_pipeline_id=worker_source_pipeline_id,
+        )
+        self.session.add(debug_entry)
+        self.session.commit()
+        return debug_entry
+
+    def get_debug_entries_with_data(self) -> list[WorkerSourcePipelineDebugEntity]:
+        """
+        Fetch all debug entries that have non-null data, ordered by creation date (oldest first).
+        
+        Returns:
+            list[WorkerSourcePipelineDebugEntity]: List of debug entries with data
+        """
+        self.session.expire_all()
+        cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=1) 
+
+        # Delete old entries
+        old_entries = self.session.query(WorkerSourcePipelineDebugEntity)\
+            .filter(
+                WorkerSourcePipelineDebugEntity.data.isnot(None),
+                WorkerSourcePipelineDebugEntity.created_at < cutoff_time
+            ).all()
+
+        for entry in old_entries:
+            if entry.image_path and os.path.exists(entry.image_path):
+                try:
+                    os.remove(entry.image_path)
+                except Exception as e:
+                    print(f"Warning: Failed to delete image at {entry.image_path} - {e}")
+            self.session.delete(entry)
+
+        self.session.commit()
+
+        # Fetch new entries
+        entries = self.session.query(WorkerSourcePipelineDebugEntity)\
+            .filter(WorkerSourcePipelineDebugEntity.data.isnot(None))\
+            .order_by(WorkerSourcePipelineDebugEntity.created_at.asc())\
+            .all()
+
+        return entries
+
+    def delete_entry_by_id(self, id: str):
+        """
+        Delete a debug entry by its ID, including the associated image file (if it exists).
+
+        :param entry_id: The ID of the entry to delete.
+        :return: True if the entry was found and deleted, False otherwise.
+        """
+        entry = self.session.query(WorkerSourcePipelineDebugEntity).filter_by(id=id).first()
+
+        if not entry:
+            return
+
+        # Delete image file if it exists
+        if entry.image_path and os.path.exists(entry.image_path):
+            try:
+                os.remove(entry.image_path)
+            except Exception as e:
+                # Optional: Log or handle file deletion error
+                print(f"Failed to delete image file: {entry.image_path}, error: {e}")
+
+        # Delete DB entry
+        self.session.delete(entry)
+        self.session.commit()
+

nedo_vision_worker/repositories/WorkerSourcePipelineDetectionRepository.py

@@ -0,0 +1,48 @@
+import os
+from sqlalchemy.orm import Session
+from ..database.DatabaseManager import _get_storage_paths, DatabaseManager
+from ..models.worker_source_pipeline_detection import WorkerSourcePipelineDetectionEntity
+
+
+class WorkerSourcePipelineDetectionRepository:
+    def __init__(self):
+        self.db_manager = DatabaseManager()
+        self.session: Session = self.db_manager.get_session("default")
+        storage_paths = _get_storage_paths()
+        self.storage_dir = storage_paths["files"] / "detection_image"
+        os.makedirs(self.storage_dir, exist_ok=True)
+
+    def get_entries(self) -> list[WorkerSourcePipelineDetectionEntity]:
+        self.session.expire_all()
+
+        # Fetch new entries
+        entries = self.session.query(WorkerSourcePipelineDetectionEntity)\
+            .order_by(WorkerSourcePipelineDetectionEntity.created_at.asc())\
+            .all()
+
+        return entries
+
+    def delete_entry_by_id(self, id: str):
+        """
+        Delete a debug entry by its ID, including the associated image file (if it exists).
+
+        :param entry_id: The ID of the entry to delete.
+        :return: True if the entry was found and deleted, False otherwise.
+        """
+        entry = self.session.query(WorkerSourcePipelineDetectionEntity).filter_by(id=id).first()
+
+        if not entry:
+            return
+
+        # Delete image file if it exists
+        if entry.image_path and os.path.exists(entry.image_path):
+            try:
+                os.remove(entry.image_path)
+            except Exception as e:
+                # Optional: Log or handle file deletion error
+                print(f"Failed to delete image file: {entry.image_path}, error: {e}")
+
+        # Delete DB entry
+        self.session.delete(entry)
+        self.session.commit()
+

nedo_vision_worker/repositories/WorkerSourcePipelineRepository.py

@@ -0,0 +1,174 @@
+import logging
+from sqlalchemy.orm import Session 
+from sqlalchemy.exc import SQLAlchemyError
+from ..database.DatabaseManager import DatabaseManager
+from ..protos.WorkerSourcePipelineService_pb2 import WorkerSourcePipelineListResponse
+from ..models.worker_source_pipeline import WorkerSourcePipelineEntity
+from ..models.worker_source_pipeline_config import WorkerSourcePipelineConfigEntity
+
+logger = logging.getLogger(__name__)
+
+class WorkerSourcePipelineRepository:
+    def __init__(self):
+        self.db_manager = DatabaseManager()
+        self.session: Session = self.db_manager.get_session("config")
+
+    def sync_worker_source_pipelines(self, response: WorkerSourcePipelineListResponse, on_status_update):
+        """
+        Synchronize worker source pipelines from the server with the local database.
+        This includes both WorkerSourcePipelineEntity and WorkerSourcePipelineConfigEntity.
+
+        Args:
+            response (WorkerSourcePipelineListResponse): The gRPC response containing worker source pipeline data.
+        """
+        try:
+            local_pipelines = {pipeline.id: pipeline for pipeline in self.session.query(WorkerSourcePipelineEntity).all()}
+            local_pipeline_configs = {config.id: config for config in self.session.query(WorkerSourcePipelineConfigEntity).all()}
+
+            new_pipelines = []
+            updated_pipelines = []
+            new_pipeline_configs = []
+            updated_pipeline_configs = []
+            changed_pipelines = []
+            changed_configs = []
+            server_pipeline_ids = set()
+            server_pipeline_config_ids = set()
+
+            for pipeline in response.data:
+                server_pipeline_ids.add(pipeline.id)
+                existing_pipeline = local_pipelines.get(pipeline.id)
+                changes = []
+
+                if existing_pipeline:
+                    if existing_pipeline.name != pipeline.name:
+                        changes.append(f"name: '{existing_pipeline.name}' → '{pipeline.name}'")
+                    if existing_pipeline.worker_source_id != pipeline.worker_source_id:
+                        changes.append(f"worker_source_id: {existing_pipeline.worker_source_id} → {pipeline.worker_source_id}")
+                    if existing_pipeline.worker_id != pipeline.worker_id:
+                        changes.append(f"worker_id: {existing_pipeline.worker_id} → {pipeline.worker_id}")
+                    if existing_pipeline.ai_model_id != pipeline.ai_model_id:
+                        changes.append(f"ai_model_id: {existing_pipeline.ai_model_id} → {pipeline.ai_model_id}")
+                    if existing_pipeline.location_name != pipeline.location_name:
+                        changes.append(f"location_name: {existing_pipeline.location_name} → {pipeline.location_name}")
+                    if existing_pipeline.pipeline_status_code != pipeline.pipeline_status_code and existing_pipeline.pipeline_status_code != "restart":
+                        on_status_update(pipeline.id, existing_pipeline.pipeline_status_code)
+
+                    if changes:
+                        existing_pipeline.name = pipeline.name
+                        existing_pipeline.worker_source_id = pipeline.worker_source_id
+                        existing_pipeline.worker_id = pipeline.worker_id
+                        existing_pipeline.ai_model_id = pipeline.ai_model_id
+                        existing_pipeline.location_name = pipeline.location_name
+                        updated_pipelines.append(existing_pipeline)
+                        changed_pipelines.append(f"🔄 [APP] [UPDATE] Worker Source Pipeline ID {pipeline.id}: " + ", ".join(changes))
+                else:
+                    new_pipelines.append(WorkerSourcePipelineEntity(
+                        id=pipeline.id,
+                        name=pipeline.name,
+                        worker_source_id=pipeline.worker_source_id,
+                        worker_id=pipeline.worker_id,
+                        ai_model_id=pipeline.ai_model_id,
+                        pipeline_status_code=pipeline.pipeline_status_code,
+                        location_name=pipeline.location_name
+                    ))
+                    logger.info(f"🆕 [APP] [INSERT] Added Worker Source Pipeline ID {pipeline.id} - {pipeline.name}")
+
+                for config in pipeline.worker_source_pipeline_configs:
+                    server_pipeline_config_ids.add(config.id)
+                    existing_config = local_pipeline_configs.get(config.id)
+                    config_changes = []
+
+                    if existing_config:
+                        if existing_config.worker_source_pipeline_id != config.worker_source_pipeline_id:
+                            config_changes.append(f"worker_source_pipeline_id: {existing_config.worker_source_pipeline_id} → {config.worker_source_pipeline_id}")
+                        if existing_config.pipeline_config_id != config.pipeline_config_id:
+                            config_changes.append(f"pipeline_config_id: {existing_config.pipeline_config_id} → {config.pipeline_config_id}")
+                        if existing_config.is_enabled != config.is_enabled:
+                            config_changes.append(f"is_enabled: {existing_config.is_enabled} → {config.is_enabled}")
+                        if existing_config.value != config.value:
+                            config_changes.append(f"value: '{existing_config.value}' → '{config.value}'")
+                        if existing_config.pipeline_config_name != config.pipeline_config.name:
+                            config_changes.append(f"pipeline_config_name: '{existing_config.pipeline_config_name}' → '{config.pipeline_config.name}'")
+                        if existing_config.pipeline_config_code != config.pipeline_config.code:
+                            config_changes.append(f"pipeline_config_code: '{existing_config.pipeline_config_code}' → '{config.pipeline_config.code}'")
+
+                        if config_changes:
+                            existing_config.worker_source_pipeline_id = config.worker_source_pipeline_id
+                            existing_config.pipeline_config_id = config.pipeline_config_id
+                            existing_config.is_enabled = config.is_enabled
+                            existing_config.value = config.value
+                            existing_config.pipeline_config_name = config.pipeline_config.name
+                            existing_config.pipeline_config_code = config.pipeline_config.code
+                            updated_pipeline_configs.append(existing_config)
+                            changed_configs.append(f"🔄 [APP] [UPDATE] Worker Source Pipeline Config ID {config.id}: " + ", ".join(config_changes))
+                    else:
+                        new_pipeline_configs.append(WorkerSourcePipelineConfigEntity(
+                            id=config.id,
+                            worker_source_pipeline_id=config.worker_source_pipeline_id,
+                            pipeline_config_id=config.pipeline_config_id,
+                            is_enabled=config.is_enabled,
+                            value=config.value,
+                            pipeline_config_name=config.pipeline_config.name,
+                            pipeline_config_code=config.pipeline_config.code
+                        ))
+                        logger.info(f"🆕 [APP] [INSERT] Added Worker Source Pipeline Config ID {config.id}")
+
+            self.session.commit()
+
+            for change in changed_pipelines:
+                logger.info(change)
+            for change in changed_configs:
+                logger.info(change)
+
+            # Identify and delete pipelines not in the server response
+            records_to_delete = [
+                pipeline for pipeline_id, pipeline in local_pipelines.items()
+                if pipeline_id not in server_pipeline_ids
+            ]
+
+            # Identify and delete pipeline configs not in the server response
+            configs_to_delete = [
+                config for config_id, config in local_pipeline_configs.items()
+                if config_id not in server_pipeline_config_ids
+            ]
+
+            # Perform batch insert, update, and delete in a single transaction
+            if new_pipelines:
+                self.session.bulk_save_objects(new_pipelines)  # Bulk insert
+
+            if updated_pipelines:
+                self.session.bulk_save_objects(updated_pipelines)  # Bulk update
+
+            if new_pipeline_configs:
+                self.session.bulk_save_objects(new_pipeline_configs)  # Bulk insert configs
+
+            if updated_pipeline_configs:
+                self.session.bulk_save_objects(updated_pipeline_configs)  # Bulk update configs
+
+            if records_to_delete:
+                for record in records_to_delete:
+                    self.session.delete(record)  # Mark for deletion
+                    logger.info(f"❌ [APP] [DELETE] Worker Source Pipeline ID {record.id} - {record.name}")
+
+            if configs_to_delete:
+                for config in configs_to_delete:
+                    self.session.delete(config)  # Mark for deletion
+
+            self.session.commit()  # Commit once (reducing DB round trips)
+        
+        except SQLAlchemyError as e:
+            self.session.rollback()
+            logger.error(f"❌ [APP] [DATABASE ERROR] Error during sync: {e}", exc_info=True)
+
+    def get_worker_source_pipelines(self):
+        try:
+            return self.session.query(WorkerSourcePipelineEntity).all()
+        except Exception as e:
+            logger.error(f"🚨 [APP] Database error while fetching worker source pipelines: {e}", exc_info=True)
+            return []
+        
+    def get_worker_source_pipeline(self, pipeline_id):
+        try:
+            return self.session.query(WorkerSourcePipelineEntity).filter_by(id=pipeline_id).first()
+        except Exception as e:
+            logger.error(f"🚨 [APP] Database error while fetching worker source pipeline: {e}", exc_info=True)