endoreg-db 0.4.5__py3-none-any.whl → 0.8.6.3__py3-none-any.whl

endoreg_db/services/polling_coordinator.py

@@ -0,0 +1,288 @@
+# endoreg_db/services/polling_coordinator.py
+
+import logging
+import threading
+from typing import Dict, Optional
+from django.core.cache import cache
+from django.utils import timezone
+from datetime import timedelta
+
+logger = logging.getLogger(__name__)
+
+class PollingCoordinator:
+    """
+    Service to prevent duplicate polling operations on the same media items.
+    Uses Django cache and thread-safe operations to coordinate polling requests.
+    """
+    
+    # Class-level lock for thread safety
+    _lock = threading.Lock()
+    
+    # Cache key prefixes
+    PROCESSING_PREFIX = "polling_processing:"
+    LAST_CHECK_PREFIX = "polling_last_check:"
+    
+    # Default timeouts
+    PROCESSING_TIMEOUT = 300  # 5 minutes
+    CHECK_COOLDOWN = 10       # 10 seconds minimum between checks
+    
+    @classmethod
+    def acquire_processing_lock(cls, file_id: int, file_type: str = "video", timeout: Optional[int] = None) -> bool:
+        """
+        Acquire a processing lock for a media file to prevent duplicate processing.
+        
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+            timeout: Lock timeout in seconds (default: 5 minutes)
+            
+        Returns:
+            True if lock acquired, False if already locked
+        """
+        if timeout is None:
+            timeout = cls.PROCESSING_TIMEOUT
+            
+        cache_key = f"{cls.PROCESSING_PREFIX}{file_type}:{file_id}"
+        
+        with cls._lock:
+            # Try to acquire lock atomically
+            lock_acquired = cache.add(cache_key, {
+                "locked_at": timezone.now().isoformat(),
+                "file_id": file_id,
+                "file_type": file_type,
+                "thread_id": threading.get_ident()
+            }, timeout)
+            
+            if lock_acquired:
+                logger.info(f"Processing lock acquired for {file_type}:{file_id}")
+                return True
+            else:
+                # Check if existing lock is stale
+                existing_lock = cache.get(cache_key)
+                if existing_lock:
+                    logger.warning(f"Processing lock already exists for {file_type}:{file_id}: {existing_lock}")
+                else:
+                    logger.warning(f"Failed to acquire processing lock for {file_type}:{file_id}")
+                return False
+    
+    @classmethod
+    def release_processing_lock(cls, file_id: int, file_type: str = "video") -> bool:
+        """
+        Release a processing lock for a media file.
+        
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+            
+        Returns:
+            True if lock released, False if lock didn't exist
+        """
+        cache_key = f"{cls.PROCESSING_PREFIX}{file_type}:{file_id}"
+        
+        with cls._lock:
+            if cache.delete(cache_key):
+                logger.info(f"Processing lock released for {file_type}:{file_id}")
+                return True
+            else:
+                logger.warning(f"No processing lock found to release for {file_type}:{file_id}")
+                return False
+    
+    @classmethod
+    def is_processing_locked(cls, file_id: int, file_type: str = "video") -> bool:
+        """
+        Check if a media file is currently processing locked.
+        
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+            
+        Returns:
+            True if locked, False otherwise
+        """
+        cache_key = f"{cls.PROCESSING_PREFIX}{file_type}:{file_id}"
+        return cache.get(cache_key) is not None
+    
+    @classmethod
+    def can_check_status(cls, file_id: int, file_type: str = "video") -> bool:
+        """
+        Check if enough time has passed since last status check to prevent spam.
+        
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+            
+        Returns:
+            True if status check is allowed, False if still in cooldown
+        """
+        cache_key = f"{cls.LAST_CHECK_PREFIX}{file_type}:{file_id}"
+        last_check = cache.get(cache_key)
+        
+        if last_check is None:
+            # First check or cooldown expired - allowed
+            cls._record_status_check(file_id, file_type)
+            return True
+        
+        # Check if cooldown period has passed
+        last_check_time = timezone.datetime.fromisoformat(last_check.replace('Z', '+00:00'))
+        cooldown_end = last_check_time + timedelta(seconds=cls.CHECK_COOLDOWN)
+        
+        if timezone.now() > cooldown_end:
+            cls._record_status_check(file_id, file_type)
+            return True
+        else:
+            remaining_cooldown = (cooldown_end - timezone.now()).total_seconds()
+            logger.debug(f"Status check cooldown active for {file_type}:{file_id}, {remaining_cooldown:.1f}s remaining")
+            return False
+    
+    @classmethod
+    def get_remaining_cooldown_seconds(cls, file_id: int, file_type: str = "video") -> int:
+        """
+        Get the remaining cooldown seconds for a status check.
+        
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+            
+        Returns:
+            Remaining cooldown in seconds (0 if no cooldown active)
+        """
+        cache_key = f"{cls.LAST_CHECK_PREFIX}{file_type}:{file_id}"
+        last_check = cache.get(cache_key)
+        
+        if last_check is None:
+            return 0
+        
+        # Check if cooldown period has passed
+        last_check_time = timezone.datetime.fromisoformat(last_check.replace('Z', '+00:00'))
+        cooldown_end = last_check_time + timedelta(seconds=cls.CHECK_COOLDOWN)
+        
+        if timezone.now() > cooldown_end:
+            return 0
+        else:
+            remaining_cooldown = (cooldown_end - timezone.now()).total_seconds()
+            # Round up to at least 1 second
+            return max(1, int(remaining_cooldown) + 1)
+
+    @classmethod
+    def _record_status_check(cls, file_id: int, file_type: str = "video") -> None:
+        """Record the time of a status check"""
+        cache_key = f"{cls.LAST_CHECK_PREFIX}{file_type}:{file_id}"
+        cache.set(cache_key, timezone.now().isoformat(), cls.CHECK_COOLDOWN + 5)
+    
+    @classmethod
+    def get_processing_locks_info(cls) -> Dict[str, any]:
+        """
+        Get information about all currently active processing locks.
+        Useful for debugging and monitoring.
+        
+        Returns:
+            Dictionary with lock information
+        """
+        # Note: This is a simplified version since Django cache doesn't support pattern scanning
+        # In production, consider using Redis with SCAN command for better performance
+        
+        info = {
+            "coordinator_status": "active",
+            "config": {
+                "processing_timeout": cls.PROCESSING_TIMEOUT,
+                "check_cooldown": cls.CHECK_COOLDOWN
+            },
+            "note": "Active locks info requires Redis backend for pattern scanning"
+        }
+        
+        return info
+    
+    @classmethod
+    def clear_all_locks(cls, file_type: Optional[str] = None) -> int:
+        """
+        Emergency function to clear all processing locks.
+        Use with caution - only for debugging/recovery scenarios.
+        
+        Args:
+            file_type: Optionally clear locks only for specific file type
+            
+        Returns:
+            Number of locks cleared (approximation)
+        """
+        logger.warning("clear_all_locks called - this should only be used for debugging/recovery")
+        
+        # This is a simplified implementation
+        # In production with Redis, you'd use SCAN to find and delete matching keys
+        if hasattr(cache, 'clear'):
+            cache.clear()
+            return -1  # Unknown count
+        else:
+            logger.warning("Cache backend doesn't support pattern deletion")
+            return 0
+
+
+# Decorator for views that need processing coordination
+def processing_coordination(file_id_param: str = "file_id", file_type: str = "video"):
+    """
+    Decorator to add automatic processing coordination to views.
+    
+    Args:
+        file_id_param: Name of the parameter containing file ID
+        file_type: Type of media file
+    """
+    def decorator(view_func):
+        def wrapper(request, *args, **kwargs):
+            # Extract file_id from kwargs or request
+            file_id = kwargs.get(file_id_param) or request.data.get(file_id_param)
+            
+            if file_id is None:
+                logger.error(f"No {file_id_param} found in request for processing coordination")
+                from rest_framework.response import Response
+                from rest_framework import status
+                return Response(
+                    {"error": f"Missing {file_id_param} parameter"}, 
+                    status=status.HTTP_400_BAD_REQUEST
+                )
+            
+            # Check if processing is already locked
+            if PollingCoordinator.is_processing_locked(file_id, file_type):
+                from rest_framework.response import Response
+                from rest_framework import status
+                return Response(
+                    {"detail": "File is currently being processed by another request"}, 
+                    status=status.HTTP_409_CONFLICT
+                )
+            
+            # Proceed with the view
+            return view_func(request, *args, **kwargs)
+        
+        return wrapper
+    return decorator
+
+
+# Context manager for automatic lock management
+class ProcessingLockContext:
+    """
+    Context manager for automatic processing lock acquisition and release.
+    
+    Usage:
+        with ProcessingLockContext(file_id, "video") as lock:
+            if lock.acquired:
+                # Perform processing
+                pass
+            else:
+                # Handle lock acquisition failure
+                pass
+    """
+    
+    def __init__(self, file_id: int, file_type: str = "video", timeout: Optional[int] = None):
+        self.file_id = file_id
+        self.file_type = file_type
+        self.timeout = timeout
+        self.acquired = False
+    
+    def __enter__(self):
+        self.acquired = PollingCoordinator.acquire_processing_lock(
+            self.file_id, self.file_type, self.timeout
+        )
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.acquired:
+            PollingCoordinator.release_processing_lock(self.file_id, self.file_type)
+        return False  # Don't suppress exceptions

endoreg_db/services/pseudonym_service.py

@@ -0,0 +1,89 @@
+"""
+Service for generating patient pseudonyms using SensitiveMeta logic.
+"""
+import logging
+from typing import Tuple, List
+from django.db import transaction
+
+from endoreg_db.models.administration.person.patient.patient import Patient
+from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
+from endoreg_db.models.metadata import sensitive_meta_logic as logic
+
+logger = logging.getLogger(__name__)
+
+
+def generate_patient_pseudonym(patient: Patient) -> Tuple[str, bool]:
+    """
+    Generate a pseudonym hash for an existing Patient using SensitiveMeta logic.
+    
+    Args:
+        patient: The Patient instance to generate a pseudonym for
+        
+    Returns:
+        Tuple of (patient_hash, persisted_flag)
+        
+    Raises:
+        ValueError: If required fields are missing for hash calculation
+    """
+    # Validate required fields for hash calculation
+    if not patient.dob:
+        raise ValueError("Patient date of birth (dob) is required for pseudonym generation")
+    
+    if not patient.center:
+        raise ValueError("Patient center is required for pseudonym generation")
+    
+    # Use existing patient_hash if it exists and is valid
+    if patient.patient_hash and len(patient.patient_hash.strip()) > 0:
+        logger.info(f"Patient {patient.id} already has a hash: {patient.patient_hash}")
+        return patient.patient_hash, True
+    
+    # Create a transient SensitiveMeta instance to calculate the hash
+    # We don't save this instance, just use it for hash calculation
+    sensitive_meta = SensitiveMeta(
+        patient_first_name=patient.first_name or "",
+        patient_last_name=patient.last_name or "",
+        patient_dob=patient.dob,
+        center=patient.center,
+        patient_gender=patient.gender  # Optional, but include if available
+    )
+    
+    try:
+        # Calculate the hash using the existing logic
+        patient_hash = logic.calculate_patient_hash(sensitive_meta, salt=logic.SECRET_SALT)
+        
+        # Persist the hash to the Patient model
+        with transaction.atomic():
+            patient.patient_hash = patient_hash
+            patient.save(update_fields=['patient_hash'])
+        
+        logger.info(f"Generated and persisted pseudonym for patient {patient.id}")
+        
+        return patient_hash, True
+        
+    except Exception as e:
+        logger.error(f"Error generating pseudonym for patient {patient.id}: {str(e)}")
+        raise ValueError(f"Failed to generate pseudonym: {str(e)}")
+
+
+def validate_patient_for_pseudonym(patient: Patient) -> list[str]:
+    """
+    Validate that a patient has all required fields for pseudonym generation.
+    
+    Args:
+        patient: The Patient instance to validate
+        
+    Returns:
+        List of missing required fields (empty if all fields present)
+    """
+    missing_fields = []
+    
+    if not patient.dob:
+        missing_fields.append("dob")
+    
+    if not patient.center:
+        missing_fields.append("center")
+    
+    # Note: first_name and last_name can be empty strings according to the logic,
+    # so we don't require them to be non-empty
+    
+    return missing_fields

endoreg_db/services/requirements_object.py

@@ -0,0 +1,147 @@
+from ast import List
+import pickle
+from endoreg_db.models import Requirement, Examination, PatientExamination
+from endoreg_db.serializers import PatientExaminationSerializer, ExaminationSerializer, FindingClassificationSerializer, RequirementSerializer
+
+
+class LookupService:
+    def __init__(self, db_session):
+        self.db_session = db_session
+        self.hexdict = int()
+
+    def hash_lookup(self, key, value):
+        """
+        Store a key-value pair in the lookup dictionary.
+        """
+        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
+        lookup_dict[key] = value
+        self.hexdict = pickle.dumps(dict).hex()
+        return True
+    def hash_retrieve(self, key):
+        """
+        Retrieve a value by key from the lookup dictionary.
+        """
+        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
+        return lookup_dict.get(key, None)
+    
+    def hash_add_key_value(self, key, value):
+        """
+        Add a key-value pair to the lookup dictionary.
+        """
+        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
+        lookup_dict[key] = value
+        self.hexdict = pickle.dumps(lookup_dict).hex()
+        return True
+    
+    def hash_dehash(self):
+        """
+        Retrieve the entire lookup dictionary.
+        """
+        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
+        return lookup_dict
+    
+    def fetch_examination_dict(self, patient_id, tags: List[str] = None):
+        """
+        Fetch examinations for a given patient ID, optionally filtered by tags.
+        """
+        query = self.db_session.query(Examination).filter(Examination.patient_id == patient_id)
+        if tags:
+            query = query.filter(Examination.tags.any(Examination.tags.in_(tags)))
+        examinations = query.all()
+        return [ExaminationSerializer(exam).data for exam in examinations]  
+    
+    def fetch_patient_examination_by_id(self, patient_id, examination_id):
+        """
+        Fetch a patient examination object from the database by its ID.
+        """
+        return self.db_session.query(PatientExamination).filter(PatientExamination.id == patient_examination_id).first()
+    
+    def fetch_patient_examinations(self, patient_id):
+        """
+        Fetch all patient examinations for a given patient ID.
+        """
+        examinations = self.db_session.query(PatientExamination).filter(PatientExamination.patient_id == patient_id).all()
+        return [PatientExaminationSerializer(exam).data for exam in examinations]
+
+    def fetch_findings_by_patient_examination(self, patient_examination_id):
+        """
+        Fetch findings associated with a given patient examination ID.
+        """
+        patient_examination = self.db_session.query(PatientExamination).filter(PatientExamination.id == patient_examination_id).first()
+        if not patient_examination:
+            return []
+        return [finding.serialize() for finding in patient_examination.findings]
+    
+    def fetch_classifications_by_finding(self, finding_id):
+        """
+        Fetch classifications associated with a given finding ID.
+        """
+        finding = self.db_session.query(Finding).filter(Finding.id == finding_id).first()
+        if not finding:
+            return []
+        return [classification.serialize() for classification in finding.classifications]
+    
+    def fetch_default_finding_classification(self):
+        """
+        Fetch the default finding classification from the database.
+        """
+        return self.db_session.query(FindingClassification).filter(FindingClassification.is_default == True).first()
+    
+    def fetch_finding_classification_choice_by_requirement(self, requirement_id):
+        """
+        Retrieve finding classification choices based on the provided requirement ID.
+        """
+        requirement = self.get_requirement_by_id(requirement_id)
+        if not requirement:
+            return []
+        return [choice.serialize() for choice in requirement.finding_classification_choices]
+    
+    def fetch_requirement_choices_by_requirement(self, requirement_id):
+        """
+        Retrieve requirement choices based on the provided requirement ID.
+        """
+        requirement = self.get_requirement_by_id(requirement_id)
+        if not requirement:
+            return []
+        return [choice.serialize() for choice in requirement.requirement_choices]
+    
+    def available_finding_classification_choices_by_finding_classification(self, finding_classification):
+        """
+        Retrieve available finding classification choices based on the provided finding classification.
+        """
+        return self.db_session.query(FindingClassificationChoice).filter(
+            FindingClassificationChoice.finding_classification_id == finding_classification.id
+        ).all()
+
+    def get_requirement_by_id(self, requirement_id):
+        """
+        Fetch a requirement object from the database by its ID.
+        """
+        return self.db_session.query(Requirement).filter(Requirement.id == requirement_id).first()
+
+    def get_all_requirements_for_requirement_sets(self, requirement_set_ids: List[int]):
+        """
+        Fetch all requirements associated with the given requirement set IDs.
+        """
+        return self.db_session.query(Requirement).filter(Requirement.requirement_set_id.in_(requirement_set_ids)).all()
+    
+    def get_all_requirement_sets_for_examination(self, examination_id):
+        """
+        Fetch all requirement sets associated with the given examination ID.
+        """
+        examination = self.db_session.query(Examination).filter(Examination.id == examination_id).first()
+        if not examination:
+            return []
+        return [req_set.serialize() for req_set in examination.requirement_sets]
+
+    def serialize_requirement(self, requirement):
+        """
+        Serialize a requirement object to a byte stream.
+        """
+        return pickle.dumps(requirement)
+
+    def deserialize_requirement(self, serialized_requirement):
+        """
+        Deserialize a byte stream back to a requirement object.
+        """
+        return pickle.loads(serialized_requirement)

endoreg_db/services/segment_sync.py

@@ -0,0 +1,155 @@
+"""
+Service for synchronizing annotation updates with LabelVideoSegment creation.
+
+This module provides functionality to create user-source LabelVideoSegments
+when segment annotations are created or updated.
+"""
+
+import logging
+from typing import Optional, Dict, Any
+from django.contrib.auth.models import User
+from django.db import transaction
+
+from ..models import VideoFile, Label, LabelVideoSegment, InformationSource
+
+logger = logging.getLogger(__name__)
+
+
+def create_user_segment_from_annotation(annotation: Dict[str, Any], request_user: User) -> Optional[LabelVideoSegment]:
+    """
+    Create a user-source LabelVideoSegment from a segment annotation.
+    
+    This function:
+    1. Locates the original LabelVideoSegment (if segmentId is present)
+    2. Clones all its DB fields 
+    3. Overwrites with new data from annotation
+    4. Sets information_source = user
+    5. Saves via model manager
+    
+    Args:
+        annotation: Annotation data containing segment information
+        request_user: The authenticated user making the request
+        
+    Returns:
+        New LabelVideoSegment instance or None if creation failed/skipped
+    """
+    # Validate this is a segment annotation
+    if annotation.get('type') != 'segment':
+        logger.debug("Annotation is not a segment type, skipping user segment creation")
+        return None
+    
+    # Get required data from annotation
+    video_id = annotation.get('videoId')
+    start_time = annotation.get('startTime')
+    end_time = annotation.get('endTime')
+    label_text = annotation.get('text', '').strip()
+    metadata = annotation.get('metadata', {})
+    original_segment_id = metadata.get('segmentId')
+    
+    if not all([video_id, start_time is not None, end_time is not None]):
+        logger.warning("Missing required segment data in annotation, skipping user segment creation")
+        return None
+    
+    try:
+        # Get the video file
+        video_file = VideoFile.objects.get(id=video_id)
+        
+        # Get video FPS for frame calculation
+        fps = video_file.get_fps()
+        if not fps or fps <= 0:
+            logger.warning(f"Invalid FPS ({fps}) for video {video_id}, using default 25")
+            fps = 25.0
+        
+        # Calculate frame numbers
+        start_frame_number = int(start_time * fps)
+        end_frame_number = int(end_time * fps)
+        
+        # Get or create user information source
+        user_source, _ = InformationSource.objects.get_or_create(
+            name="user",
+            defaults={'description': 'User-generated annotations'}
+        )
+        
+        # Try to find label by name from annotation text
+        label = None
+        if label_text:
+            try:
+                label = Label.objects.filter(name__iexact=label_text).first()
+                if not label:
+                    # Try to extract label from tags
+                    tags = annotation.get('tags', [])
+                    for tag in tags:
+                        label = Label.objects.filter(name__iexact=tag).first()
+                        if label:
+                            break
+            except Exception as e:
+                logger.warning(f"Error finding label '{label_text}': {e}")
+        
+        # Start with default segment data
+        segment_data = {
+            'video_file': video_file,
+            'start_frame_number': start_frame_number,
+            'end_frame_number': end_frame_number,
+            'source': user_source,
+            'label': label,
+            'prediction_meta': None,  # User segments don't have prediction meta
+        }
+        
+        # If original segment ID is provided, try to clone from original
+        if original_segment_id:
+            try:
+                original_segment = LabelVideoSegment.objects.get(id=original_segment_id)
+                
+                # Check if timing or label actually changed
+                original_start_time = original_segment.start_frame_number / fps
+                original_end_time = original_segment.end_frame_number / fps
+                
+                timing_changed = (
+                    abs(original_start_time - start_time) > 0.1 or  # Allow small floating point differences
+                    abs(original_end_time - end_time) > 0.1
+                )
+                
+                label_changed = (
+                    (label and original_segment.label != label) or
+                    (not label and original_segment.label is not None)
+                )
+                
+                # Only create new segment if something actually changed
+                if not timing_changed and not label_changed:
+                    logger.debug(f"No changes detected in segment {original_segment_id}, skipping user segment creation")
+                    return None
+                
+                # Clone relevant fields from original segment
+                segment_data.update({
+                    'prediction_meta': original_segment.prediction_meta,
+                    'label': label or original_segment.label,  # Use new label if provided, otherwise keep original
+                })
+                
+                logger.info(f"Cloning segment {original_segment_id} with user modifications")
+                
+            except LabelVideoSegment.DoesNotExist:
+                logger.warning(f"Original segment {original_segment_id} not found, creating new user segment")
+        
+        # Create the new user segment using the manager
+        with transaction.atomic():
+            new_segment = LabelVideoSegment.create_from_video(
+                source=video_file,
+                prediction_meta=segment_data.get('prediction_meta'),
+                label=segment_data['label'],
+                start_frame_number=start_frame_number,
+                end_frame_number=end_frame_number,
+            )
+            
+            # Set the user information source
+            new_segment.source = user_source
+            new_segment.save()
+            
+            logger.info(f"Created user segment {new_segment.id} for video {video_id} by user {request_user.username}")
+            return new_segment
+            
+    except VideoFile.DoesNotExist:
+        logger.error(f"Video {video_id} not found, cannot create user segment")
+        return None
+    except Exception as e:
+        logger.error(f"Error creating user segment from annotation: {e}")
+        return None