endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/services/model_meta_from_hf.py

@@ -0,0 +1,76 @@
+# endoreg_db/services/model_meta_from_hf.py
+
+from django.core.files.base import ContentFile
+from huggingface_hub import hf_hub_download
+
+from endoreg_db.models import AiModel, LabelSet, ModelMeta
+
+
+def ensure_model_meta_from_hf(
+    *,
+    model_id: str,
+    model_name: str,
+    labelset_name: str,
+    meta_version: str = "1",
+    labelset_version: int | None = None,
+) -> ModelMeta:
+    """
+    Download weights from Hugging Face (if needed) and ensure a ModelMeta
+    exists for the given configuration. Returns the ModelMeta.
+    """
+    # Download the model weights
+    weights_path = hf_hub_download(
+        repo_id=model_id,
+        filename="colo_segmentation_RegNetX800MF_base.safetensors",
+        local_dir="/tmp",
+    )
+
+    # Get or create AI model
+    ai_model, _ = AiModel.objects.get_or_create(
+        name=model_name, defaults={"description": f"Model from {model_id}"}
+    )
+
+    # Get labelset
+    labelset_qs = LabelSet.objects.filter(name=labelset_name)
+    if labelset_version is not None:
+        labelset_qs = labelset_qs.filter(version=labelset_version)
+    labelset = labelset_qs.order_by("-version").first()
+    if labelset is None:
+        raise ValueError(
+            f"LabelSet '{labelset_name}'"
+            + (f" v{labelset_version}" if labelset_version is not None else "")
+            + " not found"
+        )
+
+    # Create or get ModelMeta
+    model_meta, _ = ModelMeta.objects.get_or_create(
+        name=model_name,
+        model=ai_model,
+        version=meta_version,
+        defaults={
+            "labelset": labelset,
+            "activation": "sigmoid",
+            "mean": "0.45211223,0.27139644,0.19264949",
+            "std": "0.31418097,0.21088019,0.16059452",
+            "size_x": 716,
+            "size_y": 716,
+            "axes": "2,0,1",
+            "batchsize": 16,
+            "num_workers": 0,
+            "description": f"Downloaded from {model_id}",
+        },
+    )
+
+    # If weights file not yet saved, save it
+    if not model_meta.weights:
+        with open(weights_path, "rb") as f:
+            model_meta.weights.save(
+                f"{model_name}_v{meta_version}.safetensors",
+                ContentFile(f.read()),
+            )
+
+    # Set as active meta
+    ai_model.active_meta = model_meta
+    ai_model.save(update_fields=["active_meta"])
+
+    return model_meta

endoreg_db/services/polling_coordinator.py

@@ -0,0 +1,319 @@
+# endoreg_db/services/polling_coordinator.py
+
+import logging
+import threading
+from typing import Dict, Optional
+from django.core.cache import cache
+from django.utils import timezone
+from datetime import timedelta
+
+logger = logging.getLogger(__name__)
+
+
+class PollingCoordinator:
+    """
+    Service to prevent duplicate polling operations on the same media items.
+    Uses Django cache and thread-safe operations to coordinate polling requests.
+    """
+
+    # Class-level lock for thread safety
+    _lock = threading.Lock()
+
+    # Cache key prefixes
+    PROCESSING_PREFIX = "polling_processing:"
+    LAST_CHECK_PREFIX = "polling_last_check:"
+
+    # Default timeouts
+    PROCESSING_TIMEOUT = 300  # 5 minutes
+    CHECK_COOLDOWN = 10  # 10 seconds minimum between checks
+
+    @classmethod
+    def acquire_processing_lock(
+        cls, file_id: int, file_type: str = "video", timeout: Optional[int] = None
+    ) -> bool:
+        """
+        Acquire a processing lock for a media file to prevent duplicate processing.
+
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+            timeout: Lock timeout in seconds (default: 5 minutes)
+
+        Returns:
+            True if lock acquired, False if already locked
+        """
+        if timeout is None:
+            timeout = cls.PROCESSING_TIMEOUT
+
+        cache_key = f"{cls.PROCESSING_PREFIX}{file_type}:{file_id}"
+
+        with cls._lock:
+            # Try to acquire lock atomically
+            lock_acquired = cache.add(
+                cache_key,
+                {
+                    "locked_at": timezone.now().isoformat(),
+                    "file_id": file_id,
+                    "file_type": file_type,
+                    "thread_id": threading.get_ident(),
+                },
+                timeout,
+            )
+
+            if lock_acquired:
+                logger.info(f"Processing lock acquired for {file_type}:{file_id}")
+                return True
+            else:
+                # Check if existing lock is stale
+                existing_lock = cache.get(cache_key)
+                if existing_lock:
+                    logger.warning(
+                        f"Processing lock already exists for {file_type}:{file_id}: {existing_lock}"
+                    )
+                else:
+                    logger.warning(
+                        f"Failed to acquire processing lock for {file_type}:{file_id}"
+                    )
+                return False
+
+    @classmethod
+    def release_processing_lock(cls, file_id: int, file_type: str = "video") -> bool:
+        """
+        Release a processing lock for a media file.
+
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+
+        Returns:
+            True if lock released, False if lock didn't exist
+        """
+        cache_key = f"{cls.PROCESSING_PREFIX}{file_type}:{file_id}"
+
+        with cls._lock:
+            if cache.delete(cache_key):
+                logger.info(f"Processing lock released for {file_type}:{file_id}")
+                return True
+            else:
+                logger.warning(
+                    f"No processing lock found to release for {file_type}:{file_id}"
+                )
+                return False
+
+    @classmethod
+    def is_processing_locked(cls, file_id: int, file_type: str = "video") -> bool:
+        """
+        Check if a media file is currently processing locked.
+
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+
+        Returns:
+            True if locked, False otherwise
+        """
+        cache_key = f"{cls.PROCESSING_PREFIX}{file_type}:{file_id}"
+        return cache.get(cache_key) is not None
+
+    @classmethod
+    def can_check_status(cls, file_id: int, file_type: str = "video") -> bool:
+        """
+        Check if enough time has passed since last status check to prevent spam.
+
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+
+        Returns:
+            True if status check is allowed, False if still in cooldown
+        """
+        cache_key = f"{cls.LAST_CHECK_PREFIX}{file_type}:{file_id}"
+        last_check = cache.get(cache_key)
+
+        if last_check is None:
+            # First check or cooldown expired - allowed
+            cls._record_status_check(file_id, file_type)
+            return True
+
+        # Check if cooldown period has passed
+        last_check_time = timezone.datetime.fromisoformat(
+            last_check.replace("Z", "+00:00")
+        )
+        cooldown_end = last_check_time + timedelta(seconds=cls.CHECK_COOLDOWN)
+
+        if timezone.now() > cooldown_end:
+            cls._record_status_check(file_id, file_type)
+            return True
+        else:
+            remaining_cooldown = (cooldown_end - timezone.now()).total_seconds()
+            logger.debug(
+                f"Status check cooldown active for {file_type}:{file_id}, {remaining_cooldown:.1f}s remaining"
+            )
+            return False
+
+    @classmethod
+    def get_remaining_cooldown_seconds(
+        cls, file_id: int, file_type: str = "video"
+    ) -> int:
+        """
+        Get the remaining cooldown seconds for a status check.
+
+        Args:
+            file_id: ID of the media file
+            file_type: Type of media (video, pdf)
+
+        Returns:
+            Remaining cooldown in seconds (0 if no cooldown active)
+        """
+        cache_key = f"{cls.LAST_CHECK_PREFIX}{file_type}:{file_id}"
+        last_check = cache.get(cache_key)
+
+        if last_check is None:
+            return 0
+
+        # Check if cooldown period has passed
+        last_check_time = timezone.datetime.fromisoformat(
+            last_check.replace("Z", "+00:00")
+        )
+        cooldown_end = last_check_time + timedelta(seconds=cls.CHECK_COOLDOWN)
+
+        if timezone.now() > cooldown_end:
+            return 0
+        else:
+            remaining_cooldown = (cooldown_end - timezone.now()).total_seconds()
+            # Round up to at least 1 second
+            return max(1, int(remaining_cooldown) + 1)
+
+    @classmethod
+    def _record_status_check(cls, file_id: int, file_type: str = "video") -> None:
+        """Record the time of a status check"""
+        cache_key = f"{cls.LAST_CHECK_PREFIX}{file_type}:{file_id}"
+        cache.set(cache_key, timezone.now().isoformat(), cls.CHECK_COOLDOWN + 5)
+
+    @classmethod
+    def get_processing_locks_info(cls) -> Dict[str, any]:
+        """
+        Get information about all currently active processing locks.
+        Useful for debugging and monitoring.
+
+        Returns:
+            Dictionary with lock information
+        """
+        # Note: This is a simplified version since Django cache doesn't support pattern scanning
+        # In production, consider using Redis with SCAN command for better performance
+
+        info = {
+            "coordinator_status": "active",
+            "config": {
+                "processing_timeout": cls.PROCESSING_TIMEOUT,
+                "check_cooldown": cls.CHECK_COOLDOWN,
+            },
+            "note": "Active locks info requires Redis backend for pattern scanning",
+        }
+
+        return info
+
+    @classmethod
+    def clear_all_locks(cls, file_type: Optional[str] = None) -> int:
+        """
+        Emergency function to clear all processing locks.
+        Use with caution - only for debugging/recovery scenarios.
+
+        Args:
+            file_type: Optionally clear locks only for specific file type
+
+        Returns:
+            Number of locks cleared (approximation)
+        """
+        logger.warning(
+            "clear_all_locks called - this should only be used for debugging/recovery"
+        )
+
+        # This is a simplified implementation
+        # In production with Redis, you'd use SCAN to find and delete matching keys
+        if hasattr(cache, "clear"):
+            cache.clear()
+            return -1  # Unknown count
+        else:
+            logger.warning("Cache backend doesn't support pattern deletion")
+            return 0
+
+
+# Decorator for views that need processing coordination
+def processing_coordination(file_id_param: str = "file_id", file_type: str = "video"):
+    """
+    Decorator to add automatic processing coordination to views.
+
+    Args:
+        file_id_param: Name of the parameter containing file ID
+        file_type: Type of media file
+    """
+
+    def decorator(view_func):
+        def wrapper(request, *args, **kwargs):
+            # Extract file_id from kwargs or request
+            file_id = kwargs.get(file_id_param) or request.data.get(file_id_param)
+
+            if file_id is None:
+                logger.error(
+                    f"No {file_id_param} found in request for processing coordination"
+                )
+                from rest_framework.response import Response
+                from rest_framework import status
+
+                return Response(
+                    {"error": f"Missing {file_id_param} parameter"},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+
+            # Check if processing is already locked
+            if PollingCoordinator.is_processing_locked(file_id, file_type):
+                from rest_framework.response import Response
+                from rest_framework import status
+
+                return Response(
+                    {"detail": "File is currently being processed by another request"},
+                    status=status.HTTP_409_CONFLICT,
+                )
+
+            # Proceed with the view
+            return view_func(request, *args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+# Context manager for automatic lock management
+class ProcessingLockContext:
+    """
+    Context manager for automatic processing lock acquisition and release.
+
+    Usage:
+        with ProcessingLockContext(file_id, "video") as lock:
+            if lock.acquired:
+                # Perform processing
+                pass
+            else:
+                # Handle lock acquisition failure
+                pass
+    """
+
+    def __init__(
+        self, file_id: int, file_type: str = "video", timeout: Optional[int] = None
+    ):
+        self.file_id = file_id
+        self.file_type = file_type
+        self.timeout = timeout
+        self.acquired = False
+
+    def __enter__(self):
+        self.acquired = PollingCoordinator.acquire_processing_lock(
+            self.file_id, self.file_type, self.timeout
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.acquired:
+            PollingCoordinator.release_processing_lock(self.file_id, self.file_type)
+        return False  # Don't suppress exceptions

endoreg_db/services/pseudonym_service.py

@@ -0,0 +1,94 @@
+"""
+Service for generating patient pseudonyms using SensitiveMeta logic.
+"""
+
+import logging
+from typing import Tuple
+from django.db import transaction
+
+from endoreg_db.models.administration.person.patient.patient import Patient
+from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
+from endoreg_db.models.metadata import sensitive_meta_logic as logic
+
+logger = logging.getLogger(__name__)
+
+
+def generate_patient_pseudonym(patient: Patient) -> Tuple[str, bool]:
+    """
+    Generate a pseudonym hash for an existing Patient using SensitiveMeta logic.
+
+    Args:
+        patient: The Patient instance to generate a pseudonym for
+
+    Returns:
+        Tuple of (patient_hash, persisted_flag)
+
+    Raises:
+        ValueError: If required fields are missing for hash calculation
+    """
+    # Validate required fields for hash calculation
+    if not patient.dob:
+        raise ValueError(
+            "Patient date of birth (dob) is required for pseudonym generation"
+        )
+
+    if not patient.center:
+        raise ValueError("Patient center is required for pseudonym generation")
+
+    # Use existing patient_hash if it exists and is valid
+    if patient.patient_hash and len(patient.patient_hash.strip()) > 0:
+        logger.info(f"Patient {patient.id} already has a hash: {patient.patient_hash}")
+        return patient.patient_hash, True
+
+    # Create a transient SensitiveMeta instance to calculate the hash
+    # We don't save this instance, just use it for hash calculation
+    sensitive_meta = SensitiveMeta(
+        patient_first_name=patient.first_name or "",
+        patient_last_name=patient.last_name or "",
+        patient_dob=patient.dob,
+        center=patient.center,
+        patient_gender=patient.gender,  # Optional, but include if available
+    )
+
+    try:
+        # Calculate the hash using the existing logic
+        patient_hash = logic.calculate_patient_hash(
+            sensitive_meta, salt=logic.SECRET_SALT
+        )
+
+        # Persist the hash to the Patient model
+        with transaction.atomic():
+            patient.patient_hash = patient_hash
+            patient.save(update_fields=["patient_hash"])
+
+        logger.info(f"Generated and persisted pseudonym for patient {patient.id}")
+
+        return patient_hash, True
+
+    except Exception as e:
+        logger.error(f"Error generating pseudonym for patient {patient.id}: {str(e)}")
+        raise ValueError(f"Failed to generate pseudonym: {str(e)}")
+
+
+def validate_patient_for_pseudonym(patient: Patient) -> list[str]:
+    """
+    Validate that a patient has all required fields for pseudonym generation.
+
+    Args:
+        patient: The Patient instance to validate
+
+    Returns:
+        List of missing required fields (empty if all fields present)
+    """
+    missing_fields = []
+
+    if not patient.dob:
+        missing_fields.append("dob")
+
+    if not patient.center:
+        missing_fields.append("center")
+
+    # Note: first_name and last_name can be empty strings according to the logic,
+    # so we don't require them to be non-empty
+
+    return missing_fields

endoreg_db/services/report_import.py

@@ -0,0 +1,13 @@
+from endoreg_db.import_files.report_import_service import ReportImportService as ris
+
+
+class ReportImportService(ris):
+    __doc__ = (
+        (ris.__doc__ or "")
+        + """
+    Wrapper for known import of concurrency safe ReportImportService, 
+    here celery will be implemented in the future.
+    12.02.2025
+    """
+    )
+    pass

endoreg_db/services/segment_sync.py

@@ -0,0 +1,171 @@
+"""
+Service for synchronizing annotation updates with LabelVideoSegment creation.
+
+This module provides functionality to create user-source LabelVideoSegments
+when segment annotations are created or updated.
+"""
+
+import logging
+from typing import Optional, Dict, Any
+from django.contrib.auth.models import User
+from django.db import transaction
+
+from ..models import VideoFile, Label, LabelVideoSegment, InformationSource
+
+logger = logging.getLogger(__name__)
+
+
+def create_user_segment_from_annotation(
+    annotation: Dict[str, Any], request_user: User
+) -> Optional[LabelVideoSegment]:
+    """
+    Create a user-source LabelVideoSegment from a segment annotation.
+
+    This function:
+    1. Locates the original LabelVideoSegment (if segmentId is present)
+    2. Clones all its DB fields
+    3. Overwrites with new data from annotation
+    4. Sets information_source = user
+    5. Saves via model manager
+
+    Args:
+        annotation: Annotation data containing segment information
+        request_user: The authenticated user making the request
+
+    Returns:
+        New LabelVideoSegment instance or None if creation failed/skipped
+    """
+    # Validate this is a segment annotation
+    if annotation.get("type") != "segment":
+        logger.debug("Annotation is not a segment type, skipping user segment creation")
+        return None
+
+    # Get required data from annotation
+    video_id = annotation.get("videoId")
+    start_time = annotation.get("startTime")
+    end_time = annotation.get("endTime")
+    label_text = annotation.get("text", "").strip()
+    metadata = annotation.get("metadata", {})
+    original_segment_id = metadata.get("segmentId")
+
+    if not all([video_id, start_time is not None, end_time is not None]):
+        logger.warning(
+            "Missing required segment data in annotation, skipping user segment creation"
+        )
+        return None
+
+    try:
+        # Get the video file
+        video_file = VideoFile.objects.get(id=video_id)
+
+        # Get video FPS for frame calculation
+        fps = video_file.get_fps()
+        if not fps or fps <= 0:
+            logger.warning(
+                f"Invalid FPS ({fps}) for video {video_id}, using default 25"
+            )
+            fps = 25.0
+
+        # Calculate frame numbers
+        start_frame_number = int(start_time * fps)
+        end_frame_number = int(end_time * fps)
+
+        # Get or create user information source
+        user_source, _ = InformationSource.objects.get_or_create(
+            name="user", defaults={"description": "User-generated annotations"}
+        )
+
+        # Try to find label by name from annotation text
+        label = None
+        if label_text:
+            try:
+                label = Label.objects.filter(name__iexact=label_text).first()
+                if not label:
+                    # Try to extract label from tags
+                    tags = annotation.get("tags", [])
+                    for tag in tags:
+                        label = Label.objects.filter(name__iexact=tag).first()
+                        if label:
+                            break
+            except Exception as e:
+                logger.warning(f"Error finding label '{label_text}': {e}")
+
+        # Start with default segment data
+        segment_data = {
+            "video_file": video_file,
+            "start_frame_number": start_frame_number,
+            "end_frame_number": end_frame_number,
+            "source": user_source,
+            "label": label,
+            "prediction_meta": None,  # User segments don't have prediction meta
+        }
+
+        # If original segment ID is provided, try to clone from original
+        if original_segment_id:
+            try:
+                original_segment = LabelVideoSegment.objects.get(id=original_segment_id)
+
+                # Check if timing or label actually changed
+                original_start_time = original_segment.start_frame_number / fps
+                original_end_time = original_segment.end_frame_number / fps
+
+                timing_changed = (
+                    abs(original_start_time - start_time)
+                    > 0.1  # Allow small floating point differences
+                    or abs(original_end_time - end_time) > 0.1
+                )
+
+                label_changed = (label and original_segment.label != label) or (
+                    not label and original_segment.label is not None
+                )
+
+                # Only create new segment if something actually changed
+                if not timing_changed and not label_changed:
+                    logger.debug(
+                        f"No changes detected in segment {original_segment_id}, skipping user segment creation"
+                    )
+                    return None
+
+                # Clone relevant fields from original segment
+                segment_data.update(
+                    {
+                        "prediction_meta": original_segment.prediction_meta,
+                        "label": label
+                        or original_segment.label,  # Use new label if provided, otherwise keep original
+                    }
+                )
+
+                logger.info(
+                    f"Cloning segment {original_segment_id} with user modifications"
+                )
+
+            except LabelVideoSegment.DoesNotExist:
+                logger.warning(
+                    f"Original segment {original_segment_id} not found, creating new user segment"
+                )
+
+        # Create the new user segment using the manager
+        with transaction.atomic():
+            new_segment = LabelVideoSegment.create_from_video(
+                source=video_file,
+                prediction_meta=segment_data.get("prediction_meta"),
+                label=segment_data["label"],
+                start_frame_number=start_frame_number,
+                end_frame_number=end_frame_number,
+            )
+
+            # Set the user information source
+            new_segment.source = user_source
+            new_segment.save()
+
+            logger.info(
+                f"Created user segment {new_segment.id} for video {video_id} by user {request_user.username}"
+            )
+            return new_segment
+
+    except VideoFile.DoesNotExist:
+        logger.error(f"Video {video_id} not found, cannot create user segment")
+        return None
+    except Exception as e:
+        logger.error(f"Error creating user segment from annotation: {e}")
+        return None

endoreg_db/services/video_import.py

@@ -0,0 +1,9 @@
+from endoreg_db.import_files.video_import_service import VideoImportService as vis
+
+
+class VideoImportService(vis):
+    __doc__ = (vis.__doc__ or "") + (
+        "Wrapper for service import of concurrency safe VideoImportService.\n"
+        "Implemented for future Celery integration (12.02.2025).\n\n"
+    )
+    pass