endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/views/anonymization/overview.py

@@ -0,0 +1,307 @@
+# endoreg_db/api/views/anonymization_overview.py
+
+from pickle import GET
+from jwt.algorithms import NoneAlgorithm
+from rest_framework import status
+from rest_framework.decorators import api_view, permission_classes
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from django.db import transaction
+from endoreg_db.utils.permissions import DEBUG_PERMISSIONS
+from endoreg_db.services.anonymization import AnonymizationService
+from endoreg_db.services.polling_coordinator import PollingCoordinator, ProcessingLockContext
+from rest_framework.generics import ListAPIView
+from rest_framework.pagination import PageNumberPagination
+from endoreg_db.models import VideoFile, RawPdfFile
+from ...serializers import FileOverviewSerializer, VoPPatientDataSerializer
+from django.http import JsonResponse
+from endoreg_db.utils.operation_log import (
+    record_operation,
+    ACTION_ANONYMIZATION_START,
+    STATUS_NOT_STARTED,
+    STATUS_PROCESSING,
+)
+
+
+from endoreg_db.authz.permissions import PolicyPermission  #  import RBAC
+import logging
+logger = logging.getLogger(__name__)
+PERMS = DEBUG_PERMISSIONS   # shorten
+
+# ---------- overview ----------------------------------------------------
+class NoPagination(PageNumberPagination):
+    page_size = None
+
+
+class AnonymizationOverviewView(ListAPIView):
+    """
+    GET /api/anonymization/items/overview/
+    --------------------------------------
+    Returns a flat list (Video + PDF) ordered by newest upload first.
+    """
+    serializer_class = FileOverviewSerializer
+    #permission_classes = DEBUG_PERMISSIONS   
+    permission_classes = [PolicyPermission]
+    pagination_class = NoPagination
+
+    def get_queryset(self):
+        """
+        Returns a combined queryset of VideoFile and RawPdfFile instances.
+        """
+        # 1) VideoFile queryset - only fields that exist on VideoFile
+        qs_video = (
+            VideoFile.objects
+            .select_related("state", "sensitive_meta")
+            .prefetch_related("label_video_segments__state")
+            .only("id", "original_file_name", "raw_file", "uploaded_at", "state", "sensitive_meta")
+        )
+        # 2) RawPdfFile queryset - only fields that exist on RawPdfFile
+        qs_pdf = (
+            RawPdfFile.objects
+            .select_related("sensitive_meta")
+            .only("id", "file", "date_created", 
+                "text", "anonymized_text",     
+                "sensitive_meta")
+
+        )
+
+        return list(qs_video) + list(qs_pdf)
+    
+class AnonymizationValidateView(APIView):
+    """
+    POST /api/anonymization/<int:item_id>/validate/
+    Body: {
+      // common SensitiveMeta fields (snake_case):
+      "patient_first_name": "...",
+      "patient_last_name":  "...",
+      "patient_dob":        "YYYY-MM-DD",
+      "examination_date":   "YYYY-MM-DD",
+      "casenumber":         "...",
+      "anonymized_text":    "...",   # only for PDFs; ignored by videos
+      "is_verified": true            # optional; defaults to true here
+    }
+    """
+
+    @transaction.atomic
+    def post(self, request, item_id: int):
+        payload = request.data or {}
+        payload.setdefault("is_verified", True)
+
+        # Try Video first
+        video = VideoFile.objects.filter(pk=item_id).first()
+        if video:
+            ok = video.validate_metadata_annotation(payload)
+            if not ok:
+                return Response({"error": "Video validation failed."}, status=status.HTTP_400_BAD_REQUEST)
+            return Response({"message": "Video validated."}, status=status.HTTP_200_OK)
+
+        # Then PDF
+        pdf = RawPdfFile.objects.filter(pk=item_id).first()
+        if pdf:
+            ok = pdf.validate_metadata_annotation(payload)
+            if not ok:
+                return Response({"error": "PDF validation failed."}, status=status.HTTP_400_BAD_REQUEST)
+            return Response({"message": "PDF validated."}, status=status.HTTP_200_OK)
+
+        return Response({"error": f"Item {item_id} not found as video or pdf."}, status=status.HTTP_404_NOT_FOUND)
+
+
+# ---------- status with polling protection ------------------------------
+@api_view(["GET"])
+@permission_classes(PERMS)
+def anonymization_status(request, file_id: int):
+    """
+    Get anonymization status with polling rate limiting.
+    """
+    # Ermittele erst den echten Typ und Status
+    info = AnonymizationService.get_status(file_id)
+    if not info:
+        return Response({"detail": "File not found"}, status=status.HTTP_404_NOT_FOUND)
+
+    file_type = info.get("mediaType") or info.get("type") or "video"
+
+    # Wende Rate-Limiting auf den echten Typ an (nicht auf einen evtl. falschen request-Parameter)
+    if not PollingCoordinator.can_check_status(file_id, file_type):
+        return Response(
+            {
+                "detail": "Status check rate limited. Please wait before checking again.",
+                "file_id": file_id,
+                "cooldown_active": True
+            }, 
+            status=status.HTTP_429_TOO_MANY_REQUESTS
+        )
+
+    status_val = info.get("anonymizationStatus") or info.get("status") or "not_started"
+
+    # processing_locked als Ableitung des Status interpretieren
+    processing_statuses = {"processing_anonymization", "extracting_frames", "predicting_segments"}
+    processing_locked_derived = status_val in processing_statuses
+
+    return Response({
+        "file_id": file_id,
+        "file_type": file_type,
+        "anonymizationStatus": status_val,
+        "processing_locked": processing_locked_derived,
+    })
+
+# ---------- start with processing lock ----------------------------------
+@api_view(["POST"])
+@permission_classes(PERMS)
+def start_anonymization(request, file_id: int):
+    """
+    Start anonymization with processing lock to prevent duplicates.
+    """
+    # First check what type of file this is
+    info = AnonymizationService.get_status(file_id)
+    if not info:
+        return Response({"detail": "File not found"}, status=status.HTTP_404_NOT_FOUND)
+
+    file_type = info.get("mediaType") or "unknown"
+    status_before = info.get("anonymizationStatus") or info.get("status") or "not_started"
+
+    # Use processing lock context to prevent duplicate processing
+    with ProcessingLockContext(file_id, file_type) as lock:
+        if not lock.acquired:
+            return Response(
+                {
+                    "detail": "File is already being processed by another request",
+                    "file_id": file_id,
+                    "file_type": file_type,
+                    "processing_locked": True,
+                },
+                status=status.HTTP_409_CONFLICT,
+            )
+
+        # Proceed with starting anonymization
+        service = AnonymizationService()
+        kind = service.start(file_id)
+        if not kind:
+            return Response(
+                {"detail": "Failed to start anonymization"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+        # Re-read status AFTER starting
+        try:
+            info_after = AnonymizationService.get_status(file_id) or {}
+        except Exception:
+            logger.exception("Failed to refresh anonymization status for file %s", file_id)
+            info_after = {}
+
+        status_after = (
+            info_after.get("anonymizationStatus")
+            or info_after.get("status")
+            or status_before
+        )
+
+        # 🔐 Write operation log
+        record_operation(
+            request,
+            action=ACTION_ANONYMIZATION_START,
+            resource_type=kind,          # 'video' or 'pdf' as returned by service.start
+            resource_id=file_id,
+            status_before=STATUS_NOT_STARTED,
+            status_after=STATUS_PROCESSING,
+            meta={
+                "file_type_from_status": file_type,
+            },
+        )
+
+        return Response(
+            {
+                "detail": f"Anonymization started for {kind} file",
+                "file_id": file_id,
+                "file_type": kind,
+                "processing_locked": True,
+            }
+        )
+
+
+
+# ---------- current with coordination ------------------------------------
+@api_view(['GET', 'POST', 'PUT'])
+@permission_classes(DEBUG_PERMISSIONS)
+def anonymization_current(request, file_id):
+    """
+    Set current file for validation and return patient data
+    """
+    # Try to find the file in VideoFile first
+    try:
+        video_file = VideoFile.objects.select_related('sensitive_meta').get(id=file_id)
+        serializer = VoPPatientDataSerializer(video_file, context={'request': request})
+        return Response(serializer.data)
+    except VideoFile.DoesNotExist:
+        pass
+    # Try to find the file in RawPdfFile
+    try:
+        pdf_file = RawPdfFile.objects.select_related('sensitive_meta').get(id=file_id)
+        serializer = VoPPatientDataSerializer(pdf_file, context={'request': request})
+        return Response(serializer.data)
+
+    except RawPdfFile.DoesNotExist:
+        pass
+
+    except (ValueError, TypeError, AttributeError) as e:
+        logger.error(f"Error in set_current_for_validation: {e}")
+        return JsonResponse({'status': 'error', 'message': str(e)}, status=500)
+
+    return JsonResponse({'status': 'error', 'message': 'File not found'}, status=404)
+
+# ---------- polling coordinator info ------------------------------------
+@api_view(['GET'])
+@permission_classes(DEBUG_PERMISSIONS)
+def polling_coordinator_info(request):
+    """
+    GET /api/anonymization/polling-info/
+    Get information about polling coordinator status
+    """
+    try:
+        info = PollingCoordinator.get_processing_locks_info()
+        return Response(info)
+    except Exception as e:
+        logger.error(f"Error getting polling coordinator info: {e}")
+        return Response(
+            {"error": "Failed to get coordinator info"}, 
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )
+
+# ---------- emergency lock management -----------------------------------
+@api_view(['DELETE'])
+@permission_classes(DEBUG_PERMISSIONS)
+def clear_processing_locks(request):
+    """
+    DELETE /api/anonymization/clear-locks/
+    Emergency endpoint to clear all processing locks
+    """
+    try:
+        file_type = request.query_params.get('type', None)
+        cleared_count = PollingCoordinator.clear_all_locks(file_type)
+        
+        return Response({
+            "detail": "Processing locks cleared",
+            "cleared_count": cleared_count,
+            "file_type_filter": file_type
+        })
+    except Exception as e:
+        logger.error(f"Error clearing processing locks: {e}")
+        return Response(
+            {"error": "Failed to clear locks"}, 
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )
+   
+@api_view(["GET"])
+@permission_classes(PERMS)
+def has_raw_video_file(request, file_id: int):
+    """
+    Return whether the video still has a raw video file.
+    """
+    try:
+        video = VideoFile.get_video_by_pk(pk=file_id)
+    except VideoFile.DoesNotExist:
+        return Response(
+            {"detail": "Video not found", "file_id": file_id},
+            status=status.HTTP_404_NOT_FOUND,
+        )
+
+    return Response({"file_id": file_id, "has_raw": video.has_raw})
+        

endoreg_db/views/anonymization/validate.py

@@ -0,0 +1,310 @@
+import logging
+from typing import Any, Dict, cast
+
+from django.db import transaction
+from rest_framework import status
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from endoreg_db.models import RawPdfFile, VideoFile
+from endoreg_db.models.metadata import SensitiveMeta
+from endoreg_db.serializers.anonymization import SensitiveMetaValidateSerializer
+from endoreg_db.utils.operation_log import (
+    record_operation,
+    ACTION_ANONYMIZATION_VALIDATED,
+    STATUS_PROCESSING,
+    STATUS_ANONYMIZED,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class AnonymizationValidateView(APIView):
+    """
+    POST /api/anonymization/<int:file_id>/validate/
+
+    Validiert und aktualisiert SensitiveMeta-Felder für Videos oder reports.
+
+    DATA HERE IS COMING FROM THE ANONYIZATION VALIDATION COMPONENT
+
+    Body (Datumsfelder bevorzugt in deutschem Format DD.MM.YYYY; ISO YYYY-MM-DD ebenfalls akzeptiert):
+    {
+      "patient_first_name": "Max",
+      "patient_last_name":  "Mustermann",
+      "patient_dob":        "21.03.1994",      // DD.MM.YYYY bevorzugt
+      "patient_gender":     "male"
+      "examination_date":   "15.02.2024",      // DD.MM.YYYY bevorzugt
+
+      "casenumber":         "12345",
+      "anonymized_text":    "...",             // nur für reports; Videos ignorieren
+      "is_verified":        true               // optional; default true
+      "file_type":        "video"            // optional; "video" oder "pdf"; wenn nicht angegeben, wird zuerst Video, dann report versucht
+      "center_name":       editedPatient.value.centerName || '',
+      "external_id":       editedPatient.value.externalId || '',
+      "external_id_origin":editedPatient.value.externalIdOrigin || '',
+    }
+
+    Rückwärtskompatibilität: ISO-Format (YYYY-MM-DD) wird ebenfalls akzeptiert.
+    """
+
+    @transaction.atomic
+    def post(self, request, file_id: int):
+        # Serializer-Validierung mit deutscher Datums-Priorität
+        serializer = SensitiveMetaValidateSerializer(data=request.data or {})
+        serializer.is_valid(raise_exception=True)
+        validated_data = cast(Dict[str, Any], serializer.validated_data)
+        payload: Dict[str, Any] = dict(validated_data)
+
+        # Default ist_verified = True
+        if "is_verified" not in payload:
+            payload["is_verified"] = True
+
+        file_type = payload.get("file_type")
+        status_before = None
+
+        with transaction.atomic():
+            # Try Video first (unless explicitly requesting report)
+            if file_type in (None, "video"):
+                video = (
+                    VideoFile.objects.select_related(
+                        "center", "sensitive_meta", "state"
+                    )
+                    .filter(pk=file_id)
+                    .first()
+                )
+                # TODO: The state for video will be none when no state is set and the state for pdf will always be none. After status needs to be inferred after calling the sensitive meta state update functions
+                if video is not None:
+                    prepared_payload = self._prepare_payload(payload, video)
+                    try:
+                        ok = video.validate_metadata_annotation(prepared_payload)
+                    except Exception:  # pragma: no cover - defensive safety net
+                        logger.exception("Video validation crashed for id=%s", file_id)
+                        return Response(
+                            {
+                                "error": "Video validation encountered an unexpected error."
+                            },
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                        )
+
+                    if not ok:
+                        return Response(
+                            {"error": "Video validation failed."},
+                            status=status.HTTP_400_BAD_REQUEST,
+                        )
+
+                    # this is here for tests!
+                    if video.sensitive_meta is None:
+                        sm = SensitiveMeta.objects.create(center=video.center)
+                        video.sensitive_meta = sm
+
+                    video.save(update_fields=["sensitive_meta"])
+                    video.sensitive_meta.get_or_create_state()
+                    if video.sensitive_meta.state is not None:
+                        video.sensitive_meta.state.refresh_from_db()
+                        video.sensitive_meta.state.mark_dob_verified()
+                        video.sensitive_meta.state.mark_names_verified()
+                        video.sensitive_meta.create_anonymized_record()
+                    else:
+                        return Response(
+                            {"message": "Video not validated, failed to create State."},
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                        )
+
+                    if video.state is not None:
+                        video.state.anonymized = True
+                        video.sensitive_meta.state.save()
+                    try:
+                        if video.state is not None:
+                            st = getattr(video.state, "anonymization_status", None)
+                            if st is not None:
+                                status_before = str(getattr(st, "value", st))
+                    except Exception:
+                        logger.exception(
+                            "Failed to read video anonymization_status before validation"
+                        )
+
+                        # --- NEW: status AFTER validation ---
+                    status_after = status_before
+                    try:
+                        if video.state is not None:
+                            video.state.refresh_from_db()
+                            st_after = getattr(
+                                video.state, "anonymization_status", None
+                            )
+                            if st_after is not None:
+                                status_after = str(getattr(st_after, "value", st_after))
+                    except Exception:
+                        logger.exception(
+                            "Failed to read video anonymization_status after validation"
+                        )
+
+                    # --- write operation log ---
+                    # TODO: update the function call bases on the status , once merged
+                    record_operation(
+                        request,
+                        action=ACTION_ANONYMIZATION_VALIDATED,
+                        resource_type="video",
+                        resource_id=file_id,
+                        status_before=STATUS_PROCESSING,
+                        status_after=STATUS_ANONYMIZED,
+                    )
+
+                    return Response(
+                        {"message": "Video validated."},
+                        status=status.HTTP_200_OK,
+                    )
+
+                if file_type == "video":
+                    return Response(
+                        {"error": f"Video {file_id} not found."},
+                        status=status.HTTP_404_NOT_FOUND,
+                    )
+
+            # Then report (unless explicitly requesting Video)
+            if file_type in (None, "pdf"):
+                pdf = (
+                    RawPdfFile.objects.select_related(
+                        "center", "sensitive_meta", "state"
+                    )
+                    .filter(pk=file_id)
+                    .first()
+                )
+                if pdf is not None:
+                    prepared_payload = self._prepare_payload(payload, pdf)
+                    try:
+                        ok = pdf.validate_metadata_annotation(prepared_payload)
+                    except Exception:  # pragma: no cover - defensive safety net
+                        logger.exception("report validation crashed for id=%s", file_id)
+                        return Response(
+                            {
+                                "error": "report validation encountered an unexpected error."
+                            },
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                        )
+
+                    # sanity logging – but don't break flow
+                    try:
+                        assert pdf.sensitive_meta is not None
+                        assert pdf.sensitive_meta.state is not None
+                    except AssertionError as e:
+                        logger.error("%s", e)
+
+                    if not ok:
+                        return Response(
+                            {"error": "report validation failed."},
+                            status=status.HTTP_400_BAD_REQUEST,
+                        )
+                    else:
+                        # this is here for tests!
+                        if pdf.sensitive_meta is None:
+                            sm = SensitiveMeta.objects.create(center=pdf.center)
+                            pdf.sensitive_meta = sm
+
+                        pdf.save(update_fields=["sensitive_meta"])
+                        pdf.sensitive_meta.get_or_create_state()
+                        if (
+                                pdf.sensitive_meta
+                                and pdf.sensitive_meta.state
+                            ):
+                                pdf.sensitive_meta.state.refresh_from_db()
+                                pdf.sensitive_meta.state.mark_dob_verified()
+                                pdf.sensitive_meta.state.mark_names_verified()
+                                pdf.sensitive_meta.create_anonymized_record()
+                                
+                                if pdf.state:
+                                    pdf.state.mark_anonymized()
+                                    pdf.state.save(update_fields=["anonymized"])
+                                    
+                                pdf.sensitive_meta.state.save()
+                        else:
+                            return Response(
+                                {"message": "report not validated, failed to create State."},
+                                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                            )
+
+                    status_after = status_before
+                    try:
+                        if pdf.state is not None:
+                            pdf.state.refresh_from_db()
+                            st_after = getattr(pdf.state, "anonymization_status", None)
+                            if st_after is not None:
+                                status_after = str(getattr(st_after, "value", st_after))
+                    except Exception:
+                        logger.exception(
+                            "Failed to read pdf anonymization_status after validation"
+                        )
+
+                    # --- NEW: write operation log ---
+                    record_operation(
+                        request,
+                        action=ACTION_ANONYMIZATION_VALIDATED,
+                        resource_type="pdf",
+                        resource_id=file_id,
+                        status_before=STATUS_PROCESSING,
+                        status_after=STATUS_ANONYMIZED,
+                    )
+
+                    return Response(
+                        {"message": "report validated."},
+                        status=status.HTTP_200_OK,
+                    )
+
+                if file_type == "pdf":
+                    return Response(
+                        {"error": f"report {file_id} not found."},
+                        status=status.HTTP_404_NOT_FOUND,
+                    )
+
+        return Response(
+            {"error": f"Item {file_id} not found as video or pdf."},
+            status=status.HTTP_404_NOT_FOUND,
+        )
+
+    @staticmethod
+    def _prepare_payload(base_payload: Dict[str, Any], file_obj: Any) -> Dict[str, Any]:
+        """
+        Return a fresh payload tailored for the given file object.
+
+        - Strips `file_type` before forwarding to validators.
+        - Injects `center_name` from the file's center if not already present.
+        - Normalizes `patient_gender` if present, but does NOT require it.
+        """
+        prepared: Dict[str, Any] = dict(base_payload)
+
+        # never send file_type to validators
+        prepared.pop("file_type", None)
+
+        # center_name from file.center if not already set
+        center = getattr(file_obj, "center", None)
+        center_name = getattr(center, "name", None)
+        if center_name and not prepared.get("center_name"):
+            prepared["center_name"] = center_name
+
+        # Gender normalization: optional, robust against missing or unknown values
+        raw_gender = base_payload.get("patient_gender", None)
+        if raw_gender is None:
+            # nothing provided → don't touch gender
+            return prepared
+
+        gender = str(raw_gender).strip().lower()
+
+        # empty string behaves as "not set" – don't override anything
+        if gender == "":
+            return prepared
+
+        male_values = {"m", "male", "männlich"}
+        female_values = {"w", "f", "female", "weiblich"}
+
+        if gender in male_values:
+            prepared["patient_gender"] = "male"
+        elif gender in female_values:
+            prepared["patient_gender"] = "female"
+        else:
+            # keep existing semantics: unknown values default to "male"
+            logger.warning(
+                "Unsupported patient_gender value %r; defaulting to 'male'", raw_gender
+            )
+            prepared["patient_gender"] = "male"
+
+        return prepared

endoreg_db/views/auth/__init__.py

@@ -0,0 +1,13 @@
+from .keycloak import (
+    KeycloakVideoView,
+    keycloak_login,
+    keycloak_callback,
+    public_home,
+)
+
+__all__ = [
+    "KeycloakVideoView",
+    "keycloak_login",
+    "keycloak_callback",
+    "public_home",
+]