endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

endoreg_db/views/anonymization/overview.py

@@ -1,19 +1,28 @@
 # endoreg_db/api/views/anonymization_overview.py
 
+import logging
+
+from django.http import JsonResponse
 from rest_framework import status
 from rest_framework.decorators import api_view, permission_classes
-from rest_framework.response import Response
-from endoreg_db.utils.permissions import DEBUG_PERMISSIONS
-from endoreg_db.services.anonymization import AnonymizationService
-from endoreg_db.services.polling_coordinator import PollingCoordinator, ProcessingLockContext
 from rest_framework.generics import ListAPIView
 from rest_framework.pagination import PageNumberPagination
-from endoreg_db.models import VideoFile, RawPdfFile
+from rest_framework.response import Response
+
+from endoreg_db.authz.permissions import PolicyPermission  # import RBAC
+from endoreg_db.models import RawPdfFile, VideoFile
+from endoreg_db.services.anonymization import AnonymizationService
+from endoreg_db.services.polling_coordinator import (
+    PollingCoordinator,
+    ProcessingLockContext,
+)
+from endoreg_db.utils.permissions import DEBUG_PERMISSIONS
+
 from ...serializers import FileOverviewSerializer, VoPPatientDataSerializer
-from django.http import JsonResponse
-import logging
+
 logger = logging.getLogger(__name__)
-PERMS = DEBUG_PERMISSIONS   # shorten
+PERMS = DEBUG_PERMISSIONS  # shorten
+
 
 # ---------- overview ----------------------------------------------------
 class NoPagination(PageNumberPagination):
@@ -24,10 +33,12 @@ class AnonymizationOverviewView(ListAPIView):
     """
     GET /api/anonymization/items/overview/
     --------------------------------------
-    Returns a flat list (Video + PDF) ordered by newest upload first.
+    Returns a flat list (Video + report) ordered by newest upload first.
     """
+
     serializer_class = FileOverviewSerializer
-    permission_classes = DEBUG_PERMISSIONS   
+    # permission_classes = DEBUG_PERMISSIONS
+    permission_classes = [PolicyPermission]
     pagination_class = NoPagination
 
     def get_queryset(self):
@@ -36,23 +47,25 @@ class AnonymizationOverviewView(ListAPIView):
         """
         # 1) VideoFile queryset - only fields that exist on VideoFile
         qs_video = (
-            VideoFile.objects
-            .select_related("state", "sensitive_meta")
+            VideoFile.objects.select_related("state", "sensitive_meta")
             .prefetch_related("label_video_segments__state")
-            .only("id", "original_file_name", "raw_file", "uploaded_at", "state", "sensitive_meta")
+            .only(
+                "id",
+                "original_file_name",
+                "raw_file",
+                "uploaded_at",
+                "state",
+                "sensitive_meta",
+            )
         )
         # 2) RawPdfFile queryset - only fields that exist on RawPdfFile
-        qs_pdf = (
-            RawPdfFile.objects
-            .select_related("sensitive_meta")
-            .only("id", "file", "date_created", 
-                "text", "anonymized_text",     
-                "sensitive_meta")
-
+        qs_pdf = RawPdfFile.objects.select_related("sensitive_meta").only(
+            "id", "file", "date_created", "text", "anonymized_text", "sensitive_meta"
         )
 
         return list(qs_video) + list(qs_pdf)
-    
+
+
 # ---------- status with polling protection ------------------------------
 @api_view(["GET"])
 @permission_classes(PERMS)
@@ -69,12 +82,14 @@ def anonymization_status(request, file_id: int):
 
     # Wende Rate-Limiting auf den echten Typ an (nicht auf einen evtl. falschen request-Parameter)
     if not PollingCoordinator.can_check_status(file_id, file_type):
-        remaining_seconds = PollingCoordinator.get_remaining_cooldown_seconds(file_id, file_type)
+        remaining_seconds = PollingCoordinator.get_remaining_cooldown_seconds(
+            file_id, file_type
+        )
         response_data = {
             "detail": "Status check rate limited. Please wait before checking again.",
             "file_id": file_id,
             "cooldown_active": True,
-            "retry_after": remaining_seconds
+            "retry_after": remaining_seconds,
         }
         response = Response(response_data, status=status.HTTP_429_TOO_MANY_REQUESTS)
         response["Retry-After"] = str(remaining_seconds)
@@ -83,15 +98,22 @@ def anonymization_status(request, file_id: int):
     status_val = info.get("anonymizationStatus") or info.get("status") or "not_started"
 
     # processing_locked als Ableitung des Status interpretieren
-    processing_statuses = {"processing_anonymization", "extracting_frames", "predicting_segments"}
+    processing_statuses = {
+        "processing_anonymization",
+        "extracting_frames",
+        "predicting_segments",
+    }
     processing_locked_derived = status_val in processing_statuses
 
-    return Response({
-        "file_id": file_id,
-        "file_type": file_type,
-        "anonymizationStatus": status_val,
-        "processing_locked": processing_locked_derived,
-    })
+    return Response(
+        {
+            "file_id": file_id,
+            "file_type": file_type,
+            "anonymizationStatus": status_val,
+            "processing_locked": processing_locked_derived,
+        }
+    )
+
 
 # ---------- start with processing lock ----------------------------------
 @api_view(["POST"])
@@ -104,9 +126,9 @@ def start_anonymization(request, file_id: int):
     info = AnonymizationService.get_status(file_id)
     if not info:
         return Response({"detail": "File not found"}, status=status.HTTP_404_NOT_FOUND)
-    
+
     file_type = info["mediaType"]
-    
+
     # Use processing lock context to prevent duplicate processing
     with ProcessingLockContext(file_id, file_type) as lock:
         if not lock.acquired:
@@ -115,27 +137,32 @@ def start_anonymization(request, file_id: int):
                     "detail": "File is already being processed by another request",
                     "file_id": file_id,
                     "file_type": file_type,
-                    "processing_locked": True
-                }, 
-                status=status.HTTP_409_CONFLICT
+                    "processing_locked": True,
+                },
+                status=status.HTTP_409_CONFLICT,
             )
-        
+
         # Proceed with starting anonymization
         service = AnonymizationService()
         kind = service.start(file_id)
         if not kind:
-            return Response({"detail": "Failed to start anonymization"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-        
-        return Response({
-            "detail": f"Anonymization started for {kind} file",
-            "file_id": file_id,
-            "file_type": kind,
-            "processing_locked": True
-        })
+            return Response(
+                {"detail": "Failed to start anonymization"},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+        return Response(
+            {
+                "detail": f"Anonymization started for {kind} file",
+                "file_id": file_id,
+                "file_type": kind,
+                "processing_locked": True,
+            }
+        )
 
 
 # ---------- current with coordination ------------------------------------
-@api_view(['GET', 'POST', 'PUT'])
+@api_view(["GET", "POST", "PUT"])
 @permission_classes(DEBUG_PERMISSIONS)
 def anonymization_current(request, file_id):
     """
@@ -143,15 +170,15 @@ def anonymization_current(request, file_id):
     """
     # Try to find the file in VideoFile first
     try:
-        video_file = VideoFile.objects.select_related('sensitive_meta').get(id=file_id)
-        serializer = VoPPatientDataSerializer(video_file, context={'request': request})
+        video_file = VideoFile.objects.select_related("sensitive_meta").get(id=file_id)
+        serializer = VoPPatientDataSerializer(video_file, context={"request": request})
         return Response(serializer.data)
     except VideoFile.DoesNotExist:
         pass
     # Try to find the file in RawPdfFile
     try:
-        pdf_file = RawPdfFile.objects.select_related('sensitive_meta').get(id=file_id)
-        serializer = VoPPatientDataSerializer(pdf_file, context={'request': request})
+        pdf_file = RawPdfFile.objects.select_related("sensitive_meta").get(id=file_id)
+        serializer = VoPPatientDataSerializer(pdf_file, context={"request": request})
         return Response(serializer.data)
 
     except RawPdfFile.DoesNotExist:
@@ -159,12 +186,13 @@ def anonymization_current(request, file_id):
 
     except (ValueError, TypeError, AttributeError) as e:
         logger.error(f"Error in set_current_for_validation: {e}")
-        return JsonResponse({'status': 'error', 'message': str(e)}, status=500)
+        return JsonResponse({"status": "error", "message": str(e)}, status=500)
+
+    return JsonResponse({"status": "error", "message": "File not found"}, status=404)
 
-    return JsonResponse({'status': 'error', 'message': 'File not found'}, status=404)
 
 # ---------- polling coordinator info ------------------------------------
-@api_view(['GET'])
+@api_view(["GET"])
 @permission_classes(DEBUG_PERMISSIONS)
 def polling_coordinator_info(request):
     """
@@ -177,12 +205,13 @@ def polling_coordinator_info(request):
     except Exception as e:
         logger.error(f"Error getting polling coordinator info: {e}")
         return Response(
-            {"error": "Failed to get coordinator info"}, 
-            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+            {"error": "Failed to get coordinator info"},
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
         )
 
+
 # ---------- emergency lock management -----------------------------------
-@api_view(['DELETE'])
+@api_view(["DELETE"])
 @permission_classes(DEBUG_PERMISSIONS)
 def clear_processing_locks(request):
     """
@@ -190,22 +219,25 @@ def clear_processing_locks(request):
     Emergency endpoint to clear all processing locks
     """
     try:
-        file_type = request.query_params.get('type', None)
+        file_type = request.query_params.get("type", None)
         cleared_count = PollingCoordinator.clear_all_locks(file_type)
-        
-        return Response({
-            "detail": "Processing locks cleared",
-            "cleared_count": cleared_count,
-            "file_type_filter": file_type
-        })
+
+        return Response(
+            {
+                "detail": "Processing locks cleared",
+                "cleared_count": cleared_count,
+                "file_type_filter": file_type,
+            }
+        )
     except Exception as e:
         logger.error(f"Error clearing processing locks: {e}")
         return Response(
-            {"error": "Failed to clear locks"}, 
-            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+            {"error": "Failed to clear locks"},
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
         )
-        
-@api_view(['GET'])
+
+
+@api_view(["GET"])
 @permission_classes(DEBUG_PERMISSIONS)
 def has_raw_video_file(request, file_id):
     """
@@ -213,4 +245,4 @@ def has_raw_video_file(request, file_id):
     Check if a raw video file exists for the given file ID
     """
     exists = VideoFile.objects.filter(id=file_id, raw_file__isnull=False).exists()
-    return Response({"file_id": file_id, "has_raw_file": exists})
+    return Response({"file_id": file_id, "has_raw_file": exists})

endoreg_db/views/anonymization/validate.py

@@ -7,30 +7,37 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 
 from endoreg_db.models import RawPdfFile, VideoFile
+from endoreg_db.models.metadata import SensitiveMeta
 from endoreg_db.serializers.anonymization import SensitiveMetaValidateSerializer
 
-
 logger = logging.getLogger(__name__)
 
 
 class AnonymizationValidateView(APIView):
     """
     POST /api/anonymization/<int:file_id>/validate/
-    
+
     Validiert und aktualisiert SensitiveMeta-Felder für Videos oder PDFs.
-    
+
+    DATA HERE IS COMING FROM THE ANONYIZATION VALIDATION COMPONENT
+
     Body (Datumsfelder bevorzugt in deutschem Format DD.MM.YYYY; ISO YYYY-MM-DD ebenfalls akzeptiert):
     {
       "patient_first_name": "Max",
       "patient_last_name":  "Mustermann",
       "patient_dob":        "21.03.1994",      // DD.MM.YYYY bevorzugt
+      "patient_gender":     "male"
       "examination_date":   "15.02.2024",      // DD.MM.YYYY bevorzugt
+
       "casenumber":         "12345",
       "anonymized_text":    "...",             // nur für PDFs; Videos ignorieren
       "is_verified":        true               // optional; default true
-      "file_type":        "video"            // optional; "video" oder "pdf"; wenn nicht angegeben, wird zuerst Video, dann PDF versucht
+      "file_type":        "video"            // optional; "video" oder "pdf"; wenn nicht angegeben, wird zuerst Video, dann report versucht
+      "center_name":       editedPatient.value.centerName || '',
+      "external_id":       editedPatient.value.externalId || '',
+      "external_id_origin":editedPatient.value.externalIdOrigin || '',
     }
-    
+
     Rückwärtskompatibilität: ISO-Format (YYYY-MM-DD) wird ebenfalls akzeptiert.
     """
 
@@ -41,67 +48,198 @@ class AnonymizationValidateView(APIView):
         serializer.is_valid(raise_exception=True)
         validated_data = cast(Dict[str, Any], serializer.validated_data)
         payload: Dict[str, Any] = dict(validated_data)
+
+        # Default ist_verified = True
         if "is_verified" not in payload:
             payload["is_verified"] = True
 
         file_type = payload.get("file_type")
 
-        # Try Video first (unless explicitly requesting PDF)
-        if file_type in (None, "video"):
-            video = VideoFile.objects.select_related("center").filter(pk=file_id).first()
-            if video is not None:
-                prepared_payload = self._prepare_payload(payload, video)
-                try:
-                    ok = video.validate_metadata_annotation(prepared_payload)
-                except Exception:  # pragma: no cover - defensive safety net
-                    logger.exception("Video validation crashed for id=%s", file_id)
-                    return Response(
-                        {"error": "Video validation encountered an unexpected error."},
-                        status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        with transaction.atomic():
+            # Try Video first (unless explicitly requesting report)
+            if file_type in (None, "video"):
+                video = (
+                    VideoFile.objects.select_related(
+                        "center", "sensitive_meta", "state"
                     )
+                    .filter(pk=file_id)
+                    .first()
+                )
+                if video is not None:
+                    prepared_payload = self._prepare_payload(payload, video)
+                    try:
+                        ok = video.validate_metadata_annotation(prepared_payload)
+                    except Exception:  # pragma: no cover - defensive safety net
+                        logger.exception("Video validation crashed for id=%s", file_id)
+                        return Response(
+                            {
+                                "error": "Video validation encountered an unexpected error."
+                            },
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                        )
+
+                    if not ok:
+                        return Response(
+                            {"error": "Video validation failed."},
+                            status=status.HTTP_400_BAD_REQUEST,
+                        )
+
+                    # this is here for tests!
+                    if video.sensitive_meta is None:
+                        sm = SensitiveMeta.objects.create(center=video.center)
+                        video.sensitive_meta = sm
+
+                    video.save(update_fields=["sensitive_meta"])
+                    video.sensitive_meta.get_or_create_state()
+                    if video.sensitive_meta.state is not None:
+                        video.sensitive_meta.state.refresh_from_db()
+                        video.sensitive_meta.state.mark_dob_verified()
+                        video.sensitive_meta.state.mark_names_verified()
+                        video.sensitive_meta.create_anonymized_record()
+                    else:
+                        return Response(
+                            {"message": "Video not validated, failed to create State."},
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                        )
+
+                    if video.state is not None:
+                        video.state.state.anonymization_status.mark_anonymized()
+                        video.state.save(update_fields=["anonymized"])
+                        video.sensitive_meta.state.save()
 
-                if not ok:
-                    return Response({"error": "Video validation failed."}, status=status.HTTP_400_BAD_REQUEST)
-
-                return Response({"message": "Video validated."}, status=status.HTTP_200_OK)
-
-            if file_type == "video":
-                return Response({"error": f"Video {file_id} not found."}, status=status.HTTP_404_NOT_FOUND)
+                    return Response(
+                        {"message": "Video validated."},
+                        status=status.HTTP_200_OK,
+                    )
 
-        # Then PDF (unless explicitly requesting Video)
-        if file_type in (None, "pdf"):
-            pdf = RawPdfFile.objects.select_related("center").filter(pk=file_id).first()
-            if pdf is not None:
-                prepared_payload = self._prepare_payload(payload, pdf)
-                try:
-                    ok = pdf.validate_metadata_annotation(prepared_payload)
-                except Exception:  # pragma: no cover - defensive safety net
-                    logger.exception("PDF validation crashed for id=%s", file_id)
+                if file_type == "video":
                     return Response(
-                        {"error": "PDF validation encountered an unexpected error."},
-                        status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                        {"error": f"Video {file_id} not found."},
+                        status=status.HTTP_404_NOT_FOUND,
                     )
 
-                if not ok:
-                    return Response({"error": "PDF validation failed."}, status=status.HTTP_400_BAD_REQUEST)
+            # Then report (unless explicitly requesting Video)
+            if file_type in (None, "pdf"):
+                pdf = (
+                    RawPdfFile.objects.select_related(
+                        "center", "sensitive_meta", "state"
+                    )
+                    .filter(pk=file_id)
+                    .first()
+                )
+                if pdf is not None:
+                    prepared_payload = self._prepare_payload(payload, pdf)
+                    try:
+                        ok = pdf.validate_metadata_annotation(prepared_payload)
+                    except Exception:  # pragma: no cover - defensive safety net
+                        logger.exception("report validation crashed for id=%s", file_id)
+                        return Response(
+                            {
+                                "error": "report validation encountered an unexpected error."
+                            },
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                        )
+
+                    # sanity logging – but don't break flow
+                    try:
+                        assert pdf.sensitive_meta is not None
+                        assert pdf.sensitive_meta.state is not None
+                    except AssertionError as e:
+                        logger.error("%s", e)
+
+                    if not ok:
+                        return Response(
+                            {"error": "report validation failed."},
+                            status=status.HTTP_400_BAD_REQUEST,
+                        )
+                    else:
+                        # this is here for tests!
+                        if pdf.sensitive_meta is None:
+                            sm = SensitiveMeta.objects.create(center=pdf.center)
+                            pdf.sensitive_meta = sm
+
+                        pdf.save(update_fields=["sensitive_meta"])
+                        pdf.sensitive_meta.get_or_create_state()
+                        if (
+                            pdf.sensitive_meta
+                            and pdf.sensitive_meta.state
+                            and pdf.state
+                        ):
+                            pdf.sensitive_meta.state.refresh_from_db()
+                            pdf.sensitive_meta.state.mark_dob_verified()
+                            pdf.sensitive_meta.state.mark_names_verified()
+                            pdf.sensitive_meta.create_anonymized_record()
+                            pdf.state.state.anonymization_status.mark_anonymized()
+                            pdf.state.save(update_fields=["anonymized"])
+                            pdf.sensitive_meta.state.save()
+                        else:
+                            return Response(
+                                {
+                                    "message": "report not validated, failed to create State."
+                                },
+                                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                            )
 
-                return Response({"message": "PDF validated."}, status=status.HTTP_200_OK)
+                    return Response(
+                        {"message": "report validated."},
+                        status=status.HTTP_200_OK,
+                    )
 
-            if file_type == "pdf":
-                return Response({"error": f"PDF {file_id} not found."}, status=status.HTTP_404_NOT_FOUND)
+                if file_type == "pdf":
+                    return Response(
+                        {"error": f"report {file_id} not found."},
+                        status=status.HTTP_404_NOT_FOUND,
+                    )
 
-        return Response({"error": f"Item {file_id} not found as video or pdf."}, status=status.HTTP_404_NOT_FOUND)
+        return Response(
+            {"error": f"Item {file_id} not found as video or pdf."},
+            status=status.HTTP_404_NOT_FOUND,
+        )
 
     @staticmethod
     def _prepare_payload(base_payload: Dict[str, Any], file_obj: Any) -> Dict[str, Any]:
-        """Return a fresh payload tailored for the given file object."""
+        """
+        Return a fresh payload tailored for the given file object.
+
+        - Strips `file_type` before forwarding to validators.
+        - Injects `center_name` from the file's center if not already present.
+        - Normalizes `patient_gender` if present, but does NOT require it.
+        """
+        prepared: Dict[str, Any] = dict(base_payload)
 
-        prepared = dict(base_payload)
+        # never send file_type to validators
         prepared.pop("file_type", None)
 
+        # center_name from file.center if not already set
         center = getattr(file_obj, "center", None)
         center_name = getattr(center, "name", None)
         if center_name and not prepared.get("center_name"):
             prepared["center_name"] = center_name
 
-        return prepared
+        # Gender normalization: optional, robust against missing or unknown values
+        raw_gender = base_payload.get("patient_gender", None)
+        if raw_gender is None:
+            # nothing provided → don't touch gender
+            return prepared
+
+        gender = str(raw_gender).strip().lower()
+
+        # empty string behaves as "not set" – don't override anything
+        if gender == "":
+            return prepared
+
+        male_values = {"m", "male", "männlich"}
+        female_values = {"w", "f", "female", "weiblich"}
+
+        if gender in male_values:
+            prepared["patient_gender"] = "male"
+        elif gender in female_values:
+            prepared["patient_gender"] = "female"
+        else:
+            # keep existing semantics: unknown values default to "male"
+            logger.warning(
+                "Unsupported patient_gender value %r; defaulting to 'male'", raw_gender
+            )
+            prepared["patient_gender"] = "male"
+
+        return prepared

endoreg_db/views/media/__init__.py

@@ -3,18 +3,11 @@
 from .video_media import VideoMediaView
 from .pdf_media import PdfMediaView
 from ..video.reimport import VideoReimportView
-from ..pdf.reimport import PdfReimportView
-from .segments import video_segments_by_pk
-from .video_segments import (
-    video_segments_collection,
-    video_segments_by_video,
-    video_segment_detail,
-    video_segments_stats,
-    video_segment_validate,
-    video_segments_validate_bulk,
-    video_segments_validation_status,
-)
+from ..report.reimport import ReportReimportView
+from ..ai.label import label_list
+
 from .sensitive_metadata import (
+    get_sensitive_metadata_pk,
     video_sensitive_metadata,
     video_sensitive_metadata_verify,
     pdf_sensitive_metadata,
@@ -27,19 +20,13 @@ __all__ = [
     'VideoMediaView',
     'PdfMediaView',
     'VideoReimportView',
-    'PdfReimportView',
-    'video_segments_by_pk',
-    'video_segments_collection',
-    'video_segments_by_video',
-    'video_segment_detail',
-    'video_segments_stats',
-    'video_segment_validate',
-    'video_segments_validate_bulk',
-    'video_segments_validation_status',
+    'ReportReimportView',
+    'get_sensitive_metadata_pk',
     'video_sensitive_metadata',
     'video_sensitive_metadata_verify',
     'pdf_sensitive_metadata',
     'pdf_sensitive_metadata_verify',
     'sensitive_metadata_list',
     'pdf_sensitive_metadata_list',
+    'label_list',
 ]