endoreg-db 0.8.6.4__py3-none-any.whl

endoreg_db/views/anonymization/media_management.py

@@ -0,0 +1,454 @@
+# endoreg_db/views/media_management.py
+
+import logging
+from typing import Dict, Any
+from django.db import transaction
+from django.db.models import Q
+from numpy import delete
+from rest_framework import status
+from rest_framework.decorators import api_view, permission_classes
+from rest_framework.response import Response
+from rest_framework.views import APIView
+from django.utils import timezone
+from datetime import timedelta
+
+from endoreg_db.models import VideoFile, RawPdfFile, VideoState
+from endoreg_db.utils.permissions import DEBUG_PERMISSIONS
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Media Cleanup and Management API
+# ---------------------------------------------------------------------------
+
+class MediaManagementView(APIView):
+    """
+    Comprehensive Media Management API for cleanup and maintenance operations
+    """
+    permission_classes = DEBUG_PERMISSIONS
+
+    def get(self, request):
+        """
+        GET /api/media-management/status/
+        Get overview of media status and cleanup opportunities
+        """
+        try:
+            status_overview = self._get_status_overview()
+            return Response(status_overview)
+        except Exception as e:
+            logger.error(f"Error getting media status overview: {e}")
+            return Response(
+                {"error": "Failed to get status overview"}, 
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
+
+    def delete(self, request):
+        """
+        DELETE /api/media-management/cleanup/
+        Cleanup unfinished, failed, or stale media processing entries
+        """
+        cleanup_type = request.query_params.get('type', 'unfinished')
+        force = request.query_params.get('force', 'false').lower() == 'true'
+        media_type = request.query_params.get('file_type', 'all')
+        file_id = request.query_params.get('file_id', None)
+
+        try:
+            result = self._perform_cleanup(cleanup_type, force, media_type, file_id)
+
+            return Response(result)
+        except Exception as e:
+            logger.error(f"Error during media cleanup: {e}")
+            return Response(
+                {"error": "Cleanup operation failed"}, 
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
+
+    def _get_status_overview(self) -> Dict[str, Any]:
+        """Get comprehensive status overview"""
+        video_stats = self._get_video_stats()
+        pdf_stats = self._get_pdf_stats()
+        
+        # Stale processing detection (older than 2 hours)
+        stale_threshold = timezone.now() - timedelta(hours=2)
+        # Use VideoState boolean fields instead of non-existent name field
+        stale_videos = VideoFile.objects.filter(
+            uploaded_at__lt=stale_threshold,
+            state__frames_extracted=True,
+            state__sensitive_meta_processed=False
+        ).count()
+        
+        return {
+            "videos": video_stats,
+            "pdfs": pdf_stats,
+            "cleanup_opportunities": {
+                "stale_processing": stale_videos,
+                "failed_videos": video_stats["failed"],
+                "unfinished_total": video_stats["unfinished"] + pdf_stats["unfinished"]
+            },
+            "total_files": video_stats["total"] + pdf_stats["total"],
+            "timestamp": timezone.now().isoformat()
+        }
+
+    def _get_video_stats(self) -> Dict[str, int]:
+        """Get video file statistics using VideoState boolean fields"""
+        videos = VideoFile.objects.select_related('state').all()
+        
+        stats = {
+            "total": videos.count(),
+            "not_started": 0,
+            "processing": 0,
+            "done": 0,
+            "failed": 0,
+            "validated": 0,
+            "unfinished": 0
+        }
+        
+        for video in videos:
+            if not video.state:
+                stats["not_started"] += 1
+                stats["unfinished"] += 1
+                continue
+                
+            # Use the anonymization_status property
+            video_status = video.state.anonymization_status
+            
+            if video_status.value == 'not_started':
+                stats["not_started"] += 1
+                stats["unfinished"] += 1
+            elif video_status.value in ['extracting_frames', 'processing_anonymization']:
+                stats["processing"] += 1
+                stats["unfinished"] += 1
+            elif video_status.value == 'done':
+                stats["done"] += 1
+            elif video_status.value == 'failed':
+                stats["failed"] += 1
+                stats["unfinished"] += 1
+            elif video_status.value == 'validated':
+                stats["validated"] += 1
+            else:
+                stats["unfinished"] += 1
+        return stats
+
+    def _get_pdf_stats(self) -> Dict[str, int]:
+        """Get PDF file statistics"""
+        pdfs = RawPdfFile.objects.all()
+        
+        stats = {
+            "total": pdfs.count(),
+            "not_started": 0,
+            "processing": 0,
+            "done": 0,
+            "failed": 0,
+            "validated": 0,
+            "unfinished": 0
+        }
+        
+        for pdf in pdfs:
+            # PDF status logic based on anonymized_text presence and validation
+            has_anonymized = bool(pdf.anonymized_text and pdf.anonymized_text.strip())
+            is_validated = getattr(pdf.sensitive_meta, 'is_verified', False) if pdf.sensitive_meta else False
+            
+            if not has_anonymized:
+                stats["not_started"] += 1
+                stats["unfinished"] += 1
+            elif has_anonymized and is_validated:
+                stats["validated"] += 1
+            elif has_anonymized and not is_validated:
+                stats["done"] += 1
+            else:
+                stats["unfinished"] += 1
+                
+        return stats
+
+    def _perform_cleanup(self, cleanup_type: str, force: bool, media_type: str, file_id: int) -> Dict[str, Any]:
+        """Perform the actual cleanup operations"""
+        
+        result = {
+            "cleanup_type": cleanup_type,
+            "force": force,
+            "removed_items": [],
+            "summary": {}
+        }
+        
+        if media_type not in ['video', 'pdf', 'all']:
+            raise ValueError(f"Unknown media type: {media_type}")
+        
+        if file_id is not None:
+            try:
+                file_id = int(file_id)
+            except ValueError:
+                raise ValueError(f"Invalid file ID: {file_id}")
+
+        video_file_obj = None
+        pdf_file_obj = None
+            
+        if media_type == 'video':
+            video_file_obj = VideoFile.get_video_by_id(self, video_id=file_id) if file_id else None
+        elif media_type == 'pdf':
+            pdf_file_obj = RawPdfFile.get_pdf_by_id(file_id) if file_id else None
+
+
+        with transaction.atomic():
+            if video_file_obj:
+                video_file_obj.delete()
+            if pdf_file_obj:
+                pdf_file_obj.delete()
+            
+            if cleanup_type == 'unfinished':
+                result.update(self._cleanup_unfinished_media(force))
+            elif cleanup_type == 'failed':
+                result.update(self._cleanup_failed_media(force))
+            elif cleanup_type == 'stale':
+                result.update(self._cleanup_stale_processing(force))
+            elif cleanup_type == 'all':
+                unfinished = self._cleanup_unfinished_media(force)
+                failed = self._cleanup_failed_media(force)
+                stale = self._cleanup_stale_processing(force)
+                
+                result["removed_items"] = (
+                    unfinished.get("removed_items", []) + 
+                    failed.get("removed_items", []) + 
+                    stale.get("removed_items", [])
+                )
+                result["summary"] = {
+                    "unfinished": unfinished.get("summary", {}),
+                    "failed": failed.get("summary", {}),
+                    "stale": stale.get("summary", {})
+                }
+            else:
+                raise ValueError(f"Unknown cleanup type: {cleanup_type}")
+                
+        return result
+
+    def _cleanup_unfinished_media(self, force: bool) -> Dict[str, Any]:
+        """Remove unfinished media processing entries"""
+        removed_videos = []
+        removed_pdfs = []
+        
+        # Find unfinished videos using VideoState boolean fields
+        unfinished_videos = VideoFile.objects.select_related('state').all()
+        
+        for video in unfinished_videos:
+            if not video.state:
+                if force:  # Only remove videos without state if force=True
+                    removed_videos.append({
+                        "id": video.id,
+                        "type": "video",
+                        "filename": video.original_file_name,
+                        "status": "no_state",
+                        "uploaded_at": video.uploaded_at.isoformat()
+                    })
+                    video.delete()
+                continue
+                
+            video_status = video.state.anonymization_status
+            is_unfinished = video_status.value in ['not_started', 'extracting_frames', 'processing_anonymization', 'failed']
+            
+            # Remove unfinished videos
+            if is_unfinished and (force or video_status.value != 'not_started'):
+                removed_videos.append({
+                    "id": video.id,
+                    "type": "video",
+                    "filename": video.original_file_name,
+                    "status": video_status.value,
+                    "uploaded_at": video.uploaded_at.isoformat()
+                })
+                if force:
+                    video.delete()
+        
+        # Return the results
+        return {
+            "removed_items": removed_videos + removed_pdfs,
+            "summary": {
+                "videos_removed": len(removed_videos),
+                "pdfs_removed": len(removed_pdfs),
+                "total_removed": len(removed_videos) + len(removed_pdfs),
+                "dry_run": not force
+            }
+        }
+
+    def _cleanup_failed_media(self, force: bool) -> Dict[str, Any]:
+        """Remove failed media processing entries"""
+        removed_items = []
+        
+        # Find failed videos using VideoState boolean fields
+        failed_videos = VideoFile.objects.select_related('state').all()
+        
+        for video in failed_videos:
+            if video.state and video.state.anonymization_status.value == 'failed':
+                removed_items.append({
+                    "id": video.id,
+                    "type": "video",
+                    "filename": video.original_file_name,
+                    "status": "failed",
+                    "uploaded_at": video.uploaded_at.isoformat()
+                })
+                if force:
+                    video.delete()
+            
+        if force:
+            # Count actual deletions
+            videos_deleted = len([v for v in removed_items if v["type"] == "video"])
+            return {
+                "removed_items": removed_items,
+                "summary": {
+                    "videos_removed": videos_deleted,
+                    "total_removed": videos_deleted,
+                    "dry_run": False
+                }
+            }
+        else:
+            return {
+                "removed_items": removed_items,
+                "summary": {
+                    "videos_removed": len(removed_items),
+                    "total_removed": len(removed_items),
+                    "dry_run": True
+                }
+            }
+
+    def _cleanup_stale_processing(self, force: bool) -> Dict[str, Any]:
+        """Remove stale processing entries (older than 2 hours)"""
+        stale_threshold = timezone.now() - timedelta(hours=2)
+        removed_items = []
+        
+        # Find stale videos using VideoState boolean fields
+        stale_videos = VideoFile.objects.filter(
+            uploaded_at__lt=stale_threshold,
+            state__frames_extracted=True,
+            state__sensitive_meta_processed=False
+        ).select_related('state')
+        
+        for video in stale_videos:
+            video_status = video.state.anonymization_status if video.state else "no_state"
+            removed_items.append({
+                "id": video.id,
+                "type": "video",
+                "filename": video.original_file_name,
+                "status": f"stale_{video_status.value if hasattr(video_status, 'value') else video_status}",
+                "uploaded_at": video.uploaded_at.isoformat(),
+                "stale_duration_hours": (timezone.now() - video.uploaded_at).total_seconds() / 3600
+            })
+            
+        if force:
+            videos_deleted = stale_videos.delete()[0]
+            return {
+                "removed_items": removed_items,
+                "summary": {
+                    "stale_videos_removed": videos_deleted,
+                    "total_removed": videos_deleted,
+                    "dry_run": False
+                }
+            }
+        else:
+            return {
+                "removed_items": removed_items,
+                "summary": {
+                    "stale_videos_removed": len(removed_items),
+                    "total_removed": len(removed_items),
+                    "dry_run": True
+                }
+            }
+
+
+@api_view(['DELETE'])
+@permission_classes(DEBUG_PERMISSIONS)
+def force_remove_media(request, file_id: int):
+    """
+    DELETE /api/media-management/force-remove/{file_id}/
+    Force remove a specific media item regardless of status
+    """
+    try:
+        # Try to find and delete from VideoFile first
+        try:
+            video = VideoFile.objects.get(id=file_id)
+            filename = video.original_file_name
+            video.delete()
+            
+            return Response({
+                "detail": f"Video file '{filename}' (ID: {file_id}) removed successfully",
+                "file_type": "video",
+                "file_id": file_id
+            })
+        except VideoFile.DoesNotExist:
+            pass
+            
+        # Try to find and delete from RawPdfFile
+        try:
+            pdf = RawPdfFile.objects.get(id=file_id)
+            filename = getattr(pdf.file, 'name', 'Unknown')
+            pdf.delete()
+            
+            return Response({
+                "detail": f"PDF file '{filename}' (ID: {file_id}) removed successfully",
+                "file_type": "pdf", 
+                "file_id": file_id
+            })
+        except RawPdfFile.DoesNotExist:
+            pass
+            
+        return Response(
+            {"detail": "File not found"}, 
+            status=status.HTTP_404_NOT_FOUND
+        )
+        
+    except Exception as e:
+        logger.error(f"Error force removing media {file_id}: {e}")
+        return Response(
+            {"error": "Force removal failed"}, 
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )
+
+
+@api_view(['POST'])
+@permission_classes(DEBUG_PERMISSIONS)
+def reset_processing_status(request, file_id: int):
+    """
+    POST /api/media-management/reset-status/{file_id}/
+    Reset processing status for a stuck/failed media item
+    """
+    try:
+        # Try VideoFile first
+        try:
+            video = VideoFile.objects.get(id=file_id)
+            
+            # Reset to 'not_started' state
+            not_started_state, created = VideoState.objects.get_or_create(name='not_started')
+            video.state = not_started_state
+            video.save()
+            
+            return Response({
+                "detail": "Video file status reset to 'not_started'",
+                "file_type": "video",
+                "file_id": file_id,
+                "new_status": "not_started"
+            })
+        except VideoFile.DoesNotExist:
+            pass
+            
+        # PDF files don't have state, but we can clear anonymized_text
+        try:
+            pdf = RawPdfFile.objects.get(id=file_id)
+            pdf.anonymized_text = ""
+            pdf.save()
+            
+            return Response({
+                "detail": "PDF file processing reset",
+                "file_type": "pdf",
+                "file_id": file_id,
+                "new_status": "not_started"
+            })
+        except RawPdfFile.DoesNotExist:
+            pass
+            
+        return Response(
+            {"detail": "File not found"}, 
+            status=status.HTTP_404_NOT_FOUND
+        )
+        
+    except Exception as e:
+        logger.error(f"Error resetting status for media {file_id}: {e}")
+        return Response(
+            {"error": "Status reset failed"}, 
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )

endoreg_db/views/anonymization/overview.py

@@ -0,0 +1,216 @@
+# endoreg_db/api/views/anonymization_overview.py
+
+from rest_framework import status
+from rest_framework.decorators import api_view, permission_classes
+from rest_framework.response import Response
+from endoreg_db.utils.permissions import DEBUG_PERMISSIONS
+from endoreg_db.services.anonymization import AnonymizationService
+from endoreg_db.services.polling_coordinator import PollingCoordinator, ProcessingLockContext
+from rest_framework.generics import ListAPIView
+from rest_framework.pagination import PageNumberPagination
+from endoreg_db.models import VideoFile, RawPdfFile
+from ...serializers import FileOverviewSerializer, VoPPatientDataSerializer
+from django.http import JsonResponse
+import logging
+logger = logging.getLogger(__name__)
+PERMS = DEBUG_PERMISSIONS   # shorten
+
+# ---------- overview ----------------------------------------------------
+class NoPagination(PageNumberPagination):
+    page_size = None
+
+
+class AnonymizationOverviewView(ListAPIView):
+    """
+    GET /api/anonymization/items/overview/
+    --------------------------------------
+    Returns a flat list (Video + PDF) ordered by newest upload first.
+    """
+    serializer_class = FileOverviewSerializer
+    permission_classes = DEBUG_PERMISSIONS   
+    pagination_class = NoPagination
+
+    def get_queryset(self):
+        """
+        Returns a combined queryset of VideoFile and RawPdfFile instances.
+        """
+        # 1) VideoFile queryset - only fields that exist on VideoFile
+        qs_video = (
+            VideoFile.objects
+            .select_related("state", "sensitive_meta")
+            .prefetch_related("label_video_segments__state")
+            .only("id", "original_file_name", "raw_file", "uploaded_at", "state", "sensitive_meta")
+        )
+        # 2) RawPdfFile queryset - only fields that exist on RawPdfFile
+        qs_pdf = (
+            RawPdfFile.objects
+            .select_related("sensitive_meta")
+            .only("id", "file", "date_created", 
+                "text", "anonymized_text",     
+                "sensitive_meta")
+
+        )
+
+        return list(qs_video) + list(qs_pdf)
+    
+# ---------- status with polling protection ------------------------------
+@api_view(["GET"])
+@permission_classes(PERMS)
+def anonymization_status(request, file_id: int):
+    """
+    Get anonymization status with polling rate limiting.
+    """
+    # Ermittele erst den echten Typ und Status
+    info = AnonymizationService.get_status(file_id)
+    if not info:
+        return Response({"detail": "File not found"}, status=status.HTTP_404_NOT_FOUND)
+
+    file_type = info.get("mediaType") or info.get("type") or "video"
+
+    # Wende Rate-Limiting auf den echten Typ an (nicht auf einen evtl. falschen request-Parameter)
+    if not PollingCoordinator.can_check_status(file_id, file_type):
+        remaining_seconds = PollingCoordinator.get_remaining_cooldown_seconds(file_id, file_type)
+        response_data = {
+            "detail": "Status check rate limited. Please wait before checking again.",
+            "file_id": file_id,
+            "cooldown_active": True,
+            "retry_after": remaining_seconds
+        }
+        response = Response(response_data, status=status.HTTP_429_TOO_MANY_REQUESTS)
+        response["Retry-After"] = str(remaining_seconds)
+        return response
+
+    status_val = info.get("anonymizationStatus") or info.get("status") or "not_started"
+
+    # processing_locked als Ableitung des Status interpretieren
+    processing_statuses = {"processing_anonymization", "extracting_frames", "predicting_segments"}
+    processing_locked_derived = status_val in processing_statuses
+
+    return Response({
+        "file_id": file_id,
+        "file_type": file_type,
+        "anonymizationStatus": status_val,
+        "processing_locked": processing_locked_derived,
+    })
+
+# ---------- start with processing lock ----------------------------------
+@api_view(["POST"])
+@permission_classes(PERMS)
+def start_anonymization(request, file_id: int):
+    """
+    Start anonymization with processing lock to prevent duplicates.
+    """
+    # First check what type of file this is
+    info = AnonymizationService.get_status(file_id)
+    if not info:
+        return Response({"detail": "File not found"}, status=status.HTTP_404_NOT_FOUND)
+    
+    file_type = info["mediaType"]
+    
+    # Use processing lock context to prevent duplicate processing
+    with ProcessingLockContext(file_id, file_type) as lock:
+        if not lock.acquired:
+            return Response(
+                {
+                    "detail": "File is already being processed by another request",
+                    "file_id": file_id,
+                    "file_type": file_type,
+                    "processing_locked": True
+                }, 
+                status=status.HTTP_409_CONFLICT
+            )
+        
+        # Proceed with starting anonymization
+        service = AnonymizationService()
+        kind = service.start(file_id)
+        if not kind:
+            return Response({"detail": "Failed to start anonymization"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        
+        return Response({
+            "detail": f"Anonymization started for {kind} file",
+            "file_id": file_id,
+            "file_type": kind,
+            "processing_locked": True
+        })
+
+
+# ---------- current with coordination ------------------------------------
+@api_view(['GET', 'POST', 'PUT'])
+@permission_classes(DEBUG_PERMISSIONS)
+def anonymization_current(request, file_id):
+    """
+    Set current file for validation and return patient data
+    """
+    # Try to find the file in VideoFile first
+    try:
+        video_file = VideoFile.objects.select_related('sensitive_meta').get(id=file_id)
+        serializer = VoPPatientDataSerializer(video_file, context={'request': request})
+        return Response(serializer.data)
+    except VideoFile.DoesNotExist:
+        pass
+    # Try to find the file in RawPdfFile
+    try:
+        pdf_file = RawPdfFile.objects.select_related('sensitive_meta').get(id=file_id)
+        serializer = VoPPatientDataSerializer(pdf_file, context={'request': request})
+        return Response(serializer.data)
+
+    except RawPdfFile.DoesNotExist:
+        pass
+
+    except (ValueError, TypeError, AttributeError) as e:
+        logger.error(f"Error in set_current_for_validation: {e}")
+        return JsonResponse({'status': 'error', 'message': str(e)}, status=500)
+
+    return JsonResponse({'status': 'error', 'message': 'File not found'}, status=404)
+
+# ---------- polling coordinator info ------------------------------------
+@api_view(['GET'])
+@permission_classes(DEBUG_PERMISSIONS)
+def polling_coordinator_info(request):
+    """
+    GET /api/anonymization/polling-info/
+    Get information about polling coordinator status
+    """
+    try:
+        info = PollingCoordinator.get_processing_locks_info()
+        return Response(info)
+    except Exception as e:
+        logger.error(f"Error getting polling coordinator info: {e}")
+        return Response(
+            {"error": "Failed to get coordinator info"}, 
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )
+
+# ---------- emergency lock management -----------------------------------
+@api_view(['DELETE'])
+@permission_classes(DEBUG_PERMISSIONS)
+def clear_processing_locks(request):
+    """
+    DELETE /api/anonymization/clear-locks/
+    Emergency endpoint to clear all processing locks
+    """
+    try:
+        file_type = request.query_params.get('type', None)
+        cleared_count = PollingCoordinator.clear_all_locks(file_type)
+        
+        return Response({
+            "detail": "Processing locks cleared",
+            "cleared_count": cleared_count,
+            "file_type_filter": file_type
+        })
+    except Exception as e:
+        logger.error(f"Error clearing processing locks: {e}")
+        return Response(
+            {"error": "Failed to clear locks"}, 
+            status=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )
+        
+@api_view(['GET'])
+@permission_classes(DEBUG_PERMISSIONS)
+def has_raw_video_file(request, file_id):
+    """
+    GET /api/anonymization/{file_id}/has-raw/
+    Check if a raw video file exists for the given file ID
+    """
+    exists = VideoFile.objects.filter(id=file_id, raw_file__isnull=False).exists()
+    return Response({"file_id": file_id, "has_raw_file": exists})