endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.9.2__py3-none-any.whl

endoreg_db/services/storage_aware_video_processor.py

@@ -1,370 +0,0 @@
-import logging
-import shutil
-from datetime import datetime
-from pathlib import Path
-from typing import Optional
-
-from django.db import transaction
-
-from endoreg_db.exceptions import InsufficientStorageError
-from endoreg_db.models import AnonymizationTask, VideoFile
-
-# from endoreg_db.models.media.video.create_from_file import check_storage_capacity
-
-logger = logging.getLogger(__name__)
-
-
-class StorageAwareVideoProcessor:
-    """
-    Enhanced video processor with built-in storage management and automatic cleanup.
-    """
-
-    def __init__(self, auto_cleanup: bool = True, min_free_space_gb: float = 10.0):
-        self.auto_cleanup = auto_cleanup
-        self.min_free_space_gb = min_free_space_gb
-
-    def check_and_ensure_storage(self, required_space_estimate: int = None) -> bool:
-        """
-        Check storage capacity and perform cleanup if needed.
-
-        Args:
-            required_space_estimate: Estimated space needed in bytes
-
-        Returns:
-            True if enough space is available, False otherwise
-
-        Raises:
-            InsufficientStorageError: If space cannot be freed
-        """
-        try:
-            # Get current storage stats
-            total, used, free = shutil.disk_usage("/")
-            free_gb = free / (1024**3)
-            usage_percent = (used / total) * 100
-
-            logger.info(
-                f"Storage check: {free_gb:.1f}GB free ({usage_percent:.1f}% used)"
-            )
-
-            # Check if we need emergency cleanup
-            needs_cleanup = (
-                usage_percent >= 95.0
-                or free_gb < self.min_free_space_gb
-                or (required_space_estimate and free < required_space_estimate * 1.2)
-            )
-
-            if needs_cleanup and self.auto_cleanup:
-                logger.warning(
-                    f"Storage critically low ({usage_percent:.1f}%), performing automatic cleanup"
-                )
-                self._perform_emergency_cleanup()
-
-                # Re-check after cleanup
-                total, used, free = shutil.disk_usage("/")
-                free_gb = free / (1024**3)
-                usage_percent = (used / total) * 100
-
-                if usage_percent >= 98.0:
-                    raise InsufficientStorageError(
-                        f"Storage critically low even after cleanup: {usage_percent:.1f}% used, "
-                        f"only {free_gb:.1f}GB free",
-                        required_space=required_space_estimate
-                        or int(self.min_free_space_gb * 1024**3),
-                        available_space=free,
-                    )
-
-            return True
-
-        except Exception as e:
-            logger.error(f"Storage check failed: {e}")
-            raise
-
-    def _perform_emergency_cleanup(self) -> int:
-        """
-        Perform emergency storage cleanup.
-
-        Returns:
-            Bytes freed
-        """
-        total_freed = 0
-
-        try:
-            # 1. Clean up extracted frames for completed videos
-            frames_freed = self._cleanup_completed_video_frames()
-            total_freed += frames_freed
-
-            # 2. Clean up upload cache older than 1 day
-            uploads_freed = self._cleanup_old_uploads(max_age_hours=24)
-            total_freed += uploads_freed
-
-            # 3. Clean up large log files
-            logs_freed = self._cleanup_large_logs()
-            total_freed += logs_freed
-
-            # 4. Clean up temporary files
-            temp_freed = self._cleanup_temp_files()
-            total_freed += temp_freed
-
-            logger.info(
-                f"Emergency cleanup completed: {total_freed / (1024**3):.2f}GB freed"
-            )
-
-        except Exception as e:
-            logger.error(f"Emergency cleanup failed: {e}")
-
-        return total_freed
-
-    def _cleanup_completed_video_frames(self) -> int:
-        """Clean up frames for videos that have completed processing."""
-        total_freed = 0
-
-        try:
-            from django.conf import settings
-
-            frames_dir = Path(settings.BASE_DIR).parent / "storage" / "frames"
-
-            if not frames_dir.exists():
-                return 0
-
-            # Find videos with completed anonymization
-            completed_videos = VideoFile.objects.filter(
-                anonymization_tasks__status="done_processing_anonymization"
-            ).distinct()
-
-            for video in completed_videos:
-                # Find frame directories for this video
-                video_frame_dirs = list(frames_dir.glob(f"*{video.uuid}*"))
-
-                for frame_dir in video_frame_dirs:
-                    if frame_dir.is_dir():
-                        try:
-                            # Calculate directory size
-                            dir_size = sum(
-                                f.stat().st_size
-                                for f in frame_dir.rglob("*")
-                                if f.is_file()
-                            )
-
-                            # Remove the directory
-                            shutil.rmtree(frame_dir, ignore_errors=True)
-                            total_freed += dir_size
-
-                            logger.info(
-                                f"Cleaned frames for {video.uuid}: {dir_size / (1024**2):.1f}MB"
-                            )
-
-                        except Exception as e:
-                            logger.warning(
-                                f"Failed to clean frames for {video.uuid}: {e}"
-                            )
-
-        except Exception as e:
-            logger.error(f"Frame cleanup failed: {e}")
-
-        return total_freed
-
-    def _cleanup_old_uploads(self, max_age_hours: int = 24) -> int:
-        """Clean up old upload cache files."""
-        total_freed = 0
-
-        try:
-            from django.conf import settings
-
-            uploads_dir = Path(settings.BASE_DIR).parent / "storage" / "uploads"
-
-            if not uploads_dir.exists():
-                return 0
-
-            cutoff_time = datetime.now().timestamp() - (max_age_hours * 3600)
-
-            for file_path in uploads_dir.rglob("*"):
-                if file_path.is_file():
-                    try:
-                        if file_path.stat().st_mtime < cutoff_time:
-                            file_size = file_path.stat().st_size
-                            file_path.unlink()
-                            total_freed += file_size
-
-                    except Exception as e:
-                        logger.warning(f"Failed to clean upload file {file_path}: {e}")
-
-        except Exception as e:
-            logger.error(f"Upload cleanup failed: {e}")
-
-        return total_freed
-
-    def _cleanup_large_logs(self, max_size_mb: int = 50) -> int:
-        """Clean up large log files by truncating them."""
-        total_freed = 0
-
-        try:
-            from django.conf import settings
-
-            project_root = Path(settings.BASE_DIR).parent
-            max_size_bytes = max_size_mb * 1024 * 1024
-
-            for log_file in project_root.rglob("*.log"):
-                try:
-                    if log_file.stat().st_size > max_size_bytes:
-                        original_size = log_file.stat().st_size
-
-                        # Truncate the log file
-                        with open(log_file, "w") as f:
-                            f.write(
-                                f"# Log truncated at {datetime.now()} due to size limit\n"
-                            )
-
-                        total_freed += original_size
-                        logger.info(
-                            f"Truncated log {log_file}: {original_size / (1024**2):.1f}MB"
-                        )
-
-                except Exception as e:
-                    logger.warning(f"Failed to truncate log {log_file}: {e}")
-
-        except Exception as e:
-            logger.error(f"Log cleanup failed: {e}")
-
-        return total_freed
-
-    def _cleanup_temp_files(self) -> int:
-        """Clean up temporary files."""
-        total_freed = 0
-
-        try:
-            temp_patterns = [
-                "/tmp/tmp*",
-                "/tmp/temp*",
-                "/tmp/django*",
-                "/tmp/ffmpeg*",
-            ]
-
-            for pattern in temp_patterns:
-                import glob
-
-                for temp_file in glob.glob(pattern):
-                    try:
-                        temp_path = Path(temp_file)
-                        if temp_path.is_file():
-                            file_size = temp_path.stat().st_size
-                            temp_path.unlink()
-                            total_freed += file_size
-
-                    except Exception as e:
-                        logger.warning(f"Failed to clean temp file {temp_file}: {e}")
-
-        except Exception as e:
-            logger.error(f"Temp file cleanup failed: {e}")
-
-        return total_freed
-
-    def process_video_with_storage_management(
-        self,
-        video_file: VideoFile,
-        cleanup_frames_after: bool = True,
-        progress_callback: Optional[callable] = None,
-    ) -> bool:
-        """
-        Process a video with automatic storage management.
-
-        Args:
-            video_file: VideoFile instance to process
-            cleanup_frames_after: Whether to clean up frames after processing
-            progress_callback: Optional progress callback function
-
-        Returns:
-            True if processing succeeded, False otherwise
-        """
-        try:
-            # Estimate required space (rough calculation)
-            video_size = 0
-            if video_file.raw_file and hasattr(video_file.raw_file, "size"):
-                video_size = video_file.raw_file.size
-
-            # Estimate: frames ~= 3x video size, processing ~= 2x video size
-            estimated_space_needed = video_size * 5
-
-            # Check storage before starting
-            self.check_and_ensure_storage(estimated_space_needed)
-
-            # Update task status
-            task = self._get_or_create_task(video_file)
-            task.start_processing()
-
-            if progress_callback:
-                progress_callback(10, "Starting video processing...")
-
-            # Run the actual video processing pipeline
-            success = video_file.pipe_1(
-                model_name="image_multilabel_classification_colonoscopy_default",
-                delete_frames_after=cleanup_frames_after,
-            )
-
-            if success:
-                if progress_callback:
-                    progress_callback(90, "Processing completed, cleaning up...")
-
-                # Mark as completed
-                task.mark_completed("Video processing completed successfully")
-
-                # Additional cleanup if requested
-                if cleanup_frames_after:
-                    self._cleanup_video_frames(video_file)
-
-                if progress_callback:
-                    progress_callback(100, "Processing completed successfully")
-
-                logger.info(
-                    f"Video processing completed successfully: {video_file.uuid}"
-                )
-                return True
-
-            task.mark_failed("Video processing pipeline failed")
-            logger.error(f"Video processing failed: {video_file.uuid}")
-            return False
-
-        except InsufficientStorageError as e:
-            logger.error(f"Storage error during video processing: {e}")
-            task = self._get_or_create_task(video_file)
-            task.mark_failed(f"Insufficient storage: {e}")
-            return False
-
-        except Exception as e:
-            logger.error(f"Video processing failed: {e}")
-            task = self._get_or_create_task(video_file)
-            task.mark_failed(f"Processing error: {e}")
-            return False
-
-    def _get_or_create_task(self, video_file: VideoFile) -> AnonymizationTask:
-        """Get or create an anonymization task for the video."""
-        task, created = AnonymizationTask.objects.get_or_create(
-            video_file=video_file,
-            defaults={
-                "status": "not_started",
-                "progress": 0,
-                "message": "Initializing...",
-            },
-        )
-        return task
-
-    def _cleanup_video_frames(self, video_file: VideoFile):
-        """Clean up frames for a specific video."""
-        try:
-            from django.conf import settings
-
-            frames_dir = Path(settings.BASE_DIR).parent / "storage" / "frames"
-
-            # Find frame directories for this video
-            video_frame_dirs = list(frames_dir.glob(f"*{video_file.uuid}*"))
-
-            for frame_dir in video_frame_dirs:
-                if frame_dir.is_dir():
-                    shutil.rmtree(frame_dir, ignore_errors=True)
-                    logger.info(f"Cleaned up frames for video {video_file.uuid}")
-
-        except Exception as e:
-            logger.warning(f"Failed to clean up frames for {video_file.uuid}: {e}")
-
-
-# Global instance for easy access
-storage_aware_processor = StorageAwareVideoProcessor()

endoreg_db/urls/files.py

@@ -1,6 +0,0 @@
-from django.urls import path
-from endoreg_db.views import AvailableFilesListView
-
-urlpatterns = [
-    path('available-files/', AvailableFilesListView.as_view(), name='available_files_list'),
-]

endoreg_db/urls/label_video_segment_validate.py

@@ -1,33 +0,0 @@
-from django.urls import path
-
-from endoreg_db.views.label_video_segment.validate import (
-    LabelVideoSegmentValidateView,
-    BulkSegmentValidateView,
-    VideoSegmentValidationCompleteView
-)
-
-url_patterns = [
-    # Einzelne Segment-Validierung
-    # POST /api/label-video-segment/<int:segment_id>/validate/
-    path(
-        'label-video-segment/<int:segment_id>/validate/',
-        LabelVideoSegmentValidateView.as_view(),
-        name='label_video_segment_validate'
-    ),
-    
-    # Bulk-Validierung mehrerer Segmente
-    # POST /api/label-video-segments/validate-bulk/
-    path(
-        'label-video-segments/validate-bulk/',
-        BulkSegmentValidateView.as_view(),
-        name='label_video_segments_validate_bulk'
-    ),
-    
-    # Alle Segmente eines Videos als validiert markieren
-    # POST /api/videos/<int:video_id>/segments/validate-complete/
-    path(
-        'videos/<int:video_id>/segments/validate-complete/',
-        VideoSegmentValidationCompleteView.as_view(),
-        name='video_segments_validate_complete'
-    ),
-]

endoreg_db/urls/label_video_segments.py

@@ -1,46 +0,0 @@
-from django.urls import path
-from endoreg_db.views import (
-    create_video_segment_annotation, 
-    update_label_video_segment,
-    SensitiveMetaDetailView,
-    video_segments_view,
-    video_segment_detail_view,
-    get_lvs_by_name_and_video_id
-)
-
-
-
-url_patterns = [
-    path(
-        "lvs/by-label-name/<str:label_name>/by-video-id/<int:video_id>/",
-        get_lvs_by_name_and_video_id,
-        name='get_lvs_by_name_and_video_id'
-    ),
-    path(
-        'annotations/',
-        create_video_segment_annotation,
-        name='create_annotation'
-    ),
-    path(
-        'annotations/<int:annotation_id>/',
-        update_label_video_segment,
-        name='update_annotation'
-    ),
-    path(
-        'save-anonymization-annotation-video/<int:annotation_id>/',
-        SensitiveMetaDetailView.as_view(),
-        name='save_anonymization_annotation_video'
-    ),
-    # Label Video Segment API endpoints
-    path(
-        'video-segments/',
-        video_segments_view,
-        name='video_segments'
-    ),
-    path(
-        'video-segments/<int:segment_id>/',
-        video_segment_detail_view,
-        name='video_segment_detail'
-    ),
-
-]

endoreg_db/views/label/__init__.py

@@ -1,5 +0,0 @@
-from .label import LabelViewSet
-
-__all__ = [
-    "LabelViewSet",
-]

endoreg_db/views/label/label.py

@@ -1,15 +0,0 @@
-
-from rest_framework.viewsets import ReadOnlyModelViewSet
-from rest_framework.permissions import IsAuthenticatedOrReadOnly
-from endoreg_db.models import (
-    Label,
-)
-from endoreg_db.serializers.label import LabelSerializer
-
-
-
-class LabelViewSet(ReadOnlyModelViewSet):
-    queryset = Label.objects.all()
-    serializer_class = LabelSerializer
-    permission_classes = [IsAuthenticatedOrReadOnly]
-

endoreg_db/views/label_video_segment/__init__.py

@@ -1,16 +0,0 @@
-from .create_lvs_from_annotation import create_video_segment_annotation
-from .label_video_segment_by_label import video_segments_by_label_id_view, video_segments_by_label_name_view
-from .label_video_segment_detail import video_segment_detail_view
-from .label_video_segment import video_segments_view
-from .update_lvs_from_annotation import update_label_video_segment
-from .get_lvs_by_name_and_video import get_lvs_by_name_and_video_id
-
-__all__ = [
-    'create_video_segment_annotation',
-    'video_segments_by_label_id_view',
-    'video_segments_by_label_name_view',
-    'video_segment_detail_view',
-    'video_segments_view',
-    'update_label_video_segment',
-    "get_lvs_by_name_and_video_id"
-]

endoreg_db/views/label_video_segment/create_lvs_from_annotation.py

@@ -1,44 +0,0 @@
-from endoreg_db.models import LabelVideoSegment
-from endoreg_db.serializers.label_video_segment.label_video_segment_annotation import LabelVideoSegmentAnnotationSerializer
-
-from django.db import transaction
-from rest_framework import status
-from rest_framework.decorators import api_view, permission_classes
-from rest_framework.permissions import IsAuthenticated
-from rest_framework.response import Response
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-@api_view(['POST'])
-@permission_classes([IsAuthenticated])
-def create_video_segment_annotation(request):
-    """
-    Create a new annotation.
-    If annotation type is 'segment', creates a user-source LabelVideoSegment.
-    """
-    logger.info(f"Creating annotation with data: {request.data}")
-
-    with transaction.atomic():
-        serializer = LabelVideoSegmentAnnotationSerializer(data=request.data)
-        if serializer.is_valid():
-            annotation = serializer.save()
-
-            # If the annotation type is 'segment', create a new LabelVideoSegment
-            if annotation.type == 'segment':
-                metadata = annotation.metadata or {}
-                #TODO @coderabbitai we should create a followup issue for @Hamzaukw so we adress this naming inconsistency in future
-                LabelVideoSegment.objects.create(
-                    video_id=annotation.video_id,
-                    start_frame=metadata.get('start_frame'),
-                    end_frame=metadata.get('end_frame'),
-                    label=metadata.get('label'),
-                    source='user'
-                )
-            
-            logger.info(f"Successfully created annotation {annotation.id}")
-            return Response(serializer.data, status=status.HTTP_201_CREATED)
-    
-    logger.error(f"Failed to create annotation with data: {request.data}, errors: {serializer.errors}")
-    return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py

@@ -1,50 +0,0 @@
-from endoreg_db.models import Label, LabelVideoSegment, VideoFile
-from endoreg_db.serializers.label_video_segment.label_video_segment import LabelVideoSegmentSerializer
-
-from rest_framework import status
-from rest_framework.decorators import api_view, permission_classes
-from rest_framework.response import Response
-
-from endoreg_db.utils.permissions import EnvironmentAwarePermission
-import logging
-logger = logging.getLogger(__name__)
-
-@api_view(['GET'])
-@permission_classes([EnvironmentAwarePermission])
-def get_lvs_by_name_and_video_id(request, label_name, video_id):
-    """
-    Handles creation and retrieval of labeled video segments.
-    GET lists all labeled video segments, filtered by video ID and label name.
-    Returns appropriate error responses for invalid input or missing referenced objects.
-    """
-
-    # Check if label exists
-    try:
-        _label = Label.objects.get(name=label_name)
-    except Label.DoesNotExist:
-        logger.error(f"Label with name '{label_name}' does not exist.")
-        return Response(
-            {"error": f"Label with name '{label_name}' does not exist."},
-            status=status.HTTP_404_NOT_FOUND
-        )
-
-    # Check if video exists
-    try:
-        _video = VideoFile.objects.get(id=video_id)
-    except VideoFile.DoesNotExist:
-        logger.error(f"Video with ID '{video_id}' does not exist.")
-        return Response(
-            {"error": f"Video with ID '{video_id}' does not exist."},
-            status=status.HTTP_404_NOT_FOUND
-        )
-
-    # get all segments for the given video and label name
-    queryset = LabelVideoSegment.objects.filter(
-        video_file__id=video_id,
-        label__name=label_name
-    )
-
-    # Order by video and start time for consistent results
-    segments = queryset.order_by('video_file__id', 'start_frame_number')
-    serializer = LabelVideoSegmentSerializer(segments, many=True)
-    return Response(serializer.data)

endoreg_db/views/label_video_segment/label_video_segment.py

@@ -1,77 +0,0 @@
-from endoreg_db.models import Label, LabelVideoSegment, VideoFile
-from endoreg_db.serializers.label_video_segment.label_video_segment import LabelVideoSegmentSerializer
-
-from django.db import transaction
-from rest_framework import status
-from rest_framework.decorators import api_view, permission_classes
-from rest_framework.response import Response
-
-from endoreg_db.utils.permissions import EnvironmentAwarePermission
-import logging
-logger = logging.getLogger(__name__)
-
-@api_view(['POST', 'GET'])
-@permission_classes([EnvironmentAwarePermission])
-def video_segments_view(request):
-    """
-    Handles creation and retrieval of labeled video segments.
-
-    POST creates a new labeled video segment with validated data and returns the created segment. GET lists all labeled video segments, optionally filtered by video ID and/or label ID. Returns appropriate error responses for invalid input or missing referenced objects.
-    """
-    if request.method == 'POST':
-        logger.info(f"Creating new video segment with data: {request.data}")
-
-        with transaction.atomic():
-            serializer = LabelVideoSegmentSerializer(data=request.data)
-            if serializer.is_valid():
-                try:
-                    segment = serializer.save()
-                    logger.info(f"Successfully created video segment {segment.pk}")
-                    return Response(
-                        LabelVideoSegmentSerializer(segment).data,
-                        status=status.HTTP_201_CREATED
-                    )
-                except Exception as e:
-                    logger.error(f"Error creating video segment: {str(e)}")
-                    return Response(
-                        {'error': f'Failed to create segment: {str(e)}'},
-                        status=status.HTTP_500_INTERNAL_SERVER_ERROR
-                    )
-            else:
-                logger.warning(f"Invalid data for video segment creation: {serializer.errors}")
-                return Response(
-                    {'error': 'Invalid data', 'details': serializer.errors},
-                    status=status.HTTP_400_BAD_REQUEST
-                )
-
-    elif request.method == 'GET':
-        # Optional filtering by video_id
-        video_id = request.GET.get('video_id')
-        label_id = request.GET.get('label_id')
-
-        queryset = LabelVideoSegment.objects.all()
-
-        if video_id:
-            try:
-                video = VideoFile.objects.get(id=video_id)
-                queryset = queryset.filter(video_file=video)
-            except VideoFile.DoesNotExist:
-                return Response(
-                    {'error': f'Video with id {video_id} not found'},
-                    status=status.HTTP_404_NOT_FOUND
-                )
-
-        if label_id:
-            try:
-                label = Label.objects.get(id=label_id)
-                queryset = queryset.filter(label=label)
-            except Label.DoesNotExist:
-                return Response(
-                    {'error': f'Label with id {label_id} not found'},
-                    status=status.HTTP_404_NOT_FOUND
-                )
-
-        # Order by video and start time for consistent results
-        segments = queryset.order_by('video_file__id', 'start_frame_number')
-        serializer = LabelVideoSegmentSerializer(segments, many=True)
-        return Response(serializer.data)