endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

endoreg_db/serializers/video/video_metadata.py

@@ -1,105 +0,0 @@
-"""
-Video Metadata Serializer
-
-Serializes VideoMetadata model for API responses.
-Created as part of Phase 1.1: Video Correction API Endpoints.
-"""
-from rest_framework import serializers
-from endoreg_db.models.media.video import VideoMetadata
-import json
-
-
-class VideoMetadataSerializer(serializers.ModelSerializer):
-    """
-    Serializer for VideoMetadata model.
-    
-    Provides analysis results (sensitive frame count, ratio, frame IDs)
-    for the correction UI.
-    """
-    sensitive_frame_ids_list = serializers.SerializerMethodField()
-    sensitive_percentage = serializers.ReadOnlyField()
-    has_analysis = serializers.ReadOnlyField()
-    
-    class Meta:
-        model = VideoMetadata
-        fields = [
-            'id',
-            'video',
-            'sensitive_frame_count',
-            'sensitive_ratio',
-            'sensitive_frame_ids',
-            'sensitive_frame_ids_list',
-            'sensitive_percentage',
-            'has_analysis',
-            'analyzed_at',
-        ]
-        read_only_fields = ['id', 'analyzed_at']
-    
-    def get_sensitive_frame_ids_list(self, obj) -> list:
-        """
-        Parse sensitive_frame_ids from JSON string to Python list.
-        
-        Returns:
-            list: Frame indices (0-based), or empty list if no analysis
-        """
-        if not obj.sensitive_frame_ids:
-            return []
-        
-        try:
-            return json.loads(obj.sensitive_frame_ids)
-        except (json.JSONDecodeError, TypeError):
-            return []
-    
-    def validate_sensitive_frame_ids(self, value):
-        """
-        Validate that sensitive_frame_ids is a valid JSON array.
-        
-        Args:
-            value: JSON string representing frame IDs
-            
-        Returns:
-            str: Validated JSON string
-            
-        Raises:
-            ValidationError: If value is not valid JSON or not an array
-        """
-        if not value:
-            return value
-        
-        try:
-            parsed = json.loads(value)
-            if not isinstance(parsed, list):
-                raise serializers.ValidationError(
-                    "sensitive_frame_ids must be a JSON array"
-                )
-            
-            # Validate all elements are integers
-            if not all(isinstance(x, int) for x in parsed):
-                raise serializers.ValidationError(
-                    "All frame IDs must be integers"
-                )
-            
-            return value
-        except json.JSONDecodeError:
-            raise serializers.ValidationError(
-                "sensitive_frame_ids must be valid JSON"
-            )
-    
-    def validate_sensitive_ratio(self, value):
-        """
-        Validate that sensitive_ratio is between 0.0 and 1.0.
-        
-        Args:
-            value: Ratio value
-            
-        Returns:
-            float: Validated ratio
-            
-        Raises:
-            ValidationError: If value is outside valid range
-        """
-        if value is not None and (value < 0.0 or value > 1.0):
-            raise serializers.ValidationError(
-                "sensitive_ratio must be between 0.0 and 1.0"
-            )
-        return value

endoreg_db/services/requirements_object.py

@@ -1,147 +0,0 @@
-from ast import List
-import pickle
-from endoreg_db.models import Requirement, Examination, PatientExamination
-from endoreg_db.serializers import PatientExaminationSerializer, ExaminationSerializer, FindingClassificationSerializer, RequirementSerializer
-
-
-class LookupService:
-    def __init__(self, db_session):
-        self.db_session = db_session
-        self.hexdict = int()
-
-    def hash_lookup(self, key, value):
-        """
-        Store a key-value pair in the lookup dictionary.
-        """
-        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
-        lookup_dict[key] = value
-        self.hexdict = pickle.dumps(dict).hex()
-        return True
-    def hash_retrieve(self, key):
-        """
-        Retrieve a value by key from the lookup dictionary.
-        """
-        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
-        return lookup_dict.get(key, None)
-    
-    def hash_add_key_value(self, key, value):
-        """
-        Add a key-value pair to the lookup dictionary.
-        """
-        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
-        lookup_dict[key] = value
-        self.hexdict = pickle.dumps(lookup_dict).hex()
-        return True
-    
-    def hash_dehash(self):
-        """
-        Retrieve the entire lookup dictionary.
-        """
-        lookup_dict = pickle.loads(bytes.fromhex(self.hexdict))
-        return lookup_dict
-    
-    def fetch_examination_dict(self, patient_id, tags: List[str] = None):
-        """
-        Fetch examinations for a given patient ID, optionally filtered by tags.
-        """
-        query = self.db_session.query(Examination).filter(Examination.patient_id == patient_id)
-        if tags:
-            query = query.filter(Examination.tags.any(Examination.tags.in_(tags)))
-        examinations = query.all()
-        return [ExaminationSerializer(exam).data for exam in examinations]  
-    
-    def fetch_patient_examination_by_id(self, patient_id, examination_id):
-        """
-        Fetch a patient examination object from the database by its ID.
-        """
-        return self.db_session.query(PatientExamination).filter(PatientExamination.id == patient_examination_id).first()
-    
-    def fetch_patient_examinations(self, patient_id):
-        """
-        Fetch all patient examinations for a given patient ID.
-        """
-        examinations = self.db_session.query(PatientExamination).filter(PatientExamination.patient_id == patient_id).all()
-        return [PatientExaminationSerializer(exam).data for exam in examinations]
-
-    def fetch_findings_by_patient_examination(self, patient_examination_id):
-        """
-        Fetch findings associated with a given patient examination ID.
-        """
-        patient_examination = self.db_session.query(PatientExamination).filter(PatientExamination.id == patient_examination_id).first()
-        if not patient_examination:
-            return []
-        return [finding.serialize() for finding in patient_examination.findings]
-    
-    def fetch_classifications_by_finding(self, finding_id):
-        """
-        Fetch classifications associated with a given finding ID.
-        """
-        finding = self.db_session.query(Finding).filter(Finding.id == finding_id).first()
-        if not finding:
-            return []
-        return [classification.serialize() for classification in finding.classifications]
-    
-    def fetch_default_finding_classification(self):
-        """
-        Fetch the default finding classification from the database.
-        """
-        return self.db_session.query(FindingClassification).filter(FindingClassification.is_default == True).first()
-    
-    def fetch_finding_classification_choice_by_requirement(self, requirement_id):
-        """
-        Retrieve finding classification choices based on the provided requirement ID.
-        """
-        requirement = self.get_requirement_by_id(requirement_id)
-        if not requirement:
-            return []
-        return [choice.serialize() for choice in requirement.finding_classification_choices]
-    
-    def fetch_requirement_choices_by_requirement(self, requirement_id):
-        """
-        Retrieve requirement choices based on the provided requirement ID.
-        """
-        requirement = self.get_requirement_by_id(requirement_id)
-        if not requirement:
-            return []
-        return [choice.serialize() for choice in requirement.requirement_choices]
-    
-    def available_finding_classification_choices_by_finding_classification(self, finding_classification):
-        """
-        Retrieve available finding classification choices based on the provided finding classification.
-        """
-        return self.db_session.query(FindingClassificationChoice).filter(
-            FindingClassificationChoice.finding_classification_id == finding_classification.id
-        ).all()
-
-    def get_requirement_by_id(self, requirement_id):
-        """
-        Fetch a requirement object from the database by its ID.
-        """
-        return self.db_session.query(Requirement).filter(Requirement.id == requirement_id).first()
-
-    def get_all_requirements_for_requirement_sets(self, requirement_set_ids: List[int]):
-        """
-        Fetch all requirements associated with the given requirement set IDs.
-        """
-        return self.db_session.query(Requirement).filter(Requirement.requirement_set_id.in_(requirement_set_ids)).all()
-    
-    def get_all_requirement_sets_for_examination(self, examination_id):
-        """
-        Fetch all requirement sets associated with the given examination ID.
-        """
-        examination = self.db_session.query(Examination).filter(Examination.id == examination_id).first()
-        if not examination:
-            return []
-        return [req_set.serialize() for req_set in examination.requirement_sets]
-
-    def serialize_requirement(self, requirement):
-        """
-        Serialize a requirement object to a byte stream.
-        """
-        return pickle.dumps(requirement)
-
-    def deserialize_requirement(self, serialized_requirement):
-        """
-        Deserialize a byte stream back to a requirement object.
-        """
-        return pickle.loads(serialized_requirement)

endoreg_db/services/storage_aware_video_processor.py

@@ -1,344 +0,0 @@
-from django.db import transaction
-from pathlib import Path
-import logging
-import shutil
-from datetime import datetime
-from typing import Optional
-
-from endoreg_db.models import VideoFile, AnonymizationTask
-from endoreg_db.exceptions import InsufficientStorageError
-# from endoreg_db.models.media.video.create_from_file import check_storage_capacity
-
-logger = logging.getLogger(__name__)
-
-
-class StorageAwareVideoProcessor:
-    """
-    Enhanced video processor with built-in storage management and automatic cleanup.
-    """
-    
-    def __init__(self, auto_cleanup: bool = True, min_free_space_gb: float = 10.0):
-        self.auto_cleanup = auto_cleanup
-        self.min_free_space_gb = min_free_space_gb
-        
-    def check_and_ensure_storage(self, required_space_estimate: int = None) -> bool:
-        """
-        Check storage capacity and perform cleanup if needed.
-        
-        Args:
-            required_space_estimate: Estimated space needed in bytes
-            
-        Returns:
-            True if enough space is available, False otherwise
-            
-        Raises:
-            InsufficientStorageError: If space cannot be freed
-        """
-        try:
-            # Get current storage stats
-            total, used, free = shutil.disk_usage('/')
-            free_gb = free / (1024**3)
-            usage_percent = (used / total) * 100
-            
-            logger.info(f"Storage check: {free_gb:.1f}GB free ({usage_percent:.1f}% used)")
-            
-            # Check if we need emergency cleanup
-            needs_cleanup = (
-                usage_percent >= 95.0 or 
-                free_gb < self.min_free_space_gb or
-                (required_space_estimate and free < required_space_estimate * 1.2)
-            )
-            
-            if needs_cleanup and self.auto_cleanup:
-                logger.warning(f"Storage critically low ({usage_percent:.1f}%), performing automatic cleanup")
-                self._perform_emergency_cleanup()
-                
-                # Re-check after cleanup
-                total, used, free = shutil.disk_usage('/')
-                free_gb = free / (1024**3)
-                usage_percent = (used / total) * 100
-                
-                if usage_percent >= 98.0:
-                    raise InsufficientStorageError(
-                        f"Storage critically low even after cleanup: {usage_percent:.1f}% used, "
-                        f"only {free_gb:.1f}GB free",
-                        required_space=required_space_estimate or int(self.min_free_space_gb * 1024**3),
-                        available_space=free
-                    )
-            
-            return True
-            
-        except Exception as e:
-            logger.error(f"Storage check failed: {e}")
-            raise
-    
-    def _perform_emergency_cleanup(self) -> int:
-        """
-        Perform emergency storage cleanup.
-        
-        Returns:
-            Bytes freed
-        """
-        total_freed = 0
-        
-        try:
-            # 1. Clean up extracted frames for completed videos
-            frames_freed = self._cleanup_completed_video_frames()
-            total_freed += frames_freed
-            
-            # 2. Clean up upload cache older than 1 day
-            uploads_freed = self._cleanup_old_uploads(max_age_hours=24)
-            total_freed += uploads_freed
-            
-            # 3. Clean up large log files
-            logs_freed = self._cleanup_large_logs()
-            total_freed += logs_freed
-            
-            # 4. Clean up temporary files
-            temp_freed = self._cleanup_temp_files()
-            total_freed += temp_freed
-            
-            logger.info(f"Emergency cleanup completed: {total_freed / (1024**3):.2f}GB freed")
-            
-        except Exception as e:
-            logger.error(f"Emergency cleanup failed: {e}")
-            
-        return total_freed
-    
-    def _cleanup_completed_video_frames(self) -> int:
-        """Clean up frames for videos that have completed processing."""
-        total_freed = 0
-        
-        try:
-            from django.conf import settings
-            frames_dir = Path(settings.BASE_DIR).parent / 'storage' / 'frames'
-            
-            if not frames_dir.exists():
-                return 0
-            
-            # Find videos with completed anonymization
-            completed_videos = VideoFile.objects.filter(
-                anonymization_tasks__status='done'
-            ).distinct()
-            
-            for video in completed_videos:
-                # Find frame directories for this video
-                video_frame_dirs = list(frames_dir.glob(f"*{video.uuid}*"))
-                
-                for frame_dir in video_frame_dirs:
-                    if frame_dir.is_dir():
-                        try:
-                            # Calculate directory size
-                            dir_size = sum(
-                                f.stat().st_size for f in frame_dir.rglob('*') if f.is_file()
-                            )
-                            
-                            # Remove the directory
-                            shutil.rmtree(frame_dir, ignore_errors=True)
-                            total_freed += dir_size
-                            
-                            logger.info(f"Cleaned frames for {video.uuid}: {dir_size / (1024**2):.1f}MB")
-                            
-                        except Exception as e:
-                            logger.warning(f"Failed to clean frames for {video.uuid}: {e}")
-                            
-        except Exception as e:
-            logger.error(f"Frame cleanup failed: {e}")
-            
-        return total_freed
-    
-    def _cleanup_old_uploads(self, max_age_hours: int = 24) -> int:
-        """Clean up old upload cache files."""
-        total_freed = 0
-        
-        try:
-            from django.conf import settings
-            uploads_dir = Path(settings.BASE_DIR).parent / 'storage' / 'uploads'
-            
-            if not uploads_dir.exists():
-                return 0
-            
-            cutoff_time = datetime.now().timestamp() - (max_age_hours * 3600)
-            
-            for file_path in uploads_dir.rglob('*'):
-                if file_path.is_file():
-                    try:
-                        if file_path.stat().st_mtime < cutoff_time:
-                            file_size = file_path.stat().st_size
-                            file_path.unlink()
-                            total_freed += file_size
-                            
-                    except Exception as e:
-                        logger.warning(f"Failed to clean upload file {file_path}: {e}")
-                        
-        except Exception as e:
-            logger.error(f"Upload cleanup failed: {e}")
-            
-        return total_freed
-    
-    def _cleanup_large_logs(self, max_size_mb: int = 50) -> int:
-        """Clean up large log files by truncating them."""
-        total_freed = 0
-        
-        try:
-            from django.conf import settings
-            project_root = Path(settings.BASE_DIR).parent
-            max_size_bytes = max_size_mb * 1024 * 1024
-            
-            for log_file in project_root.rglob('*.log'):
-                try:
-                    if log_file.stat().st_size > max_size_bytes:
-                        original_size = log_file.stat().st_size
-                        
-                        # Truncate the log file
-                        with open(log_file, 'w') as f:
-                            f.write(f"# Log truncated at {datetime.now()} due to size limit\n")
-                            
-                        total_freed += original_size
-                        logger.info(f"Truncated log {log_file}: {original_size / (1024**2):.1f}MB")
-                        
-                except Exception as e:
-                    logger.warning(f"Failed to truncate log {log_file}: {e}")
-                    
-        except Exception as e:
-            logger.error(f"Log cleanup failed: {e}")
-            
-        return total_freed
-    
-    def _cleanup_temp_files(self) -> int:
-        """Clean up temporary files."""
-        total_freed = 0
-        
-        try:
-            temp_patterns = [
-                '/tmp/tmp*',
-                '/tmp/temp*',
-                '/tmp/django*',
-                '/tmp/ffmpeg*',
-            ]
-            
-            for pattern in temp_patterns:
-                import glob
-                for temp_file in glob.glob(pattern):
-                    try:
-                        temp_path = Path(temp_file)
-                        if temp_path.is_file():
-                            file_size = temp_path.stat().st_size
-                            temp_path.unlink()
-                            total_freed += file_size
-                            
-                    except Exception as e:
-                        logger.warning(f"Failed to clean temp file {temp_file}: {e}")
-                        
-        except Exception as e:
-            logger.error(f"Temp file cleanup failed: {e}")
-            
-        return total_freed
-    
-    def process_video_with_storage_management(
-        self, 
-        video_file: VideoFile, 
-        cleanup_frames_after: bool = True,
-        progress_callback: Optional[callable] = None
-    ) -> bool:
-        """
-        Process a video with automatic storage management.
-        
-        Args:
-            video_file: VideoFile instance to process
-            cleanup_frames_after: Whether to clean up frames after processing
-            progress_callback: Optional progress callback function
-            
-        Returns:
-            True if processing succeeded, False otherwise
-        """
-        try:
-            # Estimate required space (rough calculation)
-            video_size = 0
-            if video_file.raw_file and hasattr(video_file.raw_file, 'size'):
-                video_size = video_file.raw_file.size
-            
-            # Estimate: frames ~= 3x video size, processing ~= 2x video size
-            estimated_space_needed = video_size * 5
-            
-            # Check storage before starting
-            self.check_and_ensure_storage(estimated_space_needed)
-            
-            # Update task status
-            task = self._get_or_create_task(video_file)
-            task.start_processing()
-            
-            if progress_callback:
-                progress_callback(10, "Starting video processing...")
-            
-            # Run the actual video processing pipeline
-            success = video_file.pipe_1(
-                model_name="image_multilabel_classification_colonoscopy_default",
-                delete_frames_after=cleanup_frames_after
-            )
-            
-            if success:
-                if progress_callback:
-                    progress_callback(90, "Processing completed, cleaning up...")
-                
-                # Mark as completed
-                task.mark_completed("Video processing completed successfully")
-                
-                # Additional cleanup if requested
-                if cleanup_frames_after:
-                    self._cleanup_video_frames(video_file)
-                
-                if progress_callback:
-                    progress_callback(100, "Processing completed successfully")
-                
-                logger.info(f"Video processing completed successfully: {video_file.uuid}")
-                return True
-
-            task.mark_failed("Video processing pipeline failed")
-            logger.error(f"Video processing failed: {video_file.uuid}")
-            return False
-                
-        except InsufficientStorageError as e:
-            logger.error(f"Storage error during video processing: {e}")
-            task = self._get_or_create_task(video_file)
-            task.mark_failed(f"Insufficient storage: {e}")
-            return False
-            
-        except Exception as e:
-            logger.error(f"Video processing failed: {e}")
-            task = self._get_or_create_task(video_file)
-            task.mark_failed(f"Processing error: {e}")
-            return False
-    
-    def _get_or_create_task(self, video_file: VideoFile) -> AnonymizationTask:
-        """Get or create an anonymization task for the video."""
-        task, created = AnonymizationTask.objects.get_or_create(
-            video_file=video_file,
-            defaults={
-                'status': 'not_started',
-                'progress': 0,
-                'message': 'Initializing...'
-            }
-        )
-        return task
-    
-    def _cleanup_video_frames(self, video_file: VideoFile):
-        """Clean up frames for a specific video."""
-        try:
-            from django.conf import settings
-            frames_dir = Path(settings.BASE_DIR).parent / 'storage' / 'frames'
-            
-            # Find frame directories for this video
-            video_frame_dirs = list(frames_dir.glob(f"*{video_file.uuid}*"))
-            
-            for frame_dir in video_frame_dirs:
-                if frame_dir.is_dir():
-                    shutil.rmtree(frame_dir, ignore_errors=True)
-                    logger.info(f"Cleaned up frames for video {video_file.uuid}")
-                    
-        except Exception as e:
-            logger.warning(f"Failed to clean up frames for {video_file.uuid}: {e}")
-
-
-# Global instance for easy access
-storage_aware_processor = StorageAwareVideoProcessor()

endoreg_db/urls/files.py

@@ -1,6 +0,0 @@
-from django.urls import path
-from endoreg_db.views import AvailableFilesListView
-
-urlpatterns = [
-    path('available-files/', AvailableFilesListView.as_view(), name='available_files_list'),
-]

endoreg_db/urls/label_video_segment_validate.py

@@ -1,33 +0,0 @@
-from django.urls import path
-
-from endoreg_db.views.label_video_segment.validate import (
-    LabelVideoSegmentValidateView,
-    BulkSegmentValidateView,
-    VideoSegmentValidationCompleteView
-)
-
-url_patterns = [
-    # Einzelne Segment-Validierung
-    # POST /api/label-video-segment/<int:segment_id>/validate/
-    path(
-        'label-video-segment/<int:segment_id>/validate/',
-        LabelVideoSegmentValidateView.as_view(),
-        name='label_video_segment_validate'
-    ),
-    
-    # Bulk-Validierung mehrerer Segmente
-    # POST /api/label-video-segments/validate-bulk/
-    path(
-        'label-video-segments/validate-bulk/',
-        BulkSegmentValidateView.as_view(),
-        name='label_video_segments_validate_bulk'
-    ),
-    
-    # Alle Segmente eines Videos als validiert markieren
-    # POST /api/videos/<int:video_id>/segments/validate-complete/
-    path(
-        'videos/<int:video_id>/segments/validate-complete/',
-        VideoSegmentValidationCompleteView.as_view(),
-        name='video_segments_validate_complete'
-    ),
-]

endoreg_db/urls/label_video_segments.py

@@ -1,46 +0,0 @@
-from django.urls import path
-from endoreg_db.views import (
-    create_video_segment_annotation, 
-    update_label_video_segment,
-    SensitiveMetaDetailView,
-    video_segments_view,
-    video_segment_detail_view,
-    get_lvs_by_name_and_video_id
-)
-
-
-
-url_patterns = [
-    path(
-        "lvs/by-label-name/<str:label_name>/by-video-id/<int:video_id>/",
-        get_lvs_by_name_and_video_id,
-        name='get_lvs_by_name_and_video_id'
-    ),
-    path(
-        'annotations/',
-        create_video_segment_annotation,
-        name='create_annotation'
-    ),
-    path(
-        'annotations/<int:annotation_id>/',
-        update_label_video_segment,
-        name='update_annotation'
-    ),
-    path(
-        'save-anonymization-annotation-video/<int:annotation_id>/',
-        SensitiveMetaDetailView.as_view(),
-        name='save_anonymization_annotation_video'
-    ),
-    # Label Video Segment API endpoints
-    path(
-        'video-segments/',
-        video_segments_view,
-        name='video_segments'
-    ),
-    path(
-        'video-segments/<int:segment_id>/',
-        video_segment_detail_view,
-        name='video_segment_detail'
-    ),
-
-]