endoreg-db 0.6.4__py3-none-any.whl → 0.8.2__py3-none-any.whl

endoreg_db/management/commands/fix_missing_patient_data.py

@@ -0,0 +1,172 @@
+"""
+Management command to fix missing patient data in existing videos.
+Fills in default values for videos that have incomplete SensitiveMeta.
+"""
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from datetime import date
+from endoreg_db.models import VideoFile, SensitiveMeta
+
+
+class Command(BaseCommand):
+    help = """
+    Fixes missing patient data in existing VideoFile entries.
+    This command will:
+    1. Find all videos with missing or incomplete SensitiveMeta
+    2. Create default SensitiveMeta for videos without any
+    3. Fill in missing fields (first_name, last_name, DOB, examination_date)
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Show what would be changed without making actual changes',
+        )
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Show detailed output',
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options['dry_run']
+        verbose = options['verbose']
+        
+        self.stdout.write(self.style.SUCCESS("Starting patient data repair..."))
+        
+        if dry_run:
+            self.stdout.write(self.style.WARNING("DRY RUN MODE - No changes will be made"))
+        
+        # Find videos without SensitiveMeta
+        videos_without_meta = VideoFile.objects.filter(sensitive_meta__isnull=True)
+        count_without_meta = videos_without_meta.count()
+        
+        # Find videos with incomplete SensitiveMeta
+        videos_with_incomplete_meta = VideoFile.objects.filter(
+            sensitive_meta__isnull=False
+        ).filter(
+            # At least one of these fields is missing
+            sensitive_meta__patient_first_name__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_last_name__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_dob__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__examination_date__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_first_name__exact=''
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_last_name__exact=''
+        )
+        
+        count_incomplete = videos_with_incomplete_meta.count()
+        
+        self.stdout.write(f"Found {count_without_meta} videos without SensitiveMeta")
+        self.stdout.write(f"Found {count_incomplete} videos with incomplete SensitiveMeta")
+        
+        if count_without_meta == 0 and count_incomplete == 0:
+            self.stdout.write(self.style.SUCCESS("No repairs needed - all videos have complete patient data!"))
+            return
+        
+        fixed_count = 0
+        created_count = 0
+        
+        # Process videos without SensitiveMeta
+        if count_without_meta > 0:
+            self.stdout.write(f"\nProcessing {count_without_meta} videos without SensitiveMeta...")
+            
+            for video in videos_without_meta:
+                if verbose:
+                    self.stdout.write(f"Creating SensitiveMeta for video {video.uuid}")
+                
+                if not dry_run:
+                    try:
+                        with transaction.atomic():
+                            default_data = {
+                                "patient_first_name": "Patient",
+                                "patient_last_name": "Unknown", 
+                                "patient_dob": date(1990, 1, 1),
+                                "examination_date": date.today(),
+                                "center_name": video.center.name if video.center else "university_hospital_wuerzburg"
+                            }
+                            
+                            sensitive_meta = SensitiveMeta.create_from_dict(default_data)
+                            video.sensitive_meta = sensitive_meta
+                            video.save(update_fields=['sensitive_meta'])
+                            created_count += 1
+                            
+                    except Exception as e:
+                        self.stdout.write(
+                            self.style.ERROR(f"Failed to create SensitiveMeta for video {video.uuid}: {e}")
+                        )
+                else:
+                    created_count += 1
+        
+        # Process videos with incomplete SensitiveMeta
+        if count_incomplete > 0:
+            self.stdout.write(f"\nProcessing {count_incomplete} videos with incomplete SensitiveMeta...")
+            
+            for video in videos_with_incomplete_meta:
+                if not video.sensitive_meta:
+                    continue  # Skip if somehow None (already handled above)
+                
+                update_data = {}
+                missing_fields = []
+                
+                if not video.sensitive_meta.patient_first_name:
+                    update_data["patient_first_name"] = "Patient"
+                    missing_fields.append("first_name")
+                    
+                if not video.sensitive_meta.patient_last_name:
+                    update_data["patient_last_name"] = "Unknown"
+                    missing_fields.append("last_name")
+                    
+                if not video.sensitive_meta.patient_dob:
+                    update_data["patient_dob"] = date(1990, 1, 1)
+                    missing_fields.append("dob")
+                    
+                if not video.sensitive_meta.examination_date:
+                    update_data["examination_date"] = date.today()
+                    missing_fields.append("examination_date")
+                
+                if update_data:
+                    if verbose:
+                        self.stdout.write(
+                            f"Updating video {video.uuid} - missing fields: {', '.join(missing_fields)}"
+                        )
+                    
+                    if not dry_run:
+                        try:
+                            with transaction.atomic():
+                                video.sensitive_meta.update_from_dict(update_data)
+                                fixed_count += 1
+                        except Exception as e:
+                            self.stdout.write(
+                                self.style.ERROR(f"Failed to update SensitiveMeta for video {video.uuid}: {e}")
+                            )
+                    else:
+                        fixed_count += 1
+        
+        # Summary
+        self.stdout.write("\n" + "="*50)
+        if dry_run:
+            self.stdout.write(self.style.SUCCESS("DRY RUN SUMMARY:"))
+            self.stdout.write(f"Would create SensitiveMeta for: {created_count} videos")
+            self.stdout.write(f"Would update incomplete data for: {fixed_count} videos")
+            self.stdout.write(f"Total videos that would be fixed: {created_count + fixed_count}")
+        else:
+            self.stdout.write(self.style.SUCCESS("REPAIR COMPLETED:"))
+            self.stdout.write(f"Created SensitiveMeta for: {created_count} videos")
+            self.stdout.write(f"Updated incomplete data for: {fixed_count} videos")
+            self.stdout.write(f"Total videos fixed: {created_count + fixed_count}")
+        
+        if not dry_run and (created_count > 0 or fixed_count > 0):
+            self.stdout.write(self.style.SUCCESS("\n✅ Patient data repair completed successfully!"))
+            self.stdout.write("All videos now have the minimum required patient data for annotation.")

endoreg_db/management/commands/fix_video_paths.py

@@ -0,0 +1,165 @@
+"""
+Django management command to fix video file paths in the database.
+"""
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from endoreg_db.models import VideoFile
+from pathlib import Path
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Fix video file paths in the database to match actual file locations'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--video-id',
+            type=int,
+            help='Fix specific video ID only',
+        )
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Show what would be changed without making changes',
+        )
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Enable verbose output',
+        )
+        parser.add_argument(
+            '--storage-dir',
+            type=str,
+            default=None,
+            help='Path to the storage directory (default: $ENDOREG_STORAGE_DIR or ./storage)'
+        )
+
+    def handle(self, *args, **options):
+        """
+        Synchronizes video file paths in the database with actual files on disk, updating broken or missing paths as needed.
+        
+        Scans the specified storage directory for video files, matches them to database records by UUID, and updates the `raw_file` field for videos whose stored path is missing or incorrect. Supports dry-run and verbose modes, and can process all videos or a specific video by ID.
+        """
+        dry_run = options['dry_run']
+        verbose = options['verbose']
+        video_id = options.get('video_id')
+
+        # Determine storage_dir from argument, env, or fallback
+        storage_dir = options.get('storage_dir') or \
+            os.environ.get('ENDOREG_STORAGE_DIR') or \
+            './storage'
+        storage_dir = Path(storage_dir)
+        
+        # Find all actual video files
+        actual_files = {}
+        for pattern in ['**/*.mp4', '**/*.avi', '**/*.mov', '**/*.mkv']:
+            for file_path in storage_dir.glob(pattern):
+                if file_path.is_file() and file_path.stat().st_size > 0:
+                    # Extract UUID from filename
+                    filename = file_path.name
+                    # UUID is typically the first part before underscore or the whole name
+                    if '_' in filename:
+                        uuid_part = filename.split('_')[0]
+                    else:
+                        uuid_part = filename.split('.')[0]
+                    
+                    # Store relative path from storage directory
+                    relative_path = file_path.relative_to(storage_dir)
+                    actual_files[uuid_part] = {
+                        'absolute_path': file_path,
+                        'relative_path': relative_path,
+                        'size_mb': file_path.stat().st_size / (1024*1024)
+                    }
+
+        self.stdout.write(f"Found {len(actual_files)} video files in storage")
+
+        # Query videos to fix
+        if video_id:
+            try:
+                videos = [VideoFile.objects.get(pk=video_id)]
+                self.stdout.write(f"Processing specific video ID: {video_id}")
+            except VideoFile.DoesNotExist:
+                self.stdout.write(self.style.ERROR(f"Video with ID {video_id} not found"))
+                return
+        else:
+            videos = VideoFile.objects.all()
+            self.stdout.write(f"Processing {videos.count()} videos...")
+
+        fixed_count = 0
+        skipped_count = 0
+        error_count = 0
+
+        for video in videos:
+            try:
+                uuid_str = str(video.uuid)
+                
+                # Check if we have a matching file
+                if uuid_str in actual_files:
+                    file_info = actual_files[uuid_str]
+                    
+                    # Check current file path
+                    current_path_exists = False
+                    current_path = None
+                    
+                    if hasattr(video, 'raw_file') and video.raw_file:
+                        try:
+                            current_path = Path(video.raw_file.path)
+                            current_path_exists = current_path.exists()
+                        except (ValueError, AttributeError, OSError):
+                            current_path_exists = False
+                    
+                    if not current_path_exists:
+                        # File path is broken, fix it
+                        if verbose:
+                            self.stdout.write(f"Video {video.id} ({uuid_str}):")
+                            self.stdout.write(f"  Current: {current_path or 'None'} (broken)")
+                            self.stdout.write(f"  Found: {file_info['absolute_path']} ({file_info['size_mb']:.1f} MB)")
+                        
+                        if not dry_run:
+                            with transaction.atomic():
+                                # Update the raw_file path
+                                video.raw_file.name = str(file_info['relative_path'])
+                                video.save(update_fields=['raw_file'])
+                                
+                            self.stdout.write(
+                                self.style.SUCCESS(f"✅ Fixed video {video.id}: {file_info['relative_path']}")
+                            )
+                        else:
+                            self.stdout.write(
+                                self.style.WARNING(f"🔄 Would fix video {video.id}: {file_info['relative_path']}")
+                            )
+                        
+                        fixed_count += 1
+                    else:
+                        if verbose:
+                            self.stdout.write(f"✅ Video {video.id} ({uuid_str}) already has correct path")
+                        skipped_count += 1
+                else:
+                    if verbose:
+                        self.stdout.write(f"❌ Video {video.id} ({uuid_str}): No matching file found")
+                    error_count += 1
+
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(f"❌ Error processing video {video.id}: {e}")
+                )
+                error_count += 1
+
+        # Summary
+        self.stdout.write(f"\n{'='*50}")
+        self.stdout.write(self.style.SUCCESS("SUMMARY"))
+        self.stdout.write(f"{'='*50}")
+        
+        action_word = "Would fix" if dry_run else "Fixed"
+        self.stdout.write(f"🔧 {action_word}: {fixed_count} videos")
+        self.stdout.write(f"✅ Already correct: {skipped_count} videos")
+        self.stdout.write(f"❌ Errors/Missing files: {error_count} videos")
+        
+        if dry_run and fixed_count > 0:
+            self.stdout.write("\n💡 Run without --dry-run to apply changes")
+        elif not dry_run and fixed_count > 0:
+            self.stdout.write(f"\n🎉 Successfully fixed {fixed_count} video file paths!")
+            self.stdout.write("🔄 Restart your Django server to reload file paths")

endoreg_db/management/commands/import_fallback_video.py

@@ -0,0 +1,203 @@
+from pathlib import Path
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from endoreg_db.models import (
+    VideoFile, 
+    Center, 
+    Label, 
+    VideoSegmentationLabelSet,
+    VideoSegmentationLabel,
+    EndoscopyProcessor
+)
+
+class Command(BaseCommand):
+    help = 'Import fallback test video and create default labels'
+
+    def _ensure_default_objects(self):
+        """
+        Ensures that default Center and EndoscopyProcessor objects exist, creating them if necessary.
+        
+        Returns:
+            tuple: A tuple containing the Center and EndoscopyProcessor objects.
+        """
+        center, created = Center.objects.get_or_create(
+            name="Default Center",
+            defaults={
+                'description': 'Fallback center for test videos',
+                'location': 'Test Location'
+            }
+        )
+        if created:
+            self.stdout.write(f"Created center: {center.name}")
+
+        processor, created = EndoscopyProcessor.objects.get_or_create(
+            name="Default Processor",
+            defaults={
+                'description': 'Fallback processor for test videos'
+            }
+        )
+        if created:
+            self.stdout.write(f"Created processor: {processor.name}")
+        
+        return center, processor
+
+    def add_arguments(self, parser):
+        """
+        Adds the --video-path argument to specify the path of the test video file.
+        
+        This optional argument allows users to provide a custom path to the test video file to be imported. If not specified, a default path is used.
+        """
+        parser.add_argument(
+            '--video-path',
+            type=str,
+            default='~/test-data/video/lux-gastro-video.mp4',
+            help='Path to the test video file'
+        )
+
+    def handle(self, *args, **options):
+        """
+        Handles the import of a test video and setup of default labels for frontend testing.
+        
+        Checks for the existence of the specified video file. If found, ensures default Center and EndoscopyProcessor objects exist, creates default annotation labels, and imports the video into the database. If the video file is missing or import fails, creates fallback database entries to enable frontend testing.
+        """
+        video_path_str = options['video_path']
+        video_path = Path(video_path_str).expanduser()
+        
+        self.stdout.write(f"Looking for video at: {video_path}")
+        
+        if not video_path.exists():
+            self.stdout.write(
+                self.style.WARNING(f"Video file not found at {video_path}")
+            )
+            # Create a fallback entry anyway for frontend testing
+            self.create_fallback_entries()
+            return
+        
+        # Create or get default center and processor
+        center, processor = self._ensure_default_objects()
+        
+        # Create default labels
+        self.create_default_labels()
+        
+        # Import the video if it exists
+        try:
+            video_file = VideoFile.create_from_file_initialized(
+                file_path=video_path,
+                center_name=center.name,
+                processor_name=processor.name
+            )
+            self.stdout.write(
+                self.style.SUCCESS(f"Successfully imported video: {video_file.uuid}")
+            )
+        except Exception as e:
+            self.stdout.write(
+                self.style.ERROR(f"Failed to import video: {e}")
+            )
+            # Create fallback entries for frontend testing
+            self.create_fallback_entries()
+
+    def create_default_labels(self):
+        """
+        Creates a default set of video segmentation labels and associates them with a label set.
+        
+        This method ensures that a predefined group of labels, each with multilingual names, colors, and order priorities, exists in the database for endoscopy video annotation. It creates or retrieves both specialized segmentation labels and general labels for compatibility, and links the segmentation labels to a default label set named "Default Endoscopy Labels."
+        """
+        
+        # Create default labelset
+        labelset, created = VideoSegmentationLabelSet.objects.get_or_create(
+            name="Default Endoscopy Labels",
+            defaults={
+                'description': 'Default labels for endoscopy video annotation'
+            }
+        )
+        if created:
+            self.stdout.write(f"Created labelset: {labelset.name}")
+        
+        # Fetch existing labels in the labelset to avoid N+1 queries
+        existing_labels_in_set = set(labelset.labels.values_list('pk', flat=True))
+        
+        # Default labels for endoscopy
+        default_labels = [
+            {'name': 'outside', 'name_de': 'Außerhalb', 'name_en': 'Outside', 'color': '#00bcd4', 'order_priority': 1},
+            {'name': 'appendix', 'name_de': 'Appendix', 'name_en': 'Appendix', 'color': '#ff9800', 'order_priority': 2},
+            {'name': 'blood', 'name_de': 'Blut', 'name_en': 'Blood', 'color': '#f44336', 'order_priority': 3},
+            {'name': 'diverticule', 'name_de': 'Divertikel', 'name_en': 'Diverticule', 'color': '#9c27b0', 'order_priority': 4},
+            {'name': 'grasper', 'name_de': 'Greifer', 'name_en': 'Grasper', 'color': '#CBEDCA', 'order_priority': 5},
+            {'name': 'ileocaecalvalve', 'name_de': 'Ileozäkalklappe', 'name_en': 'Ileocaecal Valve', 'color': '#3f51b5', 'order_priority': 6},
+            {'name': 'ileum', 'name_de': 'Ileum', 'name_en': 'Ileum', 'color': '#2196f3', 'order_priority': 7},
+            {'name': 'low_quality', 'name_de': 'Niedrige Bildqualität', 'name_en': 'Low Quality', 'color': '#9e9e9e', 'order_priority': 8},
+            {'name': 'nbi', 'name_de': 'Narrow Band Imaging', 'name_en': 'NBI', 'color': '#795548', 'order_priority': 9},
+            {'name': 'needle', 'name_de': 'Nadel', 'name_en': 'Needle', 'color': '#e91e63', 'order_priority': 10},
+            {'name': 'polyp', 'name_de': 'Polyp', 'name_en': 'Polyp', 'color': '#8bc34a', 'order_priority': 11},
+            {'name': 'snare', 'name_de': 'Snare', 'name_en': 'Snare', 'color': '#ff5722', 'order_priority': 12},
+            {'name': 'water_jet', 'name_de': 'Wasserstrahl', 'name_en': 'Water Jet', 'color': '#03a9f4', 'order_priority': 13},
+            {'name': 'wound', 'name_de': 'Wunde', 'name_en': 'Wound', 'color': '#607d8b', 'order_priority': 14},
+        ]
+        
+        for label_data in default_labels:
+            # Create VideoSegmentationLabel
+            vs_label, created = VideoSegmentationLabel.objects.get_or_create(
+                name=label_data['name'],
+                defaults={
+                    'name_de': label_data['name_de'],
+                    'name_en': label_data['name_en'],
+                    'color': label_data['color'],
+                    'order_priority': label_data['order_priority'],
+                    'description': f"Default {label_data['name_en']} label"
+                }
+            )
+            
+            # Create general Label (for compatibility)
+            general_label, created = Label.objects.get_or_create(
+                name=label_data['name'],
+                defaults={
+                    'description': f"Default {label_data['name_en']} label"
+                }
+            )
+            
+            if created:
+                self.stdout.write(f"Created label: {label_data['name']}")
+            
+            # Link to labelset
+            if vs_label.pk not in existing_labels_in_set:
+                labelset.labels.add(vs_label)
+                existing_labels_in_set.add(vs_label.pk) # Keep the set in sync
+        
+        labelset.save()
+        self.stdout.write(f"Labelset configured with {labelset.labels.count()} labels")
+
+    def create_fallback_entries(self):
+        """
+        Creates fallback database entries for testing when the video file is unavailable.
+        
+        This method ensures that default annotation labels, a default center, and a default endoscopy processor exist. If a fallback video entry named "lux-gastro-video.mp4" does not already exist, it creates a minimal `VideoFile` record with preset metadata and initializes its state and frame directory for frontend testing.
+        """
+        with transaction.atomic():
+            # Create default labels anyway
+            self.create_default_labels()
+            
+            # Create a placeholder VideoFile entry for frontend testing
+            center, processor = self._ensure_default_objects()
+            
+            # Check if we already have a fallback video
+            if not VideoFile.objects.filter(original_file_name="lux-gastro-video.mp4").exists():
+                # Create a minimal VideoFile entry for frontend testing
+                video_file = VideoFile.objects.create(
+                    original_file_name="lux-gastro-video.mp4",
+                    video_hash="fallback_hash_12345",
+                    center=center,
+                    processor=processor,
+                    fps=30.0,
+                    duration=120.0,  # 2 minutes fallback
+                    frame_count=3600,
+                    width=1920,
+                    height=1080
+                )
+                
+                # Initialize the video file
+                video_file.get_or_create_state()
+                video_file.set_frame_dir()
+                
+                self.stdout.write(
+                    self.style.SUCCESS(f"Created fallback video entry: {video_file.uuid}")
+                )