endoreg-db 0.8.6.4__py3-none-any.whl

endoreg_db/management/commands/create_multilabel_model_meta.py

@@ -0,0 +1,214 @@
+"""
+This script creates a new ModelMeta object for a multilabel classification model.
+"""
+
+from pathlib import Path
+from django.core.management import BaseCommand
+from endoreg_db.models import LabelSet, AiModel, ModelMeta
+
+
+# Example usage:
+# python manage.py create_multilabel_model_meta --model_path "./data/models/colo_segmentation_RegNetX800MF_6.ckpt"
+# python manage.py create_multilabel_model_meta --model_path "./libs/endoreg-db/tests/assets/colo_segmentation_RegNetX800MF_6.ckpt"
+
+FPS = 50
+SMOOTH_WINDOW_SIZE_S = 1
+MIN_SEQ_LEN_S = 0.5
+crop_template = [0, 1080, 550, 1920 - 20]  # [top, bottom, left, right]
+
+
+class Command(BaseCommand):
+    help = """
+        Creates a new ModelMeta object for a multilabel classification model.
+        1. Validates the existence of the specified LabelSet and AiModel
+        2. Checks that the model file exists on disk
+        3. Creates or updates a ModelMeta entry with the specified parameters
+    """
+
+    def add_arguments(self, parser):
+        """
+        Adds command-line arguments for configuring model meta creation.
+        
+        This method defines all necessary arguments for specifying model details, label set selection, normalization parameters, image dimensions, axes configuration, batch size, worker count, and versioning options for the Django management command.
+        """
+        parser.add_argument(
+            "--model_name",
+            type=str,
+            default="image_multilabel_classification_colonoscopy_default",
+            help="Name of the model to use for segmentation",
+        )
+
+        # model_path
+        parser.add_argument(
+            "--model_path",
+            type=str,
+            #default="./data/models/colo_segmentation_RegNetX800MF_6.ckpt", #model not yet here
+            default="./tests/assets/colo_segmentation_RegNetX800MF_6.ckpt", # model currently still here
+            help="Path to the model file",
+        )
+
+        # model_meta_version: int = 1
+        parser.add_argument(
+            "--model_meta_version",
+            type=int,
+            default=1,
+            help="Version of the model meta",
+        )
+
+        parser.add_argument(
+            "--image_classification_labelset_name",
+            type=str,
+            default="multilabel_classification_colonoscopy_default",
+            help="Name of the LabelSet for image classification",
+        )
+
+        parser.add_argument(
+            "--image_classification_labelset_version",
+            type=int,
+            default=-1,
+            help="Version of the LabelSet for image classification, default is -1 which invokes the highest version",
+        )
+
+        # activation: str = "sigmoid"
+        parser.add_argument(
+            "--activation_function_name",
+            type=str,
+            default="sigmoid",
+            help="Activation function for the model",
+        )
+
+        # mean: str = "0.45211223,0.27139644,0.19264949"
+        parser.add_argument(
+            "--mean",
+            type=str,
+            default="0.45211223,0.27139644,0.19264949",
+            help="Mean for normalization",
+        )
+
+        parser.add_argument(
+            "--std",
+            type=str,
+            default="0.31418097,0.21088019,0.16059452",
+            help="Std for normalization",
+        )
+
+        # size_x: int = 716
+        parser.add_argument(
+            "--size_x",
+            type=int,
+            default=716,
+            help="Size of the image in x direction",
+        )
+
+        # size_y: int = 716
+        parser.add_argument(
+            "--size_y",
+            type=int,
+            default=716,
+            help="Size of the image in y direction",
+        )
+
+        parser.add_argument(
+            "--axes",
+            type=str,
+            default="2,0,1",
+            help="Axes of the image",
+        )
+
+        # batchsize: int = 16
+        parser.add_argument(
+            "--batchsize",
+            type=int,
+            default=16,
+            help="Batchsize for the model",
+        )
+
+        # num_workers: int = 0
+        parser.add_argument(
+            "--num_workers",
+            type=int,
+            default=0,
+            help="Number of workers for the model",
+        )
+
+        # bump_version: bool = False
+        parser.add_argument(
+            "--bump_version",
+            action="store_true",
+            help="Bump the version of the model",
+        )
+
+    def handle(self, *args, **options):
+        """
+        Processes command-line arguments to create or update a ModelMeta object for a multilabel classification model.
+        
+        Retrieves and validates the specified LabelSet and AiModel, ensures the model file exists, and invokes ModelMeta.create_from_file with the provided configuration. Raises a ValueError if the requested LabelSet does not exist.
+        """
+        model_name = options["model_name"]
+
+        model_path = options["model_path"]
+        activation_function_name = options["activation_function_name"]
+
+        mean = options["mean"]
+        std = options["std"]
+
+        size_x = options["size_x"]
+        size_y = options["size_y"]
+
+        axes = options["axes"]
+
+        image_classification_labelset_name = options[
+            "image_classification_labelset_name"
+        ]
+
+        image_classification_labelset_version = options[
+            "image_classification_labelset_version"
+        ]
+        if image_classification_labelset_version == -1:
+            # If version is -1, we want the latest version of the labelset
+            labelset = LabelSet.objects.filter(
+                name=image_classification_labelset_name
+            ).order_by("-version").first()
+            if labelset:
+                image_classification_labelset_version = labelset.version
+            else:
+                raise ValueError(
+                    f"No LabelSet found with name: {image_classification_labelset_name}"
+                )
+        else:
+            # If a specific version is provided, we use that
+            labelset = LabelSet.objects.filter(
+                name=image_classification_labelset_name,
+                version=image_classification_labelset_version
+            ).first()
+            if not labelset:
+                raise ValueError(
+                    f"No LabelSet found with name: {image_classification_labelset_name} and version: {image_classification_labelset_version}"
+                )
+
+        # assert model exists
+        db_ai_model = AiModel.objects.filter(name=model_name).first()
+        assert db_ai_model, f"Model not found: {model_name}"
+
+        model_path = Path(model_path).expanduser().resolve().as_posix()
+
+        assert Path(model_path).exists(), f"Model file not found: {model_path}"
+
+        _model_meta = ModelMeta.create_from_file(
+            meta_name=model_name,  # Name of the new metadata object
+            model_name=model_name,  # name of the ai_model to use; This also defines the models VideoSegmentationLabelSet
+            labelset_name=image_classification_labelset_name,
+            weights_file=model_path,
+            # Use the correct keyword arguments matching the method signature
+            requested_version=options["model_meta_version"],
+            bump_if_exists=options["bump_version"],
+            # Pass other options via **kwargs
+            activation=activation_function_name,
+            mean=mean,
+            std=std,
+            size_x=size_x,
+            size_y=size_y,
+            axes=axes,
+            batchsize=options["batchsize"],
+            num_workers=options["num_workers"],
+        )

endoreg_db/management/commands/fix_missing_patient_data.py

@@ -0,0 +1,172 @@
+"""
+Management command to fix missing patient data in existing videos.
+Fills in default values for videos that have incomplete SensitiveMeta.
+"""
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from datetime import date
+from endoreg_db.models import VideoFile, SensitiveMeta
+
+
+class Command(BaseCommand):
+    help = """
+    Fixes missing patient data in existing VideoFile entries.
+    This command will:
+    1. Find all videos with missing or incomplete SensitiveMeta
+    2. Create default SensitiveMeta for videos without any
+    3. Fill in missing fields (first_name, last_name, DOB, examination_date)
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Show what would be changed without making actual changes',
+        )
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Show detailed output',
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options['dry_run']
+        verbose = options['verbose']
+        
+        self.stdout.write(self.style.SUCCESS("Starting patient data repair..."))
+        
+        if dry_run:
+            self.stdout.write(self.style.WARNING("DRY RUN MODE - No changes will be made"))
+        
+        # Find videos without SensitiveMeta
+        videos_without_meta = VideoFile.objects.filter(sensitive_meta__isnull=True)
+        count_without_meta = videos_without_meta.count()
+        
+        # Find videos with incomplete SensitiveMeta
+        videos_with_incomplete_meta = VideoFile.objects.filter(
+            sensitive_meta__isnull=False
+        ).filter(
+            # At least one of these fields is missing
+            sensitive_meta__patient_first_name__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_last_name__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_dob__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__examination_date__isnull=True
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_first_name__exact=''
+        ) | VideoFile.objects.filter(
+            sensitive_meta__isnull=False,
+            sensitive_meta__patient_last_name__exact=''
+        )
+        
+        count_incomplete = videos_with_incomplete_meta.count()
+        
+        self.stdout.write(f"Found {count_without_meta} videos without SensitiveMeta")
+        self.stdout.write(f"Found {count_incomplete} videos with incomplete SensitiveMeta")
+        
+        if count_without_meta == 0 and count_incomplete == 0:
+            self.stdout.write(self.style.SUCCESS("No repairs needed - all videos have complete patient data!"))
+            return
+        
+        fixed_count = 0
+        created_count = 0
+        
+        # Process videos without SensitiveMeta
+        if count_without_meta > 0:
+            self.stdout.write(f"\nProcessing {count_without_meta} videos without SensitiveMeta...")
+            
+            for video in videos_without_meta:
+                if verbose:
+                    self.stdout.write(f"Creating SensitiveMeta for video {video.uuid}")
+                
+                if not dry_run:
+                    try:
+                        with transaction.atomic():
+                            default_data = {
+                                "patient_first_name": "Patient",
+                                "patient_last_name": "Unknown", 
+                                "patient_dob": date(1990, 1, 1),
+                                "examination_date": date.today(),
+                                "center_name": video.center.name if video.center else "university_hospital_wuerzburg"
+                            }
+                            
+                            sensitive_meta = SensitiveMeta.create_from_dict(default_data)
+                            video.sensitive_meta = sensitive_meta
+                            video.save(update_fields=['sensitive_meta'])
+                            created_count += 1
+                            
+                    except Exception as e:
+                        self.stdout.write(
+                            self.style.ERROR(f"Failed to create SensitiveMeta for video {video.uuid}: {e}")
+                        )
+                else:
+                    created_count += 1
+        
+        # Process videos with incomplete SensitiveMeta
+        if count_incomplete > 0:
+            self.stdout.write(f"\nProcessing {count_incomplete} videos with incomplete SensitiveMeta...")
+            
+            for video in videos_with_incomplete_meta:
+                if not video.sensitive_meta:
+                    continue  # Skip if somehow None (already handled above)
+                
+                update_data = {}
+                missing_fields = []
+                
+                if not video.sensitive_meta.patient_first_name:
+                    update_data["patient_first_name"] = "Patient"
+                    missing_fields.append("first_name")
+                    
+                if not video.sensitive_meta.patient_last_name:
+                    update_data["patient_last_name"] = "Unknown"
+                    missing_fields.append("last_name")
+                    
+                if not video.sensitive_meta.patient_dob:
+                    update_data["patient_dob"] = date(1990, 1, 1)
+                    missing_fields.append("dob")
+                    
+                if not video.sensitive_meta.examination_date:
+                    update_data["examination_date"] = date.today()
+                    missing_fields.append("examination_date")
+                
+                if update_data:
+                    if verbose:
+                        self.stdout.write(
+                            f"Updating video {video.uuid} - missing fields: {', '.join(missing_fields)}"
+                        )
+                    
+                    if not dry_run:
+                        try:
+                            with transaction.atomic():
+                                video.sensitive_meta.update_from_dict(update_data)
+                                fixed_count += 1
+                        except Exception as e:
+                            self.stdout.write(
+                                self.style.ERROR(f"Failed to update SensitiveMeta for video {video.uuid}: {e}")
+                            )
+                    else:
+                        fixed_count += 1
+        
+        # Summary
+        self.stdout.write("\n" + "="*50)
+        if dry_run:
+            self.stdout.write(self.style.SUCCESS("DRY RUN SUMMARY:"))
+            self.stdout.write(f"Would create SensitiveMeta for: {created_count} videos")
+            self.stdout.write(f"Would update incomplete data for: {fixed_count} videos")
+            self.stdout.write(f"Total videos that would be fixed: {created_count + fixed_count}")
+        else:
+            self.stdout.write(self.style.SUCCESS("REPAIR COMPLETED:"))
+            self.stdout.write(f"Created SensitiveMeta for: {created_count} videos")
+            self.stdout.write(f"Updated incomplete data for: {fixed_count} videos")
+            self.stdout.write(f"Total videos fixed: {created_count + fixed_count}")
+        
+        if not dry_run and (created_count > 0 or fixed_count > 0):
+            self.stdout.write(self.style.SUCCESS("\n✅ Patient data repair completed successfully!"))
+            self.stdout.write("All videos now have the minimum required patient data for annotation.")

endoreg_db/management/commands/fix_video_paths.py

@@ -0,0 +1,165 @@
+"""
+Django management command to fix video file paths in the database.
+"""
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from endoreg_db.models import VideoFile
+from pathlib import Path
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Fix video file paths in the database to match actual file locations'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--video-id',
+            type=int,
+            help='Fix specific video ID only',
+        )
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            help='Show what would be changed without making changes',
+        )
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Enable verbose output',
+        )
+        parser.add_argument(
+            '--storage-dir',
+            type=str,
+            default=None,
+            help='Path to the storage directory (default: $ENDOREG_STORAGE_DIR or ./storage)'
+        )
+
+    def handle(self, *args, **options):
+        """
+        Synchronizes video file paths in the database with actual files on disk, updating broken or missing paths as needed.
+        
+        Scans the specified storage directory for video files, matches them to database records by UUID, and updates the `raw_file` field for videos whose stored path is missing or incorrect. Supports dry-run and verbose modes, and can process all videos or a specific video by ID.
+        """
+        dry_run = options['dry_run']
+        verbose = options['verbose']
+        video_id = options.get('video_id')
+
+        # Determine storage_dir from argument, env, or fallback
+        storage_dir = options.get('storage_dir') or \
+            os.environ.get('ENDOREG_STORAGE_DIR') or \
+            './storage'
+        storage_dir = Path(storage_dir)
+        
+        # Find all actual video files
+        actual_files = {}
+        for pattern in ['**/*.mp4', '**/*.avi', '**/*.mov', '**/*.mkv']:
+            for file_path in storage_dir.glob(pattern):
+                if file_path.is_file() and file_path.stat().st_size > 0:
+                    # Extract UUID from filename
+                    filename = file_path.name
+                    # UUID is typically the first part before underscore or the whole name
+                    if '_' in filename:
+                        uuid_part = filename.split('_')[0]
+                    else:
+                        uuid_part = filename.split('.')[0]
+                    
+                    # Store relative path from storage directory
+                    relative_path = file_path.relative_to(storage_dir)
+                    actual_files[uuid_part] = {
+                        'absolute_path': file_path,
+                        'relative_path': relative_path,
+                        'size_mb': file_path.stat().st_size / (1024*1024)
+                    }
+
+        self.stdout.write(f"Found {len(actual_files)} video files in storage")
+
+        # Query videos to fix
+        if video_id:
+            try:
+                videos = [VideoFile.objects.get(pk=video_id)]
+                self.stdout.write(f"Processing specific video ID: {video_id}")
+            except VideoFile.DoesNotExist:
+                self.stdout.write(self.style.ERROR(f"Video with ID {video_id} not found"))
+                return
+        else:
+            videos = VideoFile.objects.all()
+            self.stdout.write(f"Processing {videos.count()} videos...")
+
+        fixed_count = 0
+        skipped_count = 0
+        error_count = 0
+
+        for video in videos:
+            try:
+                uuid_str = str(video.uuid)
+                
+                # Check if we have a matching file
+                if uuid_str in actual_files:
+                    file_info = actual_files[uuid_str]
+                    
+                    # Check current file path
+                    current_path_exists = False
+                    current_path = None
+                    
+                    if hasattr(video, 'raw_file') and video.raw_file:
+                        try:
+                            current_path = Path(video.raw_file.path)
+                            current_path_exists = current_path.exists()
+                        except (ValueError, AttributeError, OSError):
+                            current_path_exists = False
+                    
+                    if not current_path_exists:
+                        # File path is broken, fix it
+                        if verbose:
+                            self.stdout.write(f"Video {video.id} ({uuid_str}):")
+                            self.stdout.write(f"  Current: {current_path or 'None'} (broken)")
+                            self.stdout.write(f"  Found: {file_info['absolute_path']} ({file_info['size_mb']:.1f} MB)")
+                        
+                        if not dry_run:
+                            with transaction.atomic():
+                                # Update the raw_file path
+                                video.raw_file.name = str(file_info['relative_path'])
+                                video.save(update_fields=['raw_file'])
+                                
+                            self.stdout.write(
+                                self.style.SUCCESS(f"✅ Fixed video {video.id}: {file_info['relative_path']}")
+                            )
+                        else:
+                            self.stdout.write(
+                                self.style.WARNING(f"🔄 Would fix video {video.id}: {file_info['relative_path']}")
+                            )
+                        
+                        fixed_count += 1
+                    else:
+                        if verbose:
+                            self.stdout.write(f"✅ Video {video.id} ({uuid_str}) already has correct path")
+                        skipped_count += 1
+                else:
+                    if verbose:
+                        self.stdout.write(f"❌ Video {video.id} ({uuid_str}): No matching file found")
+                    error_count += 1
+
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(f"❌ Error processing video {video.id}: {e}")
+                )
+                error_count += 1
+
+        # Summary
+        self.stdout.write(f"\n{'='*50}")
+        self.stdout.write(self.style.SUCCESS("SUMMARY"))
+        self.stdout.write(f"{'='*50}")
+        
+        action_word = "Would fix" if dry_run else "Fixed"
+        self.stdout.write(f"🔧 {action_word}: {fixed_count} videos")
+        self.stdout.write(f"✅ Already correct: {skipped_count} videos")
+        self.stdout.write(f"❌ Errors/Missing files: {error_count} videos")
+        
+        if dry_run and fixed_count > 0:
+            self.stdout.write("\n💡 Run without --dry-run to apply changes")
+        elif not dry_run and fixed_count > 0:
+            self.stdout.write(f"\n🎉 Successfully fixed {fixed_count} video file paths!")
+            self.stdout.write("🔄 Restart your Django server to reload file paths")