endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/management/commands/import_fallback_video.py

@@ -1,203 +0,0 @@
-from pathlib import Path
-from django.core.management.base import BaseCommand
-from django.db import transaction
-from endoreg_db.models import (
-    VideoFile, 
-    Center, 
-    Label, 
-    VideoSegmentationLabelSet,
-    VideoSegmentationLabel,
-    EndoscopyProcessor
-)
-
-class Command(BaseCommand):
-    help = 'Import fallback test video and create default labels'
-
-    def _ensure_default_objects(self):
-        """
-        Ensures that default Center and EndoscopyProcessor objects exist, creating them if necessary.
-        
-        Returns:
-            tuple: A tuple containing the Center and EndoscopyProcessor objects.
-        """
-        center, created = Center.objects.get_or_create(
-            name="Default Center",
-            defaults={
-                'description': 'Fallback center for test videos',
-                'location': 'Test Location'
-            }
-        )
-        if created:
-            self.stdout.write(f"Created center: {center.name}")
-
-        processor, created = EndoscopyProcessor.objects.get_or_create(
-            name="Default Processor",
-            defaults={
-                'description': 'Fallback processor for test videos'
-            }
-        )
-        if created:
-            self.stdout.write(f"Created processor: {processor.name}")
-        
-        return center, processor
-
-    def add_arguments(self, parser):
-        """
-        Adds the --video-path argument to specify the path of the test video file.
-        
-        This optional argument allows users to provide a custom path to the test video file to be imported. If not specified, a default path is used.
-        """
-        parser.add_argument(
-            '--video-path',
-            type=str,
-            default='~/test-data/video/lux-gastro-video.mp4',
-            help='Path to the test video file'
-        )
-
-    def handle(self, *args, **options):
-        """
-        Handles the import of a test video and setup of default labels for frontend testing.
-        
-        Checks for the existence of the specified video file. If found, ensures default Center and EndoscopyProcessor objects exist, creates default annotation labels, and imports the video into the database. If the video file is missing or import fails, creates fallback database entries to enable frontend testing.
-        """
-        video_path_str = options['video_path']
-        video_path = Path(video_path_str).expanduser()
-        
-        self.stdout.write(f"Looking for video at: {video_path}")
-        
-        if not video_path.exists():
-            self.stdout.write(
-                self.style.WARNING(f"Video file not found at {video_path}")
-            )
-            # Create a fallback entry anyway for frontend testing
-            self.create_fallback_entries()
-            return
-        
-        # Create or get default center and processor
-        center, processor = self._ensure_default_objects()
-        
-        # Create default labels
-        self.create_default_labels()
-        
-        # Import the video if it exists
-        try:
-            video_file = VideoFile.create_from_file_initialized(
-                file_path=video_path,
-                center_name=center.name,
-                processor_name=processor.name
-            )
-            self.stdout.write(
-                self.style.SUCCESS(f"Successfully imported video: {video_file.uuid}")
-            )
-        except Exception as e:
-            self.stdout.write(
-                self.style.ERROR(f"Failed to import video: {e}")
-            )
-            # Create fallback entries for frontend testing
-            self.create_fallback_entries()
-
-    def create_default_labels(self):
-        """
-        Creates a default set of video segmentation labels and associates them with a label set.
-        
-        This method ensures that a predefined group of labels, each with multilingual names, colors, and order priorities, exists in the database for endoscopy video annotation. It creates or retrieves both specialized segmentation labels and general labels for compatibility, and links the segmentation labels to a default label set named "Default Endoscopy Labels."
-        """
-        
-        # Create default labelset
-        labelset, created = VideoSegmentationLabelSet.objects.get_or_create(
-            name="Default Endoscopy Labels",
-            defaults={
-                'description': 'Default labels for endoscopy video annotation'
-            }
-        )
-        if created:
-            self.stdout.write(f"Created labelset: {labelset.name}")
-        
-        # Fetch existing labels in the labelset to avoid N+1 queries
-        existing_labels_in_set = set(labelset.labels.values_list('pk', flat=True))
-        
-        # Default labels for endoscopy
-        default_labels = [
-            {'name': 'outside', 'name_de': 'Außerhalb', 'name_en': 'Outside', 'color': '#00bcd4', 'order_priority': 1},
-            {'name': 'appendix', 'name_de': 'Appendix', 'name_en': 'Appendix', 'color': '#ff9800', 'order_priority': 2},
-            {'name': 'blood', 'name_de': 'Blut', 'name_en': 'Blood', 'color': '#f44336', 'order_priority': 3},
-            {'name': 'diverticule', 'name_de': 'Divertikel', 'name_en': 'Diverticule', 'color': '#9c27b0', 'order_priority': 4},
-            {'name': 'grasper', 'name_de': 'Greifer', 'name_en': 'Grasper', 'color': '#CBEDCA', 'order_priority': 5},
-            {'name': 'ileocaecalvalve', 'name_de': 'Ileozäkalklappe', 'name_en': 'Ileocaecal Valve', 'color': '#3f51b5', 'order_priority': 6},
-            {'name': 'ileum', 'name_de': 'Ileum', 'name_en': 'Ileum', 'color': '#2196f3', 'order_priority': 7},
-            {'name': 'low_quality', 'name_de': 'Niedrige Bildqualität', 'name_en': 'Low Quality', 'color': '#9e9e9e', 'order_priority': 8},
-            {'name': 'nbi', 'name_de': 'Narrow Band Imaging', 'name_en': 'NBI', 'color': '#795548', 'order_priority': 9},
-            {'name': 'needle', 'name_de': 'Nadel', 'name_en': 'Needle', 'color': '#e91e63', 'order_priority': 10},
-            {'name': 'polyp', 'name_de': 'Polyp', 'name_en': 'Polyp', 'color': '#8bc34a', 'order_priority': 11},
-            {'name': 'snare', 'name_de': 'Snare', 'name_en': 'Snare', 'color': '#ff5722', 'order_priority': 12},
-            {'name': 'water_jet', 'name_de': 'Wasserstrahl', 'name_en': 'Water Jet', 'color': '#03a9f4', 'order_priority': 13},
-            {'name': 'wound', 'name_de': 'Wunde', 'name_en': 'Wound', 'color': '#607d8b', 'order_priority': 14},
-        ]
-        
-        for label_data in default_labels:
-            # Create VideoSegmentationLabel
-            vs_label, created = VideoSegmentationLabel.objects.get_or_create(
-                name=label_data['name'],
-                defaults={
-                    'name_de': label_data['name_de'],
-                    'name_en': label_data['name_en'],
-                    'color': label_data['color'],
-                    'order_priority': label_data['order_priority'],
-                    'description': f"Default {label_data['name_en']} label"
-                }
-            )
-            
-            # Create general Label (for compatibility)
-            general_label, created = Label.objects.get_or_create(
-                name=label_data['name'],
-                defaults={
-                    'description': f"Default {label_data['name_en']} label"
-                }
-            )
-            
-            if created:
-                self.stdout.write(f"Created label: {label_data['name']}")
-            
-            # Link to labelset
-            if vs_label.pk not in existing_labels_in_set:
-                labelset.labels.add(vs_label)
-                existing_labels_in_set.add(vs_label.pk) # Keep the set in sync
-        
-        labelset.save()
-        self.stdout.write(f"Labelset configured with {labelset.labels.count()} labels")
-
-    def create_fallback_entries(self):
-        """
-        Creates fallback database entries for testing when the video file is unavailable.
-        
-        This method ensures that default annotation labels, a default center, and a default endoscopy processor exist. If a fallback video entry named "lux-gastro-video.mp4" does not already exist, it creates a minimal `VideoFile` record with preset metadata and initializes its state and frame directory for frontend testing.
-        """
-        with transaction.atomic():
-            # Create default labels anyway
-            self.create_default_labels()
-            
-            # Create a placeholder VideoFile entry for frontend testing
-            center, processor = self._ensure_default_objects()
-            
-            # Check if we already have a fallback video
-            if not VideoFile.objects.filter(original_file_name="lux-gastro-video.mp4").exists():
-                # Create a minimal VideoFile entry for frontend testing
-                video_file = VideoFile.objects.create(
-                    original_file_name="lux-gastro-video.mp4",
-                    video_hash="fallback_hash_12345",
-                    center=center,
-                    processor=processor,
-                    fps=30.0,
-                    duration=120.0,  # 2 minutes fallback
-                    frame_count=3600,
-                    width=1920,
-                    height=1080
-                )
-                
-                # Initialize the video file
-                video_file.get_or_create_state()
-                video_file.set_frame_dir()
-                
-                self.stdout.write(
-                    self.style.SUCCESS(f"Created fallback video entry: {video_file.uuid}")
-                )

endoreg_db/management/commands/import_video.py

@@ -1,422 +0,0 @@
-# See Pipe 1 video file function.
-#
-
-"""
-Management command to import a video file into the database.
-This command is designed to be run from the command line and takes various arguments
-to specify the video file, center name, and other options.
-"""
-
-from django.core.management import BaseCommand
-from django.core.files.base import ContentFile
-from django.db import connection
-from pathlib import Path
-from endoreg_db.models import VideoFile
-from endoreg_db.models.administration.center import Center
-from endoreg_db.models.medical.hardware import EndoscopyProcessor
-# #FIXME
-# from endoreg_db.management.commands import validate_video
-
-from endoreg_db.utils.video.ffmpeg_wrapper import check_ffmpeg_availability # ADDED
-
-import logging
-logger = logging.getLogger(__name__)
-
-# Import frame cleaning functionality - simplified approach
-FRAME_CLEANING_AVAILABLE = False
-
-# Try to import lx_anonymizer using the existing working import method from create_from_file
-try:
-    # Check if we can find the lx-anonymizer directory
-    current_file = Path(__file__)
-    endoreg_db_root = current_file.parent.parent.parent.parent
-    lx_anonymizer_path = endoreg_db_root / "lx-anonymizer"
-    
-    if lx_anonymizer_path.exists():
-        # Add to Python path temporarily
-        import sys
-        if str(lx_anonymizer_path) not in sys.path:
-            sys.path.insert(0, str(lx_anonymizer_path))
-        
-        # Try simple import
-        from lx_anonymizer import FrameCleaner, ReportReader
-        
-        FRAME_CLEANING_AVAILABLE = True
-        logger.debug("Successfully imported lx_anonymizer modules")
-        
-        # Remove from path to avoid conflicts
-        if str(lx_anonymizer_path) in sys.path:
-            sys.path.remove(str(lx_anonymizer_path))
-            
-except Exception as e:
-    logger.debug(f"Frame cleaning not available: {e}")
-    FRAME_CLEANING_AVAILABLE = False
-
-IMPORT_MODELS = [
-    VideoFile.__name__,
-]
-
-IMPORT_METADATA = {
-    VideoFile.__name__: {
-        "uuid": VideoFile.uuid,
-        "raw_file": VideoFile.raw_file,
-        "processed_file": VideoFile.processed_file,
-        "foreign_keys": [],
-        "foreign_key_models": [],
-    },
-}
-
-class Command(BaseCommand):
-    help = """
-        Creates a new VideoFile object in the database.
-        1. Validates the existence of the specified center and processor
-        2. Checks that the video file is saved and anonymized
-        3. Creates or updates a ModelMeta entry with the specified parameters
-    """
-
-    def add_arguments(self, parser):
-        
-        """
-        Adds command-line arguments for the video import management command.
-        
-        Defines options for specifying the video file path, associated center and processor names, directory roots for frames and videos, deletion and saving behavior, model path, segmentation usage, and verbosity.
-        """
-        parser.add_argument(
-            "--verbose",
-            action="store_true",
-            help="Display verbose output",
-        )
-        parser.add_argument(
-            "--",
-            type=str,
-            default="university_hospital_wuerzburg",
-            help="Name of the center to associate with video",
-        )
-        
-        # add the path to the video file
-        parser.add_argument(
-            "video_file",
-            type=Path,
-            help="Path to the video file to import",
-        )
-
-        # frame dir parent
-        parser.add_argument(
-            "--frame_dir_root",
-            type=str,
-            default="~/test-data/raw_frame_dir",
-            help="Path to the frame directory",
-        )
-
-        # video dir
-        parser.add_argument(
-            "--video_dir_root",
-            type=str,
-            default="~/test-data/raw_video_dir",
-            help="Path to the video directory",
-        )
-
-        # delete source
-        parser.add_argument(
-            "--delete_source",
-            action="store_true",
-            default=False,
-            help="Delete the source video file after importing",
-        )
-
-        # save video file
-        parser.add_argument(
-            "--save_video_file",
-            action="store_true",
-            default=True,
-            help="Save the video file to the video directory",
-        )
-
-        # model_path
-        parser.add_argument(
-            "--model_name",
-            type=str,
-            default="image_multilabel_classification_colonoscopy_default",
-            help="AiModel Name",
-        )
-        
-        # 
-        parser.add_argument(
-            "--segmentation",
-            action="store_true",
-            help="Whether to use segmentation",
-        )
-        
-        parser.add_argument(
-            "--processor_name",
-            type=str,
-            default="olympus_cv_1500",
-            help="Name of the processor to associate with video",
-        )
-
-    def handle(self, *args, **options):  
-        
-        """
-        Imports a video file into the database, associating it with a specified medical center and endoscopy processor, and optionally applies AI-based segmentation.
-        
-        Checks for required dependencies, loads reference data, validates the existence of the specified center and processor, and processes the video file. If segmentation is enabled, retrieves the latest segmentation model metadata. Handles interactive processor selection if multiple are available, creates a new `VideoFile` entry, and invokes the processing pipeline. Can optionally delete the source file or save the video to a specified directory. Reports the outcome of the import and processing steps.
-        """
-        try: # ADDED
-            check_ffmpeg_availability() # ADDED
-            self.stdout.write(self.style.SUCCESS("FFMPEG is available")) # ADDED
-        except FileNotFoundError as e: # ADDED
-            self.stderr.write(self.style.ERROR(str(e))) # ADDED
-            # Decide if the command should exit or if FFMPEG is optional for some operations
-            # For this command, it seems FFMPEG is critical for VideoFile.pipe_1 and VideoFile.create_from_file
-            return # ADDED
-        
-        self.stdout.write(f"Current database: {connection.alias}")
-        self.stdout.write(self.style.SUCCESS("Starting video import..."))      
-
-        # Should not be invoked here but in a previous db setup step
-        # load_gender_data()
-        # load_disease_data()
-        # load_event_data()
-        # load_information_source()
-        # load_examination_data()
-        # load_center_data()
-        # load_endoscope_data()
-
-        segmentation = options["segmentation"]
-
-        verbose = options["verbose"]
-        center_name = options["center_name"]
-        video_file = options["video_file"]
-        frame_dir_root = options["frame_dir_root"]
-        delete_source = options["delete_source"]
-        save_video_file = options["save_video_file"]
-        model_name = options["model_name"]
-        processor_name = options["processor_name"]
-        video_file = Path(video_file).expanduser()
-
-        
-        assert isinstance(delete_source, bool), "delete_source must be a boolean"
-        assert isinstance(save_video_file, bool), "save_video_file must be a boolean"
-        assert isinstance(verbose, bool), "verbose must be a boolean"
-        assert isinstance(center_name, str), "center_name must be a string"
-        assert isinstance(video_file, Path), "video_file must be a Path"
-        assert isinstance(frame_dir_root, str), "frame_dir_root must be a string"
-                # Assert Center exists -> Does not exist methods are deprecated
-        try:
-            center = Center.objects.get(name=center_name)
-            self.stdout.write(self.style.SUCCESS(f"Using center: {center.name}"))
-        except Center.DoesNotExist:
-            self.stdout.write(self.style.ERROR(f"Center not found: {center_name}"))
-            return
-
-        # Assert Processor Exists
-        if processor_name is None:
-            processors_qs = center.endoscopy_processors.all()
-            proc_count = processors_qs.count()
-            if proc_count == 0:
-                raise AssertionError(
-                    f"No processors linked to '{center.name}' and no processor called '{processor_name}' exists. "
-                    "Fallback from default Processor applied."
-                )
-            elif proc_count == 1:
-                processor = processors_qs.first()
-            else:
-
-                processor = self._choose_processor_interactively(processors_qs)
-                self.stdout.write(self.style.SUCCESS(f"Using processor: {processor.name}"))
-        
-        else:
-            processor = EndoscopyProcessor.objects.get(name=processor_name)
-
-            cns = processor.centers.values_list("name", flat=True)
-            if center_name not in cns:
-                self.stdout.write(
-                    self.style.ERROR(
-                        f"Processor '{processor_name}' is not linked to center '{center_name}'."
-                    )
-                )
-                return
-
-        if not Path(video_file).exists():
-            self.stdout.write(self.style.ERROR(f"Video file not found: {video_file} saving unsuccessful."))
-            return AssertionError(f"Video file not found: {video_file}")
-        
-        # Create VideoFile instance first
-        video_file_obj = VideoFile.create_from_file_initialized(
-            file_path=video_file,
-            center_name=center_name,
-            processor_name=processor_name,
-            delete_source=delete_source,
-            save_video_file=save_video_file, # Add this line
-        )
-        
-        if not video_file_obj:
-            self.stdout.write(self.style.ERROR("Failed to create VideoFile instance"))
-            return
-        
-        self.stdout.write(self.style.SUCCESS(f"Created VideoFile with UUID: {video_file_obj.uuid}"))
-        
-        # Initialize video specs and frames
-        video_file_obj.initialize_video_specs()
-        video_file_obj.initialize_frames()
-        
-        # Run Pipe 1 for OCR and AI processing
-        self.stdout.write(self.style.SUCCESS("Starting Pipe 1 processing (OCR + AI)..."))
-        success = video_file_obj.pipe_1(
-            model_name=model_name,
-            delete_frames_after=True,
-            ocr_frame_fraction=0.01,
-            ocr_cap=5
-        )
-        
-        if not success:
-            self.stdout.write(self.style.ERROR("Pipe 1 processing failed"))
-            return
-            
-        self.stdout.write(self.style.SUCCESS("Pipe 1 processing completed"))
-        
-        # Ensure minimum patient data is available
-        self.stdout.write(self.style.SUCCESS("Ensuring minimum patient data..."))
-        self._ensure_default_patient_data(video_file_obj)
-        
-        # Frame-level anonymization integration
-        if FRAME_CLEANING_AVAILABLE and video_file_obj.raw_file:
-            try:
-                self.stdout.write(self.style.SUCCESS("Starting frame-level anonymization..."))
-                # Properly instantiate FrameCleaner and ReportReader with correct arguments
-                frame_cleaner = FrameCleaner()
-                report_reader = ReportReader(
-                    report_root_path=str(video_file_obj.raw_file.path),
-                    locale="de_DE",  # Default German locale for medical data
-                    text_date_format="%d.%m.%Y"  # Common German date format
-                )
-                
-                # Updated to handle new return signature (path, metadata)
-                cleaned_video_path, extracted_metadata = frame_cleaner.clean_video(
-                    video_path=Path(video_file_obj.raw_file.path),
-                    endoscope_image_roi=video_file_obj.processor.get_roi_endoscope_image() if video_file_obj.processor else None,
-                    endoscope_data_roi_nested=video_file_obj.processor.get_rois() if video_file_obj.processor else None,
-                    output_path=video_file_obj.get_processed_file_path(),
-                    technique="mask_overlay"  # Use mask overlay technique as default, if not set this will be inferred.
-                )
-                
-                # Save the cleaned video using Django's FileField
-                with open(cleaned_video_path, 'rb') as f:
-                    video_file_obj.raw_file.save(cleaned_video_path.name, ContentFile(f.read()))
-                video_file_obj.save()
-                
-                self.stdout.write(self.style.SUCCESS(f"Frame cleaning completed: {cleaned_video_path.name}"))
-                self.stdout.write(self.style.SUCCESS(f"Extracted metadata: {extracted_metadata}"))
-                
-            except Exception as e:
-                self.stdout.write(self.style.WARNING(f"Frame cleaning failed, continuing with original video: {e}"))
-        elif not FRAME_CLEANING_AVAILABLE:
-            self.stdout.write(self.style.WARNING("Frame cleaning not available (lx_anonymizer not found)"))
-        
-        # Now call pipe_1 on the VideoFile instance
-        if segmentation:
-            success = video_file_obj.pipe_1(model_name=model_name, model_meta_version = None)
-
-            if success:
-                self.stdout.write(self.style.SUCCESS("Pipeline 1 completed successfully"))
-            else:
-                self.stdout.write(self.style.ERROR("Pipeline 1 failed"))
-    
-    def _ensure_default_patient_data(self, video_file):
-        """
-        Ensure video has minimum required patient data in SensitiveMeta.
-        Creates default values if data is missing after OCR processing.
-        """
-        from endoreg_db.models import SensitiveMeta
-        from datetime import date
-        
-        if not video_file.sensitive_meta:
-            self.stdout.write(self.style.WARNING(f"No SensitiveMeta found for video {video_file.uuid}, creating default"))
-            
-            # Create default SensitiveMeta with placeholder data
-            default_data = {
-                "patient_first_name": None,
-                "patient_last_name": None, 
-                "patient_dob": None,
-                "examination_date": None,
-                "center_name": video_file.center.name if video_file.center else "unknown",
-            }
-            
-            try:
-                sensitive_meta = SensitiveMeta.create_from_dict(default_data)
-                video_file.sensitive_meta = sensitive_meta
-                video_file.save(update_fields=['sensitive_meta'])
-                self.stdout.write(self.style.SUCCESS(f"Created default SensitiveMeta for video {video_file.uuid}"))
-            except Exception as e:
-                self.stdout.write(self.style.ERROR(f"Failed to create default SensitiveMeta for video {video_file.uuid}: {e}"))
-                
-        else:
-            # Update existing SensitiveMeta with missing fields
-            update_needed = False
-            update_data = {}
-            
-            if not video_file.sensitive_meta.patient_first_name:
-                update_data["patient_first_name"] = "Patient"
-                update_needed = True
-                
-            if not video_file.sensitive_meta.patient_last_name:
-                update_data["patient_last_name"] = "Unknown"
-                update_needed = True
-                
-            if not video_file.sensitive_meta.patient_dob:
-                update_data["patient_dob"] = date(1990, 1, 1)
-                update_needed = True
-                
-            if not video_file.sensitive_meta.examination_date:
-                update_data["examination_date"] = date.today()
-                update_needed = True
-                
-            if update_needed:
-                try:
-                    video_file.sensitive_meta.update_from_dict(update_data)
-                    self.stdout.write(self.style.SUCCESS(f"Updated missing SensitiveMeta fields for video {video_file.uuid}: {list(update_data.keys())}"))
-                except Exception as e:
-                    self.stdout.write(self.style.ERROR(f"Failed to update SensitiveMeta for video {video_file.uuid}: {e}"))
-            else:
-                self.stdout.write(self.style.SUCCESS(f"SensitiveMeta for video {video_file.uuid} already has all required fields"))
-    
-    def _choose_processor_interactively(
-        self, processors_qs
-    ) -> EndoscopyProcessor:
-        """
-        Interactively prompts the user to select an endoscopy processor from a list.
-        
-        Displays all available processors and requests user input until a valid selection is made. Aborts the process if the user interrupts input.
-        
-        Args:
-            processors_qs: Queryset of EndoscopyProcessor objects to present for selection.
-        
-        Returns:
-            The EndoscopyProcessor object chosen by the user.
-        """
-        # turn the QS into a concrete list so we can index it later
-        processors = list(processors_qs)           # -> [EndoscopyProcessor, …]
-
-        self.stdout.write(self.style.ERROR(
-            f"\nThe centre has {len(processors)} endoscopy processors.\n"
-            "Choose one for this import:\n"
-        ))
-        for idx, proc in enumerate(processors, 1):
-            self.stdout.write(f"  [{idx}] {proc.name}")
-
-        while True:                                # keep prompting until valid
-            try:
-                choice = input("Processor number › ").strip()
-            except (EOFError, KeyboardInterrupt):
-                self.stderr.write("\nAborted.")
-                raise SystemExit(1)
-
-            try:
-                index = int(choice) - 1
-                if not 0 <= index < len(processors):
-                    raise ValueError
-            except ValueError:
-                self.stderr.write("❌  Please enter a number in the list above.\n")
-                continue
-
-            return processors[index]               # ← the chosen object