endoreg-db 0.6.3__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/management/commands/import_video_with_classification.py

@@ -0,0 +1,367 @@
+"""
+Management command to import a video file and automatically run NICE/PARIS classifications.
+This command extends the basic import_video functionality with automatic classification.
+"""
+from django.core.management import BaseCommand, call_command
+from django.db import connection
+from pathlib import Path
+from endoreg_db.models import VideoFile, ModelMeta
+from endoreg_db.models.administration.center import Center
+from endoreg_db.models.medical.hardware import EndoscopyProcessor
+
+# TODO Migrate
+from endoreg_db.serializers._old.Frames_NICE_and_PARIS_classifications import (
+    ForNiceClassificationSerializer,
+    ForParisClassificationSerializer
+)
+
+from ...helpers.default_objects import (
+    get_latest_segmentation_model
+)
+
+from endoreg_db.utils.video.ffmpeg_wrapper import check_ffmpeg_availability
+
+from endoreg_db.helpers.data_loader import (
+    load_disease_data,
+    load_gender_data,
+    load_event_data,
+    load_information_source,
+    load_examination_data,
+    load_center_data,
+    load_endoscope_data,
+    load_ai_model_label_data,
+    load_ai_model_data,
+    load_default_ai_model
+)
+
+
+class Command(BaseCommand):
+    help = """
+        Imports a video file and automatically runs NICE/PARIS classifications.
+        
+        Workflow:
+        1. Validates dependencies (FFMPEG, center, processor)
+        2. Loads reference data
+        3. Imports video using VideoFile.create_from_file_initialized()
+        4. Runs VideoFile.pipe_1() for AI segmentation
+        5. Automatically runs NICE classification
+        6. Automatically runs PARIS classification (if frame_dir available)
+        7. Reports results of all steps
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--verbose",
+            action="store_true",
+            help="Display verbose output",
+        )
+        parser.add_argument(
+            "--center_name",
+            type=str,
+            default="university_hospital_wuerzburg",
+            help="Name of the center to associate with video",
+        )
+        
+        parser.add_argument(
+            "video_file",
+            type=Path,
+            help="Path to the video file to import",
+        )
+
+        parser.add_argument(
+            "--frame_dir_root",
+            type=str,
+            default="~/test-data/raw_frame_dir",
+            help="Path to the frame directory",
+        )
+
+        parser.add_argument(
+            "--video_dir_root",
+            type=str,
+            default="~/test-data/raw_video_dir",
+            help="Path to the video directory",
+        )
+
+        parser.add_argument(
+            "--delete_source",
+            action="store_true",
+            default=False,
+            help="Delete the source video file after importing",
+        )
+
+        parser.add_argument(
+            "--save_video_file",
+            action="store_true",
+            default=True,
+            help="Save the video file to the video directory",
+        )
+
+        parser.add_argument(
+            "--model_path",
+            type=str,
+            default="./data/models/colo_segmentation_RegNetX800MF_6.ckpt",
+            help="Path to the model file",
+        )
+        
+        parser.add_argument(
+            "--segmentation",
+            action="store_true",
+            help="Whether to use segmentation",
+        )
+        
+        parser.add_argument(
+            "--processor_name",
+            type=str,
+            default="olympus_cv_1500",
+            help="Name of the processor to associate with video",
+        )
+        
+        # NEW: Classification options
+        parser.add_argument(
+            "--skip_classification",
+            action="store_true",
+            default=False,
+            help="Skip automatic NICE/PARIS classification after import",
+        )
+        
+        parser.add_argument(
+            "--classification_only",
+            type=str,
+            choices=['nice', 'paris', 'both'],
+            default='both',
+            help="Which classification types to run (default: both)",
+        )
+
+    def handle(self, *args, **options):
+        """
+        Main command handler that orchestrates the complete workflow.
+        """
+        # Check FFMPEG availability
+        try:
+            check_ffmpeg_availability()
+            self.stdout.write(self.style.SUCCESS("✓ FFMPEG is available"))
+        except FileNotFoundError as e:
+            self.stderr.write(self.style.ERROR(f"✗ FFMPEG not found: {str(e)}"))
+            return
+
+        self.stdout.write(f"Database: {connection.alias}")
+        self.stdout.write(self.style.SUCCESS("=== Starting Video Import and Classification Workflow ==="))
+
+        # Load reference data
+        self._load_reference_data()
+
+        # Parse options
+        segmentation = options["segmentation"]
+        skip_classification = options["skip_classification"]
+        classification_only = options["classification_only"]
+        
+        # Get AI model if segmentation is enabled
+        self.ai_model_meta = None
+        if segmentation:
+            load_ai_model_label_data()
+            load_ai_model_data()
+            load_default_ai_model()
+            try:
+                self.ai_model_meta = get_latest_segmentation_model()
+            except ModelMeta.DoesNotExist as exc:
+                raise AssertionError("No ModelMeta found for the latest default segmentation AiModel") from exc
+
+        # Validate input parameters
+        video_file = Path(options["video_file"]).expanduser()
+        if not video_file.exists():
+            self.stdout.write(self.style.ERROR(f"✗ Video file not found: {video_file}"))
+            return
+
+        # Validate center and processor
+        center, processor = self._validate_center_and_processor(options)
+        if not center or not processor:
+            return
+
+        # Step 1: Import Video
+        self.stdout.write(self.style.SUCCESS("\n=== Step 1: Importing Video ==="))
+        video_file_obj = self._import_video(options, video_file)
+        if not video_file_obj:
+            return
+
+        # Step 2: Run Pipeline 1 (AI Segmentation)
+        self.stdout.write(self.style.SUCCESS("\n=== Step 2: Running AI Segmentation (Pipeline 1) ==="))
+        pipeline_success = self._run_pipeline_1(video_file_obj)
+        if not pipeline_success:
+            self.stdout.write(self.style.WARNING("⚠ Pipeline 1 failed, but continuing with classification..."))
+
+        # Step 3: Run Classifications (if not skipped)
+        if not skip_classification:
+            self.stdout.write(self.style.SUCCESS("\n=== Step 3: Running Polyp Classifications ==="))
+            self._run_classifications(video_file_obj, classification_only)
+        else:
+            self.stdout.write(self.style.WARNING("⚠ Skipping classifications as requested"))
+
+        self.stdout.write(self.style.SUCCESS("\n=== Workflow Complete ==="))
+        self.stdout.write(f"✓ Video imported with ID: {video_file_obj.id}")
+        self.stdout.write(f"✓ Video available at: {video_file_obj.processed_file}")
+
+    def _load_reference_data(self):
+        """Load all necessary reference data."""
+        self.stdout.write("Loading reference data...")
+        load_gender_data()
+        load_disease_data()
+        load_event_data()
+        load_information_source()
+        load_examination_data()
+        load_center_data()
+        load_endoscope_data()
+        self.stdout.write(self.style.SUCCESS("✓ Reference data loaded"))
+
+    def _validate_center_and_processor(self, options):
+        """Validate that center and processor exist."""
+        center_name = options["center_name"]
+        processor_name = options["processor_name"]
+
+        # Validate center
+        try:
+            center = Center.objects.get(name=center_name)
+            self.stdout.write(self.style.SUCCESS(f"✓ Using center: {center.name_en}"))
+        except Center.DoesNotExist:
+            self.stdout.write(self.style.ERROR(f"✗ Center not found: {center_name}"))
+            return None, None
+
+        # Validate processor
+        try:
+            processors_qs = EndoscopyProcessor.objects.filter(centers=center)
+            proc_count = processors_qs.count()
+            
+            if proc_count == 0:
+                processor = EndoscopyProcessor.objects.filter(name=processor_name).first()
+                if processor is None:
+                    self.stderr.write(self.style.ERROR(f"✗ No processors found for center or fallback"))
+                    return None, None
+                processor.centers.add(center)
+                self.stdout.write(self.style.WARNING(f"⚠ Linked fallback processor '{processor.name}' to center"))
+            elif proc_count == 1:
+                processor = processors_qs.first()
+            else:
+                processor = self._choose_processor_interactively(processors_qs)
+
+            self.stdout.write(self.style.SUCCESS(f"✓ Using processor: {processor.name}"))
+            return center, processor
+
+        except EndoscopyProcessor.DoesNotExist:
+            self.stdout.write(self.style.ERROR(f"✗ Processor not found: {processor_name}"))
+            return None, None
+
+    def _import_video(self, options, video_file):
+        """Import the video file into the database."""
+        try:
+            video_file_obj = VideoFile.create_from_file_initialized(
+                file_path=video_file,
+                center_name=options["center_name"],
+                processor_name=options["processor_name"],
+                delete_source=options["delete_source"],
+                save_video_file=options["save_video_file"],
+            )
+            
+            if video_file_obj:
+                self.stdout.write(self.style.SUCCESS(f"✓ Video imported successfully (ID: {video_file_obj.id})"))
+                return video_file_obj
+            else:
+                self.stdout.write(self.style.ERROR("✗ Failed to create VideoFile instance"))
+                return None
+                
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"✗ Error importing video: {str(e)}"))
+            return None
+
+    def _run_pipeline_1(self, video_file_obj):
+        """Run the AI segmentation pipeline."""
+        try:
+            if self.ai_model_meta:
+                success = video_file_obj.pipe_1(model_name=self.ai_model_meta.model.name)
+            else:
+                # Get the default model meta if segmentation is not enabled
+                ai_model_meta = ModelMeta.objects.filter(
+                    model__name="colo_segmentation_RegNetX800MF_6"
+                ).first()
+                
+                if ai_model_meta:
+                    success = video_file_obj.pipe_1(model_name=ai_model_meta.model.name)
+                else:
+                    success = video_file_obj.pipe_1(model_name="colo_segmentation_RegNetX800MF_6")
+
+            if success:
+                self.stdout.write(self.style.SUCCESS("✓ Pipeline 1 completed successfully"))
+                return True
+            else:
+                self.stdout.write(self.style.ERROR("✗ Pipeline 1 failed"))
+                return False
+                
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"✗ Pipeline 1 error: {str(e)}"))
+            return False
+
+    def _run_classifications(self, video_file_obj, classification_only):
+        """Run NICE and/or PARIS classifications."""
+        video_list = [video_file_obj]
+        
+        # Run NICE Classification
+        if classification_only in ['nice', 'both']:
+            self.stdout.write("Running NICE classification...")
+            try:
+                nice_serializer = ForNiceClassificationSerializer()
+                nice_results = nice_serializer.to_representation(video_list)
+                
+                if nice_results and isinstance(nice_results, dict) and 'data' in nice_results:
+                    segments_count = sum(len(item.get('frames', [])) for item in nice_results['data'] if 'frames' in item)
+                    self.stdout.write(self.style.SUCCESS(f"✓ NICE classification completed ({segments_count} segments found)"))
+                else:
+                    self.stdout.write(self.style.WARNING("⚠ NICE classification completed but no valid segments found"))
+                    
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"✗ NICE classification failed: {str(e)}"))
+
+        # Run PARIS Classification
+        if classification_only in ['paris', 'both']:
+            self.stdout.write("Running PARIS classification...")
+            try:
+                # Check if video has frame_dir (required for PARIS)
+                if not getattr(video_file_obj, "frame_dir", None):
+                    self.stdout.write(self.style.WARNING("⚠ PARIS classification skipped: no frame_dir available"))
+                else:
+                    paris_serializer = ForParisClassificationSerializer()
+                    paris_results = paris_serializer.to_representation(video_list)
+                    
+                    if paris_results and isinstance(paris_results, dict) and 'data' in paris_results:
+                        segments_count = sum(len(item.get('frames', [])) for item in paris_results['data'] if 'frames' in item)
+                        self.stdout.write(self.style.SUCCESS(f"✓ PARIS classification completed ({segments_count} segments found)"))
+                    else:
+                        self.stdout.write(self.style.WARNING("⚠ PARIS classification completed but no valid segments found"))
+                        
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"✗ PARIS classification failed: {str(e)}"))
+
+    def _choose_processor_interactively(self, processors_qs):
+        """Interactively choose a processor from multiple options."""
+        processors = list(processors_qs)
+
+        self.stdout.write(self.style.ERROR(
+            f"\nThe center has {len(processors)} endoscopy processors.\n"
+            "Choose one for this import:\n"
+        ))
+        for idx, proc in enumerate(processors, 1):
+            self.stdout.write(f"  [{idx}] {proc.name}")
+
+        while True:
+            try:
+                choice = input("Processor number › ").strip()
+            except (EOFError, KeyboardInterrupt):
+                self.stderr.write("\nAborted.")
+                raise SystemExit(1)
+
+            try:
+                index = int(choice) - 1
+                if not 0 <= index < len(processors):
+                    raise ValueError
+            except ValueError:
+                self.stderr.write("❌ Please enter a number in the list above.\n")
+                continue
+
+            return processors[index]

endoreg_db/management/commands/init_default_ai_model.py

@@ -0,0 +1,112 @@
+"""
+Django management command to initialize the default AI model with metadata.
+This command ensures that a default AI model exists with proper ModelMeta records.
+"""
+
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+from pathlib import Path
+#FIXME
+from endoreg_db.data.ai_model_meta import (
+    default_multilabel_classification
+)
+
+from endoreg_db.models import AiModel
+from endoreg_db.helpers.data_loader import (
+    load_ai_model_label_data,
+    load_ai_model_data,
+)
+
+
+class Command(BaseCommand):
+    help = """
+    Initialize the default AI model with metadata.
+    This command ensures that the default segmentation model exists with proper ModelMeta records.
+    """
+
+    def add_arguments(self, parser):
+        """
+        Adds the --force command-line argument to control metadata recreation.
+        
+        The --force flag, when specified, forces the recreation of model metadata even if it already exists.
+        """
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            help='Force recreation of the model metadata even if it exists',
+        )
+
+    def handle(self, *args, **options):
+        """
+        Initializes the default AI model and its metadata, creating or updating as needed.
+        
+        Loads required AI model label and model data, ensures the presence of a default AI model, and creates associated model metadata if it does not already exist or if the `--force` flag is specified. Generates a dummy weights file if necessary and invokes the metadata creation command. Verifies successful creation and outputs status messages throughout the process.
+        """
+        force = options.get('force', False)
+        default_multilabel_classification()
+        # First ensure the basic AI model data is loaded
+        self.stdout.write("Loading AI model label data...")
+        load_ai_model_label_data()
+        
+        self.stdout.write("Loading AI model data...")
+        load_ai_model_data()
+
+        # Check if default model exists
+        default_model_name = "image_multilabel_classification_colonoscopy_default"
+        
+        try:
+            ai_model = AiModel.objects.get(name=default_model_name)
+            self.stdout.write(f"Found AI model: {ai_model.name}")
+        except AiModel.DoesNotExist:
+            self.stdout.write(
+                self.style.ERROR(f"AI model '{default_model_name}' not found. Make sure AI model data is loaded.")
+            )
+            return
+
+        # Check if model metadata exists
+        existing_meta = ai_model.metadata_versions.first()
+        if existing_meta and not force:
+            self.stdout.write(
+                self.style.SUCCESS(f"Model metadata already exists for {ai_model.name}. Use --force to recreate.")
+            )
+            return
+
+        # Create default model metadata
+        self.stdout.write("Creating default model metadata...")
+        
+        # Use a dummy weights file path for now - in production this should point to actual model weights
+        dummy_weights_path = Path(__file__).parent.parent.parent / "assets" / "dummy_model.ckpt"
+        
+        # Create the dummy weights file if it doesn't exist
+        dummy_weights_path.parent.mkdir(parents=True, exist_ok=True)
+        if not dummy_weights_path.exists():
+            dummy_weights_path.write_bytes(b"dummy weights content")
+            self.stdout.write(f"Created dummy weights file at {dummy_weights_path}")
+
+        try:
+            # Create ModelMeta using the create_multilabel_model_meta command
+            call_command(
+                "create_multilabel_model_meta",
+                "--model_path", str(dummy_weights_path),
+                "--model_name", default_model_name,
+                "--image_classification_labelset_name", "multilabel_classification_colonoscopy",
+                "--activation_function_name", "sigmoid",
+                "--model_meta_version", "1",
+            )
+            
+            self.stdout.write(
+                self.style.SUCCESS(f"Successfully created model metadata for {default_model_name}")
+            )
+            
+            # Verify the model can be retrieved
+            from endoreg_db.helpers.default_objects import get_latest_segmentation_model
+            model_meta = get_latest_segmentation_model()
+            self.stdout.write(
+                self.style.SUCCESS(f"Verified: Model metadata can be retrieved: {model_meta}")
+            )
+            
+        except Exception as e:
+            self.stdout.write(
+                self.style.ERROR(f"Failed to create model metadata: {e}")
+            )
+            raise

endoreg_db/management/commands/load_ai_model_data.py

@@ -1,10 +1,6 @@
-from django.conf import settings
-import os
 from django.core.management.base import BaseCommand
 from ...models import (
-    ModelMeta,
     ModelType,
-    LabelSet,
     VideoSegmentationLabel,
     VideoSegmentationLabelSet,
     AiModel,
@@ -15,7 +11,6 @@ from ...data import (
     VIDEO_SEGMENTATION_LABEL_DATA_DIR,
     VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
     AI_MODEL_DATA_DIR,
-    AI_MODEL_META_DATA_DIR,
 )
 
 
@@ -55,8 +50,8 @@ IMPORT_METADATA = {
     AiModel.__name__: {
         "dir": AI_MODEL_DATA_DIR,  # e.g. "intervention_types"
         "model": AiModel,  # e.g. InterventionType
-        "foreign_keys": ["video_segmentation_labelset"],  # e.g. ["interventions"]
-        "foreign_key_models": [VideoSegmentationLabelSet],  # e.g. [Intervention]
+        "foreign_keys": ["video_segmentation_labelset", "model_type"],  # e.g. ["interventions"]
+        "foreign_key_models": [VideoSegmentationLabelSet, ModelType],  # e.g. [Intervention]
     },
 }
 

endoreg_db/management/commands/load_base_db_data.py

@@ -14,12 +14,24 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         # verbose = options['verbose']
+        """
+        Orchestrates the sequential execution of data loading commands to populate base database models.
+        
+        This management command displays an initial message and then runs a series of data loading routines 
+        (via call_command) in a specified order. It ignores any verbose setting from the command-line options 
+        and forces verbose output. A final success message is printed after all commands complete.
+        """
         verbose = True
 
         self.stdout.write(self.style.SUCCESS("Populating base db models with data..."))
 
         out = self.stdout
 
+        call_command("load_tag_data", stdout=out, verbose=verbose)
+        call_command("load_information_source", stdout=out, verbose=verbose)
+
+        call_command("load_risk_data", stdout=out, verbose=verbose)
+
         # Load Center Data
         call_command("load_center_data", stdout=out, verbose=verbose)
         call_command("load_endoscope_data", stdout=out, verbose=verbose)
@@ -34,11 +46,13 @@ class Command(BaseCommand):
         call_command("load_organ_data", stdout=out, verbose=verbose)
         call_command("load_contraindication_data", stdout=out, verbose=verbose)
         call_command("load_examination_data", stdout=out, verbose=verbose)
-        call_command("load_examination_indication_data", stdout=out, verbose=verbose)
         call_command("load_lab_value_data", stdout=out, verbose=verbose)
         call_command("load_finding_data", stdout=out, verbose=verbose)
+        call_command("load_examination_indication_data", stdout=out, verbose=verbose)
         call_command("load_medication_data", stdout=out, verbose=verbose)
 
+        call_command("load_requirement_data", stdout=out, verbose=verbose)
+
         # Load AI Model Data
         call_command("load_ai_model_label_data", stdout=out, verbose=verbose)
         call_command("load_ai_model_data", stdout=out, verbose=verbose)

endoreg_db/management/commands/load_center_data.py

@@ -1,43 +1,68 @@
 from django.core.management.base import BaseCommand
 from ...utils import load_model_data_from_yaml
 from ...models import Center, FirstName, LastName
-from ...data import CENTER_DATA_DIR
+from ...data import CENTER_DATA_DIR, NAMES_FIRST_DATA_DIR, NAMES_LAST_DATA_DIR
 
-SOURCE_DIR = CENTER_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
 
-MODEL_0 = Center
+SOURCE_DIR = CENTER_DATA_DIR  # e.g. settings.DATA_DIR_INTERVENTION
 
-IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
-    MODEL_0.__name__,
+IMPORT_MODELS = [  # string as model key, serves as key in IMPORT_METADATA
+    FirstName.__name__,
+    LastName.__name__,
+    Center.__name__,
 ]
 
 IMPORT_METADATA = {
-    MODEL_0.__name__: {
-        "dir": SOURCE_DIR, # e.g. "interventions"
-        "model": MODEL_0, # e.g. Intervention
-        "foreign_keys": ["first_names", "last_names"], # e.g. ["intervention_types"]
-        "foreign_key_models": [FirstName, LastName] # e.g. [InterventionType]
-    }
+    FirstName.__name__: {
+        "dir": NAMES_FIRST_DATA_DIR,  # e.g. "interventions"
+        "model": FirstName,  # e.g. Intervention
+        "foreign_keys": [],  # e.g. ["intervention_types"]
+        "foreign_key_models": [],  # e.g. [InterventionType]
+    },
+    LastName.__name__: {
+        "dir": NAMES_LAST_DATA_DIR,  # e.g. "interventions"
+        "model": LastName,  # e.g. Intervention
+        "foreign_keys": [],  # e.g. ["intervention_types"]
+        "foreign_key_models": [],  # e.g. [InterventionType]
+    },
+    Center.__name__: {
+        "dir": SOURCE_DIR,  # e.g. "interventions"
+        "model": Center,  # e.g. Intervention
+        "foreign_keys": ["first_names", "last_names"],  # e.g. ["intervention_types"]
+        "foreign_key_models": [FirstName, LastName],  # e.g. [InterventionType]
+    },
 }
 
+
 class Command(BaseCommand):
     help = """Load all .yaml files in the data/intervention directory
     into the Intervention and InterventionType model"""
 
     def add_arguments(self, parser):
+        """
+        Adds the '--verbose' flag to the argument parser.
+        
+        When specified, this flag enables verbose output for the management command.
+        """
         parser.add_argument(
-            '--verbose',
-            action='store_true',
-            help='Display verbose output',
+            "--verbose",
+            action="store_true",
+            help="Display verbose output",
         )
 
     def handle(self, *args, **options):
-        verbose = options['verbose']
+        """
+        Load YAML data for each predefined model.
+        
+        Iterates over the models specified in IMPORT_MODELS, retrieving each model's metadata from
+        IMPORT_METADATA and invoking load_model_data_from_yaml to load YAML data. The verbosity of
+        the output is determined by the 'verbose' flag in the command options.
+        
+        Args:
+            *args: Additional positional arguments.
+            **options: Command options; must include a 'verbose' key to control output detail.
+        """
+        verbose = options["verbose"]
         for model_name in IMPORT_MODELS:
             _metadata = IMPORT_METADATA[model_name]
-            load_model_data_from_yaml(
-                self,
-                model_name,
-                _metadata,
-                verbose
-            )
+            load_model_data_from_yaml(self, model_name, _metadata, verbose)

endoreg_db/management/commands/load_endoscope_data.py

@@ -34,8 +34,8 @@ IMPORT_METADATA = {
     EndoscopyProcessor.__name__: {
         "dir": ENDOSCOPY_PROCESSOR_DATA_DIR,
         "model": EndoscopyProcessor,
-        "foreign_keys": [],
-        "foreign_key_models": []
+        "foreign_keys": ["centers"],
+        "foreign_key_models": [Center]
     },
     Endoscope.__name__: {
         "dir": ENDOSCOPE_DATA_DIR,