endoreg-db 0.8.4.1__py3-none-any.whl → 0.8.4.3__py3-none-any.whl

endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml

@@ -1,5 +1,27 @@
+# Model metadata configuration with setup hints
 - model: endoreg_db.model_meta
   fields:
     name: "image_multilabel_classification_colonoscopy_default"
-    version: 1
+    version: "1"
+    model: "image_multilabel_classification_colonoscopy_default"
     labelset: "multilabel_classification_colonoscopy_default"
+    activation: "sigmoid"
+    mean: "0.485,0.456,0.406" # ImageNet defaults
+    std: "0.229,0.224,0.225" # ImageNet defaults
+    size_x: 224
+    size_y: 224
+    axes: "CHW"
+    batchsize: 32
+    num_workers: 4
+    description: "Default colonoscopy image multilabel classification model from YAML"
+    # Note: weights field intentionally left empty - will be set by setup command or model meta logic
+
+  # Setup configuration hints for this model
+  setup_config:
+    is_primary_model: true # Mark this as the primary model for setup
+    weight_filenames: # Specific weight file patterns for this model
+      - "colo_segmentation_RegNetX800MF_6.ckpt"
+      - "image_multilabel_classification_colonoscopy_default_v1_*.ckpt"
+    huggingface_fallback: # HF configuration for this model
+      repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
+      filename: "colo_segmentation_RegNetX800MF_base.ckpt"

endoreg_db/data/setup_config.yaml

@@ -0,0 +1,38 @@
+# EndoReg DB Setup Configuration
+# This file defines which models should be used for setup and their fallback configurations
+
+default_models:
+  primary_classification_model: "image_multilabel_classification_colonoscopy_default"
+  primary_labelset: "multilabel_classification_colonoscopy_default"
+
+# HuggingFace fallback configuration
+huggingface_fallback:
+  enabled: true
+  repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
+  filename: "colo_segmentation_RegNetX800MF_base.ckpt"
+  labelset_name: "multilabel_classification_colonoscopy_default"
+
+# Local weights search patterns
+weights_search_patterns:
+  # Primary model weight filename patterns (supports wildcards)
+  - "colo_segmentation_RegNetX800MF_*.ckpt"
+  - "image_multilabel_classification_colonoscopy_default_*.ckpt"
+  - "*_colonoscopy_*.ckpt"
+
+# Search directories for local weights (relative to project root)
+weights_search_dirs:
+  - "tests/assets"
+  - "assets"
+  - "data/storage/model_weights"
+  - "${STORAGE_DIR}/model_weights" # Environment variable substitution
+
+# Auto-generation defaults when creating metadata
+auto_generation_defaults:
+  activation: "sigmoid"
+  mean: "0.485,0.456,0.406" # ImageNet defaults
+  std: "0.229,0.224,0.225" # ImageNet defaults
+  size_x: 224
+  size_y: 224
+  axes: "CHW"
+  batchsize: 32
+  num_workers: 4

endoreg_db/management/commands/load_ai_model_data.py

@@ -1,25 +1,27 @@
 from django.core.management.base import BaseCommand
+
+from ...data import (
+    AI_MODEL_DATA_DIR,
+    AI_MODEL_META_DATA_DIR,  # Add this import
+    MODEL_TYPE_DATA_DIR,
+    VIDEO_SEGMENTATION_LABEL_DATA_DIR,
+    VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
+)
 from ...models import (
+    AiModel,
+    ModelMeta,  # Add ModelMeta back to imports
     ModelType,
     VideoSegmentationLabel,
     VideoSegmentationLabelSet,
-    AiModel,
 )
 from ...utils import load_model_data_from_yaml
-from ...data import (
-    MODEL_TYPE_DATA_DIR,
-    VIDEO_SEGMENTATION_LABEL_DATA_DIR,
-    VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
-    AI_MODEL_DATA_DIR,
-)
-
 
 IMPORT_MODELS = [  # string as model key, serves as key in IMPORT_METADATA
     ModelType.__name__,
-    # ModelMeta.__name__,
     VideoSegmentationLabel.__name__,
     VideoSegmentationLabelSet.__name__,
     AiModel.__name__,
+    ModelMeta.__name__,  # Re-enable ModelMeta loading
 ]
 
 IMPORT_METADATA = {
@@ -29,12 +31,12 @@ IMPORT_METADATA = {
         "foreign_keys": [],  # e.g. ["interventions"]
         "foreign_key_models": [],  # e.g. [Intervention]
     },
-    # ModelMeta.__name__: {
-    #     "dir": AI_MODEL_META_DATA_DIR,  # e.g. "intervention_types"
-    #     "model": ModelMeta,  # e.g. InterventionType
-    #     "foreign_keys": ["labelset", "type"],  # e.g. ["interventions"]
-    #     "foreign_key_models": [LabelSet, ModelType],  # e.g. [Intervention]
-    # },
+    ModelMeta.__name__: {
+        "dir": AI_MODEL_META_DATA_DIR,  # e.g. "ai_model_meta"
+        "model": ModelMeta,  # e.g. ModelMeta
+        "foreign_keys": ["labelset", "model"],  # Foreign key relationships
+        "foreign_key_models": [VideoSegmentationLabelSet, AiModel],  # Actual model classes
+    },
     VideoSegmentationLabel.__name__: {
         "dir": VIDEO_SEGMENTATION_LABEL_DATA_DIR,  # e.g. "interventions"
         "model": VideoSegmentationLabel,  # e.g. Intervention

endoreg_db/management/commands/setup_endoreg_db.py

@@ -3,9 +3,6 @@ Django management command to perform complete setup for EndoReg DB when used as
 This command ensures all necessary data and configurations are initialized.
 """
 
-import os
-from pathlib import Path
-
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
 
@@ -17,9 +14,10 @@ class Command(BaseCommand):
     Complete setup for EndoReg DB when used as an embedded app.
     This command performs all necessary initialization steps:
     1. Loads base database data
-    2. Sets up AI models and labels
-    3. Creates cache table
-    4. Initializes model metadata
+    2. Sets up caching (if using db cache)
+    3. Loads default models from setup configuration file (setup_config.yaml)
+    4. Loads models according to fallback chain (Local Files -> HuggingFace -> graceful failure)
+    5. Initializes model metadata
     """
 
     def add_arguments(self, parser):
@@ -33,13 +31,22 @@ class Command(BaseCommand):
             action="store_true",
             help="Force recreation of AI model metadata even if it exists",
         )
+        parser.add_argument(
+            "--yaml-only",
+            action="store_true",
+            help="Only use YAML-defined models, don't auto-generate missing metadata",
+        )
 
     def handle(self, *args, **options):
         skip_ai = options.get("skip_ai_setup", False)
         force_recreate = options.get("force_recreate", False)
+        yaml_only = options.get("yaml_only", False)
 
         self.stdout.write(self.style.SUCCESS("🚀 Starting EndoReg DB embedded app setup..."))
 
+        if yaml_only:
+            self.stdout.write(self.style.WARNING("📋 YAML-only mode: Will not auto-generate missing metadata"))
+
         # Step 1: Load base database data
         self.stdout.write("\n📊 Step 1: Loading base database data...")
         try:
@@ -49,7 +56,7 @@ class Command(BaseCommand):
             self.stdout.write(self.style.ERROR(f"❌ Failed to load base data: {e}"))
             return
 
-            # Step 2: Create cache table (only if using database caching)
+        # Step 2: Create cache table (only if using database caching)
         self.stdout.write("\n💾 Step 2: Setting up caching...")
         from django.conf import settings
 
@@ -89,10 +96,16 @@ class Command(BaseCommand):
             # Step 5: Create model metadata
             self.stdout.write("\n📋 Step 5: Creating AI model metadata...")
             try:
+                # Load setup configuration
+                from endoreg_db.utils.setup_config import setup_config
+
+                # Get primary model from configuration
+                default_model_name = setup_config.get_primary_model_name()
+                primary_labelset = setup_config.get_primary_labelset_name()
+
                 # Check if model metadata already exists
                 from endoreg_db.models import AiModel
 
-                default_model_name = "image_multilabel_classification_colonoscopy_default"
                 ai_model = AiModel.objects.filter(name=default_model_name).first()
 
                 if not ai_model:
@@ -103,14 +116,14 @@ class Command(BaseCommand):
                 if existing_meta and not force_recreate:
                     self.stdout.write(self.style.SUCCESS("✅ Model metadata already exists (use --force-recreate to recreate)"))
                 else:
-                    # Try to create model metadata
+                    # Try to create model metadata using configurable approach
                     model_path = self._find_model_weights_file()
                     if model_path:
                         call_command(
                             "create_multilabel_model_meta",
                             model_name=default_model_name,
                             model_meta_version=1,
-                            image_classification_labelset_name="multilabel_classification_colonoscopy_default",
+                            image_classification_labelset_name=primary_labelset,
                             model_path=str(model_path),
                         )
                         self.stdout.write(self.style.SUCCESS("✅ AI model metadata created successfully"))
@@ -124,7 +137,7 @@ class Command(BaseCommand):
             # Step 5.5: Validate and fix AI model active metadata
             self.stdout.write("\n🔧 Step 5.5: Validating AI model active metadata...")
             try:
-                self._validate_and_fix_ai_model_metadata()
+                self._validate_and_fix_ai_model_metadata(yaml_only)
                 self.stdout.write(self.style.SUCCESS("✅ AI model metadata validation completed"))
             except Exception as e:
                 self.stdout.write(self.style.ERROR(f"❌ Failed to validate AI model metadata: {e}"))
@@ -146,33 +159,37 @@ class Command(BaseCommand):
         self.stdout.write("3. Start development server: python manage.py runserver")
 
     def _find_model_weights_file(self):
-        """Find the model weights file in various possible locations."""
-        # Check common locations for model weights
-
-        if not ModelMeta.objects.exists():
-            print("📦 No model metadata found — creating from Hugging Face...")
-            ModelMeta.setup_default_from_huggingface(
-                "wg-lux/colo_segmentation_RegNetX800MF_base", labelset_name="multilabel_classification_colonoscopy_default"
-            )
-            print("✅ Default ModelMeta created.")
-        possible_paths = [
-            # Test assets (for development)
-            Path("tests/assets/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Project root assets
-            Path("assets/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Storage directory
-            Path("data/storage/model_weights/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Absolute paths based on environment
-            Path(os.getenv("STORAGE_DIR", "storage")) / "model_weights" / "colo_segmentation_RegNetX800MF_6.ckpt",
-        ]
-
-        for path in possible_paths:
-            if path.exists():
-                self.stdout.write(f"Found model weights at: {path}")
-                return path
+        """Find the model weights file using configurable search patterns and directories."""
+        # Load setup configuration
+        from endoreg_db.utils.setup_config import setup_config
+
+        # First try to find weights using configured patterns
+        found_files = setup_config.find_model_weights_files()
+        if found_files:
+            self.stdout.write(f"Found model weights at: {found_files[0]}")
+            return found_files[0]
+
+        # If no local weights found and HuggingFace fallback is enabled
+        hf_config = setup_config.get_huggingface_config()
+        if hf_config.get("enabled", True):
+            self.stdout.write("📦 No local model weights found — attempting HuggingFace download...")
+            try:
+                if not ModelMeta.objects.exists():
+                    ModelMeta.setup_default_from_huggingface(
+                        hf_config.get("repo_id", "wg-lux/colo_segmentation_RegNetX800MF_base"),
+                        labelset_name=hf_config.get("labelset_name", "multilabel_classification_colonoscopy_default"),
+                    )
+                    self.stdout.write("✅ Default ModelMeta created from HuggingFace.")
+
+                    # Try to find the downloaded weights
+                    found_files = setup_config.find_model_weights_files()
+                    if found_files:
+                        return found_files[0]
 
-        self.stdout.write("Model weights file not found in standard locations")
+            except Exception as e:
+                self.stdout.write(f"⚠️  HuggingFace download failed: {e}")
 
+        self.stdout.write("Model weights file not found in configured locations")
         return None
 
     def _verify_setup(self):
@@ -214,16 +231,24 @@ class Command(BaseCommand):
 
         self.stdout.write("Setup verification passed")
 
-    def _validate_and_fix_ai_model_metadata(self):
+    def _validate_and_fix_ai_model_metadata(self, yaml_only=False):
         """
         Validate that all AI models have proper active metadata and fix if necessary.
         This addresses the "No model metadata found for this model" error.
+
+        Args:
+            yaml_only (bool): If True, only set active metadata but don't create new metadata
         """
         from endoreg_db.models import AiModel, LabelSet, ModelMeta
+        from endoreg_db.utils.setup_config import setup_config
 
         all_models = AiModel.objects.all()
         fixed_count = 0
 
+        # Get configurable defaults
+        defaults = setup_config.get_auto_generation_defaults()
+        primary_labelset_name = setup_config.get_primary_labelset_name()
+
         for model in all_models:
             self.stdout.write(f"Checking model: {model.name}")
 
@@ -232,28 +257,59 @@ class Command(BaseCommand):
             self.stdout.write(f"  Metadata versions: {metadata_count}")
 
             if metadata_count == 0:
+                if yaml_only:
+                    self.stdout.write(f"  ⚠️  YAML-only mode: Skipping auto-generation for {model.name}")
+                    continue
+
                 # Create metadata for models that don't have any
                 self.stdout.write(f"  Creating metadata for {model.name}...")
 
-                # Use existing labelset or create default
-                labelset = LabelSet.objects.first()
-                if not labelset:
-                    labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
-
-                # Create basic metadata
+                # Use configured labelset or create default
+                labelset = None
+                try:
+                    labelset = LabelSet.objects.get(name=primary_labelset_name)
+                except LabelSet.DoesNotExist:
+                    labelset = LabelSet.objects.first()
+                    if not labelset:
+                        labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
+
+                # Create basic metadata WITH weights if available
+                weights_file = self._find_model_weights_file()
+                weights_path = ""
+                if weights_file:
+                    # If we have weights, set up the relative path
+                    from pathlib import Path
+
+                    from endoreg_db.utils.paths import STORAGE_DIR
+
+                    try:
+                        weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                    except ValueError:
+                        # If file is not in storage dir, copy it there
+                        import shutil
+
+                        weights_dir = STORAGE_DIR / "model_weights"
+                        weights_dir.mkdir(parents=True, exist_ok=True)
+                        dest_path = weights_dir / Path(weights_file).name
+                        shutil.copy2(weights_file, dest_path)
+                        weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                        self.stdout.write(f"    Copied weights to: {dest_path}")
+
+                # Create basic metadata using configurable defaults
                 meta = ModelMeta.objects.create(
                     name=model.name,
                     version="1.0",
                     model=model,
                     labelset=labelset,
-                    activation="sigmoid" if "classification" in model.name else "sigmoid",
-                    mean="0.485,0.456,0.406",  # ImageNet defaults
-                    std="0.229,0.224,0.225",  # ImageNet defaults
-                    size_x=224,
-                    size_y=224,
-                    axes="CHW",
-                    batchsize=32,
-                    num_workers=4,
+                    weights=weights_path,  # Set weights if available
+                    activation=defaults.get("activation", "sigmoid"),
+                    mean=defaults.get("mean", "0.485,0.456,0.406"),
+                    std=defaults.get("std", "0.229,0.224,0.225"),
+                    size_x=defaults.get("size_x", 224),
+                    size_y=defaults.get("size_y", 224),
+                    axes=defaults.get("axes", "CHW"),
+                    batchsize=defaults.get("batchsize", 32),
+                    num_workers=defaults.get("num_workers", 4),
                     description=f"Auto-generated metadata for {model.name}",
                 )
 
@@ -267,6 +323,34 @@ class Command(BaseCommand):
                 first_meta = model.metadata_versions.first()
                 if first_meta:
                     self.stdout.write(f"  Setting active metadata for {model.name}...")
+
+                    # Check if the metadata has weights - if not, try to assign them
+                    if not first_meta.weights:
+                        self.stdout.write("    Metadata exists but no weights assigned, attempting to add weights...")
+                        weights_file = self._find_model_weights_file()
+                        if weights_file:
+                            from pathlib import Path
+
+                            from endoreg_db.utils.paths import STORAGE_DIR
+
+                            try:
+                                weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                            except ValueError:
+                                # Copy weights to storage if not already there
+                                import shutil
+
+                                weights_dir = STORAGE_DIR / "model_weights"
+                                weights_dir.mkdir(parents=True, exist_ok=True)
+                                dest_path = weights_dir / Path(weights_file).name
+                                shutil.copy2(weights_file, dest_path)
+                                weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                                self.stdout.write(f"      Copied weights to: {dest_path}")
+
+                            # Assign the relative path to the FileField
+                            first_meta.weights.name = weights_path
+                            first_meta.save(update_fields=["weights"])
+                            self.stdout.write(f"      Added weights to existing metadata: {weights_path}")
+
                     model.active_meta = first_meta
                     model.save()
                     fixed_count += 1

endoreg_db/models/medical/hardware/endoscopy_processor.py

@@ -160,7 +160,7 @@ class EndoscopyProcessor(models.Model):
             "height": self.endoscope_sn_height,
         }
 
-    def get_rois(self) -> dict[ str, dict[str, int | None] | None]:
+    def get_rois(self) -> dict[str, dict[str, int | None] | None]:
         return {
             "endoscope_image": self.get_roi_endoscope_image(),
             "examination_date": self.get_roi_examination_date(),
@@ -171,3 +171,12 @@ class EndoscopyProcessor(models.Model):
             "endoscope_type": self.get_roi_endoscope_type(),
             "endoscope_sn": self.get_roi_endoscopy_sn(),
         }
+
+    def get_sensitive_rois(self) -> dict[str, dict[str, int | None] | None]:
+        return {
+            "examination_date": self.get_roi_examination_date(),
+            "examination_time": self.get_roi_examination_time(),
+            "patient_first_name": self.get_roi_patient_first_name(),
+            "patient_last_name": self.get_roi_patient_last_name(),
+            "patient_dob": self.get_roi_patient_dob(),
+        }

endoreg_db/models/metadata/model_meta_logic.py

@@ -139,8 +139,8 @@ def create_from_file_logic(
     # --- Optionally update AiModel's active_meta ---
     # You might want to add logic here to automatically set the newly created/updated
     # meta as the active one for the AiModel, e.g.:
-    # ai_model.active_meta = model_meta
-    # ai_model.save()
+    ai_model.active_meta = model_meta
+    ai_model.save()
 
     return model_meta
 
@@ -308,6 +308,10 @@ def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str
 
     ai_model, _ = AiModel.objects.get_or_create(name=meta["name"])
     labelset = LabelSet.objects.first() if not labelset_name else LabelSet.objects.get(name=labelset_name)
+    model_meta = ModelMeta.objects.filter(name=meta["name"], model=ai_model).first()
+    if model_meta:
+        logger.info(f"ModelMeta {meta['name']} for model {ai_model.name} already exists. Skipping creation.")
+        return model_meta
 
     return create_from_file_logic(
         cls,