endoreg-db 0.8.4.2__py3-none-any.whl → 0.8.4.3__py3-none-any.whl

endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml

@@ -1,5 +1,27 @@
+# Model metadata configuration with setup hints
 - model: endoreg_db.model_meta
   fields:
     name: "image_multilabel_classification_colonoscopy_default"
-    version: 1
+    version: "1"
+    model: "image_multilabel_classification_colonoscopy_default"
     labelset: "multilabel_classification_colonoscopy_default"
+    activation: "sigmoid"
+    mean: "0.485,0.456,0.406" # ImageNet defaults
+    std: "0.229,0.224,0.225" # ImageNet defaults
+    size_x: 224
+    size_y: 224
+    axes: "CHW"
+    batchsize: 32
+    num_workers: 4
+    description: "Default colonoscopy image multilabel classification model from YAML"
+    # Note: weights field intentionally left empty - will be set by setup command or model meta logic
+
+  # Setup configuration hints for this model
+  setup_config:
+    is_primary_model: true # Mark this as the primary model for setup
+    weight_filenames: # Specific weight file patterns for this model
+      - "colo_segmentation_RegNetX800MF_6.ckpt"
+      - "image_multilabel_classification_colonoscopy_default_v1_*.ckpt"
+    huggingface_fallback: # HF configuration for this model
+      repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
+      filename: "colo_segmentation_RegNetX800MF_base.ckpt"

endoreg_db/data/setup_config.yaml

@@ -0,0 +1,38 @@
+# EndoReg DB Setup Configuration
+# This file defines which models should be used for setup and their fallback configurations
+
+default_models:
+  primary_classification_model: "image_multilabel_classification_colonoscopy_default"
+  primary_labelset: "multilabel_classification_colonoscopy_default"
+
+# HuggingFace fallback configuration
+huggingface_fallback:
+  enabled: true
+  repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
+  filename: "colo_segmentation_RegNetX800MF_base.ckpt"
+  labelset_name: "multilabel_classification_colonoscopy_default"
+
+# Local weights search patterns
+weights_search_patterns:
+  # Primary model weight filename patterns (supports wildcards)
+  - "colo_segmentation_RegNetX800MF_*.ckpt"
+  - "image_multilabel_classification_colonoscopy_default_*.ckpt"
+  - "*_colonoscopy_*.ckpt"
+
+# Search directories for local weights (relative to project root)
+weights_search_dirs:
+  - "tests/assets"
+  - "assets"
+  - "data/storage/model_weights"
+  - "${STORAGE_DIR}/model_weights" # Environment variable substitution
+
+# Auto-generation defaults when creating metadata
+auto_generation_defaults:
+  activation: "sigmoid"
+  mean: "0.485,0.456,0.406" # ImageNet defaults
+  std: "0.229,0.224,0.225" # ImageNet defaults
+  size_x: 224
+  size_y: 224
+  axes: "CHW"
+  batchsize: 32
+  num_workers: 4

endoreg_db/management/commands/load_ai_model_data.py

@@ -1,25 +1,27 @@
 from django.core.management.base import BaseCommand
+
+from ...data import (
+    AI_MODEL_DATA_DIR,
+    AI_MODEL_META_DATA_DIR,  # Add this import
+    MODEL_TYPE_DATA_DIR,
+    VIDEO_SEGMENTATION_LABEL_DATA_DIR,
+    VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
+)
 from ...models import (
+    AiModel,
+    ModelMeta,  # Add ModelMeta back to imports
     ModelType,
     VideoSegmentationLabel,
     VideoSegmentationLabelSet,
-    AiModel,
 )
 from ...utils import load_model_data_from_yaml
-from ...data import (
-    MODEL_TYPE_DATA_DIR,
-    VIDEO_SEGMENTATION_LABEL_DATA_DIR,
-    VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
-    AI_MODEL_DATA_DIR,
-)
-
 
 IMPORT_MODELS = [  # string as model key, serves as key in IMPORT_METADATA
     ModelType.__name__,
-    # ModelMeta.__name__,
     VideoSegmentationLabel.__name__,
     VideoSegmentationLabelSet.__name__,
     AiModel.__name__,
+    ModelMeta.__name__,  # Re-enable ModelMeta loading
 ]
 
 IMPORT_METADATA = {
@@ -29,12 +31,12 @@ IMPORT_METADATA = {
         "foreign_keys": [],  # e.g. ["interventions"]
         "foreign_key_models": [],  # e.g. [Intervention]
     },
-    # ModelMeta.__name__: {
-    #     "dir": AI_MODEL_META_DATA_DIR,  # e.g. "intervention_types"
-    #     "model": ModelMeta,  # e.g. InterventionType
-    #     "foreign_keys": ["labelset", "type"],  # e.g. ["interventions"]
-    #     "foreign_key_models": [LabelSet, ModelType],  # e.g. [Intervention]
-    # },
+    ModelMeta.__name__: {
+        "dir": AI_MODEL_META_DATA_DIR,  # e.g. "ai_model_meta"
+        "model": ModelMeta,  # e.g. ModelMeta
+        "foreign_keys": ["labelset", "model"],  # Foreign key relationships
+        "foreign_key_models": [VideoSegmentationLabelSet, AiModel],  # Actual model classes
+    },
     VideoSegmentationLabel.__name__: {
         "dir": VIDEO_SEGMENTATION_LABEL_DATA_DIR,  # e.g. "interventions"
         "model": VideoSegmentationLabel,  # e.g. Intervention

endoreg_db/management/commands/setup_endoreg_db.py

@@ -3,21 +3,21 @@ Django management command to perform complete setup for EndoReg DB when used as
 This command ensures all necessary data and configurations are initialized.
 """
 
-import os
-from pathlib import Path
-
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
+
 from endoreg_db.models import ModelMeta
 
+
 class Command(BaseCommand):
     help = """
     Complete setup for EndoReg DB when used as an embedded app.
     This command performs all necessary initialization steps:
     1. Loads base database data
-    2. Sets up AI models and labels
-    3. Creates cache table
-    4. Initializes model metadata
+    2. Sets up caching (if using db cache)
+    3. Loads default models from setup configuration file (setup_config.yaml)
+    4. Loads models according to fallback chain (Local Files -> HuggingFace -> graceful failure)
+    5. Initializes model metadata
     """
 
     def add_arguments(self, parser):
@@ -31,13 +31,22 @@ class Command(BaseCommand):
             action="store_true",
             help="Force recreation of AI model metadata even if it exists",
         )
+        parser.add_argument(
+            "--yaml-only",
+            action="store_true",
+            help="Only use YAML-defined models, don't auto-generate missing metadata",
+        )
 
     def handle(self, *args, **options):
         skip_ai = options.get("skip_ai_setup", False)
         force_recreate = options.get("force_recreate", False)
+        yaml_only = options.get("yaml_only", False)
 
         self.stdout.write(self.style.SUCCESS("🚀 Starting EndoReg DB embedded app setup..."))
 
+        if yaml_only:
+            self.stdout.write(self.style.WARNING("📋 YAML-only mode: Will not auto-generate missing metadata"))
+
         # Step 1: Load base database data
         self.stdout.write("\n📊 Step 1: Loading base database data...")
         try:
@@ -47,7 +56,7 @@ class Command(BaseCommand):
             self.stdout.write(self.style.ERROR(f"❌ Failed to load base data: {e}"))
             return
 
-            # Step 2: Create cache table (only if using database caching)
+        # Step 2: Create cache table (only if using database caching)
         self.stdout.write("\n💾 Step 2: Setting up caching...")
         from django.conf import settings
 
@@ -87,10 +96,16 @@ class Command(BaseCommand):
             # Step 5: Create model metadata
             self.stdout.write("\n📋 Step 5: Creating AI model metadata...")
             try:
+                # Load setup configuration
+                from endoreg_db.utils.setup_config import setup_config
+
+                # Get primary model from configuration
+                default_model_name = setup_config.get_primary_model_name()
+                primary_labelset = setup_config.get_primary_labelset_name()
+
                 # Check if model metadata already exists
                 from endoreg_db.models import AiModel
 
-                default_model_name = "image_multilabel_classification_colonoscopy_default"
                 ai_model = AiModel.objects.filter(name=default_model_name).first()
 
                 if not ai_model:
@@ -101,14 +116,14 @@ class Command(BaseCommand):
                 if existing_meta and not force_recreate:
                     self.stdout.write(self.style.SUCCESS("✅ Model metadata already exists (use --force-recreate to recreate)"))
                 else:
-                    # Try to create model metadata
+                    # Try to create model metadata using configurable approach
                     model_path = self._find_model_weights_file()
                     if model_path:
                         call_command(
                             "create_multilabel_model_meta",
                             model_name=default_model_name,
                             model_meta_version=1,
-                            image_classification_labelset_name="multilabel_classification_colonoscopy_default",
+                            image_classification_labelset_name=primary_labelset,
                             model_path=str(model_path),
                         )
                         self.stdout.write(self.style.SUCCESS("✅ AI model metadata created successfully"))
@@ -119,6 +134,15 @@ class Command(BaseCommand):
                 self.stdout.write(self.style.ERROR(f"❌ Failed to create AI model metadata: {e}"))
                 return
 
+            # Step 5.5: Validate and fix AI model active metadata
+            self.stdout.write("\n🔧 Step 5.5: Validating AI model active metadata...")
+            try:
+                self._validate_and_fix_ai_model_metadata(yaml_only)
+                self.stdout.write(self.style.SUCCESS("✅ AI model metadata validation completed"))
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to validate AI model metadata: {e}"))
+                return
+
         # Step 6: Verification
         self.stdout.write("\n🔍 Step 6: Verifying setup...")
         try:
@@ -135,37 +159,38 @@ class Command(BaseCommand):
         self.stdout.write("3. Start development server: python manage.py runserver")
 
     def _find_model_weights_file(self):
-        """Find the model weights file in various possible locations."""
-        # Check common locations for model weights
-        
-        if not ModelMeta.objects.exists():
-            print("📦 No model metadata found — creating from Hugging Face...")
-            ModelMeta.setup_default_from_huggingface(
-                "wg-lux/colo_segmentation_RegNetX800MF_base",
-                labelset_name="multilabel_classification_colonoscopy_default"
-            )
-            print("✅ Default ModelMeta created.")
-        possible_paths = [
-            # Test assets (for development)
-            Path("tests/assets/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Project root assets
-            Path("assets/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Storage directory
-            Path("data/storage/model_weights/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Absolute paths based on environment
-            Path(os.getenv("STORAGE_DIR", "storage")) / "model_weights" / "colo_segmentation_RegNetX800MF_6.ckpt",
-        ]
+        """Find the model weights file using configurable search patterns and directories."""
+        # Load setup configuration
+        from endoreg_db.utils.setup_config import setup_config
+
+        # First try to find weights using configured patterns
+        found_files = setup_config.find_model_weights_files()
+        if found_files:
+            self.stdout.write(f"Found model weights at: {found_files[0]}")
+            return found_files[0]
+
+        # If no local weights found and HuggingFace fallback is enabled
+        hf_config = setup_config.get_huggingface_config()
+        if hf_config.get("enabled", True):
+            self.stdout.write("📦 No local model weights found — attempting HuggingFace download...")
+            try:
+                if not ModelMeta.objects.exists():
+                    ModelMeta.setup_default_from_huggingface(
+                        hf_config.get("repo_id", "wg-lux/colo_segmentation_RegNetX800MF_base"),
+                        labelset_name=hf_config.get("labelset_name", "multilabel_classification_colonoscopy_default"),
+                    )
+                    self.stdout.write("✅ Default ModelMeta created from HuggingFace.")
+
+                    # Try to find the downloaded weights
+                    found_files = setup_config.find_model_weights_files()
+                    if found_files:
+                        return found_files[0]
 
-        for path in possible_paths:
-            if path.exists():
-                self.stdout.write(f"Found model weights at: {path}")
-                return path
+            except Exception as e:
+                self.stdout.write(f"⚠️  HuggingFace download failed: {e}")
 
-        self.stdout.write("Model weights file not found in standard locations")
-        
+        self.stdout.write("Model weights file not found in configured locations")
         return None
-    
-
 
     def _verify_setup(self):
         """Verify that the setup was successful."""
@@ -205,5 +230,148 @@ class Command(BaseCommand):
             self.stdout.write(f"Found {meta_count} model metadata record(s)")
 
         self.stdout.write("Setup verification passed")
-        
 
+    def _validate_and_fix_ai_model_metadata(self, yaml_only=False):
+        """
+        Validate that all AI models have proper active metadata and fix if necessary.
+        This addresses the "No model metadata found for this model" error.
+
+        Args:
+            yaml_only (bool): If True, only set active metadata but don't create new metadata
+        """
+        from endoreg_db.models import AiModel, LabelSet, ModelMeta
+        from endoreg_db.utils.setup_config import setup_config
+
+        all_models = AiModel.objects.all()
+        fixed_count = 0
+
+        # Get configurable defaults
+        defaults = setup_config.get_auto_generation_defaults()
+        primary_labelset_name = setup_config.get_primary_labelset_name()
+
+        for model in all_models:
+            self.stdout.write(f"Checking model: {model.name}")
+
+            # Check if model has metadata versions
+            metadata_count = model.metadata_versions.count()
+            self.stdout.write(f"  Metadata versions: {metadata_count}")
+
+            if metadata_count == 0:
+                if yaml_only:
+                    self.stdout.write(f"  ⚠️  YAML-only mode: Skipping auto-generation for {model.name}")
+                    continue
+
+                # Create metadata for models that don't have any
+                self.stdout.write(f"  Creating metadata for {model.name}...")
+
+                # Use configured labelset or create default
+                labelset = None
+                try:
+                    labelset = LabelSet.objects.get(name=primary_labelset_name)
+                except LabelSet.DoesNotExist:
+                    labelset = LabelSet.objects.first()
+                    if not labelset:
+                        labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
+
+                # Create basic metadata WITH weights if available
+                weights_file = self._find_model_weights_file()
+                weights_path = ""
+                if weights_file:
+                    # If we have weights, set up the relative path
+                    from pathlib import Path
+
+                    from endoreg_db.utils.paths import STORAGE_DIR
+
+                    try:
+                        weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                    except ValueError:
+                        # If file is not in storage dir, copy it there
+                        import shutil
+
+                        weights_dir = STORAGE_DIR / "model_weights"
+                        weights_dir.mkdir(parents=True, exist_ok=True)
+                        dest_path = weights_dir / Path(weights_file).name
+                        shutil.copy2(weights_file, dest_path)
+                        weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                        self.stdout.write(f"    Copied weights to: {dest_path}")
+
+                # Create basic metadata using configurable defaults
+                meta = ModelMeta.objects.create(
+                    name=model.name,
+                    version="1.0",
+                    model=model,
+                    labelset=labelset,
+                    weights=weights_path,  # Set weights if available
+                    activation=defaults.get("activation", "sigmoid"),
+                    mean=defaults.get("mean", "0.485,0.456,0.406"),
+                    std=defaults.get("std", "0.229,0.224,0.225"),
+                    size_x=defaults.get("size_x", 224),
+                    size_y=defaults.get("size_y", 224),
+                    axes=defaults.get("axes", "CHW"),
+                    batchsize=defaults.get("batchsize", 32),
+                    num_workers=defaults.get("num_workers", 4),
+                    description=f"Auto-generated metadata for {model.name}",
+                )
+
+                model.active_meta = meta
+                model.save()
+                fixed_count += 1
+                self.stdout.write(f"  ✅ Created and set metadata for {model.name}")
+
+            elif not model.active_meta:
+                # Model has metadata but no active meta set
+                first_meta = model.metadata_versions.first()
+                if first_meta:
+                    self.stdout.write(f"  Setting active metadata for {model.name}...")
+
+                    # Check if the metadata has weights - if not, try to assign them
+                    if not first_meta.weights:
+                        self.stdout.write("    Metadata exists but no weights assigned, attempting to add weights...")
+                        weights_file = self._find_model_weights_file()
+                        if weights_file:
+                            from pathlib import Path
+
+                            from endoreg_db.utils.paths import STORAGE_DIR
+
+                            try:
+                                weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                            except ValueError:
+                                # Copy weights to storage if not already there
+                                import shutil
+
+                                weights_dir = STORAGE_DIR / "model_weights"
+                                weights_dir.mkdir(parents=True, exist_ok=True)
+                                dest_path = weights_dir / Path(weights_file).name
+                                shutil.copy2(weights_file, dest_path)
+                                weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                                self.stdout.write(f"      Copied weights to: {dest_path}")
+
+                            # Assign the relative path to the FileField
+                            first_meta.weights.name = weights_path
+                            first_meta.save(update_fields=["weights"])
+                            self.stdout.write(f"      Added weights to existing metadata: {weights_path}")
+
+                    model.active_meta = first_meta
+                    model.save()
+                    fixed_count += 1
+                    self.stdout.write(f"  ✅ Set active metadata: {first_meta.name} v{first_meta.version}")
+                else:
+                    self.stdout.write(f"  ⚠️  No metadata versions available for {model.name}")
+
+            else:
+                self.stdout.write(f"  ✅ Model {model.name} has active metadata: {model.active_meta}")
+
+        # Verify all models can get latest version
+        self.stdout.write("\nTesting model metadata access...")
+        for model in all_models:
+            try:
+                latest = model.get_latest_version()
+                self.stdout.write(f"  ✅ {model.name}: {latest}")
+            except Exception as e:
+                self.stdout.write(f"  ❌ {model.name}: {e}")
+                raise Exception(f"Model {model.name} still has metadata issues: {e}")
+
+        if fixed_count > 0:
+            self.stdout.write(f"Fixed metadata for {fixed_count} model(s)")
+        else:
+            self.stdout.write("All models already had proper metadata")

endoreg_db/management/commands/validate_ai_models.py

@@ -0,0 +1,124 @@
+"""
+Django management command to validate and fix AI model metadata issues.
+This command addresses the "No model metadata found for this model" error.
+"""
+
+from django.core.management.base import BaseCommand
+
+from endoreg_db.models.metadata.model_meta_logic import validate_and_fix_ai_model_metadata_logic
+
+
+class Command(BaseCommand):
+    help = """
+    Validate and fix AI model metadata to prevent processing errors.
+    
+    This command:
+    - Checks all AI models for proper metadata configuration
+    - Creates missing metadata with sensible defaults
+    - Sets active metadata for models that don't have it
+    - Validates that all models can access their latest versions
+    
+    Use this command to fix "No model metadata found for this model" errors.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Show what would be fixed without making changes",
+        )
+        parser.add_argument(
+            "--force",
+            action="store_true",
+            help="Force recreation of existing metadata",
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options.get("dry_run", False)
+        force = options.get("force", False)
+
+        self.stdout.write(self.style.SUCCESS("🔍 Validating AI model metadata..."))
+
+        if dry_run:
+            self.stdout.write(self.style.WARNING("🧪 DRY RUN MODE - No changes will be made"))
+
+        try:
+            if dry_run:
+                # In dry run, we just check and report issues
+                summary = self._check_ai_models_dry_run()
+            else:
+                # Actually fix the issues
+                summary = validate_and_fix_ai_model_metadata_logic()
+
+            # Report results
+            self._report_summary(summary)
+
+            if summary["errors"]:
+                self.stdout.write(self.style.ERROR("❌ Validation completed with errors"))
+                for error in summary["errors"]:
+                    self.stdout.write(self.style.ERROR(f"  - {error}"))
+                return
+
+            self.stdout.write(self.style.SUCCESS("✅ AI model metadata validation completed successfully"))
+
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"❌ Validation failed: {e}"))
+            raise
+
+    def _check_ai_models_dry_run(self):
+        """Check AI models without making changes."""
+        from endoreg_db.models import AiModel
+
+        summary = {"models_checked": 0, "models_fixed": 0, "metadata_created": 0, "active_meta_set": 0, "errors": []}
+
+        all_models = AiModel.objects.all()
+        summary["models_checked"] = all_models.count()
+
+        for model in all_models:
+            self.stdout.write(f"Checking model: {model.name}")
+
+            metadata_count = model.metadata_versions.count()
+            self.stdout.write(f"  Metadata versions: {metadata_count}")
+
+            if metadata_count == 0:
+                self.stdout.write(f"  🔧 Would create metadata for {model.name}")
+                summary["models_fixed"] += 1
+                summary["metadata_created"] += 1
+
+            elif not model.active_meta:
+                self.stdout.write(f"  🔧 Would set active metadata for {model.name}")
+                summary["models_fixed"] += 1
+                summary["active_meta_set"] += 1
+
+            else:
+                self.stdout.write(f"  ✅ Model {model.name} has valid active metadata")
+
+            # Test metadata access
+            try:
+                latest = model.get_latest_version()
+                self.stdout.write(f"  ✅ Can access latest version: {latest}")
+            except Exception as e:
+                error_msg = f"Model {model.name} metadata test failed: {e}"
+                self.stdout.write(self.style.ERROR(f"  ❌ {error_msg}"))
+                summary["errors"].append(error_msg)
+
+        return summary
+
+    def _report_summary(self, summary):
+        """Report the validation summary."""
+        self.stdout.write("\n📊 Validation Summary:")
+        self.stdout.write(f"  Models checked: {summary['models_checked']}")
+
+        if summary["models_fixed"] > 0:
+            self.stdout.write(f"  Models fixed: {summary['models_fixed']}")
+
+        if summary["metadata_created"] > 0:
+            self.stdout.write(f"  Metadata created: {summary['metadata_created']}")
+
+        if summary["active_meta_set"] > 0:
+            self.stdout.write(f"  Active metadata set: {summary['active_meta_set']}")
+
+        if summary["errors"]:
+            self.stdout.write(f"  Errors encountered: {len(summary['errors'])}")
+        else:
+            self.stdout.write("  No errors found")

endoreg_db/models/medical/hardware/endoscopy_processor.py

@@ -160,7 +160,7 @@ class EndoscopyProcessor(models.Model):
             "height": self.endoscope_sn_height,
         }
 
-    def get_rois(self) -> dict[ str, dict[str, int | None] | None]:
+    def get_rois(self) -> dict[str, dict[str, int | None] | None]:
         return {
             "endoscope_image": self.get_roi_endoscope_image(),
             "examination_date": self.get_roi_examination_date(),
@@ -171,3 +171,12 @@ class EndoscopyProcessor(models.Model):
             "endoscope_type": self.get_roi_endoscope_type(),
             "endoscope_sn": self.get_roi_endoscopy_sn(),
         }
+
+    def get_sensitive_rois(self) -> dict[str, dict[str, int | None] | None]:
+        return {
+            "examination_date": self.get_roi_examination_date(),
+            "examination_time": self.get_roi_examination_time(),
+            "patient_first_name": self.get_roi_patient_first_name(),
+            "patient_last_name": self.get_roi_patient_last_name(),
+            "patient_dob": self.get_roi_patient_dob(),
+        }