endoreg-db 0.8.3.7__py3-none-any.whl → 0.8.6.3__py3-none-any.whl

endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml

@@ -1,5 +1,27 @@
+# Model metadata configuration with setup hints
 - model: endoreg_db.model_meta
   fields:
     name: "image_multilabel_classification_colonoscopy_default"
-    version: 1
+    version: "1"
+    model: "image_multilabel_classification_colonoscopy_default"
     labelset: "multilabel_classification_colonoscopy_default"
+    activation: "sigmoid"
+    mean: "0.485,0.456,0.406" # ImageNet defaults
+    std: "0.229,0.224,0.225" # ImageNet defaults
+    size_x: 224
+    size_y: 224
+    axes: "CHW"
+    batchsize: 32
+    num_workers: 4
+    description: "Default colonoscopy image multilabel classification model from YAML"
+    # Note: weights field intentionally left empty - will be set by setup command or model meta logic
+
+  # Setup configuration hints for this model
+  setup_config:
+    is_primary_model: true # Mark this as the primary model for setup
+    weight_filenames: # Specific weight file patterns for this model
+      - "colo_segmentation_RegNetX800MF_6.ckpt"
+      - "image_multilabel_classification_colonoscopy_default_v1_*.ckpt"
+    huggingface_fallback: # HF configuration for this model
+      repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
+      filename: "colo_segmentation_RegNetX800MF_base.ckpt"

endoreg_db/data/setup_config.yaml

@@ -0,0 +1,38 @@
+# EndoReg DB Setup Configuration
+# This file defines which models should be used for setup and their fallback configurations
+
+default_models:
+  primary_classification_model: "image_multilabel_classification_colonoscopy_default"
+  primary_labelset: "multilabel_classification_colonoscopy_default"
+
+# HuggingFace fallback configuration
+huggingface_fallback:
+  enabled: true
+  repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
+  filename: "colo_segmentation_RegNetX800MF_base.ckpt"
+  labelset_name: "multilabel_classification_colonoscopy_default"
+
+# Local weights search patterns
+weights_search_patterns:
+  # Primary model weight filename patterns (supports wildcards)
+  - "colo_segmentation_RegNetX800MF_*.ckpt"
+  - "image_multilabel_classification_colonoscopy_default_*.ckpt"
+  - "*_colonoscopy_*.ckpt"
+
+# Search directories for local weights (relative to project root)
+weights_search_dirs:
+  - "tests/assets"
+  - "assets"
+  - "data/storage/model_weights"
+  - "${STORAGE_DIR}/model_weights" # Environment variable substitution
+
+# Auto-generation defaults when creating metadata
+auto_generation_defaults:
+  activation: "sigmoid"
+  mean: "0.485,0.456,0.406" # ImageNet defaults
+  std: "0.229,0.224,0.225" # ImageNet defaults
+  size_x: 224
+  size_y: 224
+  axes: "CHW"
+  batchsize: 32
+  num_workers: 4

endoreg_db/management/commands/create_model_meta_from_huggingface.py

@@ -50,11 +50,17 @@ class Command(BaseCommand):
 
         try:
             # Download the model weights
-            weights_path = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin", local_dir="/tmp")
+            weights_path = hf_hub_download(
+                repo_id=model_id,
+                filename="colo_segmentation_RegNetX800MF_base.ckpt",
+                local_dir="/tmp",
+            )
             self.stdout.write(f"Downloaded weights to: {weights_path}")
 
             # Get or create AI model
-            ai_model, created = AiModel.objects.get_or_create(name=model_name, defaults={"description": f"Model from {model_id}"})
+            ai_model, created = AiModel.objects.get_or_create(
+                name=model_name, defaults={"description": f"Model from {model_id}"}
+            )
             if created:
                 self.stdout.write(f"Created AI model: {ai_model.name}")
 
@@ -62,7 +68,9 @@ class Command(BaseCommand):
             try:
                 labelset = LabelSet.objects.get(name=labelset_name)
             except LabelSet.DoesNotExist:
-                self.stdout.write(self.style.ERROR(f"LabelSet '{labelset_name}' not found"))
+                self.stdout.write(
+                    self.style.ERROR(f"LabelSet '{labelset_name}' not found")
+                )
                 return
 
             # Create ModelMeta
@@ -86,13 +94,19 @@ class Command(BaseCommand):
 
             # Save the weights file to the model
             with open(weights_path, "rb") as f:
-                model_meta.weights.save(f"{model_name}_v{version}_pytorch_model.bin", ContentFile(f.read()))
+                model_meta.weights.save(
+                    f"{model_name}_v{version}_pytorch_model.bin", ContentFile(f.read())
+                )
 
             # Set as active meta
             ai_model.active_meta = model_meta
             ai_model.save()
 
-            self.stdout.write(self.style.SUCCESS(f"Successfully {'created' if created else 'updated'} ModelMeta: {model_meta}"))
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"Successfully {'created' if created else 'updated'} ModelMeta: {model_meta}"
+                )
+            )
 
         except Exception as e:
             self.stdout.write(self.style.ERROR(f"Error creating ModelMeta: {e}"))

endoreg_db/management/commands/load_ai_model_data.py

@@ -1,25 +1,28 @@
 from django.core.management.base import BaseCommand
+
+from ...data import (
+    AI_MODEL_DATA_DIR,
+    AI_MODEL_META_DATA_DIR,  # Add this import
+    MODEL_TYPE_DATA_DIR,
+    VIDEO_SEGMENTATION_LABEL_DATA_DIR,
+    VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
+)
 from ...models import (
+    AiModel,
+    LabelSet,  # Add LabelSet import
+    ModelMeta,  # Add ModelMeta back to imports
     ModelType,
     VideoSegmentationLabel,
     VideoSegmentationLabelSet,
-    AiModel,
 )
 from ...utils import load_model_data_from_yaml
-from ...data import (
-    MODEL_TYPE_DATA_DIR,
-    VIDEO_SEGMENTATION_LABEL_DATA_DIR,
-    VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
-    AI_MODEL_DATA_DIR,
-)
-
 
 IMPORT_MODELS = [  # string as model key, serves as key in IMPORT_METADATA
     ModelType.__name__,
-    # ModelMeta.__name__,
     VideoSegmentationLabel.__name__,
     VideoSegmentationLabelSet.__name__,
     AiModel.__name__,
+    ModelMeta.__name__,  # Re-enable ModelMeta loading
 ]
 
 IMPORT_METADATA = {
@@ -29,12 +32,12 @@ IMPORT_METADATA = {
         "foreign_keys": [],  # e.g. ["interventions"]
         "foreign_key_models": [],  # e.g. [Intervention]
     },
-    # ModelMeta.__name__: {
-    #     "dir": AI_MODEL_META_DATA_DIR,  # e.g. "intervention_types"
-    #     "model": ModelMeta,  # e.g. InterventionType
-    #     "foreign_keys": ["labelset", "type"],  # e.g. ["interventions"]
-    #     "foreign_key_models": [LabelSet, ModelType],  # e.g. [Intervention]
-    # },
+    ModelMeta.__name__: {
+        "dir": AI_MODEL_META_DATA_DIR,  # e.g. "ai_model_meta"
+        "model": ModelMeta,  # e.g. ModelMeta
+        "foreign_keys": ["labelset", "model"],  # Foreign key relationships
+        "foreign_key_models": [LabelSet, AiModel],  # Actual model classes
+    },
     VideoSegmentationLabel.__name__: {
         "dir": VIDEO_SEGMENTATION_LABEL_DATA_DIR,  # e.g. "interventions"
         "model": VideoSegmentationLabel,  # e.g. Intervention

endoreg_db/management/commands/setup_endoreg_db.py

@@ -3,21 +3,21 @@ Django management command to perform complete setup for EndoReg DB when used as
 This command ensures all necessary data and configurations are initialized.
 """
 
-import os
-from pathlib import Path
-
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
 
+from endoreg_db.models import ModelMeta
+
 
 class Command(BaseCommand):
     help = """
     Complete setup for EndoReg DB when used as an embedded app.
     This command performs all necessary initialization steps:
     1. Loads base database data
-    2. Sets up AI models and labels
-    3. Creates cache table
-    4. Initializes model metadata
+    2. Sets up caching (if using db cache)
+    3. Loads default models from setup configuration file (setup_config.yaml)
+    4. Loads models according to fallback chain (Local Files -> HuggingFace -> graceful failure)
+    5. Initializes model metadata
     """
 
     def add_arguments(self, parser):
@@ -31,13 +31,22 @@ class Command(BaseCommand):
             action="store_true",
             help="Force recreation of AI model metadata even if it exists",
         )
+        parser.add_argument(
+            "--yaml-only",
+            action="store_true",
+            help="Only use YAML-defined models, don't auto-generate missing metadata",
+        )
 
     def handle(self, *args, **options):
         skip_ai = options.get("skip_ai_setup", False)
         force_recreate = options.get("force_recreate", False)
+        yaml_only = options.get("yaml_only", False)
 
         self.stdout.write(self.style.SUCCESS("🚀 Starting EndoReg DB embedded app setup..."))
 
+        if yaml_only:
+            self.stdout.write(self.style.WARNING("📋 YAML-only mode: Will not auto-generate missing metadata"))
+
         # Step 1: Load base database data
         self.stdout.write("\n📊 Step 1: Loading base database data...")
         try:
@@ -47,7 +56,7 @@ class Command(BaseCommand):
             self.stdout.write(self.style.ERROR(f"❌ Failed to load base data: {e}"))
             return
 
-            # Step 2: Create cache table (only if using database caching)
+        # Step 2: Create cache table (only if using database caching)
         self.stdout.write("\n💾 Step 2: Setting up caching...")
         from django.conf import settings
 
@@ -87,10 +96,16 @@ class Command(BaseCommand):
             # Step 5: Create model metadata
             self.stdout.write("\n📋 Step 5: Creating AI model metadata...")
             try:
+                # Load setup configuration
+                from endoreg_db.utils.setup_config import setup_config
+
+                # Get primary model from configuration
+                default_model_name = setup_config.get_primary_model_name()
+                primary_labelset = setup_config.get_primary_labelset_name()
+
                 # Check if model metadata already exists
                 from endoreg_db.models import AiModel
 
-                default_model_name = "image_multilabel_classification_colonoscopy_default"
                 ai_model = AiModel.objects.filter(name=default_model_name).first()
 
                 if not ai_model:
@@ -101,16 +116,20 @@ class Command(BaseCommand):
                 if existing_meta and not force_recreate:
                     self.stdout.write(self.style.SUCCESS("✅ Model metadata already exists (use --force-recreate to recreate)"))
                 else:
-                    # Try to create model metadata
+                    # Try to create model metadata using configurable approach
                     model_path = self._find_model_weights_file()
                     if model_path:
-                        call_command(
-                            "create_multilabel_model_meta",
-                            model_name=default_model_name,
-                            model_meta_version=1,
-                            image_classification_labelset_name="multilabel_classification_colonoscopy_default",
-                            model_path=str(model_path),
-                        )
+                        call_command_kwargs = {
+                            "model_name": default_model_name,
+                            "model_meta_version": 1,
+                            "image_classification_labelset_name": primary_labelset,
+                            "model_path": str(model_path),
+                        }
+                        # Add bump_version flag if force_recreate is enabled
+                        if force_recreate:
+                            call_command_kwargs["bump_version"] = True
+
+                        call_command("create_multilabel_model_meta", **call_command_kwargs)
                         self.stdout.write(self.style.SUCCESS("✅ AI model metadata created successfully"))
                     else:
                         self.stdout.write(self.style.WARNING("⚠️  Model weights file not found. AI features may not work properly."))
@@ -119,6 +138,15 @@ class Command(BaseCommand):
                 self.stdout.write(self.style.ERROR(f"❌ Failed to create AI model metadata: {e}"))
                 return
 
+            # Step 5.5: Validate and fix AI model active metadata
+            self.stdout.write("\n🔧 Step 5.5: Validating AI model active metadata...")
+            try:
+                self._validate_and_fix_ai_model_metadata(yaml_only)
+                self.stdout.write(self.style.SUCCESS("✅ AI model metadata validation completed"))
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to validate AI model metadata: {e}"))
+                return
+
         # Step 6: Verification
         self.stdout.write("\n🔍 Step 6: Verifying setup...")
         try:
@@ -135,25 +163,37 @@ class Command(BaseCommand):
         self.stdout.write("3. Start development server: python manage.py runserver")
 
     def _find_model_weights_file(self):
-        """Find the model weights file in various possible locations."""
-        # Check common locations for model weights
-        possible_paths = [
-            # Test assets (for development)
-            Path("tests/assets/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Project root assets
-            Path("assets/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Storage directory
-            Path("data/storage/model_weights/colo_segmentation_RegNetX800MF_6.ckpt"),
-            # Absolute paths based on environment
-            Path(os.getenv("STORAGE_DIR", "storage")) / "model_weights" / "colo_segmentation_RegNetX800MF_6.ckpt",
-        ]
+        """Find the model weights file using configurable search patterns and directories."""
+        # Load setup configuration
+        from endoreg_db.utils.setup_config import setup_config
+
+        # First try to find weights using configured patterns
+        found_files = setup_config.find_model_weights_files()
+        if found_files:
+            self.stdout.write(f"Found model weights at: {found_files[0]}")
+            return found_files[0]
+
+        # If no local weights found and HuggingFace fallback is enabled
+        hf_config = setup_config.get_huggingface_config()
+        if hf_config.get("enabled", True):
+            self.stdout.write("📦 No local model weights found — attempting HuggingFace download...")
+            try:
+                if not ModelMeta.objects.exists():
+                    ModelMeta.setup_default_from_huggingface(
+                        hf_config.get("repo_id", "wg-lux/colo_segmentation_RegNetX800MF_base"),
+                        labelset_name=hf_config.get("labelset_name", "multilabel_classification_colonoscopy_default"),
+                    )
+                    self.stdout.write("✅ Default ModelMeta created from HuggingFace.")
+
+                    # Try to find the downloaded weights
+                    found_files = setup_config.find_model_weights_files()
+                    if found_files:
+                        return found_files[0]
 
-        for path in possible_paths:
-            if path.exists():
-                self.stdout.write(f"Found model weights at: {path}")
-                return path
+            except Exception as e:
+                self.stdout.write(f"⚠️  HuggingFace download failed: {e}")
 
-        self.stdout.write("Model weights file not found in standard locations")
+        self.stdout.write("Model weights file not found in configured locations")
         return None
 
     def _verify_setup(self):
@@ -194,3 +234,148 @@ class Command(BaseCommand):
             self.stdout.write(f"Found {meta_count} model metadata record(s)")
 
         self.stdout.write("Setup verification passed")
+
+    def _validate_and_fix_ai_model_metadata(self, yaml_only=False):
+        """
+        Validate that all AI models have proper active metadata and fix if necessary.
+        This addresses the "No model metadata found for this model" error.
+
+        Args:
+            yaml_only (bool): If True, only set active metadata but don't create new metadata
+        """
+        from endoreg_db.models import AiModel, LabelSet, ModelMeta
+        from endoreg_db.utils.setup_config import setup_config
+
+        all_models = AiModel.objects.all()
+        fixed_count = 0
+
+        # Get configurable defaults
+        defaults = setup_config.get_auto_generation_defaults()
+        primary_labelset_name = setup_config.get_primary_labelset_name()
+
+        for model in all_models:
+            self.stdout.write(f"Checking model: {model.name}")
+
+            # Check if model has metadata versions
+            metadata_count = model.metadata_versions.count()
+            self.stdout.write(f"  Metadata versions: {metadata_count}")
+
+            if metadata_count == 0:
+                if yaml_only:
+                    self.stdout.write(f"  ⚠️  YAML-only mode: Skipping auto-generation for {model.name}")
+                    continue
+
+                # Create metadata for models that don't have any
+                self.stdout.write(f"  Creating metadata for {model.name}...")
+
+                # Use configured labelset or create default
+                labelset = None
+                try:
+                    labelset = LabelSet.objects.get(name=primary_labelset_name)
+                except LabelSet.DoesNotExist:
+                    labelset = LabelSet.objects.first()
+                    if not labelset:
+                        labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
+
+                # Create basic metadata WITH weights if available
+                weights_file = self._find_model_weights_file()
+                weights_path = ""
+                if weights_file:
+                    # If we have weights, set up the relative path
+                    from pathlib import Path
+
+                    from endoreg_db.utils.paths import STORAGE_DIR
+
+                    try:
+                        weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                    except ValueError:
+                        # If file is not in storage dir, copy it there
+                        import shutil
+
+                        weights_dir = STORAGE_DIR / "model_weights"
+                        weights_dir.mkdir(parents=True, exist_ok=True)
+                        dest_path = weights_dir / Path(weights_file).name
+                        shutil.copy2(weights_file, dest_path)
+                        weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                        self.stdout.write(f"    Copied weights to: {dest_path}")
+
+                # Create basic metadata using configurable defaults
+                meta = ModelMeta.objects.create(
+                    name=model.name,
+                    version="1.0",
+                    model=model,
+                    labelset=labelset,
+                    weights=weights_path,  # Set weights if available
+                    activation=defaults.get("activation", "sigmoid"),
+                    mean=defaults.get("mean", "0.485,0.456,0.406"),
+                    std=defaults.get("std", "0.229,0.224,0.225"),
+                    size_x=defaults.get("size_x", 224),
+                    size_y=defaults.get("size_y", 224),
+                    axes=defaults.get("axes", "CHW"),
+                    batchsize=defaults.get("batchsize", 32),
+                    num_workers=defaults.get("num_workers", 4),
+                    description=f"Auto-generated metadata for {model.name}",
+                )
+
+                model.active_meta = meta
+                model.save()
+                fixed_count += 1
+                self.stdout.write(f"  ✅ Created and set metadata for {model.name}")
+
+            elif not model.active_meta:
+                # Model has metadata but no active meta set
+                first_meta = model.metadata_versions.first()
+                if first_meta:
+                    self.stdout.write(f"  Setting active metadata for {model.name}...")
+
+                    # Check if the metadata has weights - if not, try to assign them
+                    if not first_meta.weights:
+                        self.stdout.write("    Metadata exists but no weights assigned, attempting to add weights...")
+                        weights_file = self._find_model_weights_file()
+                        if weights_file:
+                            from pathlib import Path
+
+                            from endoreg_db.utils.paths import STORAGE_DIR
+
+                            try:
+                                weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                            except ValueError:
+                                # Copy weights to storage if not already there
+                                import shutil
+
+                                weights_dir = STORAGE_DIR / "model_weights"
+                                weights_dir.mkdir(parents=True, exist_ok=True)
+                                dest_path = weights_dir / Path(weights_file).name
+                                shutil.copy2(weights_file, dest_path)
+                                weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                                self.stdout.write(f"      Copied weights to: {dest_path}")
+
+                            # Assign the relative path to the FileField
+                            first_meta.weights.name = weights_path
+                            first_meta.save(update_fields=["weights"])
+                            self.stdout.write(f"      Added weights to existing metadata: {weights_path}")
+
+                    model.active_meta = first_meta
+                    model.save()
+                    fixed_count += 1
+                    self.stdout.write(f"  ✅ Set active metadata: {first_meta.name} v{first_meta.version}")
+                else:
+                    self.stdout.write(f"  ⚠️  No metadata versions available for {model.name}")
+
+            else:
+                self.stdout.write(f"  ✅ Model {model.name} has active metadata: {model.active_meta}")
+
+        # Verify all models can get latest version
+        self.stdout.write("\nTesting model metadata access...")
+        for model in all_models:
+            try:
+                latest = model.get_latest_version()
+                self.stdout.write(f"  ✅ {model.name}: {latest}")
+            except Exception as e:
+                self.stdout.write(f"  ❌ {model.name}: {e}")
+                raise Exception(f"Model {model.name} still has metadata issues: {e}")
+
+        if fixed_count > 0:
+            self.stdout.write(f"Fixed metadata for {fixed_count} model(s)")
+        else:
+            self.stdout.write("All models already had proper metadata")