endoreg-db 0.4.5__py3-none-any.whl → 0.8.6.3__py3-none-any.whl

endoreg_db/management/commands/load_shift_data.py

@@ -0,0 +1,60 @@
+from django.core.management.base import BaseCommand
+from ...models import Shift, ShiftType
+from ...utils import load_model_data_from_yaml
+from ...data import REPORT_READER_FLAG_DATA_DIR
+
+SOURCE_DIR = REPORT_READER_FLAG_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
+
+model_0 = ShiftType
+model_1 = Shift
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    model_0.__name__,
+]
+
+IMPORT_METADATA = {
+    model_0.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": model_0, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    },
+    model_1.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": model_1, # e.g. Intervention
+        "foreign_keys": ["shift_types"], # e.g. ["intervention_types"]
+        "foreign_key_models": [model_0]   # e.g. [InterventionType]
+    }
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/shift directory
+    into the Shift and ShiftType model"""
+
+    def add_arguments(self, parser):
+        """
+        Adds the --verbose command-line option to enable detailed output during execution.
+        """
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        """
+        Loads YAML data files into models defined in IMPORT_MODELS using provided metadata.
+        
+        Iterates over each model specified in IMPORT_MODELS, retrieves its import metadata,
+        and calls the data loading utility to populate the database from YAML files. Supports
+        optional verbose output if enabled via command-line options.
+        """
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_tag_data.py

@@ -0,0 +1,57 @@
+from django.core.management.base import BaseCommand
+from endoreg_db.models import (
+    Tag,
+
+)
+
+from ...utils import load_model_data_from_yaml
+from ...data import (
+    TAG_DATA_DIR
+)
+
+
+IMPORT_MODELS = [  # string as model key, serves as key in IMPORT_METADATA
+    Tag.__name__,
+]
+
+IMPORT_METADATA = {
+    Tag.__name__: {
+        "dir": TAG_DATA_DIR,  # e.g. "interventions"
+        "model": Tag,  # e.g. Intervention
+        "foreign_keys": [],  # e.g. ["intervention_types"]
+        "foreign_key_models": [],  # e.g. [InterventionType]
+    },
+}
+
+
+class Command(BaseCommand):
+    help = """Load all requirement-related YAML files from their respective directories
+    into the database, including RequirementType, RequirementOperator, Requirement, 
+    RequirementSetType, and RequirementSet models"""
+
+    def add_arguments(self, parser):
+        """
+        Add command-line arguments to enable verbose output.
+        
+        Adds an optional '--verbose' flag to the command parser. When specified,
+        this flag causes the command to display detailed output during execution.
+        """
+        parser.add_argument(
+            "--verbose",
+            action="store_true",
+            help="Display verbose output",
+        )
+
+    def handle(self, *args, **options):
+        """
+        Executes data import for requirement models from YAML files.
+        
+        Retrieves the verbosity setting from the command options and iterates over each model 
+        listed in IMPORT_MODELS. For each model, it obtains the corresponding metadata from 
+        IMPORT_METADATA and calls a helper to load the YAML data into the database. Verbose mode 
+        enables detailed output during the process.
+        """
+        verbose = options["verbose"]
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(self, model_name, _metadata, verbose)

endoreg_db/management/commands/register_ai_model.py

@@ -25,7 +25,7 @@ class Command(BaseCommand):
         parser.add_argument('model_meta_path', type=str)
 
     def handle(self, *args, **options):
-        model_meta_path = Path(options['model_meta_path']).resolve()
+        model_meta_path = Path(options['model_meta_path'])
 
         with open(model_meta_path, 'r') as f:
             model_meta = json.load(f)

endoreg_db/management/commands/setup_endoreg_db.py

@@ -0,0 +1,381 @@
+"""
+Django management command to perform complete setup for EndoReg DB when used as an embedded app.
+This command ensures all necessary data and configurations are initialized.
+"""
+
+from django.core.management import call_command
+from django.core.management.base import BaseCommand
+
+from endoreg_db.models import ModelMeta
+
+
+class Command(BaseCommand):
+    help = """
+    Complete setup for EndoReg DB when used as an embedded app.
+    This command performs all necessary initialization steps:
+    1. Loads base database data
+    2. Sets up caching (if using db cache)
+    3. Loads default models from setup configuration file (setup_config.yaml)
+    4. Loads models according to fallback chain (Local Files -> HuggingFace -> graceful failure)
+    5. Initializes model metadata
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--skip-ai-setup",
+            action="store_true",
+            help="Skip AI model setup (for cases where AI features are not needed)",
+        )
+        parser.add_argument(
+            "--force-recreate",
+            action="store_true",
+            help="Force recreation of AI model metadata even if it exists",
+        )
+        parser.add_argument(
+            "--yaml-only",
+            action="store_true",
+            help="Only use YAML-defined models, don't auto-generate missing metadata",
+        )
+
+    def handle(self, *args, **options):
+        skip_ai = options.get("skip_ai_setup", False)
+        force_recreate = options.get("force_recreate", False)
+        yaml_only = options.get("yaml_only", False)
+
+        self.stdout.write(self.style.SUCCESS("🚀 Starting EndoReg DB embedded app setup..."))
+
+        if yaml_only:
+            self.stdout.write(self.style.WARNING("📋 YAML-only mode: Will not auto-generate missing metadata"))
+
+        # Step 1: Load base database data
+        self.stdout.write("\n📊 Step 1: Loading base database data...")
+        try:
+            call_command("load_base_db_data")
+            self.stdout.write(self.style.SUCCESS("✅ Base database data loaded successfully"))
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"❌ Failed to load base data: {e}"))
+            return
+
+        # Step 2: Create cache table (only if using database caching)
+        self.stdout.write("\n💾 Step 2: Setting up caching...")
+        from django.conf import settings
+
+        cache_backend = settings.CACHES.get("default", {}).get("BACKEND", "")
+        if "db" in cache_backend or "database" in cache_backend:
+            self.stdout.write("Using database caching - creating cache table...")
+            try:
+                call_command("createcachetable")
+                self.stdout.write(self.style.SUCCESS("✅ Cache table created successfully"))
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to create cache table: {e}"))
+                return
+        else:
+            self.stdout.write("Using in-memory caching - skipping cache table creation")
+
+        if skip_ai:
+            self.stdout.write(self.style.WARNING("\n⚠️  Skipping AI setup as requested"))
+        else:
+            # Step 3: Load AI model data
+            self.stdout.write("\n🤖 Step 3: Loading AI model data...")
+            try:
+                call_command("load_ai_model_data")
+                self.stdout.write(self.style.SUCCESS("✅ AI model data loaded successfully"))
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to load AI model data: {e}"))
+                return
+
+            # Step 4: Load AI model label data
+            self.stdout.write("\n🏷️  Step 4: Loading AI model label data...")
+            try:
+                call_command("load_ai_model_label_data")
+                self.stdout.write(self.style.SUCCESS("✅ AI model label data loaded successfully"))
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to load AI model label data: {e}"))
+                return
+
+            # Step 5: Create model metadata
+            self.stdout.write("\n📋 Step 5: Creating AI model metadata...")
+            try:
+                # Load setup configuration
+                from endoreg_db.utils.setup_config import setup_config
+
+                # Get primary model from configuration
+                default_model_name = setup_config.get_primary_model_name()
+                primary_labelset = setup_config.get_primary_labelset_name()
+
+                # Check if model metadata already exists
+                from endoreg_db.models import AiModel
+
+                ai_model = AiModel.objects.filter(name=default_model_name).first()
+
+                if not ai_model:
+                    self.stdout.write(self.style.ERROR(f"❌ AI model '{default_model_name}' not found"))
+                    return
+
+                existing_meta = ai_model.metadata_versions.first()
+                if existing_meta and not force_recreate:
+                    self.stdout.write(self.style.SUCCESS("✅ Model metadata already exists (use --force-recreate to recreate)"))
+                else:
+                    # Try to create model metadata using configurable approach
+                    model_path = self._find_model_weights_file()
+                    if model_path:
+                        call_command_kwargs = {
+                            "model_name": default_model_name,
+                            "model_meta_version": 1,
+                            "image_classification_labelset_name": primary_labelset,
+                            "model_path": str(model_path),
+                        }
+                        # Add bump_version flag if force_recreate is enabled
+                        if force_recreate:
+                            call_command_kwargs["bump_version"] = True
+
+                        call_command("create_multilabel_model_meta", **call_command_kwargs)
+                        self.stdout.write(self.style.SUCCESS("✅ AI model metadata created successfully"))
+                    else:
+                        self.stdout.write(self.style.WARNING("⚠️  Model weights file not found. AI features may not work properly."))
+
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to create AI model metadata: {e}"))
+                return
+
+            # Step 5.5: Validate and fix AI model active metadata
+            self.stdout.write("\n🔧 Step 5.5: Validating AI model active metadata...")
+            try:
+                self._validate_and_fix_ai_model_metadata(yaml_only)
+                self.stdout.write(self.style.SUCCESS("✅ AI model metadata validation completed"))
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"❌ Failed to validate AI model metadata: {e}"))
+                return
+
+        # Step 6: Verification
+        self.stdout.write("\n🔍 Step 6: Verifying setup...")
+        try:
+            self._verify_setup()
+            self.stdout.write(self.style.SUCCESS("✅ Setup verification completed successfully"))
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"❌ Setup verification failed: {e}"))
+            return
+
+        self.stdout.write(self.style.SUCCESS("\n🎉 EndoReg DB embedded app setup completed successfully!"))
+        self.stdout.write("\nNext steps:")
+        self.stdout.write("1. Run migrations: python manage.py migrate")
+        self.stdout.write("2. Create superuser: python manage.py createsuperuser")
+        self.stdout.write("3. Start development server: python manage.py runserver")
+
+    def _find_model_weights_file(self):
+        """Find the model weights file using configurable search patterns and directories."""
+        # Load setup configuration
+        from endoreg_db.utils.setup_config import setup_config
+
+        # First try to find weights using configured patterns
+        found_files = setup_config.find_model_weights_files()
+        if found_files:
+            self.stdout.write(f"Found model weights at: {found_files[0]}")
+            return found_files[0]
+
+        # If no local weights found and HuggingFace fallback is enabled
+        hf_config = setup_config.get_huggingface_config()
+        if hf_config.get("enabled", True):
+            self.stdout.write("📦 No local model weights found — attempting HuggingFace download...")
+            try:
+                if not ModelMeta.objects.exists():
+                    ModelMeta.setup_default_from_huggingface(
+                        hf_config.get("repo_id", "wg-lux/colo_segmentation_RegNetX800MF_base"),
+                        labelset_name=hf_config.get("labelset_name", "multilabel_classification_colonoscopy_default"),
+                    )
+                    self.stdout.write("✅ Default ModelMeta created from HuggingFace.")
+
+                    # Try to find the downloaded weights
+                    found_files = setup_config.find_model_weights_files()
+                    if found_files:
+                        return found_files[0]
+
+            except Exception as e:
+                self.stdout.write(f"⚠️  HuggingFace download failed: {e}")
+
+        self.stdout.write("Model weights file not found in configured locations")
+        return None
+
+    def _verify_setup(self):
+        """Verify that the setup was successful."""
+        from django.conf import settings
+        from django.db import connection
+
+        # Check that required tables exist
+        required_tables = [
+            "endoreg_db_aimodel",
+            "endoreg_db_modelmeta",
+        ]
+
+        # Only check for cache table if using database caching
+        cache_backend = settings.CACHES.get("default", {}).get("BACKEND", "")
+        if "db" in cache_backend or "database" in cache_backend:
+            required_tables.append("django_cache_table")
+
+        cursor = connection.cursor()
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        existing_tables = [row[0] for row in cursor.fetchall()]
+
+        missing_tables = [table for table in required_tables if table not in existing_tables]
+        if missing_tables:
+            raise Exception(f"Missing required tables: {missing_tables}")
+
+        # Check that AI models exist (if AI setup was performed)
+        from endoreg_db.models import AiModel
+
+        if AiModel.objects.exists():
+            ai_model_count = AiModel.objects.count()
+            self.stdout.write(f"Found {ai_model_count} AI model(s)")
+
+            # Check for model metadata
+            from endoreg_db.models import ModelMeta
+
+            meta_count = ModelMeta.objects.count()
+            self.stdout.write(f"Found {meta_count} model metadata record(s)")
+
+        self.stdout.write("Setup verification passed")
+
+    def _validate_and_fix_ai_model_metadata(self, yaml_only=False):
+        """
+        Validate that all AI models have proper active metadata and fix if necessary.
+        This addresses the "No model metadata found for this model" error.
+
+        Args:
+            yaml_only (bool): If True, only set active metadata but don't create new metadata
+        """
+        from endoreg_db.models import AiModel, LabelSet, ModelMeta
+        from endoreg_db.utils.setup_config import setup_config
+
+        all_models = AiModel.objects.all()
+        fixed_count = 0
+
+        # Get configurable defaults
+        defaults = setup_config.get_auto_generation_defaults()
+        primary_labelset_name = setup_config.get_primary_labelset_name()
+
+        for model in all_models:
+            self.stdout.write(f"Checking model: {model.name}")
+
+            # Check if model has metadata versions
+            metadata_count = model.metadata_versions.count()
+            self.stdout.write(f"  Metadata versions: {metadata_count}")
+
+            if metadata_count == 0:
+                if yaml_only:
+                    self.stdout.write(f"  ⚠️  YAML-only mode: Skipping auto-generation for {model.name}")
+                    continue
+
+                # Create metadata for models that don't have any
+                self.stdout.write(f"  Creating metadata for {model.name}...")
+
+                # Use configured labelset or create default
+                labelset = None
+                try:
+                    labelset = LabelSet.objects.get(name=primary_labelset_name)
+                except LabelSet.DoesNotExist:
+                    labelset = LabelSet.objects.first()
+                    if not labelset:
+                        labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
+
+                # Create basic metadata WITH weights if available
+                weights_file = self._find_model_weights_file()
+                weights_path = ""
+                if weights_file:
+                    # If we have weights, set up the relative path
+                    from pathlib import Path
+
+                    from endoreg_db.utils.paths import STORAGE_DIR
+
+                    try:
+                        weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                    except ValueError:
+                        # If file is not in storage dir, copy it there
+                        import shutil
+
+                        weights_dir = STORAGE_DIR / "model_weights"
+                        weights_dir.mkdir(parents=True, exist_ok=True)
+                        dest_path = weights_dir / Path(weights_file).name
+                        shutil.copy2(weights_file, dest_path)
+                        weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                        self.stdout.write(f"    Copied weights to: {dest_path}")
+
+                # Create basic metadata using configurable defaults
+                meta = ModelMeta.objects.create(
+                    name=model.name,
+                    version="1.0",
+                    model=model,
+                    labelset=labelset,
+                    weights=weights_path,  # Set weights if available
+                    activation=defaults.get("activation", "sigmoid"),
+                    mean=defaults.get("mean", "0.485,0.456,0.406"),
+                    std=defaults.get("std", "0.229,0.224,0.225"),
+                    size_x=defaults.get("size_x", 224),
+                    size_y=defaults.get("size_y", 224),
+                    axes=defaults.get("axes", "CHW"),
+                    batchsize=defaults.get("batchsize", 32),
+                    num_workers=defaults.get("num_workers", 4),
+                    description=f"Auto-generated metadata for {model.name}",
+                )
+
+                model.active_meta = meta
+                model.save()
+                fixed_count += 1
+                self.stdout.write(f"  ✅ Created and set metadata for {model.name}")
+
+            elif not model.active_meta:
+                # Model has metadata but no active meta set
+                first_meta = model.metadata_versions.first()
+                if first_meta:
+                    self.stdout.write(f"  Setting active metadata for {model.name}...")
+
+                    # Check if the metadata has weights - if not, try to assign them
+                    if not first_meta.weights:
+                        self.stdout.write("    Metadata exists but no weights assigned, attempting to add weights...")
+                        weights_file = self._find_model_weights_file()
+                        if weights_file:
+                            from pathlib import Path
+
+                            from endoreg_db.utils.paths import STORAGE_DIR
+
+                            try:
+                                weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
+                            except ValueError:
+                                # Copy weights to storage if not already there
+                                import shutil
+
+                                weights_dir = STORAGE_DIR / "model_weights"
+                                weights_dir.mkdir(parents=True, exist_ok=True)
+                                dest_path = weights_dir / Path(weights_file).name
+                                shutil.copy2(weights_file, dest_path)
+                                weights_path = str(dest_path.relative_to(STORAGE_DIR))
+                                self.stdout.write(f"      Copied weights to: {dest_path}")
+
+                            # Assign the relative path to the FileField
+                            first_meta.weights.name = weights_path
+                            first_meta.save(update_fields=["weights"])
+                            self.stdout.write(f"      Added weights to existing metadata: {weights_path}")
+
+                    model.active_meta = first_meta
+                    model.save()
+                    fixed_count += 1
+                    self.stdout.write(f"  ✅ Set active metadata: {first_meta.name} v{first_meta.version}")
+                else:
+                    self.stdout.write(f"  ⚠️  No metadata versions available for {model.name}")
+
+            else:
+                self.stdout.write(f"  ✅ Model {model.name} has active metadata: {model.active_meta}")
+
+        # Verify all models can get latest version
+        self.stdout.write("\nTesting model metadata access...")
+        for model in all_models:
+            try:
+                latest = model.get_latest_version()
+                self.stdout.write(f"  ✅ {model.name}: {latest}")
+            except Exception as e:
+                self.stdout.write(f"  ❌ {model.name}: {e}")
+                raise Exception(f"Model {model.name} still has metadata issues: {e}")
+
+        if fixed_count > 0:
+            self.stdout.write(f"Fixed metadata for {fixed_count} model(s)")
+        else:
+            self.stdout.write("All models already had proper metadata")

endoreg_db/management/commands/start_filewatcher.py

@@ -0,0 +1,106 @@
+"""
+Django Management Command for File Watcher Service
+
+This command provides Django integration for the file watcher service.
+"""
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+import sys
+import os
+from pathlib import Path
+
+
+class Command(BaseCommand):
+    help = """
+    Start the file watcher service for automatic video and PDF processing.
+    
+    This command monitors:
+    - data/raw_videos/ for video files (.mp4, .avi, .mov, .mkv, .webm, .m4v)
+    - data/raw_pdfs/ for PDF files (.pdf)
+    
+    When files are detected, they are automatically processed with:
+    - Video: Import, anonymization, and segmentation
+    - PDF: Import and anonymization
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--test',
+            action='store_true',
+            help='Test the file watcher configuration without starting monitoring',
+        )
+        parser.add_argument(
+            '--existing',
+            action='store_true',
+            help='Process existing files in the directories before starting monitoring',
+        )
+        parser.add_argument(
+            '--log-level',
+            choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
+            default='INFO',
+            help='Set logging level (default: INFO)',
+        )
+
+    def handle(self, *args, **options):
+        self.stdout.write(self.style.SUCCESS("Starting File Watcher Service"))
+        
+        # Set environment variables
+        os.environ['WATCHER_LOG_LEVEL'] = options['log_level']
+        
+        try:
+            # Add project root to path
+            project_root = Path(settings.BASE_DIR)
+            if str(project_root) not in sys.path:
+                sys.path.insert(0, str(project_root))
+            
+            # Import the file watcher service from the correct location
+            file_watcher_path = project_root / 'scripts' / 'file_watcher.py'
+            
+            if not file_watcher_path.exists():
+                self.stdout.write(self.style.ERROR(f"❌ File watcher script not found: {file_watcher_path}"))
+                return
+            
+            # Import the module dynamically
+            import importlib.util
+            spec = importlib.util.spec_from_file_location("file_watcher", file_watcher_path)
+            file_watcher_module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(file_watcher_module)
+            
+            FileWatcherService = file_watcher_module.FileWatcherService
+            
+            if options['test']:
+                self.stdout.write("Testing file watcher configuration...")
+                service = FileWatcherService()
+                try:
+                    service._validate_django_setup()
+                except Exception as e:
+                    self.stdout.write(self.style.ERROR(f"❌ Validation failed: {e}"))
+                    return
+                self.stdout.write(self.style.SUCCESS("✅ File watcher test passed"))
+                return
+            
+            # Create and start the service
+            service = FileWatcherService()
+            
+            if options['existing']:
+                self.stdout.write("Processing existing files...")
+                service._process_existing_files()
+                self.stdout.write(self.style.SUCCESS("✅ Existing files processed"))
+                return
+            
+            self.stdout.write("Starting file monitoring...")
+            self.stdout.write(f"Video directory: {service.video_dir}")
+            self.stdout.write(f"PDF directory: {service.pdf_dir}")
+            self.stdout.write("Press Ctrl+C to stop")
+            
+            service.start()
+            
+        except KeyboardInterrupt:
+            self.stdout.write(self.style.WARNING("\n⚠️  File watcher stopped by user"))
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"❌ Error: {str(e)}"))
+            if options['verbosity'] >= 2:
+                import traceback
+                self.stdout.write(traceback.format_exc())
+            sys.exit(1)