endoreg-db 0.8.6.4__py3-none-any.whl

endoreg_db/helpers/test_video_helper.py

@@ -0,0 +1,119 @@
+from pathlib import Path
+import random
+import re
+from typing import Optional
+
+from logging import getLogger
+logger = getLogger(__name__)
+
+
+from endoreg_db.utils.video.names import (
+    get_video_key_regex_by_examination_alias
+)
+
+from django.conf import settings
+
+
+ASSET_DIR:Path = settings.ASSET_DIR
+assert ASSET_DIR.exists(), f"ASSET_DIR does not exist: {ASSET_DIR}"
+
+_TEST_VIDEOS = {
+    # "egd-instrument-non_anonymous": ASSET_DIR / "test_instrument.mp4", # No detected segments
+    "egd-endoscope-non_anonymous": ASSET_DIR / "test_endoscope.mp4",
+    "egd-nbi-non_anonymous": ASSET_DIR / "test_nbi.mp4",
+    # "egd-outside-non_anonymous": ASSET_DIR / "test_outside.mp4",
+    "egd-small_intestine-non_anonymous": ASSET_DIR / "test_small_intestine.mp4",
+}
+
+# Only keep entries that actually exist on disk to avoid returning None paths
+TEST_VIDEOS = {key: value for key, value in _TEST_VIDEOS.items() if value.exists()}
+
+
+def get_video_path(video_key:str) -> Path:
+    """
+    Returns the file path associated with the given video key.
+    
+    Raises:
+        ValueError: If the video key does not exist in the available test videos.
+    """
+    if video_key in TEST_VIDEOS:
+        return TEST_VIDEOS[video_key]
+    else:
+        raise ValueError(f"Video key '{video_key}' not found in TEST_VIDEOS.")
+
+def get_video_keys(
+    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+):
+    """
+    Returns a list of video keys matching the specified examination alias, content type, and anonymity status.
+    
+    If no direct matches are found, falls back to suffix-based filtering for anonymity. Logs warnings and errors when fallback logic is used or no matches are found.
+    
+    Args:
+        examination_alias: The examination alias to filter by, or None for any.
+        content: The content type to filter by, or None for any.
+        is_anonymous: Whether to filter for anonymous, non-anonymous, or both.
+    
+    Returns:
+        A list of matching video keys.
+    """
+    # Build a more flexible pattern locally if content is None
+    if content is None:
+        pattern_parts = ["^"]
+        if examination_alias:
+            pattern_parts.append(re.escape(examination_alias))
+            pattern_parts.append("-.*-") # Match any content
+        else:
+            pattern_parts.append(".*-") # Match any examination alias and content
+
+        if is_anonymous is True:
+            pattern_parts.append("anonymous$")
+        elif is_anonymous is False:
+            pattern_parts.append("non_anonymous$")
+        else: # is_anonymous is None
+            pattern_parts.append("(non_)?anonymous$") # Match either
+
+        pattern = "".join(pattern_parts)
+    else:
+        # Use the imported function if content is specified
+        pattern = get_video_key_regex_by_examination_alias(examination_alias, content, is_anonymous)
+        logger.warning(f"Generated pattern (from imported function): {pattern}")
+
+
+    # Only consider keys for which the file actually exists
+    keys_to_check = list(TEST_VIDEOS.keys())
+    matched_keys = [key for key in keys_to_check if re.match(pattern, key)]
+
+    # Fallback logic remains as a safety net, but ideally shouldn't be needed now for this case
+    if not matched_keys and is_anonymous is False:
+        logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=False. Falling back to suffix check '-non_anonymous'.")
+        matched_keys = [key for key in keys_to_check if key.endswith("-non_anonymous")]
+    elif not matched_keys and is_anonymous is True:
+         logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=True. Falling back to suffix check '-anonymous'.")
+         matched_keys = [key for key in keys_to_check if key.endswith("-anonymous")]
+
+    if not matched_keys:
+        logger.error(f"No keys found matching pattern '{pattern}' or fallback logic for keys: {keys_to_check}")
+
+
+    return matched_keys
+
+
+def get_random_video_path_by_examination_alias(
+    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+):
+    """
+    Returns the file path of a randomly selected video matching the specified examination alias, content type, and anonymity status.
+    
+    Raises:
+        ValueError: If no matching video keys are found for the given criteria.
+    """
+    keys = get_video_keys(examination_alias, content, is_anonymous)
+    if keys:
+        random_video_key = random.choice(keys)
+        video_path = get_video_path(random_video_key)
+        return video_path  # Return the first match for simplicity
+    else:
+        raise ValueError("No matching video keys found for the given criteria.")
+
+

endoreg_db/logger_conf.py

@@ -0,0 +1,140 @@
+import logging
+from pathlib import Path
+import os
+
+LOG_DIR = Path("data/logs")
+DEFAULT_FILE_LOG_LEVEL = "INFO"
+DEFAULT_CONSOLE_LOG_LEVEL = "WARNING"
+
+def clear_log_files(logger_names, log_dir=None):
+    """
+    Clears specified log files in the log directory.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        log_dir (Path, optional): The directory containing log files. Defaults to LOG_DIR.
+    """
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists (though clearing implies it might)
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    # Files to manage
+    log_files_to_clear = [log_dir / f"{logger_name}.log" for logger_name in logger_names]
+    log_files_to_clear.append(log_dir / "root.log") # Add root log file
+
+    # Clear existing log files in the directory managed by this config
+    for log_file in log_files_to_clear:
+        if log_file.exists():
+            try:
+                log_file.unlink()
+                print(f"Removed log file: {log_file}")
+            except OSError as e:
+                print(f"Error removing log file {log_file}: {e}")
+        # Optionally, create the log file to ensure it exists after clearing
+        # log_file.touch() # Removed touch to just clear
+
+def get_logging_config(logger_names, file_log_level=None, console_log_level=None, log_dir=None):
+    """
+    Generates Django LOGGING configuration dynamically.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        file_log_level (str, optional): The default log level for file handlers. Defaults to DEFAULT_FILE_LOG_LEVEL.
+        console_log_level (str, optional): The default log level for the console handler. Defaults to DEFAULT_CONSOLE_LOG_LEVEL.
+        log_dir (Path, optional): The directory to store log files. Defaults to LOG_DIR.
+
+    Returns:
+        dict: The Django LOGGING configuration dictionary.
+    """
+    file_log_level = file_log_level or os.environ.get("FILE_LOG_LEVEL", DEFAULT_FILE_LOG_LEVEL)
+    console_log_level = console_log_level or os.environ.get("CONSOLE_LOG_LEVEL", DEFAULT_CONSOLE_LOG_LEVEL)
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    handlers = {
+        'console': {
+            'level': console_log_level, # Use console level
+            'class': 'logging.StreamHandler',
+            'formatter': 'standard',
+        },
+        'file_root': { # Handler for the root logger's file
+            'level': file_log_level, # Use file level
+            'class': 'logging.FileHandler',
+            'filename': log_dir / "root.log",
+            'formatter': 'standard',
+        },
+    }
+    loggers = {
+        # Root logger configuration - logs INFO+ to file, WARNING+ to console
+        '': {
+            'handlers': ['console', 'file_root'], # Use both handlers
+            'level': file_log_level, # Set to lowest level needed (INFO for file)
+            'propagate': False, # Root logger doesn't propagate further
+        },
+         # Django's default logger
+        'django': {
+            'handlers': ['console', 'file_root'], # Log to console and root file
+            'level': file_log_level, # Or your preferred level, ensure it's captured by root file
+            'propagate': False, # Don't propagate Django logs to avoid double handling by root
+        },
+    }
+
+    # Dynamically create file handlers and logger configurations
+    for name in logger_names:
+        handler_name = f"file_{name}"
+        handlers[handler_name] = {
+            'level': file_log_level, # Use file level
+            'class': 'logging.FileHandler',
+            'filename': log_dir / f"{name}.log",
+            'formatter': 'standard',
+        }
+        loggers[name] = {
+            'handlers': [handler_name], # Log only to its own file directly
+            'level': file_log_level, # Use file level
+            'propagate': True, # Propagate to root logger (which handles console and root.log)
+        }
+
+    return {
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'standard': {
+                'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            },
+        },
+        'handlers': handlers,
+        'loggers': loggers,
+    }
+
+# Example usage (optional, for testing the function itself)
+if __name__ == "__main__":
+    test_loggers = ["app1", "app2", "database"]
+    log_directory = Path("data/logs_test") # Use a test directory
+
+    # Optionally clear logs before configuring logging
+    print("Clearing logs...")
+    clear_log_files(test_loggers, log_dir=log_directory)
+    print("Log clearing complete.")
+
+    # Get logging configuration
+    logging_config = get_logging_config(test_loggers, file_log_level="DEBUG", console_log_level="WARNING", log_dir=log_directory)
+    import json
+    print("\nGenerated Logging Configuration:")
+    print(json.dumps(logging_config, indent=4))
+
+    # Example of how to use it with Python's logging
+    # import logging.config
+    # logging.config.dictConfig(logging_config)
+    # logger1 = logging.getLogger("app1")
+    # logger1.debug("This is a debug message for app1.")
+    # logger1.info("This is an info message for app1.")
+
+    # logger_db = logging.getLogger("database")
+    # logger_db.warning("This is a warning for database.")
+
+    # root_logger = logging.getLogger()
+    # root_logger.info("This is an info message from root.") # Should go to root.log and console (if level >= WARNING)
+    # root_logger.warning("This is a warning message from root.") # Should go to root.log and console

endoreg_db/management/__init__.py

@@ -0,0 +1 @@
+# Management commands package

endoreg_db/management/commands/__init__.py

@@ -0,0 +1 @@
+# Management commands

endoreg_db/management/commands/check_auth.py

@@ -0,0 +1,125 @@
+"""
+Management command to check and configure authentication settings.
+"""
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from rest_framework.permissions import IsAuthenticated, AllowAny
+import os
+
+
+class Command(BaseCommand):
+    help = 'Check and configure authentication settings based on environment'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--mode',
+            choices=['check', 'dev', 'prod'],
+            default='check',
+            help='Mode: check current settings, set dev mode, or set prod mode'
+        )
+        parser.add_argument(
+            '--show-permissions',
+            action='store_true',
+            help='Show which permission classes will be used'
+        )
+
+    def handle(self, *args, **options):
+        mode = options['mode']
+        
+        if mode == 'check':
+            self.check_current_settings()
+        elif mode == 'dev':
+            self.set_dev_mode()
+        elif mode == 'prod':
+            self.set_prod_mode()
+            
+        if options['show_permissions']:
+            self.show_permission_classes()
+
+    def check_current_settings(self):
+        """Check current authentication configuration."""
+        debug_mode = getattr(settings, 'DEBUG', False)
+        settings_module = os.environ.get('DJANGO_SETTINGS_MODULE', 'Unknown')
+        
+        self.stdout.write("\n" + "="*50)
+        self.stdout.write("🔍 AUTHENTICATION CONFIGURATION CHECK")
+        self.stdout.write("="*50)
+        
+        self.stdout.write(f"Settings Module: {settings_module}")
+        self.stdout.write(f"DEBUG Mode: {debug_mode}")
+        
+        if debug_mode:
+            self.stdout.write(self.style.WARNING("🔓 Authentication: DISABLED (AllowAny)"))
+            self.stdout.write("   - All API endpoints are accessible without login")
+            self.stdout.write("   - Suitable for development and testing")
+        else:
+            self.stdout.write(self.style.SUCCESS("🔒 Authentication: ENABLED (IsAuthenticated)"))
+            self.stdout.write("   - API endpoints require valid authentication")
+            self.stdout.write("   - Suitable for production deployment")
+            
+        # Check if Keycloak is configured
+        keycloak_server = getattr(settings, 'KEYCLOAK_SERVER_URL', None)
+        if keycloak_server:
+            self.stdout.write(f"Keycloak Server: {keycloak_server}")
+        else:
+            self.stdout.write(self.style.WARNING("⚠️  Keycloak not configured"))
+            
+        self.stdout.write("="*50 + "\n")
+
+    def set_dev_mode(self):
+        """Instructions for setting development mode."""
+        self.stdout.write(self.style.SUCCESS("\n🛠️  DEVELOPMENT MODE SETUP"))
+        self.stdout.write("To enable development mode with disabled authentication:")
+        self.stdout.write("")
+        self.stdout.write("1. Use dev settings:")
+        self.stdout.write("   export DJANGO_SETTINGS_MODULE=dev.dev_settings")
+        self.stdout.write("   python manage.py runserver")
+        self.stdout.write("")
+        self.stdout.write("2. Or run with explicit settings:")
+        self.stdout.write("   python manage.py runserver --settings=dev.dev_settings")
+        self.stdout.write("")
+        self.stdout.write("This will set DEBUG=True and disable authentication requirements.")
+
+    def set_prod_mode(self):
+        """Instructions for setting production mode."""
+        self.stdout.write(self.style.SUCCESS("\n🏭 PRODUCTION MODE SETUP"))
+        self.stdout.write("To enable production mode with authentication:")
+        self.stdout.write("")
+        self.stdout.write("1. Use production settings:")
+        self.stdout.write("   export DJANGO_SETTINGS_MODULE=prod_settings")
+        self.stdout.write("   python manage.py runserver --settings=prod_settings")
+        self.stdout.write("")
+        self.stdout.write("2. Set required environment variables:")
+        self.stdout.write("   export DJANGO_SECRET_KEY='your-secure-secret-key'")
+        self.stdout.write("   export KEYCLOAK_CLIENT_SECRET='your-keycloak-secret'")
+        self.stdout.write("")
+        self.stdout.write("This will set DEBUG=False and enable authentication requirements.")
+
+    def show_permission_classes(self):
+        """Show which permission classes are being used."""
+        debug_mode = getattr(settings, 'DEBUG', False)
+        
+        self.stdout.write("\n" + "="*40)
+        self.stdout.write("🔐 PERMISSION CLASSES IN USE")
+        self.stdout.write("="*40)
+        
+        if debug_mode:
+            permission_class = "AllowAny"
+            icon = "🔓"
+        else:
+            permission_class = "IsAuthenticated"
+            icon = "🔒"
+            
+        self.stdout.write(f"{icon} Current Mode: {permission_class}")
+        self.stdout.write("")
+        self.stdout.write("Views using dynamic permissions:")
+        self.stdout.write("  • video_segments_view")
+        self.stdout.write("  • VideoViewSet")
+        self.stdout.write("  • Other API endpoints")
+        self.stdout.write("")
+        self.stdout.write("Views always requiring authentication:")
+        self.stdout.write("  • video_segment_detail_view")
+        self.stdout.write("  • video_segments_by_label_id_view")
+        self.stdout.write("  • video_segments_by_label_name_view")
+        self.stdout.write("="*40 + "\n")

endoreg_db/management/commands/create_model_meta_from_huggingface.py

@@ -0,0 +1,115 @@
+"""
+Django management command to create ModelMeta from Hugging Face model.
+"""
+
+from pathlib import Path
+
+from django.core.files.base import ContentFile
+from django.core.management.base import BaseCommand
+from huggingface_hub import hf_hub_download
+
+from endoreg_db.models import AiModel, LabelSet, ModelMeta
+
+
+class Command(BaseCommand):
+    help = "Create ModelMeta by downloading model from Hugging Face"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--model_id",
+            type=str,
+            default="wg-lux/colo_segmentation_RegNetX800MF_base",
+            help="Hugging Face model ID",
+        )
+        parser.add_argument(
+            "--model_name",
+            type=str,
+            default="image_multilabel_classification_colonoscopy_default",
+            help="Name for the AI model",
+        )
+        parser.add_argument(
+            "--labelset_name",
+            type=str,
+            default="multilabel_classification_colonoscopy_default",
+            help="Name of the labelset",
+        )
+        parser.add_argument(
+            "--meta_version",
+            type=str,
+            default="1",
+            help="Version for the model meta",
+        )
+
+    def handle(self, *args, **options):
+        model_id = options["model_id"]
+        model_name = options["model_name"]
+        labelset_name = options["labelset_name"]
+        version = options["meta_version"]
+
+        self.stdout.write(f"Downloading model {model_id} from Hugging Face...")
+
+        try:
+            # Download the model weights
+            weights_path = hf_hub_download(
+                repo_id=model_id,
+                filename="colo_segmentation_RegNetX800MF_base.ckpt",
+                local_dir="/tmp",
+            )
+            self.stdout.write(f"Downloaded weights to: {weights_path}")
+
+            # Get or create AI model
+            ai_model, created = AiModel.objects.get_or_create(
+                name=model_name, defaults={"description": f"Model from {model_id}"}
+            )
+            if created:
+                self.stdout.write(f"Created AI model: {ai_model.name}")
+
+            # Get labelset
+            try:
+                labelset = LabelSet.objects.get(name=labelset_name)
+            except LabelSet.DoesNotExist:
+                self.stdout.write(
+                    self.style.ERROR(f"LabelSet '{labelset_name}' not found")
+                )
+                return
+
+            # Create ModelMeta
+            model_meta, created = ModelMeta.objects.get_or_create(
+                name=model_name,
+                model=ai_model,
+                version=version,
+                defaults={
+                    "labelset": labelset,
+                    "activation": "sigmoid",
+                    "mean": "0.45211223,0.27139644,0.19264949",
+                    "std": "0.31418097,0.21088019,0.16059452",
+                    "size_x": 716,
+                    "size_y": 716,
+                    "axes": "2,0,1",
+                    "batchsize": 16,
+                    "num_workers": 0,
+                    "description": f"Downloaded from {model_id}",
+                },
+            )
+
+            # Save the weights file to the model
+            with open(weights_path, "rb") as f:
+                model_meta.weights.save(
+                    f"{model_name}_v{version}_pytorch_model.bin", ContentFile(f.read())
+                )
+
+            # Set as active meta
+            ai_model.active_meta = model_meta
+            ai_model.save()
+
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"Successfully {'created' if created else 'updated'} ModelMeta: {model_meta}"
+                )
+            )
+
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"Error creating ModelMeta: {e}"))
+            import traceback
+
+            traceback.print_exc()