endoreg-db 0.6.4__py3-none-any.whl → 0.8.2__py3-none-any.whl

endoreg_db/helpers/test_video_helper.py

@@ -0,0 +1,119 @@
+from pathlib import Path
+import random
+import re
+from typing import Optional
+
+from logging import getLogger
+logger = getLogger(__name__)
+
+
+from endoreg_db.utils.video.names import (
+    get_video_key_regex_by_examination_alias
+)
+
+from django.conf import settings
+
+
+ASSET_DIR:Path = settings.ASSET_DIR
+assert ASSET_DIR.exists(), f"ASSET_DIR does not exist: {ASSET_DIR}"
+
+_TEST_VIDEOS = {
+    # "egd-instrument-non_anonymous": ASSET_DIR / "test_instrument.mp4", # No detected segments
+    "egd-endoscope-non_anonymous": ASSET_DIR / "test_endoscope.mp4",
+    "egd-nbi-non_anonymous": ASSET_DIR / "test_nbi.mp4",
+    # "egd-outside-non_anonymous": ASSET_DIR / "test_outside.mp4",
+    "egd-small_intestine-non_anonymous": ASSET_DIR / "test_small_intestine.mp4",
+}
+
+# Only keep entries that actually exist on disk to avoid returning None paths
+TEST_VIDEOS = {key: value for key, value in _TEST_VIDEOS.items() if value.exists()}
+
+
+def get_video_path(video_key:str) -> Path:
+    """
+    Returns the file path associated with the given video key.
+    
+    Raises:
+        ValueError: If the video key does not exist in the available test videos.
+    """
+    if video_key in TEST_VIDEOS:
+        return TEST_VIDEOS[video_key]
+    else:
+        raise ValueError(f"Video key '{video_key}' not found in TEST_VIDEOS.")
+
+def get_video_keys(
+    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+):
+    """
+    Returns a list of video keys matching the specified examination alias, content type, and anonymity status.
+    
+    If no direct matches are found, falls back to suffix-based filtering for anonymity. Logs warnings and errors when fallback logic is used or no matches are found.
+    
+    Args:
+        examination_alias: The examination alias to filter by, or None for any.
+        content: The content type to filter by, or None for any.
+        is_anonymous: Whether to filter for anonymous, non-anonymous, or both.
+    
+    Returns:
+        A list of matching video keys.
+    """
+    # Build a more flexible pattern locally if content is None
+    if content is None:
+        pattern_parts = ["^"]
+        if examination_alias:
+            pattern_parts.append(re.escape(examination_alias))
+            pattern_parts.append("-.*-") # Match any content
+        else:
+            pattern_parts.append(".*-") # Match any examination alias and content
+
+        if is_anonymous is True:
+            pattern_parts.append("anonymous$")
+        elif is_anonymous is False:
+            pattern_parts.append("non_anonymous$")
+        else: # is_anonymous is None
+            pattern_parts.append("(non_)?anonymous$") # Match either
+
+        pattern = "".join(pattern_parts)
+    else:
+        # Use the imported function if content is specified
+        pattern = get_video_key_regex_by_examination_alias(examination_alias, content, is_anonymous)
+        logger.warning(f"Generated pattern (from imported function): {pattern}")
+
+
+    # Only consider keys for which the file actually exists
+    keys_to_check = list(TEST_VIDEOS.keys())
+    matched_keys = [key for key in keys_to_check if re.match(pattern, key)]
+
+    # Fallback logic remains as a safety net, but ideally shouldn't be needed now for this case
+    if not matched_keys and is_anonymous is False:
+        logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=False. Falling back to suffix check '-non_anonymous'.")
+        matched_keys = [key for key in keys_to_check if key.endswith("-non_anonymous")]
+    elif not matched_keys and is_anonymous is True:
+         logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=True. Falling back to suffix check '-anonymous'.")
+         matched_keys = [key for key in keys_to_check if key.endswith("-anonymous")]
+
+    if not matched_keys:
+        logger.error(f"No keys found matching pattern '{pattern}' or fallback logic for keys: {keys_to_check}")
+
+
+    return matched_keys
+
+
+def get_random_video_path_by_examination_alias(
+    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+):
+    """
+    Returns the file path of a randomly selected video matching the specified examination alias, content type, and anonymity status.
+    
+    Raises:
+        ValueError: If no matching video keys are found for the given criteria.
+    """
+    keys = get_video_keys(examination_alias, content, is_anonymous)
+    if keys:
+        random_video_key = random.choice(keys)
+        video_path = get_video_path(random_video_key)
+        return video_path  # Return the first match for simplicity
+    else:
+        raise ValueError("No matching video keys found for the given criteria.")
+
+

endoreg_db/logger_conf.py

@@ -0,0 +1,140 @@
+import logging
+from pathlib import Path
+import os
+
+LOG_DIR = Path("data/logs")
+DEFAULT_FILE_LOG_LEVEL = "INFO"
+DEFAULT_CONSOLE_LOG_LEVEL = "WARNING"
+
+def clear_log_files(logger_names, log_dir=None):
+    """
+    Clears specified log files in the log directory.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        log_dir (Path, optional): The directory containing log files. Defaults to LOG_DIR.
+    """
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists (though clearing implies it might)
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    # Files to manage
+    log_files_to_clear = [log_dir / f"{logger_name}.log" for logger_name in logger_names]
+    log_files_to_clear.append(log_dir / "root.log") # Add root log file
+
+    # Clear existing log files in the directory managed by this config
+    for log_file in log_files_to_clear:
+        if log_file.exists():
+            try:
+                log_file.unlink()
+                print(f"Removed log file: {log_file}")
+            except OSError as e:
+                print(f"Error removing log file {log_file}: {e}")
+        # Optionally, create the log file to ensure it exists after clearing
+        # log_file.touch() # Removed touch to just clear
+
+def get_logging_config(logger_names, file_log_level=None, console_log_level=None, log_dir=None):
+    """
+    Generates Django LOGGING configuration dynamically.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        file_log_level (str, optional): The default log level for file handlers. Defaults to DEFAULT_FILE_LOG_LEVEL.
+        console_log_level (str, optional): The default log level for the console handler. Defaults to DEFAULT_CONSOLE_LOG_LEVEL.
+        log_dir (Path, optional): The directory to store log files. Defaults to LOG_DIR.
+
+    Returns:
+        dict: The Django LOGGING configuration dictionary.
+    """
+    file_log_level = file_log_level or os.environ.get("FILE_LOG_LEVEL", DEFAULT_FILE_LOG_LEVEL)
+    console_log_level = console_log_level or os.environ.get("CONSOLE_LOG_LEVEL", DEFAULT_CONSOLE_LOG_LEVEL)
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    handlers = {
+        'console': {
+            'level': console_log_level, # Use console level
+            'class': 'logging.StreamHandler',
+            'formatter': 'standard',
+        },
+        'file_root': { # Handler for the root logger's file
+            'level': file_log_level, # Use file level
+            'class': 'logging.FileHandler',
+            'filename': log_dir / "root.log",
+            'formatter': 'standard',
+        },
+    }
+    loggers = {
+        # Root logger configuration - logs INFO+ to file, WARNING+ to console
+        '': {
+            'handlers': ['console', 'file_root'], # Use both handlers
+            'level': file_log_level, # Set to lowest level needed (INFO for file)
+            'propagate': False, # Root logger doesn't propagate further
+        },
+         # Django's default logger
+        'django': {
+            'handlers': ['console', 'file_root'], # Log to console and root file
+            'level': file_log_level, # Or your preferred level, ensure it's captured by root file
+            'propagate': False, # Don't propagate Django logs to avoid double handling by root
+        },
+    }
+
+    # Dynamically create file handlers and logger configurations
+    for name in logger_names:
+        handler_name = f"file_{name}"
+        handlers[handler_name] = {
+            'level': file_log_level, # Use file level
+            'class': 'logging.FileHandler',
+            'filename': log_dir / f"{name}.log",
+            'formatter': 'standard',
+        }
+        loggers[name] = {
+            'handlers': [handler_name], # Log only to its own file directly
+            'level': file_log_level, # Use file level
+            'propagate': True, # Propagate to root logger (which handles console and root.log)
+        }
+
+    return {
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'standard': {
+                'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            },
+        },
+        'handlers': handlers,
+        'loggers': loggers,
+    }
+
+# Example usage (optional, for testing the function itself)
+if __name__ == "__main__":
+    test_loggers = ["app1", "app2", "database"]
+    log_directory = Path("data/logs_test") # Use a test directory
+
+    # Optionally clear logs before configuring logging
+    print("Clearing logs...")
+    clear_log_files(test_loggers, log_dir=log_directory)
+    print("Log clearing complete.")
+
+    # Get logging configuration
+    logging_config = get_logging_config(test_loggers, file_log_level="DEBUG", console_log_level="WARNING", log_dir=log_directory)
+    import json
+    print("\nGenerated Logging Configuration:")
+    print(json.dumps(logging_config, indent=4))
+
+    # Example of how to use it with Python's logging
+    # import logging.config
+    # logging.config.dictConfig(logging_config)
+    # logger1 = logging.getLogger("app1")
+    # logger1.debug("This is a debug message for app1.")
+    # logger1.info("This is an info message for app1.")
+
+    # logger_db = logging.getLogger("database")
+    # logger_db.warning("This is a warning for database.")
+
+    # root_logger = logging.getLogger()
+    # root_logger.info("This is an info message from root.") # Should go to root.log and console (if level >= WARNING)
+    # root_logger.warning("This is a warning message from root.") # Should go to root.log and console

endoreg_db/management/__init__.py

@@ -0,0 +1 @@
+# Management commands package

endoreg_db/management/commands/__init__.py

@@ -0,0 +1 @@
+# Management commands

endoreg_db/management/commands/check_auth.py

@@ -0,0 +1,125 @@
+"""
+Management command to check and configure authentication settings.
+"""
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from rest_framework.permissions import IsAuthenticated, AllowAny
+import os
+
+
+class Command(BaseCommand):
+    help = 'Check and configure authentication settings based on environment'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--mode',
+            choices=['check', 'dev', 'prod'],
+            default='check',
+            help='Mode: check current settings, set dev mode, or set prod mode'
+        )
+        parser.add_argument(
+            '--show-permissions',
+            action='store_true',
+            help='Show which permission classes will be used'
+        )
+
+    def handle(self, *args, **options):
+        mode = options['mode']
+        
+        if mode == 'check':
+            self.check_current_settings()
+        elif mode == 'dev':
+            self.set_dev_mode()
+        elif mode == 'prod':
+            self.set_prod_mode()
+            
+        if options['show_permissions']:
+            self.show_permission_classes()
+
+    def check_current_settings(self):
+        """Check current authentication configuration."""
+        debug_mode = getattr(settings, 'DEBUG', False)
+        settings_module = os.environ.get('DJANGO_SETTINGS_MODULE', 'Unknown')
+        
+        self.stdout.write("\n" + "="*50)
+        self.stdout.write("🔍 AUTHENTICATION CONFIGURATION CHECK")
+        self.stdout.write("="*50)
+        
+        self.stdout.write(f"Settings Module: {settings_module}")
+        self.stdout.write(f"DEBUG Mode: {debug_mode}")
+        
+        if debug_mode:
+            self.stdout.write(self.style.WARNING("🔓 Authentication: DISABLED (AllowAny)"))
+            self.stdout.write("   - All API endpoints are accessible without login")
+            self.stdout.write("   - Suitable for development and testing")
+        else:
+            self.stdout.write(self.style.SUCCESS("🔒 Authentication: ENABLED (IsAuthenticated)"))
+            self.stdout.write("   - API endpoints require valid authentication")
+            self.stdout.write("   - Suitable for production deployment")
+            
+        # Check if Keycloak is configured
+        keycloak_server = getattr(settings, 'KEYCLOAK_SERVER_URL', None)
+        if keycloak_server:
+            self.stdout.write(f"Keycloak Server: {keycloak_server}")
+        else:
+            self.stdout.write(self.style.WARNING("⚠️  Keycloak not configured"))
+            
+        self.stdout.write("="*50 + "\n")
+
+    def set_dev_mode(self):
+        """Instructions for setting development mode."""
+        self.stdout.write(self.style.SUCCESS("\n🛠️  DEVELOPMENT MODE SETUP"))
+        self.stdout.write("To enable development mode with disabled authentication:")
+        self.stdout.write("")
+        self.stdout.write("1. Use dev settings:")
+        self.stdout.write("   export DJANGO_SETTINGS_MODULE=dev.dev_settings")
+        self.stdout.write("   python manage.py runserver")
+        self.stdout.write("")
+        self.stdout.write("2. Or run with explicit settings:")
+        self.stdout.write("   python manage.py runserver --settings=dev.dev_settings")
+        self.stdout.write("")
+        self.stdout.write("This will set DEBUG=True and disable authentication requirements.")
+
+    def set_prod_mode(self):
+        """Instructions for setting production mode."""
+        self.stdout.write(self.style.SUCCESS("\n🏭 PRODUCTION MODE SETUP"))
+        self.stdout.write("To enable production mode with authentication:")
+        self.stdout.write("")
+        self.stdout.write("1. Use production settings:")
+        self.stdout.write("   export DJANGO_SETTINGS_MODULE=prod_settings")
+        self.stdout.write("   python manage.py runserver --settings=prod_settings")
+        self.stdout.write("")
+        self.stdout.write("2. Set required environment variables:")
+        self.stdout.write("   export DJANGO_SECRET_KEY='your-secure-secret-key'")
+        self.stdout.write("   export KEYCLOAK_CLIENT_SECRET='your-keycloak-secret'")
+        self.stdout.write("")
+        self.stdout.write("This will set DEBUG=False and enable authentication requirements.")
+
+    def show_permission_classes(self):
+        """Show which permission classes are being used."""
+        debug_mode = getattr(settings, 'DEBUG', False)
+        
+        self.stdout.write("\n" + "="*40)
+        self.stdout.write("🔐 PERMISSION CLASSES IN USE")
+        self.stdout.write("="*40)
+        
+        if debug_mode:
+            permission_class = "AllowAny"
+            icon = "🔓"
+        else:
+            permission_class = "IsAuthenticated"
+            icon = "🔒"
+            
+        self.stdout.write(f"{icon} Current Mode: {permission_class}")
+        self.stdout.write("")
+        self.stdout.write("Views using dynamic permissions:")
+        self.stdout.write("  • video_segments_view")
+        self.stdout.write("  • VideoViewSet")
+        self.stdout.write("  • Other API endpoints")
+        self.stdout.write("")
+        self.stdout.write("Views always requiring authentication:")
+        self.stdout.write("  • video_segment_detail_view")
+        self.stdout.write("  • video_segments_by_label_id_view")
+        self.stdout.write("  • video_segments_by_label_name_view")
+        self.stdout.write("="*40 + "\n")

endoreg_db/management/commands/create_multilabel_model_meta.py

@@ -0,0 +1,214 @@
+"""
+This script creates a new ModelMeta object for a multilabel classification model.
+"""
+
+from pathlib import Path
+from django.core.management import BaseCommand
+from endoreg_db.models import LabelSet, AiModel, ModelMeta
+
+
+# Example usage:
+# python manage.py create_multilabel_model_meta --model_path "./data/models/colo_segmentation_RegNetX800MF_6.ckpt"
+# python manage.py create_multilabel_model_meta --model_path "./libs/endoreg-db/tests/assets/colo_segmentation_RegNetX800MF_6.ckpt"
+
+FPS = 50
+SMOOTH_WINDOW_SIZE_S = 1
+MIN_SEQ_LEN_S = 0.5
+crop_template = [0, 1080, 550, 1920 - 20]  # [top, bottom, left, right]
+
+
+class Command(BaseCommand):
+    help = """
+        Creates a new ModelMeta object for a multilabel classification model.
+        1. Validates the existence of the specified LabelSet and AiModel
+        2. Checks that the model file exists on disk
+        3. Creates or updates a ModelMeta entry with the specified parameters
+    """
+
+    def add_arguments(self, parser):
+        """
+        Adds command-line arguments for configuring model meta creation.
+        
+        This method defines all necessary arguments for specifying model details, label set selection, normalization parameters, image dimensions, axes configuration, batch size, worker count, and versioning options for the Django management command.
+        """
+        parser.add_argument(
+            "--model_name",
+            type=str,
+            default="image_multilabel_classification_colonoscopy_default",
+            help="Name of the model to use for segmentation",
+        )
+
+        # model_path
+        parser.add_argument(
+            "--model_path",
+            type=str,
+            #default="./data/models/colo_segmentation_RegNetX800MF_6.ckpt", #model not yet here
+            default="./tests/assets/colo_segmentation_RegNetX800MF_6.ckpt", # model currently still here
+            help="Path to the model file",
+        )
+
+        # model_meta_version: int = 1
+        parser.add_argument(
+            "--model_meta_version",
+            type=int,
+            default=1,
+            help="Version of the model meta",
+        )
+
+        parser.add_argument(
+            "--image_classification_labelset_name",
+            type=str,
+            default="multilabel_classification_colonoscopy_default",
+            help="Name of the LabelSet for image classification",
+        )
+
+        parser.add_argument(
+            "--image_classification_labelset_version",
+            type=int,
+            default=-1,
+            help="Version of the LabelSet for image classification, default is -1 which invokes the highest version",
+        )
+
+        # activation: str = "sigmoid"
+        parser.add_argument(
+            "--activation_function_name",
+            type=str,
+            default="sigmoid",
+            help="Activation function for the model",
+        )
+
+        # mean: str = "0.45211223,0.27139644,0.19264949"
+        parser.add_argument(
+            "--mean",
+            type=str,
+            default="0.45211223,0.27139644,0.19264949",
+            help="Mean for normalization",
+        )
+
+        parser.add_argument(
+            "--std",
+            type=str,
+            default="0.31418097,0.21088019,0.16059452",
+            help="Std for normalization",
+        )
+
+        # size_x: int = 716
+        parser.add_argument(
+            "--size_x",
+            type=int,
+            default=716,
+            help="Size of the image in x direction",
+        )
+
+        # size_y: int = 716
+        parser.add_argument(
+            "--size_y",
+            type=int,
+            default=716,
+            help="Size of the image in y direction",
+        )
+
+        parser.add_argument(
+            "--axes",
+            type=str,
+            default="2,0,1",
+            help="Axes of the image",
+        )
+
+        # batchsize: int = 16
+        parser.add_argument(
+            "--batchsize",
+            type=int,
+            default=16,
+            help="Batchsize for the model",
+        )
+
+        # num_workers: int = 0
+        parser.add_argument(
+            "--num_workers",
+            type=int,
+            default=0,
+            help="Number of workers for the model",
+        )
+
+        # bump_version: bool = False
+        parser.add_argument(
+            "--bump_version",
+            action="store_true",
+            help="Bump the version of the model",
+        )
+
+    def handle(self, *args, **options):
+        """
+        Processes command-line arguments to create or update a ModelMeta object for a multilabel classification model.
+        
+        Retrieves and validates the specified LabelSet and AiModel, ensures the model file exists, and invokes ModelMeta.create_from_file with the provided configuration. Raises a ValueError if the requested LabelSet does not exist.
+        """
+        model_name = options["model_name"]
+
+        model_path = options["model_path"]
+        activation_function_name = options["activation_function_name"]
+
+        mean = options["mean"]
+        std = options["std"]
+
+        size_x = options["size_x"]
+        size_y = options["size_y"]
+
+        axes = options["axes"]
+
+        image_classification_labelset_name = options[
+            "image_classification_labelset_name"
+        ]
+
+        image_classification_labelset_version = options[
+            "image_classification_labelset_version"
+        ]
+        if image_classification_labelset_version == -1:
+            # If version is -1, we want the latest version of the labelset
+            labelset = LabelSet.objects.filter(
+                name=image_classification_labelset_name
+            ).order_by("-version").first()
+            if labelset:
+                image_classification_labelset_version = labelset.version
+            else:
+                raise ValueError(
+                    f"No LabelSet found with name: {image_classification_labelset_name}"
+                )
+        else:
+            # If a specific version is provided, we use that
+            labelset = LabelSet.objects.filter(
+                name=image_classification_labelset_name,
+                version=image_classification_labelset_version
+            ).first()
+            if not labelset:
+                raise ValueError(
+                    f"No LabelSet found with name: {image_classification_labelset_name} and version: {image_classification_labelset_version}"
+                )
+
+        # assert model exists
+        db_ai_model = AiModel.objects.filter(name=model_name).first()
+        assert db_ai_model, f"Model not found: {model_name}"
+
+        model_path = Path(model_path).expanduser().resolve().as_posix()
+
+        assert Path(model_path).exists(), f"Model file not found: {model_path}"
+
+        _model_meta = ModelMeta.create_from_file(
+            meta_name=model_name,  # Name of the new metadata object
+            model_name=model_name,  # name of the ai_model to use; This also defines the models VideoSegmentationLabelSet
+            labelset_name=image_classification_labelset_name,
+            weights_file=model_path,
+            # Use the correct keyword arguments matching the method signature
+            requested_version=options["model_meta_version"],
+            bump_if_exists=options["bump_version"],
+            # Pass other options via **kwargs
+            activation=activation_function_name,
+            mean=mean,
+            std=std,
+            size_x=size_x,
+            size_y=size_y,
+            axes=axes,
+            batchsize=options["batchsize"],
+            num_workers=options["num_workers"],
+        )