endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/import_files/video_import_service.py

@@ -0,0 +1,154 @@
+# endoreg_db/services/video_import.py
+import logging
+from pathlib import Path
+from typing import Optional, Union
+
+
+from endoreg_db.import_files.context import (
+    ImportContext,
+    file_lock,
+)
+from endoreg_db.import_files.file_storage.state_management import (
+    finalize_failure,
+    finalize_video_success,
+)
+from endoreg_db.import_files.file_storage.storage import create_sensitive_copy
+from endoreg_db.import_files.file_storage.create_video_file import (
+    create_or_retrieve_video_file,
+)
+from endoreg_db.import_files.context.validate_directories import validate_directories
+from endoreg_db.import_files.file_storage.state_management import (
+    mark_instance_processing_started,
+)
+from endoreg_db.models import VideoFile
+from endoreg_db.import_files.processing.video_processing.video_anonymization import (
+    VideoAnonymizer,
+)
+from endoreg_db.utils.paths import (
+    SENSITIVE_VIDEO_DIR,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class VideoImportService:
+    """
+    Service for importing and anonymizing video files.
+    Uses a central video instance pattern for cleaner state management.
+    Responsibility:
+        Validate path
+        Move/copy raw file into stable storage
+        Create/attach VideoFile, VideoState, default SensitiveMeta
+        Mark a single clear state like state.import_completed = True or state.ready_for_anonymization = True
+
+    It does not:
+        Run frame cleaning
+        Run ML
+        Create processed_file
+        Decide fallback anonymization
+
+    These actions are delegated to the modules inside file_import
+
+    """
+
+    def __init__(self):
+        self.logger = logger
+        self.anonymizer = VideoAnonymizer()
+        self.processing_context: Optional[ImportContext] = None
+        self.current_video: Optional[VideoFile] = None
+
+        validate_directories()
+
+    def import_and_anonymize(
+        self,
+        file_path: Union[Path, str],
+        center_name: str,
+        processor_name: str,
+        retry: bool = False,
+        delete_source: bool = True,
+    ) -> "VideoFile | None":
+        """
+        Public entrypoint: wrap import_and_anonymize logic.
+        """
+        # First, initialize import context. this will be updated during import and keep track of current paths, file type and center and processor.
+        ctx = ImportContext(
+            file_path=Path(file_path),
+            center_name=center_name,
+            processor_name=processor_name,
+            delete_source=delete_source,
+            file_type="video",
+        )
+        self.logger.info("validating and preparing file")
+        if not ctx.file_path.exists():
+            raise FileNotFoundError(f"Video file not found: {file_path}")
+
+        ctx.sensitive_path = create_sensitive_copy(ctx.file_path, SENSITIVE_VIDEO_DIR)
+
+        with file_lock(ctx.file_path):
+            logger.info("Acquired file lock for %s", ctx.file_path)
+
+            # create or retrieve VideoFile + update history
+            ctx.current_video, processed, needs_processing = (
+                create_or_retrieve_video_file(ctx)
+            )
+            ctx.current_video.get_or_create_state()
+            assert ctx.current_video.state is not None
+            ctx.current_video = ctx.current_video
+
+            ctx.retry = retry
+            # Retry is a forced overwrite of needs processing - therefore the retry will cause full deletion of processed files using finalize failure.
+
+            if (
+                retry
+                and needs_processing
+                and not ctx.current_video.state.anonymization_validated
+            ):
+                finalize_failure(ctx)
+                ctx.current_video, processed, needs_processing = (
+                    create_or_retrieve_video_file(ctx)
+                )
+                assert needs_processing is True
+            elif not needs_processing and not retry:
+                return ctx.current_video
+
+            mark_instance_processing_started(ctx.current_video, ctx)
+            try:
+                # --- Anonymization with fallback ---
+                try:
+                    ctx = self.anonymizer.anonymize_video(ctx)
+                    logger.info(
+                        "Primary video anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+                except Exception as primary_exc:
+                    logger.exception(
+                        "Primary video anonymization failed for %s: %s "
+                        "- trying basic anonymization",
+                        ctx.file_path,
+                        primary_exc,
+                    )
+                    try:
+                        ctx = self.anonymizer.anonymize_video(ctx)
+                    except Exception as e:
+                        logger.error(
+                            f"Video Extraction failed for the second time. {e}"
+                        )
+                        raise
+                    logger.info(
+                        "Secondary video anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+                logger.info(f"Anonymized Video is located at: {ctx.anonymized_path}")
+
+                # --- Finalize success: history + move anonymized file ---
+                finalize_video_success(ctx)
+
+                return ctx.current_video
+
+            except Exception as exc:
+                logger.exception(
+                    "Video import/anonymization failed for %s: %s", ctx.file_path, exc
+                )
+                finalize_failure(ctx)
+                raise

endoreg_db/logger_conf.py

@@ -0,0 +1,156 @@
+from pathlib import Path
+import os
+
+LOG_DIR = Path("data/logs")
+DEFAULT_FILE_LOG_LEVEL = "INFO"
+DEFAULT_CONSOLE_LOG_LEVEL = "WARNING"
+
+
+def clear_log_files(logger_names, log_dir=None):
+    """
+    Clears specified log files in the log directory.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        log_dir (Path, optional): The directory containing log files. Defaults to LOG_DIR.
+    """
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists (though clearing implies it might)
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    # Files to manage
+    log_files_to_clear = [
+        log_dir / f"{logger_name}.log" for logger_name in logger_names
+    ]
+    log_files_to_clear.append(log_dir / "root.log")  # Add root log file
+
+    # Clear existing log files in the directory managed by this config
+    for log_file in log_files_to_clear:
+        if log_file.exists():
+            try:
+                log_file.unlink()
+                print(f"Removed log file: {log_file}")
+            except OSError as e:
+                print(f"Error removing log file {log_file}: {e}")
+        # Optionally, create the log file to ensure it exists after clearing
+        # log_file.touch() # Removed touch to just clear
+
+
+def get_logging_config(
+    logger_names, file_log_level=None, console_log_level=None, log_dir=None
+):
+    """
+    Generates Django LOGGING configuration dynamically.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        file_log_level (str, optional): The default log level for file handlers. Defaults to DEFAULT_FILE_LOG_LEVEL.
+        console_log_level (str, optional): The default log level for the console handler. Defaults to DEFAULT_CONSOLE_LOG_LEVEL.
+        log_dir (Path, optional): The directory to store log files. Defaults to LOG_DIR.
+
+    Returns:
+        dict: The Django LOGGING configuration dictionary.
+    """
+    file_log_level = file_log_level or os.environ.get(
+        "FILE_LOG_LEVEL", DEFAULT_FILE_LOG_LEVEL
+    )
+    console_log_level = console_log_level or os.environ.get(
+        "CONSOLE_LOG_LEVEL", DEFAULT_CONSOLE_LOG_LEVEL
+    )
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    handlers = {
+        "console": {
+            "level": console_log_level,  # Use console level
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
+        },
+        "file_root": {  # Handler for the root logger's file
+            "level": file_log_level,  # Use file level
+            "class": "logging.FileHandler",
+            "filename": log_dir / "root.log",
+            "formatter": "standard",
+        },
+    }
+    loggers = {
+        # Root logger configuration - logs INFO+ to file, WARNING+ to console
+        "": {
+            "handlers": ["console", "file_root"],  # Use both handlers
+            "level": file_log_level,  # Set to lowest level needed (INFO for file)
+            "propagate": False,  # Root logger doesn't propagate further
+        },
+        # Django's default logger
+        "django": {
+            "handlers": ["console", "file_root"],  # Log to console and root file
+            "level": file_log_level,  # Or your preferred level, ensure it's captured by root file
+            "propagate": False,  # Don't propagate Django logs to avoid double handling by root
+        },
+    }
+
+    # Dynamically create file handlers and logger configurations
+    for name in logger_names:
+        handler_name = f"file_{name}"
+        handlers[handler_name] = {
+            "level": file_log_level,  # Use file level
+            "class": "logging.FileHandler",
+            "filename": log_dir / f"{name}.log",
+            "formatter": "standard",
+        }
+        loggers[name] = {
+            "handlers": [handler_name],  # Log only to its own file directly
+            "level": file_log_level,  # Use file level
+            "propagate": True,  # Propagate to root logger (which handles console and root.log)
+        }
+
+    return {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "standard": {
+                "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            },
+        },
+        "handlers": handlers,
+        "loggers": loggers,
+    }
+
+
+# Example usage (optional, for testing the function itself)
+if __name__ == "__main__":
+    test_loggers = ["app1", "app2", "database"]
+    log_directory = Path("data/logs_test")  # Use a test directory
+
+    # Optionally clear logs before configuring logging
+    print("Clearing logs...")
+    clear_log_files(test_loggers, log_dir=log_directory)
+    print("Log clearing complete.")
+
+    # Get logging configuration
+    logging_config = get_logging_config(
+        test_loggers,
+        file_log_level="DEBUG",
+        console_log_level="WARNING",
+        log_dir=log_directory,
+    )
+    import json
+
+    print("\nGenerated Logging Configuration:")
+    print(json.dumps(logging_config, indent=4))
+
+    # Example of how to use it with Python's logging
+    # import logging.config
+    # logging.config.dictConfig(logging_config)
+    # logger1 = logging.getLogger("app1")
+    # logger1.debug("This is a debug message for app1.")
+    # logger1.info("This is an info message for app1.")
+
+    # logger_db = logging.getLogger("database")
+    # logger_db.warning("This is a warning for database.")
+
+    # root_logger = logging.getLogger()
+    # root_logger.info("This is an info message from root.") # Should go to root.log and console (if level >= WARNING)
+    # root_logger.warning("This is a warning message from root.") # Should go to root.log and console

endoreg_db/management/__init__.py

@@ -0,0 +1 @@
+# Management commands package

endoreg_db/management/commands/__init__.py

@@ -0,0 +1 @@
+# Management commands

endoreg_db/management/commands/check_auth.py

@@ -0,0 +1,132 @@
+"""
+Management command to check and configure authentication settings.
+"""
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+import os
+
+
+class Command(BaseCommand):
+    help = "Check and configure authentication settings based on environment"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--mode",
+            choices=["check", "dev", "prod"],
+            default="check",
+            help="Mode: check current settings, set dev mode, or set prod mode",
+        )
+        parser.add_argument(
+            "--show-permissions",
+            action="store_true",
+            help="Show which permission classes will be used",
+        )
+
+    def handle(self, *args, **options):
+        mode = options["mode"]
+
+        if mode == "check":
+            self.check_current_settings()
+        elif mode == "dev":
+            self.set_dev_mode()
+        elif mode == "prod":
+            self.set_prod_mode()
+
+        if options["show_permissions"]:
+            self.show_permission_classes()
+
+    def check_current_settings(self):
+        """Check current authentication configuration."""
+        debug_mode = getattr(settings, "DEBUG", False)
+        settings_module = os.environ.get("DJANGO_SETTINGS_MODULE", "Unknown")
+
+        self.stdout.write("\n" + "=" * 50)
+        self.stdout.write("🔍 AUTHENTICATION CONFIGURATION CHECK")
+        self.stdout.write("=" * 50)
+
+        self.stdout.write(f"Settings Module: {settings_module}")
+        self.stdout.write(f"DEBUG Mode: {debug_mode}")
+
+        if debug_mode:
+            self.stdout.write(
+                self.style.WARNING("🔓 Authentication: DISABLED (AllowAny)")
+            )
+            self.stdout.write("   - All API endpoints are accessible without login")
+            self.stdout.write("   - Suitable for development and testing")
+        else:
+            self.stdout.write(
+                self.style.SUCCESS("🔒 Authentication: ENABLED (IsAuthenticated)")
+            )
+            self.stdout.write("   - API endpoints require valid authentication")
+            self.stdout.write("   - Suitable for production deployment")
+
+        # Check if Keycloak is configured
+        keycloak_base_url = getattr(settings, "KEYCLOAK_BASE_URL", None)
+        if keycloak_base_url:
+            self.stdout.write(f"Keycloak Base URL: {keycloak_base_url}")
+        else:
+            self.stdout.write(self.style.WARNING("⚠️  Keycloak not configured"))
+
+        self.stdout.write("=" * 50 + "\n")
+
+    def set_dev_mode(self):
+        """Instructions for setting development mode."""
+        self.stdout.write(self.style.SUCCESS("\n🛠️  DEVELOPMENT MODE SETUP"))
+        self.stdout.write("To enable development mode with disabled authentication:")
+        self.stdout.write("")
+        self.stdout.write("1. Use dev settings:")
+        self.stdout.write("   export DJANGO_SETTINGS_MODULE=dev.dev_settings")
+        self.stdout.write("   python manage.py runserver")
+        self.stdout.write("")
+        self.stdout.write("2. Or run with explicit settings:")
+        self.stdout.write("   python manage.py runserver --settings=dev.dev_settings")
+        self.stdout.write("")
+        self.stdout.write(
+            "This will set DEBUG=True and disable authentication requirements."
+        )
+
+    def set_prod_mode(self):
+        """Instructions for setting production mode."""
+        self.stdout.write(self.style.SUCCESS("\n🏭 PRODUCTION MODE SETUP"))
+        self.stdout.write("To enable production mode with authentication:")
+        self.stdout.write("")
+        self.stdout.write("1. Use production settings:")
+        self.stdout.write("   export DJANGO_SETTINGS_MODULE=prod_settings")
+        self.stdout.write("   python manage.py runserver --settings=prod_settings")
+        self.stdout.write("")
+        self.stdout.write("2. Set required environment variables:")
+        self.stdout.write("   export DJANGO_SECRET_KEY='your-secure-secret-key'")
+        self.stdout.write("   export OIDC_RP_CLIENT_SECRET='your-keycloak-secret'")
+        self.stdout.write("")
+        self.stdout.write(
+            "This will set DEBUG=False and enable authentication requirements."
+        )
+
+    def show_permission_classes(self):
+        """Show which permission classes are being used."""
+        debug_mode = getattr(settings, "DEBUG", False)
+
+        self.stdout.write("\n" + "=" * 40)
+        self.stdout.write("🔐 PERMISSION CLASSES IN USE")
+        self.stdout.write("=" * 40)
+
+        if debug_mode:
+            permission_class = "AllowAny"
+            icon = "🔓"
+        else:
+            permission_class = "IsAuthenticated"
+            icon = "🔒"
+
+        self.stdout.write(f"{icon} Current Mode: {permission_class}")
+        self.stdout.write("")
+        self.stdout.write("Views using dynamic permissions:")
+        self.stdout.write("  • video_segments_view")
+        self.stdout.write("  • VideoViewSet")
+        self.stdout.write("  • Other API endpoints")
+        self.stdout.write("")
+        self.stdout.write("Views always requiring authentication:")
+        self.stdout.write("  • video_segment_detail_view")
+        self.stdout.write("  • video_segments_by_label_id_view")
+        self.stdout.write("  • video_segments_by_label_name_view")
+        self.stdout.write("=" * 40 + "\n")

endoreg_db/management/commands/create_model_meta_from_huggingface.py

@@ -0,0 +1,177 @@
+"""
+Django management command to create ModelMeta from Hugging Face model.
+"""
+
+from django.core.files.base import ContentFile
+from django.core.management.base import BaseCommand, CommandError
+from huggingface_hub import hf_hub_download
+
+from endoreg_db.models import AiModel, LabelSet, ModelMeta
+
+
+class Command(BaseCommand):
+    help = "Create ModelMeta by downloading model from Hugging Face"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--model_id",
+            type=str,
+            default="wg-lux/colo_segmentation_RegNetX800MF_base",
+            help="Hugging Face model ID",
+        )
+        parser.add_argument(
+            "--model_name",
+            type=str,
+            default="image_multilabel_classification_colonoscopy_default",
+            help="Name for the AI model",
+        )
+        parser.add_argument(
+            "--labelset_name",
+            type=str,
+            default="multilabel_classification_colonoscopy_default",
+            help="Name of the labelset",
+        )
+        parser.add_argument(
+            "--meta_version",
+            type=str,
+            default="1",
+            help="Version for the model meta",
+        )
+        parser.add_argument(
+            "--labelset_version",
+            type=int,
+            default=None,
+            help="LabelSet version; if omitted, the latest by name is used",
+        )
+
+    def handle(self, *args, **options):
+        model_id = options["model_id"]
+        model_name = options["model_name"]
+        labelset_name = options["labelset_name"]
+        version = options["meta_version"]
+        labelset_version = options.get("labelset_version")
+
+        self.stdout.write(f"Downloading model {model_id} from Hugging Face...")
+
+        try:
+            # Download the model weights
+            weights_path = hf_hub_download(
+                repo_id=model_id,
+                filename="colo_segmentation_RegNetX800MF_base.safetensors",
+                local_dir="/tmp",
+            )
+            self.stdout.write(f"Downloaded weights to: {weights_path}")
+
+            # Get or create AI model
+            ai_model, created = AiModel.objects.get_or_create(
+                name=model_name, defaults={"description": f"Model from {model_id}"}
+            )
+            if created:
+                self.stdout.write(f"Created AI model: {ai_model.name}")
+
+            # Get labelset (optionally by version); fail with non-zero exit
+            labelset_qs = LabelSet.objects.filter(name=labelset_name)
+            if labelset_version is not None:
+                labelset_qs = labelset_qs.filter(version=labelset_version)
+            labelset = labelset_qs.order_by("-version").first()
+            if labelset is None:
+                raise CommandError(
+                    f"LabelSet '{labelset_name}'"
+                    + (f" v{labelset_version}" if labelset_version is not None else "")
+                    + " not found"
+                )
+
+            # Create ModelMeta
+            model_meta, created = ModelMeta.objects.get_or_create(
+                name=model_name,
+                model=ai_model,
+                version=version,
+                defaults={
+                    "labelset": labelset,
+                    "activation": "sigmoid",
+                    "mean": "0.45211223,0.27139644,0.19264949",
+                    "std": "0.31418097,0.21088019,0.16059452",
+                    "size_x": 716,
+                    "size_y": 716,
+                    "axes": "2,0,1",
+                    "batchsize": 16,
+                    "num_workers": 0,
+                    "description": f"Downloaded from {model_id}",
+                },
+            )
+
+            # Save the weights file to the model
+            with open(weights_path, "rb") as f:
+                model_meta.weights.save(
+                    f"{model_name}_v{version}.safetensors", ContentFile(f.read())
+                )
+
+            # Set as active meta
+            ai_model.active_meta = model_meta
+            ai_model.save()
+
+            self.stdout.write(
+                self.style.SUCCESS(
+                    f"Successfully {'created' if created else 'updated'} ModelMeta: {model_meta}"
+                )
+            )
+
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"Error creating ModelMeta: {e}"))
+            import traceback
+
+            traceback.print_exc()
+            raise CommandError("ModelMeta creation failed") from e
+
+
+# TODO Review slimmed down version using service endoreg_db/services/model_meta_from_hf.py
+# your management command file
+# from django.core.management.base import BaseCommand, CommandError
+
+# from endoreg_db.services.model_meta_from_hf import ensure_model_meta_from_hf
+
+
+# class Command(BaseCommand):
+#     help = "Create ModelMeta by downloading model from Hugging Face"
+
+#     def add_arguments(self, parser):
+#         parser.add_argument(
+#             "--model_id",
+#             type=str,
+#             default="wg-lux/colo_segmentation_RegNetX800MF_base",
+#         )
+#         parser.add_argument(
+#             "--model_name",
+#             type=str,
+#             default="image_multilabel_classification_colonoscopy_default",
+#         )
+#         parser.add_argument(
+#             "--labelset_name",
+#             type=str,
+#             default="multilabel_classification_colonoscopy_default",
+#         )
+#         parser.add_argument(
+#             "--meta_version",
+#             type=str,
+#             default="1",
+#         )
+#         parser.add_argument(
+#             "--labelset_version",
+#             type=int,
+#             default=None,
+#         )
+
+#     def handle(self, *args, **options):
+#         try:
+#             model_meta = ensure_model_meta_from_hf(
+#                 model_id=options["model_id"],
+#                 model_name=options["model_name"],
+#                 labelset_name=options["labelset_name"],
+#                 meta_version=options["meta_version"],
+#                 labelset_version=options.get("labelset_version"),
+#             )
+#             self.stdout.write(
+#                 self.style.SUCCESS(f"Successfully ensured ModelMeta: {model_meta}")
+#             )
+#         except Exception as e:
+#             raise CommandError(f"ModelMeta creation failed: {e}") from e