endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/import_files/report_import_service.py

@@ -7,33 +7,28 @@ from typing import Optional, Union
 
 from endoreg_db.import_files.context.file_lock import file_lock
 from endoreg_db.import_files.context.import_context import ImportContext
-from endoreg_db.import_files.processing.report_processing.report_anonymization import (
-    ReportAnonymizer,
-)
-
+from endoreg_db.import_files.context.validate_directories import validate_directories
 from endoreg_db.import_files.file_storage.create_report_file import (
     create_or_retrieve_report_file,
 )
 from endoreg_db.import_files.file_storage.state_management import (
-    finalize_report_success,
     finalize_failure,
+    finalize_report_success,
     mark_instance_processing_started,
 )
-from endoreg_db.import_files.context.validate_directories import validate_directories
-
 from endoreg_db.import_files.file_storage.storage import create_sensitive_copy
-from endoreg_db.models.media import RawPdfFile
-from endoreg_db.utils.paths import (
-    SENSITIVE_REPORT_DIR,
+from endoreg_db.import_files.processing.report_processing.report_anonymization import (
+    ReportAnonymizer,
 )
-
+from endoreg_db.models.media import RawPdfFile
+from endoreg_db.utils.paths import SENSITIVE_REPORT_DIR
 
 logger = logging.getLogger(__name__)
 
 
 class ReportImportService:
     """
-    Service for importing and anonymizing report (PDF) files.
+    Service for importing and anonymizing report (report) files.
 
     Responsibilities:
       - Acquire file lock
@@ -49,9 +44,8 @@ class ReportImportService:
         self.anonymizer = ReportAnonymizer()
         self.processing_context: Optional[ImportContext] = None
         self.current_report: Optional[RawPdfFile] = None
-        
-        validate_directories()
 
+        validate_directories()
 
     def import_and_anonymize(
         self,
@@ -68,35 +62,49 @@ class ReportImportService:
             file_path=Path(file_path),
             center_name=center_name,
             delete_source=delete_source,
-            file_type="report"
+            file_type="report",
+            original_path=Path(file_path),
         )
         self.logger.info("validating and preparing file")
         if not ctx.file_path.exists():
             raise FileNotFoundError(f"Video file not found: {file_path}")
 
-        ctx.sensitive_path = create_sensitive_copy(
-            ctx.file_path, 
-            SENSITIVE_REPORT_DIR
-        )
+        ctx.sensitive_path = create_sensitive_copy(ctx.file_path, SENSITIVE_REPORT_DIR)
 
         with file_lock(ctx.file_path):
             logger.info("Acquired file lock for %s", ctx.file_path)
 
             # create or retrieve RawPdfFile + update history
-            ctx.current_report, needs_processing = create_or_retrieve_report_file(ctx)
+            ctx.current_report, processed, needs_processing = (
+                create_or_retrieve_report_file(ctx)
+            )
             ctx.current_report.get_or_create_state()
-            assert(ctx.current_report.state is not None)
+            assert ctx.current_report.state is not None
             ctx.current_report = ctx.current_report
-            
-            ctx.retry = retry
+
+            if processed == True or retry == True:
+                ctx.retry = True
+
             # Retry is a forced overwrite of needs processing - therefore the retry will cause full deletion of processed files using finalize failure.
-            if retry and needs_processing and not ctx.current_report.state.anonymization_validated:
+            if (
+                ctx.retry
+                and needs_processing
+                and not ctx.current_report.state.anonymization_validated
+            ):
                 # ensure clean slate for forced reprocessing
                 finalize_failure(ctx)
-                ctx.current_report, needs_processing = create_or_retrieve_report_file(ctx)
-                assert(needs_processing is True)
-            elif not needs_processing and not retry:
+                ctx.current_report, processed, needs_processing = (
+                    create_or_retrieve_report_file(ctx)
+                )
+                assert needs_processing is True
+            elif not needs_processing and not ctx.retry:
                 return ctx.current_report
+            else:
+                finalize_failure(ctx)
+                ctx.current_report, processed, needs_processing = (
+                    create_or_retrieve_report_file(ctx)
+                )
+                assert needs_processing is True
 
             mark_instance_processing_started(ctx.current_report, ctx)
             try:
@@ -117,7 +125,7 @@ class ReportImportService:
                     try:
                         ctx = self.anonymizer.anonymize_report(ctx)
                     except Exception as e:
-                        logger.error(f"PDF Extraction failed for the second time. {e}")
+                        logger.error(f"report Extraction failed for the second time. {e}")
                         raise
 
                     logger.info(
@@ -137,5 +145,3 @@ class ReportImportService:
                 # mark failure in history
                 finalize_failure(ctx)
                 raise
-                
-                

endoreg_db/import_files/video_import_service.py

@@ -10,7 +10,7 @@ from endoreg_db.import_files.context import (
 )
 from endoreg_db.import_files.file_storage.state_management import (
     finalize_failure,
-    finalize_video_success
+    finalize_video_success,
 )
 from endoreg_db.import_files.file_storage.storage import create_sensitive_copy
 from endoreg_db.import_files.file_storage.create_video_file import (
@@ -21,7 +21,9 @@ from endoreg_db.import_files.file_storage.state_management import (
     mark_instance_processing_started,
 )
 from endoreg_db.models import VideoFile
-from endoreg_db.import_files.processing.video_processing.video_anonymization import VideoAnonymizer
+from endoreg_db.import_files.processing.video_processing.video_anonymization import (
+    VideoAnonymizer,
+)
 from endoreg_db.utils.paths import (
     SENSITIVE_VIDEO_DIR,
 )
@@ -55,16 +57,15 @@ class VideoImportService:
         self.anonymizer = VideoAnonymizer()
         self.processing_context: Optional[ImportContext] = None
         self.current_video: Optional[VideoFile] = None
-        
+
         validate_directories()
-        
 
     def import_and_anonymize(
         self,
         file_path: Union[Path, str],
         center_name: str,
         processor_name: str,
-        retry:bool = False,
+        retry: bool = False,
         delete_source: bool = True,
     ) -> "VideoFile | None":
         """
@@ -76,41 +77,43 @@ class VideoImportService:
             center_name=center_name,
             processor_name=processor_name,
             delete_source=delete_source,
-            file_type="video"
+            file_type="video",
         )
         self.logger.info("validating and preparing file")
         if not ctx.file_path.exists():
             raise FileNotFoundError(f"Video file not found: {file_path}")
 
-        ctx.sensitive_path = create_sensitive_copy(
-            ctx.file_path, 
-            SENSITIVE_VIDEO_DIR
-        )
+        ctx.sensitive_path = create_sensitive_copy(ctx.file_path, SENSITIVE_VIDEO_DIR)
 
         with file_lock(ctx.file_path):
             logger.info("Acquired file lock for %s", ctx.file_path)
 
-
             # create or retrieve VideoFile + update history
-            ctx.current_video, needs_processing = (
+            ctx.current_video, processed, needs_processing = (
                 create_or_retrieve_video_file(ctx)
             )
             ctx.current_video.get_or_create_state()
-            assert(ctx.current_video.state is not None)
+            assert ctx.current_video.state is not None
             ctx.current_video = ctx.current_video
-            
+
             ctx.retry = retry
             # Retry is a forced overwrite of needs processing - therefore the retry will cause full deletion of processed files using finalize failure.
 
-            if retry and needs_processing and not ctx.current_video.state.anonymization_validated:
+            if (
+                retry
+                and needs_processing
+                and not ctx.current_video.state.anonymization_validated
+            ):
                 finalize_failure(ctx)
-                ctx.current_video, needs_processing = create_or_retrieve_video_file(ctx)
-                assert(needs_processing is True)
+                ctx.current_video, processed, needs_processing = (
+                    create_or_retrieve_video_file(ctx)
+                )
+                assert needs_processing is True
             elif not needs_processing and not retry:
                 return ctx.current_video
 
             mark_instance_processing_started(ctx.current_video, ctx)
-            try:            
+            try:
                 # --- Anonymization with fallback ---
                 try:
                     ctx = self.anonymizer.anonymize_video(ctx)
@@ -128,20 +131,21 @@ class VideoImportService:
                     try:
                         ctx = self.anonymizer.anonymize_video(ctx)
                     except Exception as e:
-                        logger.error(f"Video Extraction failed for the second time. {e}")
+                        logger.error(
+                            f"Video Extraction failed for the second time. {e}"
+                        )
                         raise
                     logger.info(
                         "Secondary video anonymization succeeded for %s",
                         ctx.file_path,
                     )
                 logger.info(f"Anonymized Video is located at: {ctx.anonymized_path}")
-                
+
                 # --- Finalize success: history + move anonymized file ---
                 finalize_video_success(ctx)
-                
+
                 return ctx.current_video
-                
-                
+
             except Exception as exc:
                 logger.exception(
                     "Video import/anonymization failed for %s: %s", ctx.file_path, exc

endoreg_db/logger_conf.py

@@ -1,4 +1,3 @@
-import logging
 from pathlib import Path
 import os
 
@@ -6,6 +5,7 @@ LOG_DIR = Path("data/logs")
 DEFAULT_FILE_LOG_LEVEL = "INFO"
 DEFAULT_CONSOLE_LOG_LEVEL = "WARNING"
 
+
 def clear_log_files(logger_names, log_dir=None):
     """
     Clears specified log files in the log directory.
@@ -20,8 +20,10 @@ def clear_log_files(logger_names, log_dir=None):
     log_dir.mkdir(parents=True, exist_ok=True)
 
     # Files to manage
-    log_files_to_clear = [log_dir / f"{logger_name}.log" for logger_name in logger_names]
-    log_files_to_clear.append(log_dir / "root.log") # Add root log file
+    log_files_to_clear = [
+        log_dir / f"{logger_name}.log" for logger_name in logger_names
+    ]
+    log_files_to_clear.append(log_dir / "root.log")  # Add root log file
 
     # Clear existing log files in the directory managed by this config
     for log_file in log_files_to_clear:
@@ -34,7 +36,10 @@ def clear_log_files(logger_names, log_dir=None):
         # Optionally, create the log file to ensure it exists after clearing
         # log_file.touch() # Removed touch to just clear
 
-def get_logging_config(logger_names, file_log_level=None, console_log_level=None, log_dir=None):
+
+def get_logging_config(
+    logger_names, file_log_level=None, console_log_level=None, log_dir=None
+):
     """
     Generates Django LOGGING configuration dynamically.
 
@@ -47,38 +52,42 @@ def get_logging_config(logger_names, file_log_level=None, console_log_level=None
     Returns:
         dict: The Django LOGGING configuration dictionary.
     """
-    file_log_level = file_log_level or os.environ.get("FILE_LOG_LEVEL", DEFAULT_FILE_LOG_LEVEL)
-    console_log_level = console_log_level or os.environ.get("CONSOLE_LOG_LEVEL", DEFAULT_CONSOLE_LOG_LEVEL)
+    file_log_level = file_log_level or os.environ.get(
+        "FILE_LOG_LEVEL", DEFAULT_FILE_LOG_LEVEL
+    )
+    console_log_level = console_log_level or os.environ.get(
+        "CONSOLE_LOG_LEVEL", DEFAULT_CONSOLE_LOG_LEVEL
+    )
     log_dir = log_dir or LOG_DIR
 
     # Ensure log directory exists
     log_dir.mkdir(parents=True, exist_ok=True)
 
     handlers = {
-        'console': {
-            'level': console_log_level, # Use console level
-            'class': 'logging.StreamHandler',
-            'formatter': 'standard',
+        "console": {
+            "level": console_log_level,  # Use console level
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
         },
-        'file_root': { # Handler for the root logger's file
-            'level': file_log_level, # Use file level
-            'class': 'logging.FileHandler',
-            'filename': log_dir / "root.log",
-            'formatter': 'standard',
+        "file_root": {  # Handler for the root logger's file
+            "level": file_log_level,  # Use file level
+            "class": "logging.FileHandler",
+            "filename": log_dir / "root.log",
+            "formatter": "standard",
         },
     }
     loggers = {
         # Root logger configuration - logs INFO+ to file, WARNING+ to console
-        '': {
-            'handlers': ['console', 'file_root'], # Use both handlers
-            'level': file_log_level, # Set to lowest level needed (INFO for file)
-            'propagate': False, # Root logger doesn't propagate further
+        "": {
+            "handlers": ["console", "file_root"],  # Use both handlers
+            "level": file_log_level,  # Set to lowest level needed (INFO for file)
+            "propagate": False,  # Root logger doesn't propagate further
         },
-         # Django's default logger
-        'django': {
-            'handlers': ['console', 'file_root'], # Log to console and root file
-            'level': file_log_level, # Or your preferred level, ensure it's captured by root file
-            'propagate': False, # Don't propagate Django logs to avoid double handling by root
+        # Django's default logger
+        "django": {
+            "handlers": ["console", "file_root"],  # Log to console and root file
+            "level": file_log_level,  # Or your preferred level, ensure it's captured by root file
+            "propagate": False,  # Don't propagate Django logs to avoid double handling by root
         },
     }
 
@@ -86,33 +95,34 @@ def get_logging_config(logger_names, file_log_level=None, console_log_level=None
     for name in logger_names:
         handler_name = f"file_{name}"
         handlers[handler_name] = {
-            'level': file_log_level, # Use file level
-            'class': 'logging.FileHandler',
-            'filename': log_dir / f"{name}.log",
-            'formatter': 'standard',
+            "level": file_log_level,  # Use file level
+            "class": "logging.FileHandler",
+            "filename": log_dir / f"{name}.log",
+            "formatter": "standard",
         }
         loggers[name] = {
-            'handlers': [handler_name], # Log only to its own file directly
-            'level': file_log_level, # Use file level
-            'propagate': True, # Propagate to root logger (which handles console and root.log)
+            "handlers": [handler_name],  # Log only to its own file directly
+            "level": file_log_level,  # Use file level
+            "propagate": True,  # Propagate to root logger (which handles console and root.log)
         }
 
     return {
-        'version': 1,
-        'disable_existing_loggers': False,
-        'formatters': {
-            'standard': {
-                'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "standard": {
+                "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
             },
         },
-        'handlers': handlers,
-        'loggers': loggers,
+        "handlers": handlers,
+        "loggers": loggers,
     }
 
+
 # Example usage (optional, for testing the function itself)
 if __name__ == "__main__":
     test_loggers = ["app1", "app2", "database"]
-    log_directory = Path("data/logs_test") # Use a test directory
+    log_directory = Path("data/logs_test")  # Use a test directory
 
     # Optionally clear logs before configuring logging
     print("Clearing logs...")
@@ -120,8 +130,14 @@ if __name__ == "__main__":
     print("Log clearing complete.")
 
     # Get logging configuration
-    logging_config = get_logging_config(test_loggers, file_log_level="DEBUG", console_log_level="WARNING", log_dir=log_directory)
+    logging_config = get_logging_config(
+        test_loggers,
+        file_log_level="DEBUG",
+        console_log_level="WARNING",
+        log_dir=log_directory,
+    )
     import json
+
     print("\nGenerated Logging Configuration:")
     print(json.dumps(logging_config, indent=4))
 

endoreg_db/management/__init__.py

@@ -1 +1 @@
-# Management commands package
+# Management commands package

endoreg_db/management/commands/__init__.py

@@ -1 +1 @@
-# Management commands
+# Management commands

endoreg_db/management/commands/check_auth.py

@@ -4,68 +4,71 @@ Management command to check and configure authentication settings.
 
 from django.core.management.base import BaseCommand
 from django.conf import settings
-from rest_framework.permissions import IsAuthenticated, AllowAny
 import os
 
 
 class Command(BaseCommand):
-    help = 'Check and configure authentication settings based on environment'
+    help = "Check and configure authentication settings based on environment"
 
     def add_arguments(self, parser):
         parser.add_argument(
-            '--mode',
-            choices=['check', 'dev', 'prod'],
-            default='check',
-            help='Mode: check current settings, set dev mode, or set prod mode'
+            "--mode",
+            choices=["check", "dev", "prod"],
+            default="check",
+            help="Mode: check current settings, set dev mode, or set prod mode",
         )
         parser.add_argument(
-            '--show-permissions',
-            action='store_true',
-            help='Show which permission classes will be used'
+            "--show-permissions",
+            action="store_true",
+            help="Show which permission classes will be used",
         )
 
     def handle(self, *args, **options):
-        mode = options['mode']
-        
-        if mode == 'check':
+        mode = options["mode"]
+
+        if mode == "check":
             self.check_current_settings()
-        elif mode == 'dev':
+        elif mode == "dev":
             self.set_dev_mode()
-        elif mode == 'prod':
+        elif mode == "prod":
             self.set_prod_mode()
-            
-        if options['show_permissions']:
+
+        if options["show_permissions"]:
             self.show_permission_classes()
 
     def check_current_settings(self):
         """Check current authentication configuration."""
-        debug_mode = getattr(settings, 'DEBUG', False)
-        settings_module = os.environ.get('DJANGO_SETTINGS_MODULE', 'Unknown')
-        
-        self.stdout.write("\n" + "="*50)
+        debug_mode = getattr(settings, "DEBUG", False)
+        settings_module = os.environ.get("DJANGO_SETTINGS_MODULE", "Unknown")
+
+        self.stdout.write("\n" + "=" * 50)
         self.stdout.write("🔍 AUTHENTICATION CONFIGURATION CHECK")
-        self.stdout.write("="*50)
-        
+        self.stdout.write("=" * 50)
+
         self.stdout.write(f"Settings Module: {settings_module}")
         self.stdout.write(f"DEBUG Mode: {debug_mode}")
-        
+
         if debug_mode:
-            self.stdout.write(self.style.WARNING("🔓 Authentication: DISABLED (AllowAny)"))
+            self.stdout.write(
+                self.style.WARNING("🔓 Authentication: DISABLED (AllowAny)")
+            )
             self.stdout.write("   - All API endpoints are accessible without login")
             self.stdout.write("   - Suitable for development and testing")
         else:
-            self.stdout.write(self.style.SUCCESS("🔒 Authentication: ENABLED (IsAuthenticated)"))
+            self.stdout.write(
+                self.style.SUCCESS("🔒 Authentication: ENABLED (IsAuthenticated)")
+            )
             self.stdout.write("   - API endpoints require valid authentication")
             self.stdout.write("   - Suitable for production deployment")
-            
+
         # Check if Keycloak is configured
-        keycloak_server = getattr(settings, 'KEYCLOAK_SERVER_URL', None)
+        keycloak_server = getattr(settings, "KEYCLOAK_SERVER_URL", None)
         if keycloak_server:
             self.stdout.write(f"Keycloak Server: {keycloak_server}")
         else:
             self.stdout.write(self.style.WARNING("⚠️  Keycloak not configured"))
-            
-        self.stdout.write("="*50 + "\n")
+
+        self.stdout.write("=" * 50 + "\n")
 
     def set_dev_mode(self):
         """Instructions for setting development mode."""
@@ -79,7 +82,9 @@ class Command(BaseCommand):
         self.stdout.write("2. Or run with explicit settings:")
         self.stdout.write("   python manage.py runserver --settings=dev.dev_settings")
         self.stdout.write("")
-        self.stdout.write("This will set DEBUG=True and disable authentication requirements.")
+        self.stdout.write(
+            "This will set DEBUG=True and disable authentication requirements."
+        )
 
     def set_prod_mode(self):
         """Instructions for setting production mode."""
@@ -94,23 +99,25 @@ class Command(BaseCommand):
         self.stdout.write("   export DJANGO_SECRET_KEY='your-secure-secret-key'")
         self.stdout.write("   export KEYCLOAK_CLIENT_SECRET='your-keycloak-secret'")
         self.stdout.write("")
-        self.stdout.write("This will set DEBUG=False and enable authentication requirements.")
+        self.stdout.write(
+            "This will set DEBUG=False and enable authentication requirements."
+        )
 
     def show_permission_classes(self):
         """Show which permission classes are being used."""
-        debug_mode = getattr(settings, 'DEBUG', False)
-        
-        self.stdout.write("\n" + "="*40)
+        debug_mode = getattr(settings, "DEBUG", False)
+
+        self.stdout.write("\n" + "=" * 40)
         self.stdout.write("🔐 PERMISSION CLASSES IN USE")
-        self.stdout.write("="*40)
-        
+        self.stdout.write("=" * 40)
+
         if debug_mode:
             permission_class = "AllowAny"
             icon = "🔓"
         else:
             permission_class = "IsAuthenticated"
             icon = "🔒"
-            
+
         self.stdout.write(f"{icon} Current Mode: {permission_class}")
         self.stdout.write("")
         self.stdout.write("Views using dynamic permissions:")
@@ -122,4 +129,4 @@ class Command(BaseCommand):
         self.stdout.write("  • video_segment_detail_view")
         self.stdout.write("  • video_segments_by_label_id_view")
         self.stdout.write("  • video_segments_by_label_name_view")
-        self.stdout.write("="*40 + "\n")
+        self.stdout.write("=" * 40 + "\n")

endoreg_db/management/commands/create_model_meta_from_huggingface.py

@@ -2,8 +2,6 @@
 Django management command to create ModelMeta from Hugging Face model.
 """
 
-from pathlib import Path
-
 from django.core.files.base import ContentFile
 from django.core.management.base import BaseCommand, CommandError
 from huggingface_hub import hf_hub_download
@@ -124,3 +122,56 @@ class Command(BaseCommand):
 
             traceback.print_exc()
             raise CommandError("ModelMeta creation failed") from e
+
+
+# TODO Review slimmed down version using service endoreg_db/services/model_meta_from_hf.py
+# your management command file
+# from django.core.management.base import BaseCommand, CommandError
+
+# from endoreg_db.services.model_meta_from_hf import ensure_model_meta_from_hf
+
+
+# class Command(BaseCommand):
+#     help = "Create ModelMeta by downloading model from Hugging Face"
+
+#     def add_arguments(self, parser):
+#         parser.add_argument(
+#             "--model_id",
+#             type=str,
+#             default="wg-lux/colo_segmentation_RegNetX800MF_base",
+#         )
+#         parser.add_argument(
+#             "--model_name",
+#             type=str,
+#             default="image_multilabel_classification_colonoscopy_default",
+#         )
+#         parser.add_argument(
+#             "--labelset_name",
+#             type=str,
+#             default="multilabel_classification_colonoscopy_default",
+#         )
+#         parser.add_argument(
+#             "--meta_version",
+#             type=str,
+#             default="1",
+#         )
+#         parser.add_argument(
+#             "--labelset_version",
+#             type=int,
+#             default=None,
+#         )
+
+#     def handle(self, *args, **options):
+#         try:
+#             model_meta = ensure_model_meta_from_hf(
+#                 model_id=options["model_id"],
+#                 model_name=options["model_name"],
+#                 labelset_name=options["labelset_name"],
+#                 meta_version=options["meta_version"],
+#                 labelset_version=options.get("labelset_version"),
+#             )
+#             self.stdout.write(
+#                 self.style.SUCCESS(f"Successfully ensured ModelMeta: {model_meta}")
+#             )
+#         except Exception as e:
+#             raise CommandError(f"ModelMeta creation failed: {e}") from e