endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

endoreg_db/import_files/pseudonymization/k_pseudonymity.py

@@ -0,0 +1,128 @@
+from typing import Optional, Tuple
+
+from datetime import date as Date
+import datetime
+
+from .k_anonymity import _build_sensitive_meta_qi_queryset
+from .fake import fake_name_with_similar_dob_and_gender
+
+from endoreg_db.models import SensitiveMeta
+
+def k_pseudonymize(
+    instance: SensitiveMeta,
+    *,
+    k_threshold: int = 3,
+    dob_year_tolerance: int = 3,
+    qi_subset: Optional[Tuple[str, ...]] = None,
+    locale: str = "de_DE",
+    seed: Optional[int] = None,
+    save: bool = True,
+) -> Tuple[SensitiveMeta, int, bool]:
+    """
+    Ensure a SensitiveMeta instance reaches at least `k_threshold` anonymity
+    for the given quasi-identifier subset by pseudonymizing patient
+    first_name, last_name and DOB if necessary.
+
+    Args:
+        instance:
+            The SensitiveMeta instance to process.
+        k_threshold:
+            Minimal k for the chosen QI subset.
+        dob_year_tolerance:
+            Used both for k-anonymity DOB band and for Faker's DOB perturbation.
+        qi_subset:
+            Which QIs to use for k-anonymity check.
+            Elements from: {"first_name", "last_name", "center", "gender", "dob_band"}.
+            Default = all of them.
+        locale:
+            Faker locale for a realistic name.
+        seed:
+            Optional seed for reproducibility.
+        save:
+            If True, save the instance after pseudonymization.
+
+    Returns:
+        (instance, k_value_after, is_k_anonymous_after)
+    """
+    if qi_subset is None:
+        qi_subset = tuple(QI_FLAGS)
+
+    # --- 1) Compute k for the requested subset BEFORE pseudonymization ---
+    use_first_name = "first_name" in qi_subset
+    use_last_name = "last_name" in qi_subset
+    use_center = "center" in qi_subset
+    use_gender = "gender" in qi_subset
+    use_dob_band = "dob_band" in qi_subset
+
+    qs_before = _build_sensitive_meta_qi_queryset(
+        instance,
+        dob_year_tolerance=dob_year_tolerance,
+        include_self=True,
+        use_first_name=use_first_name,
+        use_last_name=use_last_name,
+        use_center=use_center,
+        use_gender=use_gender,
+        use_dob_band=use_dob_band,
+    )
+    k_before = qs_before.count()
+
+    if k_before >= k_threshold:
+        # Already sufficiently anonymous, nothing to do
+        return instance, k_before, True
+
+    # --- 2) Pseudonymize name + DOB using Faker ---
+    # Gender string for Faker
+    if instance.patient_gender and getattr(instance.patient_gender, "name", None):
+        gender_name = instance.patient_gender.name
+    else:
+        # Fallback if gender missing -> bias to 'male' but you can change that
+        gender_name = "male"
+
+    # Original DOB as date (fallback to today's date if missing)
+    if instance.patient_dob is not None:
+        orig_dob: Date = instance.patient_dob.date()
+    else:
+        orig_dob = Date.today()
+
+    first_name, last_name, fake_dob = fake_name_with_similar_dob_and_gender(
+        gender=gender_name,
+        dob=orig_dob,
+        year_tolerance=dob_year_tolerance,
+        locale=locale,
+        seed=seed,
+    )
+
+    # Assign to instance (SensitiveMeta.patient_dob is a DateTimeField)
+    instance.patient_first_name = first_name
+    instance.patient_last_name = last_name
+    instance.patient_dob = datetime(
+        fake_dob.year, fake_dob.month, fake_dob.day
+    )  # naive is usually fine for DOB
+
+    if save:
+        instance.save(update_fields=["patient_first_name", "patient_last_name", "patient_dob"])
+
+    # --- 3) Recompute k AFTER pseudonymization ---
+    qs_after = _build_sensitive_meta_qi_queryset(
+        instance,
+        dob_year_tolerance=dob_year_tolerance,
+        include_self=True,
+        use_first_name=use_first_name,
+        use_last_name=use_last_name,
+        use_center=use_center,
+        use_gender=use_gender,
+        use_dob_band=use_dob_band,
+    )
+    k_after = qs_after.count()
+    is_k_anon_after = k_after >= k_threshold
+
+    logger.info(
+        "k_pseudonymize: SensitiveMeta pk=%s, subset=%s, k_before=%s, k_after=%s, threshold=%s",
+        instance.pk,
+        qi_subset,
+        k_before,
+        k_after,
+        k_threshold,
+    )
+
+    return instance, k_after, is_k_anon_after

endoreg_db/import_files/report_import_service.py

@@ -0,0 +1,141 @@
+# endoreg_db/services/report_import_service.py
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Optional, Union
+
+from endoreg_db.import_files.context.file_lock import file_lock
+from endoreg_db.import_files.context.import_context import ImportContext
+from endoreg_db.import_files.processing.report_processing.report_anonymization import (
+    ReportAnonymizer,
+)
+
+from endoreg_db.import_files.file_storage.create_report_file import (
+    create_or_retrieve_report_file,
+)
+from endoreg_db.import_files.file_storage.state_management import (
+    finalize_report_success,
+    finalize_failure,
+    mark_instance_processing_started,
+)
+from endoreg_db.import_files.context.validate_directories import validate_directories
+
+from endoreg_db.import_files.file_storage.storage import create_sensitive_copy
+from endoreg_db.models.media import RawPdfFile
+from endoreg_db.utils.paths import (
+    SENSITIVE_REPORT_DIR,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class ReportImportService:
+    """
+    Service for importing and anonymizing report (PDF) files.
+
+    Responsibilities:
+      - Acquire file lock
+      - Create sensitive copy
+      - Create/reuse RawPdfFile (dedupe by hash) + history
+      - Run anonymization pipeline (primary + fallback)
+      - Finalize state and move anonymized file
+      - Cleanup on error
+    """
+
+    def __init__(self) -> None:
+        self.logger = logger
+        self.anonymizer = ReportAnonymizer()
+        self.processing_context: Optional[ImportContext] = None
+        self.current_report: Optional[RawPdfFile] = None
+        
+        validate_directories()
+
+
+    def import_and_anonymize(
+        self,
+        file_path: Union[Path, str],
+        center_name: str,
+        retry: bool = False,
+        delete_source: bool = True,
+    ) -> "RawPdfFile | None":
+        """
+        Public entrypoint: wrap import_and_anonymize logic.
+        """
+        # First, initialize import context. this will be updated during import and keep track of current paths, file type and center and processor.
+        ctx = ImportContext(
+            file_path=Path(file_path),
+            center_name=center_name,
+            delete_source=delete_source,
+            file_type="report"
+        )
+        self.logger.info("validating and preparing file")
+        if not ctx.file_path.exists():
+            raise FileNotFoundError(f"Video file not found: {file_path}")
+
+        ctx.sensitive_path = create_sensitive_copy(
+            ctx.file_path, 
+            SENSITIVE_REPORT_DIR
+        )
+
+        with file_lock(ctx.file_path):
+            logger.info("Acquired file lock for %s", ctx.file_path)
+
+            # create or retrieve RawPdfFile + update history
+            ctx.current_report, needs_processing = create_or_retrieve_report_file(ctx)
+            ctx.current_report.get_or_create_state()
+            assert(ctx.current_report.state is not None)
+            ctx.current_report = ctx.current_report
+            
+            ctx.retry = retry
+            # Retry is a forced overwrite of needs processing - therefore the retry will cause full deletion of processed files using finalize failure.
+            if retry and needs_processing and not ctx.current_report.state.anonymization_validated:
+                # ensure clean slate for forced reprocessing
+                finalize_failure(ctx)
+                ctx.current_report, needs_processing = create_or_retrieve_report_file(ctx)
+                assert(needs_processing is True)
+            elif not needs_processing and not retry:
+                return ctx.current_report
+
+            mark_instance_processing_started(ctx.current_report, ctx)
+            try:
+                # --- Anonymization with fallback ---
+                try:
+                    ctx = self.anonymizer.anonymize_report(ctx)
+                    logger.info(
+                        "Primary report anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+                except Exception as primary_exc:
+                    logger.exception(
+                        "Primary report anonymization failed for %s: %s "
+                        "- trying basic anonymization",
+                        ctx.file_path,
+                        primary_exc,
+                    )
+                    try:
+                        ctx = self.anonymizer.anonymize_report(ctx)
+                    except Exception as e:
+                        logger.error(f"PDF Extraction failed for the second time. {e}")
+                        raise
+
+                    logger.info(
+                        "Basic report anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+
+                # --- Finalize success: history + move anonymized file ---
+                finalize_report_success(ctx)
+
+                return ctx.current_report
+
+            except Exception as exc:
+                logger.exception(
+                    "Report import/anonymization failed for %s: %s", ctx.file_path, exc
+                )
+                # mark failure in history
+                finalize_failure(ctx)
+                raise
+                
+                

endoreg_db/import_files/video_import_service.py

@@ -0,0 +1,150 @@
+# endoreg_db/services/video_import.py
+import logging
+from pathlib import Path
+from typing import Optional, Union
+
+
+from endoreg_db.import_files.context import (
+    ImportContext,
+    file_lock,
+)
+from endoreg_db.import_files.file_storage.state_management import (
+    finalize_failure,
+    finalize_video_success
+)
+from endoreg_db.import_files.file_storage.storage import create_sensitive_copy
+from endoreg_db.import_files.file_storage.create_video_file import (
+    create_or_retrieve_video_file,
+)
+from endoreg_db.import_files.context.validate_directories import validate_directories
+from endoreg_db.import_files.file_storage.state_management import (
+    mark_instance_processing_started,
+)
+from endoreg_db.models import VideoFile
+from endoreg_db.import_files.processing.video_processing.video_anonymization import VideoAnonymizer
+from endoreg_db.utils.paths import (
+    SENSITIVE_VIDEO_DIR,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class VideoImportService:
+    """
+    Service for importing and anonymizing video files.
+    Uses a central video instance pattern for cleaner state management.
+    Responsibility:
+        Validate path
+        Move/copy raw file into stable storage
+        Create/attach VideoFile, VideoState, default SensitiveMeta
+        Mark a single clear state like state.import_completed = True or state.ready_for_anonymization = True
+
+    It does not:
+        Run frame cleaning
+        Run ML
+        Create processed_file
+        Decide fallback anonymization
+
+    These actions are delegated to the modules inside file_import
+
+    """
+
+    def __init__(self):
+        self.logger = logger
+        self.anonymizer = VideoAnonymizer()
+        self.processing_context: Optional[ImportContext] = None
+        self.current_video: Optional[VideoFile] = None
+        
+        validate_directories()
+        
+
+    def import_and_anonymize(
+        self,
+        file_path: Union[Path, str],
+        center_name: str,
+        processor_name: str,
+        retry:bool = False,
+        delete_source: bool = True,
+    ) -> "VideoFile | None":
+        """
+        Public entrypoint: wrap import_and_anonymize logic.
+        """
+        # First, initialize import context. this will be updated during import and keep track of current paths, file type and center and processor.
+        ctx = ImportContext(
+            file_path=Path(file_path),
+            center_name=center_name,
+            processor_name=processor_name,
+            delete_source=delete_source,
+            file_type="video"
+        )
+        self.logger.info("validating and preparing file")
+        if not ctx.file_path.exists():
+            raise FileNotFoundError(f"Video file not found: {file_path}")
+
+        ctx.sensitive_path = create_sensitive_copy(
+            ctx.file_path, 
+            SENSITIVE_VIDEO_DIR
+        )
+
+        with file_lock(ctx.file_path):
+            logger.info("Acquired file lock for %s", ctx.file_path)
+
+
+            # create or retrieve VideoFile + update history
+            ctx.current_video, needs_processing = (
+                create_or_retrieve_video_file(ctx)
+            )
+            ctx.current_video.get_or_create_state()
+            assert(ctx.current_video.state is not None)
+            ctx.current_video = ctx.current_video
+            
+            ctx.retry = retry
+            # Retry is a forced overwrite of needs processing - therefore the retry will cause full deletion of processed files using finalize failure.
+
+            if retry and needs_processing and not ctx.current_video.state.anonymization_validated:
+                finalize_failure(ctx)
+                ctx.current_video, needs_processing = create_or_retrieve_video_file(ctx)
+                assert(needs_processing is True)
+            elif not needs_processing and not retry:
+                return ctx.current_video
+
+            mark_instance_processing_started(ctx.current_video, ctx)
+            try:            
+                # --- Anonymization with fallback ---
+                try:
+                    ctx = self.anonymizer.anonymize_video(ctx)
+                    logger.info(
+                        "Primary video anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+                except Exception as primary_exc:
+                    logger.exception(
+                        "Primary video anonymization failed for %s: %s "
+                        "- trying basic anonymization",
+                        ctx.file_path,
+                        primary_exc,
+                    )
+                    try:
+                        ctx = self.anonymizer.anonymize_video(ctx)
+                    except Exception as e:
+                        logger.error(f"Video Extraction failed for the second time. {e}")
+                        raise
+                    logger.info(
+                        "Secondary video anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+                logger.info(f"Anonymized Video is located at: {ctx.anonymized_path}")
+                
+                # --- Finalize success: history + move anonymized file ---
+                finalize_video_success(ctx)
+                
+                return ctx.current_video
+                
+                
+            except Exception as exc:
+                logger.exception(
+                    "Video import/anonymization failed for %s: %s", ctx.file_path, exc
+                )
+                finalize_failure(ctx)
+                raise

endoreg_db/management/commands/create_model_meta_from_huggingface.py

@@ -5,7 +5,7 @@ Django management command to create ModelMeta from Hugging Face model.
 from pathlib import Path
 
 from django.core.files.base import ContentFile
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 from huggingface_hub import hf_hub_download
 
 from endoreg_db.models import AiModel, LabelSet, ModelMeta
@@ -39,12 +39,19 @@ class Command(BaseCommand):
             default="1",
             help="Version for the model meta",
         )
+        parser.add_argument(
+            "--labelset_version",
+            type=int,
+            default=None,
+            help="LabelSet version; if omitted, the latest by name is used",
+        )
 
     def handle(self, *args, **options):
         model_id = options["model_id"]
         model_name = options["model_name"]
         labelset_name = options["labelset_name"]
         version = options["meta_version"]
+        labelset_version = options.get("labelset_version")
 
         self.stdout.write(f"Downloading model {model_id} from Hugging Face...")
 
@@ -52,7 +59,7 @@ class Command(BaseCommand):
             # Download the model weights
             weights_path = hf_hub_download(
                 repo_id=model_id,
-                filename="colo_segmentation_RegNetX800MF_base.ckpt",
+                filename="colo_segmentation_RegNetX800MF_base.safetensors",
                 local_dir="/tmp",
             )
             self.stdout.write(f"Downloaded weights to: {weights_path}")
@@ -64,14 +71,17 @@ class Command(BaseCommand):
             if created:
                 self.stdout.write(f"Created AI model: {ai_model.name}")
 
-            # Get labelset
-            try:
-                labelset = LabelSet.objects.get(name=labelset_name)
-            except LabelSet.DoesNotExist:
-                self.stdout.write(
-                    self.style.ERROR(f"LabelSet '{labelset_name}' not found")
+            # Get labelset (optionally by version); fail with non-zero exit
+            labelset_qs = LabelSet.objects.filter(name=labelset_name)
+            if labelset_version is not None:
+                labelset_qs = labelset_qs.filter(version=labelset_version)
+            labelset = labelset_qs.order_by("-version").first()
+            if labelset is None:
+                raise CommandError(
+                    f"LabelSet '{labelset_name}'"
+                    + (f" v{labelset_version}" if labelset_version is not None else "")
+                    + " not found"
                 )
-                return
 
             # Create ModelMeta
             model_meta, created = ModelMeta.objects.get_or_create(
@@ -95,7 +105,7 @@ class Command(BaseCommand):
             # Save the weights file to the model
             with open(weights_path, "rb") as f:
                 model_meta.weights.save(
-                    f"{model_name}_v{version}_pytorch_model.bin", ContentFile(f.read())
+                    f"{model_name}_v{version}.safetensors", ContentFile(f.read())
                 )
 
             # Set as active meta
@@ -113,3 +123,4 @@ class Command(BaseCommand):
             import traceback
 
             traceback.print_exc()
+            raise CommandError("ModelMeta creation failed") from e