endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.9.2__py3-none-any.whl

endoreg_db/import_files/processing/sensitive_meta_adapter.py

@@ -0,0 +1,51 @@
+# endoreg_db/import_files/processing/sensitive_meta_adapter.py
+from typing import Any, Dict
+
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSensitiveMeta
+
+
+def normalize_lx_sensitive_meta(meta: LxSensitiveMeta) -> Dict[str, Any]:
+    """
+    Convert lx_anonymizer.SensitiveMeta into a dict suitable for
+    endoreg_db SensitiveMeta.update_from_dict / create_from_dict.
+
+    - Renames fields where necessary (center -> center_name, patient_gender_name -> patient_gender)
+    - Drops None/blank values (your update logic already handles blanks carefully)
+    - Leaves dates as strings; your logic layer already parses them
+    """
+    raw = meta.to_dict()
+    out: Dict[str, Any] = {}
+
+    # 1:1 fields (same names in model logic)
+    direct_keys = [
+        "file_path",
+        "patient_first_name",
+        "patient_last_name",
+        "patient_dob",          # string; logic has parsing
+        "casenumber",
+        "examination_date",     # string; logic has parsing
+        "examination_time",     # string "HH:MM" is fine
+        "examiner_first_name",
+        "examiner_last_name",
+        "text",
+        "anonymized_text",
+        "endoscope_type",
+        "endoscope_sn",
+    ]
+    for k in direct_keys:
+        v = raw.get(k)
+        if v not in (None, "", []):
+            out[k] = v
+
+    # Map patient_gender_name (interface) -> patient_gender (logic)
+    gender_name = raw.get("patient_gender_name")
+    if gender_name not in (None, ""):
+        # Your logic.update_* can handle strings for patient_gender
+        out["patient_gender"] = gender_name
+
+    # Map center (string) -> center_name (logic)
+    center_name = raw.get("center")
+    if center_name not in (None, ""):
+        out["center_name"] = center_name
+
+    return out

endoreg_db/import_files/processing/video_processing/video_anonymization.py

@@ -0,0 +1,107 @@
+from typing import List, Dict, Any, Tuple, Optional
+
+import logging
+logger = logging.getLogger(__name__)
+
+from lx_anonymizer import FrameCleaner
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
+
+
+from endoreg_db.import_files.file_storage.sensitive_meta_storage import sensitive_meta_storage
+from endoreg_db.import_files.context import ImportContext
+from endoreg_db.utils.paths import ANONYM_VIDEO_DIR
+from endoreg_db.models import EndoscopyProcessor, VideoFile
+
+
+class VideoAnonymizer:
+    def __init__(self):
+        self._ensure_frame_cleaning_available()
+        self._frame_cleaning_available = None
+        self._frame_cleaning_class = None
+        self.storage = False
+
+        
+    def anonymize_video(self, ctx: ImportContext):
+        # Setup anonymized directory
+        anonymized_dir = ANONYM_VIDEO_DIR
+        anonymized_dir.mkdir(parents=True, exist_ok=True)
+        assert ctx.current_video is not None
+        # Generate output path for anonymized report
+        
+        video_hash = ctx.current_video.video_hash
+        anonymized_output_path = anonymized_dir / f"{video_hash}.mp4"
+        
+        self._frame_cleaning_class = FrameCleaner()
+        
+        assert isinstance(self._frame_cleaning_class, FrameCleaner)
+        endoscope_roi, endoscope_roi_nested = self._get_processor_roi_info(ctx)
+        # Process with enhanced process_report method (returns 4-tuple now)
+        ctx.anonymized_path, extracted_metadata = self._frame_cleaning_class.clean_video(
+                video_path=ctx.file_path,
+                endoscope_image_roi=endoscope_roi,
+                endoscope_data_roi_nested=endoscope_roi_nested,
+                output_path=anonymized_output_path
+                
+            )
+        sm = LxSM()
+        sm.safe_update(extracted_metadata)
+        
+        self.storage = sensitive_meta_storage(sm, ctx.current_video)
+        return ctx
+                    
+    def _ensure_frame_cleaning_available(self):
+        """
+        Ensure frame cleaning modules are available by adding lx-anonymizer to path.
+
+        Returns:
+            Tuple of (availability_flag, FrameCleaner_class, ReportReader_class)
+        """
+        try:
+            from lx_anonymizer import FrameCleaner
+        except Exception as e:
+            logger.warning(
+                f"Frame cleaning not available: {e} Please install or update lx_anonymizer."
+            )
+            raise
+
+        assert FrameCleaner is not None
+        self._frame_cleaning_class = FrameCleaner()
+        self._frame_cleaning_available = True
+
+
+    def _get_processor_roi_info(
+        self,
+        ctx: ImportContext,
+    ) -> tuple[dict[str, int | None] | None,
+            dict[str, dict[str, int | None] | None] | None]:
+        """Get processor ROI information for masking and data extraction."""
+        endoscope_data_roi_nested = None
+        endoscope_image_roi = None
+
+        video = ctx.current_video
+        assert isinstance(video, VideoFile)
+
+        try:
+            processor_name = ctx.processor_name if ctx.processor_name else None
+            if processor_name:
+                pr = EndoscopyProcessor()
+                processor = pr.get_by_name(processor_name)
+                assert isinstance(processor, EndoscopyProcessor), (
+                    "Processor is not of type EndoscopyProcessor"
+                )
+                endoscope_image_roi = processor.get_roi_endoscope_image()
+                endoscope_data_roi_nested = processor.get_sensitive_rois()
+                logger.info(
+                    "Retrieved processor ROI information: endoscope_image_roi=%s",
+                    endoscope_image_roi,
+                )
+            else:
+                logger.warning(
+                    "No processor found for video %s, proceeding without ROI masking",
+                    video.uuid,
+                )
+        except Exception as exc:
+            logger.error("Failed to retrieve processor ROI information: %s", exc)
+
+        # IMPORTANT: return order must match clean_video signature
+        return endoscope_image_roi, endoscope_data_roi_nested

endoreg_db/import_files/pseudonymization/fake.py

@@ -0,0 +1,52 @@
+from datetime import date, timedelta
+from typing import Tuple, Optional
+from faker import Faker
+import random
+
+
+def fake_name_with_similar_dob_and_gender(
+    gender: str,
+    dob: date,
+    *,
+    year_tolerance: int = 3,
+    locale: str = "de_DE",
+    seed: Optional[int] = None,
+) -> Tuple[str, str, date]:
+    """
+    Generate a fake name with the same gender and a similar date of birth.
+
+    Args:
+        gender: "male" or "female"
+        dob: Original date of birth
+        year_tolerance: Maximum age difference in years
+        locale: Faker locale (default: German)
+        seed: Optional reproducible seed
+
+    Returns:
+        (full_name, fake_dob)
+    """
+
+    if gender not in {"male", "female"}:
+        raise ValueError("gender must be 'male' or 'female'")
+
+    fake = Faker(locale)
+
+    if seed is not None:
+        Faker.seed(seed)
+        random.seed(seed)
+
+    # --- Generate gender-safe name ---
+    if gender == "male":
+        first_name = fake.first_name_male()
+    else:
+        first_name = fake.first_name_female()
+
+    last_name = fake.last_name()
+    full_name = f"{first_name} {last_name}"
+
+    # --- Generate similar DOB ---
+    days_range = year_tolerance * 365
+    offset_days = random.randint(-days_range, days_range)
+    fake_dob = dob + timedelta(days=offset_days)
+
+    return first_name, last_name, fake_dob

endoreg_db/import_files/pseudonymization/k_anonymity.py

@@ -0,0 +1,182 @@
+from endoreg_db.models import Patient, SensitiveMeta, Center, Gender
+import logging
+from datetime import timedelta
+from typing import Tuple
+
+from django.db.models import QuerySet
+
+from itertools import combinations
+from typing import Dict, Tuple, List
+
+logger = logging.getLogger(__name__)
+
+
+QI_FLAGS = ["first_name", "last_name", "center", "gender", "dob_band"]
+
+
+def get_k_profile_for_instance(
+    instance: SensitiveMeta,
+    *,
+    dob_year_tolerance: int = 1,
+    include_self: bool = True,
+) -> Dict[Tuple[str, ...], int]:
+    """
+    For a given SensitiveMeta instance, compute k (equivalence class size)
+    for all non-empty subsets of the quasi-identifiers defined in QI_FLAGS.
+
+    Returns:
+        {
+          ('first_name',):  12,
+          ('center', 'gender'): 45,
+          ('first_name', 'last_name', 'dob_band'): 3,
+          ...
+        }
+    """
+    result: Dict[Tuple[str, ...], int] = {}
+
+    for r in range(1, len(QI_FLAGS) + 1):
+        for subset in combinations(QI_FLAGS, r):
+            use_first_name = "first_name" in subset
+            use_last_name = "last_name" in subset
+            use_center = "center" in subset
+            use_gender = "gender" in subset
+            use_dob_band = "dob_band" in subset
+
+            qs = _build_sensitive_meta_qi_queryset(
+                instance,
+                dob_year_tolerance=dob_year_tolerance,
+                include_self=include_self,
+                use_first_name=use_first_name,
+                use_last_name=use_last_name,
+                use_center=use_center,
+                use_gender=use_gender,
+                use_dob_band=use_dob_band,
+            )
+
+            k_value = qs.count()
+            result[subset] = k_value
+
+    return result
+
+
+def get_k_anonymity(pk, k=3):
+    """
+    How anonymized is a patient?
+    Get the k value for how many patients can be matched to the current patients attributes.
+
+    Args:
+        pk (_type_): _description_
+        k (int, optional): _description_. Defaults to 3.
+    """
+    return get_k_anonymity_for_sensitive_meta(pk=pk, k=k, dob_year_tolerance=1) 
+
+
+def _build_sensitive_meta_qi_queryset(
+    instance: SensitiveMeta,
+    *,
+    dob_year_tolerance: int = 1,
+    include_self: bool = True,
+    use_first_name: bool = True,
+    use_last_name: bool = True,
+    use_center: bool = True,
+    use_gender: bool = True,
+    use_dob_band: bool = True,
+) -> QuerySet[SensitiveMeta]:
+    """
+    Build a queryset of SensitiveMeta records that are indistinguishable from
+    `instance` on the chosen quasi-identifiers:
+
+        - same center
+        - same patient_gender
+        - patient_dob within ±dob_year_tolerance years (approx via days)
+
+    Args:
+        instance: The SensitiveMeta instance we evaluate.
+        dob_year_tolerance: Allowed +- years around patient_dob.
+        include_self: Whether to include `instance` itself in the result.
+
+    Returns:
+        A Django QuerySet for further aggregation.
+    """
+    qs = SensitiveMeta.objects.all()
+    
+    if use_first_name and instance.patient_first_name is not None:
+        qs = qs.filter(patient_first_name=instance.patient_first_name)
+            
+    if use_last_name and instance.patient_first_name is not None:
+        qs = qs.filter(patient_first_name=instance.patient_first_name)
+    
+    # --- Center ---
+    if use_center and instance.center is not None:
+        if instance.center.pk is not None:
+            qs = qs.filter(center=instance.center.pk)
+
+    # --- Gender ---
+    if use_gender and instance.patient_gender is not None
+        if instance.patient_gender.pk is not None:
+            qs = qs.filter(patient_gender_id=instance.patient_gender)
+
+    # --- DOB (approximate ±N years using days) ---
+    if use_dob_band and instance.patient_dob is not None:
+        days = dob_year_tolerance * 365
+        ref_date = instance.patient_dob.date()
+        start = ref_date - timedelta(days=days)
+        end = ref_date + timedelta(days=days)
+        qs = qs.filter(patient_dob__date__range=(start, end))
+
+    # --- Exclude self if requested ---
+    if not include_self and instance.pk is not None:
+        qs = qs.exclude(pk=instance.pk)
+
+    return qs
+
+
+def get_k_anonymity_for_sensitive_meta(
+    pk: int,
+    *,
+    k: int = 3,
+    dob_year_tolerance: int = 1,
+) -> Tuple[int, bool]:
+    """
+    Compute the k-anonymity (equivalence class size) for a SensitiveMeta record.
+
+    k-anonymity here is defined as the number of SensitiveMeta rows that share
+    the same quasi-identifiers as the given record:
+
+        - center
+        - patient_gender
+        - patient_dob within ±dob_year_tolerance years (approximate)
+
+    Args:
+        pk: Primary key of the SensitiveMeta instance to evaluate.
+        k: Desired anonymity threshold (e.g. 3 for 3-anonymity).
+        dob_year_tolerance: Allowed age window in years around patient_dob.
+
+    Returns:
+        (k_value, is_k_anonymous) where:
+            k_value       = size of the equivalence class
+            is_k_anonymous = True if k_value >= k
+    """
+    try:
+        sm = SensitiveMeta.objects.get(pk=pk)
+    except SensitiveMeta.DoesNotExist:
+        raise ValueError(f"SensitiveMeta with pk={pk} does not exist")
+
+    qs = _build_sensitive_meta_qi_queryset(
+        sm,
+        dob_year_tolerance=dob_year_tolerance,
+        include_self=True,
+    )
+
+    k_value = qs.count()
+    is_k_anon = k_value >= k
+
+    logger.info(
+        "k-anonymity for SensitiveMeta pk=%s -> k=%s (threshold=%s, dob_tol=%s years)",
+        pk,
+        k_value,
+        k,
+        dob_year_tolerance,
+    )
+
+    return k_value, is_k_anon

endoreg_db/import_files/pseudonymization/k_pseudonymity.py

@@ -0,0 +1,128 @@
+from typing import Optional, Tuple
+
+from datetime import date as Date
+import datetime
+
+from .k_anonymity import _build_sensitive_meta_qi_queryset
+from .fake import fake_name_with_similar_dob_and_gender
+
+from endoreg_db.models import SensitiveMeta
+
+def k_pseudonymize(
+    instance: SensitiveMeta,
+    *,
+    k_threshold: int = 3,
+    dob_year_tolerance: int = 3,
+    qi_subset: Optional[Tuple[str, ...]] = None,
+    locale: str = "de_DE",
+    seed: Optional[int] = None,
+    save: bool = True,
+) -> Tuple[SensitiveMeta, int, bool]:
+    """
+    Ensure a SensitiveMeta instance reaches at least `k_threshold` anonymity
+    for the given quasi-identifier subset by pseudonymizing patient
+    first_name, last_name and DOB if necessary.
+
+    Args:
+        instance:
+            The SensitiveMeta instance to process.
+        k_threshold:
+            Minimal k for the chosen QI subset.
+        dob_year_tolerance:
+            Used both for k-anonymity DOB band and for Faker's DOB perturbation.
+        qi_subset:
+            Which QIs to use for k-anonymity check.
+            Elements from: {"first_name", "last_name", "center", "gender", "dob_band"}.
+            Default = all of them.
+        locale:
+            Faker locale for a realistic name.
+        seed:
+            Optional seed for reproducibility.
+        save:
+            If True, save the instance after pseudonymization.
+
+    Returns:
+        (instance, k_value_after, is_k_anonymous_after)
+    """
+    if qi_subset is None:
+        qi_subset = tuple(QI_FLAGS)
+
+    # --- 1) Compute k for the requested subset BEFORE pseudonymization ---
+    use_first_name = "first_name" in qi_subset
+    use_last_name = "last_name" in qi_subset
+    use_center = "center" in qi_subset
+    use_gender = "gender" in qi_subset
+    use_dob_band = "dob_band" in qi_subset
+
+    qs_before = _build_sensitive_meta_qi_queryset(
+        instance,
+        dob_year_tolerance=dob_year_tolerance,
+        include_self=True,
+        use_first_name=use_first_name,
+        use_last_name=use_last_name,
+        use_center=use_center,
+        use_gender=use_gender,
+        use_dob_band=use_dob_band,
+    )
+    k_before = qs_before.count()
+
+    if k_before >= k_threshold:
+        # Already sufficiently anonymous, nothing to do
+        return instance, k_before, True
+
+    # --- 2) Pseudonymize name + DOB using Faker ---
+    # Gender string for Faker
+    if instance.patient_gender and getattr(instance.patient_gender, "name", None):
+        gender_name = instance.patient_gender.name
+    else:
+        # Fallback if gender missing -> bias to 'male' but you can change that
+        gender_name = "male"
+
+    # Original DOB as date (fallback to today's date if missing)
+    if instance.patient_dob is not None:
+        orig_dob: Date = instance.patient_dob.date()
+    else:
+        orig_dob = Date.today()
+
+    first_name, last_name, fake_dob = fake_name_with_similar_dob_and_gender(
+        gender=gender_name,
+        dob=orig_dob,
+        year_tolerance=dob_year_tolerance,
+        locale=locale,
+        seed=seed,
+    )
+
+    # Assign to instance (SensitiveMeta.patient_dob is a DateTimeField)
+    instance.patient_first_name = first_name
+    instance.patient_last_name = last_name
+    instance.patient_dob = datetime(
+        fake_dob.year, fake_dob.month, fake_dob.day
+    )  # naive is usually fine for DOB
+
+    if save:
+        instance.save(update_fields=["patient_first_name", "patient_last_name", "patient_dob"])
+
+    # --- 3) Recompute k AFTER pseudonymization ---
+    qs_after = _build_sensitive_meta_qi_queryset(
+        instance,
+        dob_year_tolerance=dob_year_tolerance,
+        include_self=True,
+        use_first_name=use_first_name,
+        use_last_name=use_last_name,
+        use_center=use_center,
+        use_gender=use_gender,
+        use_dob_band=use_dob_band,
+    )
+    k_after = qs_after.count()
+    is_k_anon_after = k_after >= k_threshold
+
+    logger.info(
+        "k_pseudonymize: SensitiveMeta pk=%s, subset=%s, k_before=%s, k_after=%s, threshold=%s",
+        instance.pk,
+        qi_subset,
+        k_before,
+        k_after,
+        k_threshold,
+    )
+
+    return instance, k_after, is_k_anon_after

endoreg_db/import_files/report_import_service.py

@@ -0,0 +1,141 @@
+# endoreg_db/services/report_import_service.py
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Optional, Union
+
+from endoreg_db.import_files.context.file_lock import file_lock
+from endoreg_db.import_files.context.import_context import ImportContext
+from endoreg_db.import_files.processing.report_processing.report_anonymization import (
+    ReportAnonymizer,
+)
+
+from endoreg_db.import_files.file_storage.create_report_file import (
+    create_or_retrieve_report_file,
+)
+from endoreg_db.import_files.file_storage.state_management import (
+    finalize_report_success,
+    finalize_failure,
+    mark_instance_processing_started,
+)
+from endoreg_db.import_files.context.validate_directories import validate_directories
+
+from endoreg_db.import_files.file_storage.storage import create_sensitive_copy
+from endoreg_db.models.media import RawPdfFile
+from endoreg_db.utils.paths import (
+    SENSITIVE_REPORT_DIR,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class ReportImportService:
+    """
+    Service for importing and anonymizing report (PDF) files.
+
+    Responsibilities:
+      - Acquire file lock
+      - Create sensitive copy
+      - Create/reuse RawPdfFile (dedupe by hash) + history
+      - Run anonymization pipeline (primary + fallback)
+      - Finalize state and move anonymized file
+      - Cleanup on error
+    """
+
+    def __init__(self) -> None:
+        self.logger = logger
+        self.anonymizer = ReportAnonymizer()
+        self.processing_context: Optional[ImportContext] = None
+        self.current_report: Optional[RawPdfFile] = None
+        
+        validate_directories()
+
+
+    def import_and_anonymize(
+        self,
+        file_path: Union[Path, str],
+        center_name: str,
+        retry: bool = False,
+        delete_source: bool = True,
+    ) -> "RawPdfFile | None":
+        """
+        Public entrypoint: wrap import_and_anonymize logic.
+        """
+        # First, initialize import context. this will be updated during import and keep track of current paths, file type and center and processor.
+        ctx = ImportContext(
+            file_path=Path(file_path),
+            center_name=center_name,
+            delete_source=delete_source,
+            file_type="report"
+        )
+        self.logger.info("validating and preparing file")
+        if not ctx.file_path.exists():
+            raise FileNotFoundError(f"Video file not found: {file_path}")
+
+        ctx.sensitive_path = create_sensitive_copy(
+            ctx.file_path, 
+            SENSITIVE_REPORT_DIR
+        )
+
+        with file_lock(ctx.file_path):
+            logger.info("Acquired file lock for %s", ctx.file_path)
+
+            # create or retrieve RawPdfFile + update history
+            ctx.current_report, needs_processing = create_or_retrieve_report_file(ctx)
+            ctx.current_report.get_or_create_state()
+            assert(ctx.current_report.state is not None)
+            ctx.current_report = ctx.current_report
+            
+            ctx.retry = retry
+            # Retry is a forced overwrite of needs processing - therefore the retry will cause full deletion of processed files using finalize failure.
+            if retry and needs_processing and not ctx.current_report.state.anonymization_validated:
+                # ensure clean slate for forced reprocessing
+                finalize_failure(ctx)
+                ctx.current_report, needs_processing = create_or_retrieve_report_file(ctx)
+                assert(needs_processing is True)
+            elif not needs_processing and not retry:
+                return ctx.current_report
+
+            mark_instance_processing_started(ctx.current_report, ctx)
+            try:
+                # --- Anonymization with fallback ---
+                try:
+                    ctx = self.anonymizer.anonymize_report(ctx)
+                    logger.info(
+                        "Primary report anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+                except Exception as primary_exc:
+                    logger.exception(
+                        "Primary report anonymization failed for %s: %s "
+                        "- trying basic anonymization",
+                        ctx.file_path,
+                        primary_exc,
+                    )
+                    try:
+                        ctx = self.anonymizer.anonymize_report(ctx)
+                    except Exception as e:
+                        logger.error(f"PDF Extraction failed for the second time. {e}")
+                        raise
+
+                    logger.info(
+                        "Basic report anonymization succeeded for %s",
+                        ctx.file_path,
+                    )
+
+                # --- Finalize success: history + move anonymized file ---
+                finalize_report_success(ctx)
+
+                return ctx.current_report
+
+            except Exception as exc:
+                logger.exception(
+                    "Report import/anonymization failed for %s: %s", ctx.file_path, exc
+                )
+                # mark failure in history
+                finalize_failure(ctx)
+                raise
+                
+