endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/import_files/file_storage/state_management.py

@@ -1,6 +1,7 @@
-from endoreg_db.models.media.processing_history.processing_history import ProcessingHistory
-from endoreg_db.utils.paths import IMPORT_REPORT_DIR, IMPORT_VIDEO_DIR, ANONYM_REPORT_DIR, ANONYM_VIDEO_DIR
-
+from endoreg_db.models.state.processing_history.processing_history import (
+    ProcessingHistory,
+)
+from endoreg_db.utils.paths import ANONYM_REPORT_DIR, ANONYM_VIDEO_DIR
 
 import logging
 import shutil
@@ -13,11 +14,24 @@ from endoreg_db.import_files.context.import_context import ImportContext
 from endoreg_db.models.media import RawPdfFile, VideoFile
 from endoreg_db.models.state import RawPdfState, VideoState
 from endoreg_db.utils import paths as path_utils
+from endoreg_db.utils.file_operations import sha256_file
 
 logger = logging.getLogger(__name__)
 
 
-def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[Union[RawPdfState, VideoState]]:
+def _get_history_filename(ctx: ImportContext) -> str:
+    """
+    Prefer original_path.name if provided, otherwise fall back to file_path.name.
+    """
+    if ctx.original_path is not None:
+        return ctx.original_path.name
+    # ctx.file_path is always present and already a Path in your tests
+    return Path(ctx.file_path).name
+
+
+def _ensure_instance_state(
+    instance: Union[VideoFile, RawPdfFile],
+) -> Optional[Union[RawPdfState, VideoState]]:
     """
     Helper: ensure instance.state exists and return it.
     Mirrors PdfImportService._ensure_state.
@@ -26,7 +40,7 @@ def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[U
         state = getattr(instance, "state", None)
     else:
         state = getattr(instance, "state", None)
-        
+
     if state is not None:
         return state
 
@@ -37,14 +51,15 @@ def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[U
 
     return None
 
-def mark_instance_processing_started(    
+
+def mark_instance_processing_started(
     instance: Union[RawPdfFile, VideoFile],
-    ctx: ImportContext,):
+    ctx: ImportContext,
+):
     state = _ensure_instance_state(instance)
 
     with transaction.atomic():
         if state is not None:
-
             # In the old code, processing_started was set earlier; we guard here
             if not getattr(state, "processing_started", False) and hasattr(
                 state, "mark_processing_started"
@@ -125,28 +140,12 @@ def finalize_report_success(
             if current_name != relative_name:
                 instance.processed_file.name = relative_name
                 logger.info("Updated processed_file to %s", relative_name)
-        try:
-            relative_name = str(ctx.anonymized_path)
-        except ValueError:
-            # Fallback: absolute path if outside STORAGE_DIR
-            relative_name = str(final_path)
-
-        current_name = getattr(instance.processed_file, "name", None)
-        if current_name != relative_name:
-            instance.processed_file.name = relative_name
-            logger.info(
-                "Updated processed_file reference to: %s",
-                instance.processed_file.name,
-            )
-
 
     # --- Update RawPdfState flags (mirrors _finalize_processing) ---
     state = _ensure_instance_state(instance)
 
     with transaction.atomic():
         if state is not None:
-
-            # In the old code, processing_started was set earlier; we guard here
             if not getattr(state, "processing_started", False) and hasattr(
                 state, "mark_processing_started"
             ):
@@ -165,17 +164,20 @@ def finalize_report_success(
     # --- ProcessingHistory entry ---
     try:
         with transaction.atomic():
-            ProcessingHistory.get_or_create_for_object(
+            if not isinstance(ctx.file_hash, str):
+                ctx.file_hash = sha256_file(ctx.file_path)
+            ProcessingHistory.get_or_create_for_hash(
                 obj=instance,
+                file_hash=ctx.file_hash,
                 success=True,
             )
     except Exception as e:
         logger.debug(
-            "Saving not possible; %s"
-            f"skipping ProcessingHistory.{e}",
+            f"Saving not possible; %sskipping ProcessingHistory.{e}",
             instance.pk,
         )
 
+
 def finalize_video_success(
     ctx: ImportContext,
 ) -> None:
@@ -187,6 +189,7 @@ def finalize_video_success(
     - Mark VideoState as anonymized + sensitive_meta_processed
     - Mark ProcessingHistory.success = True
     """
+
     instance = ctx.current_video
     if not isinstance(instance, VideoFile):
         logger.warning("finalize_video_success called with non-VideoFile instance")
@@ -263,6 +266,11 @@ def finalize_video_success(
                 instance.processed_file.name = relative_name
                 logger.info("Updated video processed_file to %s", relative_name)
 
+    if not nuke_transcoding_dir():
+        logger.warning(
+            "Transcoding directory cleanup returned False after finalize_video_success; there may be leftover files."
+        )
+
     # --- Update VideoState flags (mirrors report) ---
     state = _ensure_instance_state(instance)
 
@@ -285,8 +293,10 @@ def finalize_video_success(
     # --- ProcessingHistory entry ---
     try:
         with transaction.atomic():
-            ProcessingHistory.get_or_create_for_object(
-                obj=instance,
+            if not isinstance(ctx.file_hash, str):
+                ctx.file_hash = sha256_file(ctx.file_path)
+            ProcessingHistory.get_or_create_for_hash(
+                file_hash=ctx.file_hash,
                 success=True,
             )
     except Exception as e:
@@ -305,7 +315,9 @@ def finalize_failure(
 
     - Reset RawPdfState flags to "not processed"
     - Mark ProcessingHistory.success = False
+    - Delete all associated files
     """
+
     if ctx.instance is None:
         if isinstance(ctx.current_report, RawPdfFile):
             ctx.instance = ctx.current_report
@@ -313,6 +325,15 @@ def finalize_failure(
             ctx.instance = ctx.current_video
         else:
             raise Exception
+
+    # History entry with success=False
+    if not isinstance(ctx.file_hash, str):
+        ctx.file_hash = sha256_file(ctx.file_path)
+    ProcessingHistory.get_or_create_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
+    )
+
     # Reset state flags similar to _mark_processing_incomplete / _cleanup_on_error
     state = _ensure_instance_state(ctx.instance)
 
@@ -332,69 +353,111 @@ def finalize_failure(
                 e,
             )
 
-        try:
-            delete_associated_files(ctx)
-        except Exception as e:
-            logger.warning(f"There might be files remaining. {e}")
-
-    # History entry with success=False
-    if ctx.file_hash:
-        ProcessingHistory.get_or_create_for_object(
-            obj=ctx.instance,
-            success=False,
-        )
-    else:
-        logger.debug(
-            "No file_hash in context for instance %s when finalizing failure; "
-            "skipping ProcessingHistory.",
-            ctx.instance.pk,
-        )
+    try:
+        delete_associated_files(ctx)
+    except Exception as e:
+        logger.warning(f"There might be files remaining. {e}")
 
     logger.error(
-        "Report processing failed for %s",
+        "File processing failed for %s - state reset, ready for retry.",
         ctx.file_path,
     )
 
-def delete_associated_files(ctx:ImportContext):
-    try:
-        assert isinstance(ctx.original_path, Path)
-    except AssertionError as e:
-        logger.warning(f"Original file restored from sensitive copy. This is because the original file is gone. {e}")
-        if ctx.file_type =="video":
-            if isinstance(ctx.sensitive_path, Path):
-                try:
-                    ctx.original_path = Path(shutil.copy2(ctx.sensitive_path, IMPORT_VIDEO_DIR))
-                except Exception as e:
-                    logger.error(f"Error during safety copy: {e} Original file could not be restored!")
-                    raise
-        elif ctx.file_type == "report":
-            if isinstance(ctx.sensitive_path, Path):
-                try:
-                    ctx.original_path = Path(shutil.copy2(ctx.sensitive_path, IMPORT_REPORT_DIR))
-                except Exception as e:
-                    logger.error(f"Error during safety copy: {e} Original file could not be restored!")
-                    raise
-        
+
+def delete_associated_files(ctx: ImportContext) -> None:
+    """
+    Best-effort cleanup of anonymized, sensitive and transcoding artefacts.
+
+    - Ensure ctx.original_path points to an existing import file; if not, try to restore
+      from ctx.sensitive_path into the appropriate IMPORT_*_DIR.
+    - Delete anonymized file (if any).
+    - Nuke transcoding directory.
+    - Delete sensitive file (if any).
+
+    This function should *not* raise on non-critical cleanup errors; it logs instead.
+    Only restoration of the original import file is treated as critical.
+    """
+
+    # --- Delete anonymized file (best-effort) ---
     if isinstance(ctx.anonymized_path, Path):
         try:
-            Path.unlink(ctx.anonymized_path)
-            assert(ctx.anonymized_path is not Path)
-
+            if ctx.anonymized_path.exists():
+                ctx.anonymized_path.unlink()
+                logger.info("Deleted anonymized file %s", ctx.anonymized_path)
         except Exception as e:
-            logger.error(f"Error when unlinking anonymized path. {e}")
-            raise
-        
-    ctx.anonymized_path = None
-    
+            logger.error(
+                "Error when unlinking anonymized path %s: %s",
+                ctx.anonymized_path,
+                e,
+                exc_info=True,
+            )
+        finally:
+            ctx.anonymized_path = None
+
+    # --- Nuke transcoding directory (best-effort) ---
+    if not nuke_transcoding_dir():
+        logger.warning(
+            "Transcoding directory cleanup returned False; there may be leftover files."
+        )
+
+    # --- Delete sensitive file (best-effort) ---
     if isinstance(ctx.sensitive_path, Path):
         try:
-            Path.unlink(ctx.sensitive_path)
-            assert(ctx.sensitive_path is not Path)
-            
+            if ctx.sensitive_path.exists():
+                ctx.sensitive_path.unlink()
+                logger.info("Deleted sensitive file %s", ctx.sensitive_path)
         except Exception as e:
-            logger.error(f"Error when unlinking anonymized path. {e}")
+            logger.error(
+                "Error when unlinking sensitive path %s: %s",
+                ctx.sensitive_path,
+                e,
+                exc_info=True,
+            )
+        finally:
+            ctx.sensitive_path = None
+
+
+def nuke_transcoding_dir(transcoding_dir: Union[str, Path, None] = None) -> bool:
+    """
+    Delete all files and subdirectories inside the transcoding directory.
+
+    Returns:
+        True if the directory was either empty / successfully cleaned,
+        False if something went wrong (error is logged).
+    """
+    try:
+        if transcoding_dir is None:
+            transcoding_dir = path_utils.data_paths["transcoding"]
+
+        transcoding_dir = Path(transcoding_dir)
 
-            raise
-        
-    ctx.sensitive_path = None
-    
+        if not transcoding_dir.exists():
+            logger.info(
+                "Transcoding dir %s does not exist; nothing to clean.", transcoding_dir
+            )
+            return True
+
+        if not transcoding_dir.is_dir():
+            logger.error(
+                "Configured transcoding path %s is not a directory.", transcoding_dir
+            )
+            return False
+
+        for entry in transcoding_dir.iterdir():
+            try:
+                if entry.is_file() or entry.is_symlink():
+                    entry.unlink()
+                elif entry.is_dir():
+                    shutil.rmtree(entry)
+            except Exception as e:
+                logger.warning(
+                    "Failed to remove entry %s in transcoding dir: %s", entry, e
+                )
+                # Continue trying to delete other entries
+        return True
+
+    except Exception as e:
+        logger.error(
+            "Unexpected error while nuking transcoding dir: %s", e, exc_info=True
+        )
+        return False

endoreg_db/import_files/file_storage/storage.py

@@ -1,6 +1,10 @@
 import logging
 from pathlib import Path
-from endoreg_db.models.media.video.create_from_file import atomic_copy_with_fallback, atomic_move_with_fallback
+from endoreg_db.models.media.video.create_from_file import (
+    atomic_copy_with_fallback,
+    atomic_move_with_fallback,
+)
+
 logger = logging.getLogger(__name__)
 
 

endoreg_db/import_files/processing/report_processing/report_anonymization.py

@@ -2,13 +2,14 @@ import logging
 import os
 import sys
 from pathlib import Path
-from typing import Any, Callable, Literal, NoReturn
 
 from lx_anonymizer import ReportReader
 from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
 
 from endoreg_db.import_files.context import ImportContext
-from endoreg_db.import_files.file_storage.sensitive_meta_storage import sensitive_meta_storage
+from endoreg_db.import_files.file_storage.sensitive_meta_storage import (
+    sensitive_meta_storage,
+)
 from endoreg_db.utils.paths import ANONYM_REPORT_DIR
 
 
@@ -20,10 +21,8 @@ class ReportAnonymizer:
         self._report_reader_class = None
         self._ensure_report_reading_available()
         self.storage = False
-        
 
     def anonymize_report(self, ctx: ImportContext):
-
         # Setup anonymized directory
         anonymized_dir = ANONYM_REPORT_DIR
         anonymized_dir.mkdir(parents=True, exist_ok=True)
@@ -32,29 +31,35 @@ class ReportAnonymizer:
         pdf_hash = ctx.current_report.pdf_hash
         anonymized_output_path = anonymized_dir / f"{pdf_hash}.pdf"
         self._report_reader_class = ReportReader()
-        
+
         assert isinstance(self._report_reader_class, ReportReader)
 
         # Process with enhanced process_report method (returns 4-tuple now)
-        ctx.original_text, ctx.anonymized_text, extracted_metadata, ctx.anonymized_path = self._report_reader_class.process_report(
-                pdf_path=ctx.file_path,
-                create_anonymized_pdf=True,
-                anonymized_pdf_output_path=str(anonymized_output_path),
-            )
-        
+        (
+            ctx.original_text,
+            ctx.anonymized_text,
+            extracted_metadata,
+            ctx.anonymized_path,
+        ) = self._report_reader_class.process_report(
+            pdf_path=ctx.file_path,
+            create_anonymized_pdf=True,
+            anonymized_pdf_output_path=str(anonymized_output_path),
+        )
+
         if ctx.anonymized_path:
-            logger.info("DEBUG: after anonymizer, ctx.anonymized_path=%s (exists=%s)",
-                        ctx.anonymized_path, isinstance(ctx.anonymized_path, str))
+            logger.info(
+                "DEBUG: after anonymizer, ctx.anonymized_path=%s (exists=%s)",
+                ctx.anonymized_path,
+                isinstance(ctx.anonymized_path, str),
+            )
 
         sm = LxSM()
-        sm.safe_update(extracted_metadata)        
-        
+        sm.safe_update(extracted_metadata)
+
         self.storage = sensitive_meta_storage(sm, ctx.current_report)
         return ctx
-    
-    def _ensure_report_reading_available(
-        self
-    )  -> None:
+
+    def _ensure_report_reading_available(self) -> None:
         """
         Ensure report reading modules are available by adding lx-anonymizer to path.
 

endoreg_db/import_files/processing/sensitive_meta_adapter.py

@@ -21,10 +21,10 @@ def normalize_lx_sensitive_meta(meta: LxSensitiveMeta) -> Dict[str, Any]:
         "file_path",
         "patient_first_name",
         "patient_last_name",
-        "patient_dob",          # string; logic has parsing
+        "patient_dob",  # string; logic has parsing
         "casenumber",
-        "examination_date",     # string; logic has parsing
-        "examination_time",     # string "HH:MM" is fine
+        "examination_date",  # string; logic has parsing
+        "examination_time",  # string "HH:MM" is fine
         "examiner_first_name",
         "examiner_last_name",
         "text",

endoreg_db/import_files/processing/video_processing/video_anonymization.py

@@ -1,16 +1,16 @@
-from typing import List, Dict, Any, Tuple, Optional
-
 import logging
+
 logger = logging.getLogger(__name__)
 
 from lx_anonymizer import FrameCleaner
 from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
 
-
-from endoreg_db.import_files.file_storage.sensitive_meta_storage import sensitive_meta_storage
 from endoreg_db.import_files.context import ImportContext
-from endoreg_db.utils.paths import ANONYM_VIDEO_DIR
+from endoreg_db.import_files.file_storage.sensitive_meta_storage import (
+    sensitive_meta_storage,
+)
 from endoreg_db.models import EndoscopyProcessor, VideoFile
+from endoreg_db.utils.paths import ANONYM_VIDEO_DIR
 
 
 class VideoAnonymizer:
@@ -20,35 +20,35 @@ class VideoAnonymizer:
         self._frame_cleaning_class = None
         self.storage = False
 
-        
     def anonymize_video(self, ctx: ImportContext):
         # Setup anonymized directory
         anonymized_dir = ANONYM_VIDEO_DIR
         anonymized_dir.mkdir(parents=True, exist_ok=True)
         assert ctx.current_video is not None
         # Generate output path for anonymized report
-        
+
         video_hash = ctx.current_video.video_hash
         anonymized_output_path = anonymized_dir / f"{video_hash}.mp4"
-        
+
         self._frame_cleaning_class = FrameCleaner()
-        
+
         assert isinstance(self._frame_cleaning_class, FrameCleaner)
         endoscope_roi, endoscope_roi_nested = self._get_processor_roi_info(ctx)
         # Process with enhanced process_report method (returns 4-tuple now)
-        ctx.anonymized_path, extracted_metadata = self._frame_cleaning_class.clean_video(
+        ctx.anonymized_path, extracted_metadata = (
+            self._frame_cleaning_class.clean_video(
                 video_path=ctx.file_path,
                 endoscope_image_roi=endoscope_roi,
                 endoscope_data_roi_nested=endoscope_roi_nested,
-                output_path=anonymized_output_path
-                
+                output_path=anonymized_output_path,
             )
+        )
         sm = LxSM()
         sm.safe_update(extracted_metadata)
-        
+
         self.storage = sensitive_meta_storage(sm, ctx.current_video)
         return ctx
-                    
+
     def _ensure_frame_cleaning_available(self):
         """
         Ensure frame cleaning modules are available by adding lx-anonymizer to path.
@@ -68,12 +68,12 @@ class VideoAnonymizer:
         self._frame_cleaning_class = FrameCleaner()
         self._frame_cleaning_available = True
 
-
     def _get_processor_roi_info(
         self,
         ctx: ImportContext,
-    ) -> tuple[dict[str, int | None] | None,
-            dict[str, dict[str, int | None] | None] | None]:
+    ) -> tuple[
+        dict[str, int | None] | None, dict[str, dict[str, int | None] | None] | None
+    ]:
         """Get processor ROI information for masking and data extraction."""
         endoscope_data_roi_nested = None
         endoscope_image_roi = None
@@ -98,7 +98,7 @@ class VideoAnonymizer:
             else:
                 logger.warning(
                     "No processor found for video %s, proceeding without ROI masking",
-                    video.uuid,
+                    video.video_hash,
                 )
         except Exception as exc:
             logger.error("Failed to retrieve processor ROI information: %s", exc)

endoreg_db/import_files/pseudonymization/k_anonymity.py

@@ -1,4 +1,4 @@
-from endoreg_db.models import Patient, SensitiveMeta, Center, Gender
+from endoreg_db.models import SensitiveMeta
 import logging
 from datetime import timedelta
 from typing import Tuple
@@ -6,7 +6,7 @@ from typing import Tuple
 from django.db.models import QuerySet
 
 from itertools import combinations
-from typing import Dict, Tuple, List
+from typing import Dict
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +68,7 @@ def get_k_anonymity(pk, k=3):
         pk (_type_): _description_
         k (int, optional): _description_. Defaults to 3.
     """
-    return get_k_anonymity_for_sensitive_meta(pk=pk, k=k, dob_year_tolerance=1) 
+    return get_k_anonymity_for_sensitive_meta(pk=pk, k=k, dob_year_tolerance=1)
 
 
 def _build_sensitive_meta_qi_queryset(
@@ -99,20 +99,19 @@ def _build_sensitive_meta_qi_queryset(
         A Django QuerySet for further aggregation.
     """
     qs = SensitiveMeta.objects.all()
-    
+
     if use_first_name and instance.patient_first_name is not None:
         qs = qs.filter(patient_first_name=instance.patient_first_name)
-            
-    if use_last_name and instance.patient_first_name is not None:
-        qs = qs.filter(patient_first_name=instance.patient_first_name)
-    
+
+    if use_last_name and instance.patient_last_name is not None:
+        qs = qs.filter(patient_last_name=instance.patient_last_name)
     # --- Center ---
     if use_center and instance.center is not None:
         if instance.center.pk is not None:
             qs = qs.filter(center=instance.center.pk)
 
     # --- Gender ---
-    if use_gender and instance.patient_gender is not None
+    if use_gender and instance.patient_gender is not None:
         if instance.patient_gender.pk is not None:
             qs = qs.filter(patient_gender_id=instance.patient_gender)
 

endoreg_db/import_files/pseudonymization/k_pseudonymity.py

@@ -1,13 +1,16 @@
 from typing import Optional, Tuple
+import logging
 
 from datetime import date as Date
-import datetime
 
 from .k_anonymity import _build_sensitive_meta_qi_queryset
 from .fake import fake_name_with_similar_dob_and_gender
 
 from endoreg_db.models import SensitiveMeta
 
+logger = logging.getLogger(__name__)
+
+
 def k_pseudonymize(
     instance: SensitiveMeta,
     *,
@@ -44,10 +47,16 @@ def k_pseudonymize(
     Returns:
         (instance, k_value_after, is_k_anonymous_after)
     """
-    if qi_subset is None:
-        qi_subset = tuple(QI_FLAGS)
 
     # --- 1) Compute k for the requested subset BEFORE pseudonymization ---
+    if qi_subset is None:
+        qi_subset = ("first_name", "last_name", "center", "gender", "dob_band")
+        # --- 1) Compute k for the requested subset BEFORE pseudonymization ---
+        use_first_name = "first_name" in qi_subset
+        use_last_name = "last_name" in qi_subset
+        use_center = "center" in qi_subset
+        use_gender = "gender" in qi_subset
+        use_dob_band = "dob_band" in qi_subset
     use_first_name = "first_name" in qi_subset
     use_last_name = "last_name" in qi_subset
     use_center = "center" in qi_subset
@@ -95,12 +104,14 @@ def k_pseudonymize(
     # Assign to instance (SensitiveMeta.patient_dob is a DateTimeField)
     instance.patient_first_name = first_name
     instance.patient_last_name = last_name
-    instance.patient_dob = datetime(
+    instance.patient_dob = Date(
         fake_dob.year, fake_dob.month, fake_dob.day
     )  # naive is usually fine for DOB
 
     if save:
-        instance.save(update_fields=["patient_first_name", "patient_last_name", "patient_dob"])
+        instance.save(
+            update_fields=["patient_first_name", "patient_last_name", "patient_dob"]
+        )
 
     # --- 3) Recompute k AFTER pseudonymization ---
     qs_after = _build_sensitive_meta_qi_queryset(