endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/import_files/file_storage/state_management.py

@@ -0,0 +1,463 @@
+from endoreg_db.models.state.processing_history.processing_history import (
+    ProcessingHistory,
+)
+from endoreg_db.utils.paths import ANONYM_REPORT_DIR, ANONYM_VIDEO_DIR
+
+import logging
+import shutil
+from pathlib import Path
+from typing import Optional, Union
+
+from django.db import transaction
+
+from endoreg_db.import_files.context.import_context import ImportContext
+from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.models.state import RawPdfState, VideoState
+from endoreg_db.utils import paths as path_utils
+from endoreg_db.utils.file_operations import sha256_file
+
+logger = logging.getLogger(__name__)
+
+
+def _get_history_filename(ctx: ImportContext) -> str:
+    """
+    Prefer original_path.name if provided, otherwise fall back to file_path.name.
+    """
+    if ctx.original_path is not None:
+        return ctx.original_path.name
+    # ctx.file_path is always present and already a Path in your tests
+    return Path(ctx.file_path).name
+
+
+def _ensure_instance_state(
+    instance: Union[VideoFile, RawPdfFile],
+) -> Optional[Union[RawPdfState, VideoState]]:
+    """
+    Helper: ensure instance.state exists and return it.
+    Mirrors PdfImportService._ensure_state.
+    """
+    if isinstance(instance, RawPdfFile):
+        state = getattr(instance, "state", None)
+    else:
+        state = getattr(instance, "state", None)
+
+    if state is not None:
+        return state
+
+    if hasattr(instance, "get_or_create_state"):
+        state = instance.get_or_create_state()
+        instance.save()
+        return state
+
+    return None
+
+
+def mark_instance_processing_started(
+    instance: Union[RawPdfFile, VideoFile],
+    ctx: ImportContext,
+):
+    state = _ensure_instance_state(instance)
+
+    with transaction.atomic():
+        if state is not None:
+            # In the old code, processing_started was set earlier; we guard here
+            if not getattr(state, "processing_started", False) and hasattr(
+                state, "mark_processing_started"
+            ):
+                state.mark_processing_started()
+
+
+def finalize_report_success(
+    ctx: ImportContext,
+) -> None:
+    """
+    Finalize a successful instance import/anonymization.
+
+    - Move anonymized Report from temp to canonical anonymized dir
+    - Update RawPdfFile.processed_file and .anonymized flag
+    - Mark RawPdfState as anonymized + sensitive_meta_processed
+    - Mark ProcessingHistory.success = True
+    """
+    instance = ctx.current_report
+    if not isinstance(instance, RawPdfFile):
+        logger.warning("finalize_success called with unsaved instance")
+        return
+    if not instance.pk:
+        logger.warning("finalize_success called with unsaved instance")
+        return
+
+    # --- Move anonymized path into final storage (if we have one) ---
+    final_path: Optional[Path] = None
+    if ctx.anonymized_path is None:
+        logger.warning(
+            "No anonymized_path for instance %s (hash=%s); skipping file move.",
+            instance.pk,
+            getattr(instance, "pdf_hash", None),
+        )
+        final_path = None
+    else:
+        pdf_hash = getattr(instance, "pdf_hash", None) or instance.pk
+        expected_final_path = ANONYM_REPORT_DIR / f"{pdf_hash}.pdf"
+
+        src = Path(ctx.anonymized_path)
+
+        logger.debug(
+            "finalize_report_success: src=%s (exists=%s, resolved=%s), expected_final=%s",
+            src,
+            src.exists(),
+            src.resolve(),
+            expected_final_path,
+        )
+
+        # If anonymizer already wrote to the final path, don't move
+        if src.resolve() == expected_final_path.resolve():
+            logger.info(
+                "Anonymizer output already at final path %s; skipping move.",
+                expected_final_path,
+            )
+            final_path = expected_final_path
+        else:
+            # Only move if the source actually exists
+            if not src.exists():
+                logger.error(
+                    "Anonymized file %s does not exist; cannot move to %s",
+                    src,
+                    expected_final_path,
+                )
+                final_path = None
+            else:
+                ANONYM_REPORT_DIR.mkdir(parents=True, exist_ok=True)
+                if expected_final_path.exists():
+                    expected_final_path.unlink()
+                shutil.move(str(src), str(expected_final_path))
+                final_path = expected_final_path
+                logger.info("Moved anonymized report to %s", final_path)
+
+        # Update FileField if we have a final path
+        if final_path is not None:
+            relative_name = path_utils.to_storage_relative(final_path)
+            current_name = getattr(instance.processed_file, "name", None)
+            if current_name != relative_name:
+                instance.processed_file.name = relative_name
+                logger.info("Updated processed_file to %s", relative_name)
+
+    # --- Update RawPdfState flags (mirrors _finalize_processing) ---
+    state = _ensure_instance_state(instance)
+
+    with transaction.atomic():
+        if state is not None:
+            if not getattr(state, "processing_started", False) and hasattr(
+                state, "mark_processing_started"
+            ):
+                state.mark_processing_started()
+
+            # We consider text/meta extraction + anonymization done at this point
+            if hasattr(state, "mark_anonymized"):
+                state.mark_anonymized()
+            if hasattr(state, "mark_sensitive_meta_processed"):
+                state.mark_sensitive_meta_processed()
+
+            state.save()
+
+        instance.save()
+
+    # --- ProcessingHistory entry ---
+    try:
+        with transaction.atomic():
+            if not isinstance(ctx.file_hash, str):
+                ctx.file_hash = sha256_file(ctx.file_path)
+            ProcessingHistory.get_or_create_for_hash(
+                obj=instance,
+                file_hash=ctx.file_hash,
+                success=True,
+            )
+    except Exception as e:
+        logger.debug(
+            f"Saving not possible; %sskipping ProcessingHistory.{e}",
+            instance.pk,
+        )
+
+
+def finalize_video_success(
+    ctx: ImportContext,
+) -> None:
+    """
+    Finalize a successful video import/anonymization.
+
+    - Move anonymized video from temp to canonical anonymized dir
+    - Update VideoFile.processed_file
+    - Mark VideoState as anonymized + sensitive_meta_processed
+    - Mark ProcessingHistory.success = True
+    """
+
+    instance = ctx.current_video
+    if not isinstance(instance, VideoFile):
+        logger.warning("finalize_video_success called with non-VideoFile instance")
+        return
+    if not instance.pk:
+        logger.warning("finalize_video_success called with unsaved instance")
+        return
+
+    # --- Move anonymized path into final storage (if we have one) ---
+    final_path: Optional[Path] = None
+
+    if ctx.anonymized_path is None:
+        logger.warning(
+            "No anonymized_path for video instance %s (hash=%s); skipping file move.",
+            instance.pk,
+            getattr(instance, "video_hash", None),
+        )
+    else:
+        # Use a stable naming convention: <video_hash>.mp4
+        video_hash = getattr(instance, "video_hash", None) or instance.pk
+        expected_final_path = ANONYM_VIDEO_DIR / f"{video_hash}.mp4"
+
+        src = Path(ctx.anonymized_path)
+
+        logger.debug(
+            "finalize_video_success: src=%s (exists=%s, resolved=%s), expected_final=%s",
+            src,
+            src.exists(),
+            src.resolve(),
+            expected_final_path,
+        )
+
+        # If anonymizer already wrote to the final path, don't move
+        try:
+            same_target = src.resolve() == expected_final_path.resolve()
+        except FileNotFoundError:
+            # src might not exist anymore
+            same_target = False
+
+        if same_target:
+            logger.info(
+                "Anonymizer output already at final video path %s; skipping move.",
+                expected_final_path,
+            )
+            final_path = expected_final_path
+        else:
+            if not src.exists():
+                logger.error(
+                    "Anonymized video %s does not exist; cannot move to %s",
+                    src,
+                    expected_final_path,
+                )
+                final_path = None
+            else:
+                ANONYM_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
+                if expected_final_path.exists():
+                    try:
+                        expected_final_path.unlink()
+                    except Exception as e:
+                        logger.warning(
+                            "Could not remove existing anonymized video %s: %s",
+                            expected_final_path,
+                            e,
+                        )
+                shutil.move(str(src), str(expected_final_path))
+                final_path = expected_final_path
+                logger.info("Moved anonymized video to %s", final_path)
+
+        # Update FileField if we have a final path
+        if final_path is not None:
+            relative_name = path_utils.to_storage_relative(final_path)
+            current_name = getattr(instance.processed_file, "name", None)
+            if current_name != relative_name:
+                instance.processed_file.name = relative_name
+                logger.info("Updated video processed_file to %s", relative_name)
+
+    if not nuke_transcoding_dir():
+        logger.warning(
+            "Transcoding directory cleanup returned False after finalize_video_success; there may be leftover files."
+        )
+
+    # --- Update VideoState flags (mirrors report) ---
+    state = _ensure_instance_state(instance)
+
+    with transaction.atomic():
+        if state is not None:
+            if not getattr(state, "processing_started", False) and hasattr(
+                state, "mark_processing_started"
+            ):
+                state.mark_processing_started()
+
+            if hasattr(state, "mark_anonymized"):
+                state.mark_anonymized()
+            if hasattr(state, "mark_sensitive_meta_processed"):
+                state.mark_sensitive_meta_processed()
+
+            state.save()
+
+        instance.save()
+
+    # --- ProcessingHistory entry ---
+    try:
+        with transaction.atomic():
+            if not isinstance(ctx.file_hash, str):
+                ctx.file_hash = sha256_file(ctx.file_path)
+            ProcessingHistory.get_or_create_for_hash(
+                file_hash=ctx.file_hash,
+                success=True,
+            )
+    except Exception as e:
+        logger.debug(
+            "Saving not possible for video %s; skipping ProcessingHistory. Error: %s",
+            instance.pk,
+            e,
+        )
+
+
+def finalize_failure(
+    ctx: ImportContext,
+) -> None:
+    """
+    Finalize a failed instance import/anonymization.
+
+    - Reset RawPdfState flags to "not processed"
+    - Mark ProcessingHistory.success = False
+    - Delete all associated files
+    """
+
+    if ctx.instance is None:
+        if isinstance(ctx.current_report, RawPdfFile):
+            ctx.instance = ctx.current_report
+        elif isinstance(ctx.current_video, VideoFile):
+            ctx.instance = ctx.current_video
+        else:
+            raise Exception
+
+    # History entry with success=False
+    if not isinstance(ctx.file_hash, str):
+        ctx.file_hash = sha256_file(ctx.file_path)
+    ProcessingHistory.get_or_create_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
+    )
+
+    # Reset state flags similar to _mark_processing_incomplete / _cleanup_on_error
+    state = _ensure_instance_state(ctx.instance)
+
+    if state is not None:
+        try:
+            state.mark_processing_not_started()
+
+            state.save()
+            logger.info(
+                "Reset instance state for failed processing (instance pk=%s)",
+                ctx.instance.pk,
+            )
+        except Exception as e:
+            logger.warning(
+                "Failed to reset State for instance %s: %s",
+                ctx.instance.pk,
+                e,
+            )
+
+    try:
+        delete_associated_files(ctx)
+    except Exception as e:
+        logger.warning(f"There might be files remaining. {e}")
+
+    logger.error(
+        "File processing failed for %s - state reset, ready for retry.",
+        ctx.file_path,
+    )
+
+
+def delete_associated_files(ctx: ImportContext) -> None:
+    """
+    Best-effort cleanup of anonymized, sensitive and transcoding artefacts.
+
+    - Ensure ctx.original_path points to an existing import file; if not, try to restore
+      from ctx.sensitive_path into the appropriate IMPORT_*_DIR.
+    - Delete anonymized file (if any).
+    - Nuke transcoding directory.
+    - Delete sensitive file (if any).
+
+    This function should *not* raise on non-critical cleanup errors; it logs instead.
+    Only restoration of the original import file is treated as critical.
+    """
+
+    # --- Delete anonymized file (best-effort) ---
+    if isinstance(ctx.anonymized_path, Path):
+        try:
+            if ctx.anonymized_path.exists():
+                ctx.anonymized_path.unlink()
+                logger.info("Deleted anonymized file %s", ctx.anonymized_path)
+        except Exception as e:
+            logger.error(
+                "Error when unlinking anonymized path %s: %s",
+                ctx.anonymized_path,
+                e,
+                exc_info=True,
+            )
+        finally:
+            ctx.anonymized_path = None
+
+    # --- Nuke transcoding directory (best-effort) ---
+    if not nuke_transcoding_dir():
+        logger.warning(
+            "Transcoding directory cleanup returned False; there may be leftover files."
+        )
+
+    # --- Delete sensitive file (best-effort) ---
+    if isinstance(ctx.sensitive_path, Path):
+        try:
+            if ctx.sensitive_path.exists():
+                ctx.sensitive_path.unlink()
+                logger.info("Deleted sensitive file %s", ctx.sensitive_path)
+        except Exception as e:
+            logger.error(
+                "Error when unlinking sensitive path %s: %s",
+                ctx.sensitive_path,
+                e,
+                exc_info=True,
+            )
+        finally:
+            ctx.sensitive_path = None
+
+
+def nuke_transcoding_dir(transcoding_dir: Union[str, Path, None] = None) -> bool:
+    """
+    Delete all files and subdirectories inside the transcoding directory.
+
+    Returns:
+        True if the directory was either empty / successfully cleaned,
+        False if something went wrong (error is logged).
+    """
+    try:
+        if transcoding_dir is None:
+            transcoding_dir = path_utils.data_paths["transcoding"]
+
+        transcoding_dir = Path(transcoding_dir)
+
+        if not transcoding_dir.exists():
+            logger.info(
+                "Transcoding dir %s does not exist; nothing to clean.", transcoding_dir
+            )
+            return True
+
+        if not transcoding_dir.is_dir():
+            logger.error(
+                "Configured transcoding path %s is not a directory.", transcoding_dir
+            )
+            return False
+
+        for entry in transcoding_dir.iterdir():
+            try:
+                if entry.is_file() or entry.is_symlink():
+                    entry.unlink()
+                elif entry.is_dir():
+                    shutil.rmtree(entry)
+            except Exception as e:
+                logger.warning(
+                    "Failed to remove entry %s in transcoding dir: %s", entry, e
+                )
+                # Continue trying to delete other entries
+        return True
+
+    except Exception as e:
+        logger.error(
+            "Unexpected error while nuking transcoding dir: %s", e, exc_info=True
+        )
+        return False

endoreg_db/import_files/file_storage/storage.py

@@ -0,0 +1,42 @@
+import logging
+from pathlib import Path
+from endoreg_db.models.media.video.create_from_file import (
+    atomic_copy_with_fallback,
+    atomic_move_with_fallback,
+)
+from endoreg_db.utils import transcode_videofile_if_required
+logger = logging.getLogger(__name__)
+
+
+def ensure_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def create_sensitive_copy(src: Path, sensitive_root: Path) -> Path:
+    """
+    Create a sensitive copy of `src` in `sensitive_root`.
+
+    Returns:
+        Path to the sensitive copy.
+    """
+    ensure_dir(sensitive_root)
+    dest = sensitive_root / src.name
+    logger.info("Creating sensitive copy: %s -> %s", src, dest)
+    transcode_videofile_if_required(src, dest)
+    atomic_copy_with_fallback(src, dest)
+    return dest
+
+
+def move_to_anonymized(temp_path: Path, anonymized_root: Path) -> Path:
+    """
+    Move a (temporary) anonymized file into the canonical anonymized root.
+
+    Returns:
+        Final path inside anonymized_root.
+    """
+    ensure_dir(anonymized_root)
+    dest = anonymized_root / temp_path.name
+    logger.info("Moving anonymized file: %s -> %s", temp_path, dest)
+    transcode_videofile_if_required(temp_path, dest)
+    atomic_move_with_fallback(temp_path, dest)
+    return dest

endoreg_db/import_files/import_service.md

@@ -0,0 +1,26 @@
+# File Import and Anonymization
+
+Endoreg-Db imports are guarded by a anonymization step, that is supposed to ensure most data is redacted from the input. Here, fake patients are generated to pseudonymize the sensitive information in the data. This ensures, that videos as well as pdfs are not distributed using sensitive data, and if they are by some accident it is harder to know what data is actually real.
+
+The Import is handled by two orchestration files:
+
+Report import service (RIS)
+
+and 
+
+Video import Service (VIS)
+
+The orchestration is abstracted out by the base import service (BIS), to ensure newly implemented data imports follow the same structure and to ensure tests run agnostically of the actual media being processed.
+
+## Import Order of Execution
+
+The Import starts, when files are dropped into the corresponding media import folders. The locations need to be passed to the import service logic. To ensure atomic processing without overwhelming the server or double processing on parallelization, a file lock is added to the files that are currently processed.
+
+### File Lock
+
+File Lock is implemented as a context manager. Per default, this means during the execution the files are marked by adding a additional .lock file path inside the folder. Once the code wrapped in the context manager of file lock stops execution, the .lock file is removed only after error processing. This ensures, the full pipeline is executed on each run even when interrupted. 
+https://book.pythontips.com/en/latest/context_managers.html
+
+### Error Cleanup
+
+The ErrorCleanup class is called from inside the file lock context manager to avoid leaving half processed files laying around. It passes file type to the class instance and then runs the correct processing logic.

endoreg_db/import_files/processing/__init__.py

@@ -0,0 +1,11 @@
+# processing/__init__.py
+
+from . import sensitive_meta_adapter
+from . import report_processing
+from . import video_processing
+
+__all__ = [
+    "sensitive_meta_adapter",
+    "report_processing",
+    "video_processing",
+]

endoreg_db/import_files/processing/report_processing/report_anonymization.py

@@ -0,0 +1,99 @@
+import logging
+import os
+import sys
+from pathlib import Path
+
+from lx_anonymizer import ReportReader
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
+
+from endoreg_db.import_files.context import ImportContext
+from endoreg_db.import_files.file_storage.sensitive_meta_storage import (
+    sensitive_meta_storage,
+)
+from endoreg_db.utils.paths import ANONYM_REPORT_DIR
+
+
+logger = logging.getLogger(__name__)
+
+
+class ReportAnonymizer:
+    def __init__(self):
+        self._report_reader_class = None
+        self._ensure_report_reading_available()
+        self.storage = False
+
+    def anonymize_report(self, ctx: ImportContext):
+        # Setup anonymized directory
+        anonymized_dir = ANONYM_REPORT_DIR
+        anonymized_dir.mkdir(parents=True, exist_ok=True)
+        assert ctx.current_report is not None
+        # Generate output path for anonymized report
+        pdf_hash = ctx.current_report.pdf_hash
+        anonymized_output_path = anonymized_dir / f"{pdf_hash}.pdf"
+        self._report_reader_class = ReportReader()
+
+        assert isinstance(self._report_reader_class, ReportReader)
+
+        # Process with enhanced process_report method (returns 4-tuple now)
+        (
+            ctx.original_text,
+            ctx.anonymized_text,
+            extracted_metadata,
+            ctx.anonymized_path,
+        ) = self._report_reader_class.process_report(
+            pdf_path=ctx.file_path,
+            create_anonymized_pdf=True,
+            anonymized_pdf_output_path=str(anonymized_output_path),
+        )
+
+        if ctx.anonymized_path:
+            logger.info(
+                "DEBUG: after anonymizer, ctx.anonymized_path=%s (exists=%s)",
+                ctx.anonymized_path,
+                isinstance(ctx.anonymized_path, str),
+            )
+
+        sm = LxSM()
+        sm.safe_update(extracted_metadata)
+
+        self.storage = sensitive_meta_storage(sm, ctx.current_report)
+        return ctx
+
+    def _ensure_report_reading_available(self) -> None:
+        """
+        Ensure report reading modules are available by adding lx-anonymizer to path.
+
+        Returns:
+            Tuple of (availability_flag, ReportReader_class)
+        """
+
+        try:
+            # Try direct import first
+            from lx_anonymizer import ReportReader
+
+            logger.info("Successfully imported lx_anonymizer ReportReader module")
+            self._report_reader_available = True
+            self._report_reader_class = ReportReader
+
+        except ImportError:
+            # Optional: honor LX_ANONYMIZER_PATH=/abs/path/to/src
+            import importlib
+
+            extra = os.getenv("LX_ANONYMIZER_PATH")
+            if extra and extra not in sys.path and Path(extra).exists():
+                sys.path.insert(0, extra)
+                try:
+                    mod = importlib.import_module("lx_anonymizer")
+                    ReportReader = getattr(mod, "ReportReader")
+                    logger.info(
+                        "Imported lx_anonymizer.ReportReader via LX_ANONYMIZER_PATH"
+                    )
+                    self._report_reader_available = True
+                    self._report_reader_class = ReportReader
+                except Exception as e:
+                    logger.warning(
+                        "Failed importing lx_anonymizer via LX_ANONYMIZER_PATH: %s", e
+                    )
+
+        self._report_reader_available = False
+        self._report_reader_class = None

endoreg_db/import_files/processing/sensitive_meta_adapter.py

@@ -0,0 +1,51 @@
+# endoreg_db/import_files/processing/sensitive_meta_adapter.py
+from typing import Any, Dict
+
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSensitiveMeta
+
+
+def normalize_lx_sensitive_meta(meta: LxSensitiveMeta) -> Dict[str, Any]:
+    """
+    Convert lx_anonymizer.SensitiveMeta into a dict suitable for
+    endoreg_db SensitiveMeta.update_from_dict / create_from_dict.
+
+    - Renames fields where necessary (center -> center_name, patient_gender_name -> patient_gender)
+    - Drops None/blank values (your update logic already handles blanks carefully)
+    - Leaves dates as strings; your logic layer already parses them
+    """
+    raw = meta.to_dict()
+    out: Dict[str, Any] = {}
+
+    # 1:1 fields (same names in model logic)
+    direct_keys = [
+        "file_path",
+        "patient_first_name",
+        "patient_last_name",
+        "patient_dob",  # string; logic has parsing
+        "casenumber",
+        "examination_date",  # string; logic has parsing
+        "examination_time",  # string "HH:MM" is fine
+        "examiner_first_name",
+        "examiner_last_name",
+        "text",
+        "anonymized_text",
+        "endoscope_type",
+        "endoscope_sn",
+    ]
+    for k in direct_keys:
+        v = raw.get(k)
+        if v not in (None, "", []):
+            out[k] = v
+
+    # Map patient_gender_name (interface) -> patient_gender (logic)
+    gender_name = raw.get("patient_gender_name")
+    if gender_name not in (None, ""):
+        # Your logic.update_* can handle strings for patient_gender
+        out["patient_gender"] = gender_name
+
+    # Map center (string) -> center_name (logic)
+    center_name = raw.get("center")
+    if center_name not in (None, ""):
+        out["center_name"] = center_name
+
+    return out