endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

endoreg_db/import_files/context/validate_directories.py

@@ -0,0 +1,56 @@
+import logging
+from pathlib import Path
+from typing import Iterable
+from endoreg_db.utils.paths import (
+    ANONYM_REPORT_DIR,
+    ANONYM_VIDEO_DIR,
+    IMPORT_REPORT_DIR,
+    IMPORT_VIDEO_DIR,
+    SENSITIVE_REPORT_DIR,
+    SENSITIVE_VIDEO_DIR,
+)
+
+dirs = [
+    ANONYM_REPORT_DIR,
+    ANONYM_VIDEO_DIR,
+    IMPORT_REPORT_DIR,
+    IMPORT_VIDEO_DIR,
+    SENSITIVE_REPORT_DIR,
+    SENSITIVE_VIDEO_DIR,
+]
+
+
+logger = logging.getLogger(__name__)
+
+
+def validate_directories(dirs: Iterable[Path] = dirs) -> bool:
+    """
+    Ensure all directories in `dirs` exist.
+    Missing directories are created automatically.
+
+    Args:
+        dirs: Iterable of Path objects representing directories.
+
+    Returns:
+        bool: True if all directories exist or were created successfully,
+              False if any directory could not be created.
+    """
+    ok = True
+
+    for d in dirs:
+        try:
+            if not d.exists():
+                logger.info(f"Directory missing, creating: {d}")
+                d.mkdir(parents=True, exist_ok=True)
+
+            if not d.is_dir():
+                logger.error(f"Path exists but is not a directory: {d}")
+                ok = False
+
+        except Exception as e:
+            logger.error(f"Failed to create or validate directory '{d}': {e}")
+            ok = False
+
+    return ok
+
+validate_directories(dirs)

endoreg_db/import_files/file_storage/__init__.py

@@ -0,0 +1,15 @@
+# storage/__init__.py
+
+from . import create_report_file
+from . import create_video_file
+from . import sensitive_meta_storage
+from . import state_management
+from . import storage
+
+__all__ = [
+    "create_report_file",
+    "create_video_file",
+    "sensitive_meta_storage",
+    "state_management",
+    "storage",
+]

endoreg_db/import_files/file_storage/create_report_file.py

@@ -0,0 +1,76 @@
+# endoreg_db/import_files/storage/create_report_file.py
+import logging
+from typing import Tuple
+
+from endoreg_db.import_files.context.import_context import ImportContext
+from endoreg_db.models.media import RawPdfFile
+from endoreg_db.models.media.processing_history.processing_history import ProcessingHistory
+from endoreg_db.import_files.context.ensure_center import ensure_center
+logger = logging.getLogger(__name__)
+
+
+def create_or_retrieve_report_file(
+    ctx: ImportContext,
+) -> Tuple[RawPdfFile, bool]:
+    """
+    Create a new or retrieve an existing RawPdfFile for the given context.
+
+    Returns:
+        pdf             : RawPdfFile instance
+        needs_processing: True if the pipeline should run for this file
+                          (no successful history yet for this object/file_type key)
+    """
+    file_path = ctx.file_path
+    center_name = ctx.center_name
+    delete_source = ctx.delete_source
+    file_type = ctx.file_type  # logical key for history; can be None
+
+    # 1) Determine the RawPdfFile instance to work with
+    if ctx.current_report is not None:
+        pdf = ctx.current_report
+        logger.info("Using existing RawPdfFile from context: pk=%s", pdf.pk)
+    else:
+        logger.info("Creating new RawPdfFile from %s for center %s", file_path, center_name)
+        
+        pdf = RawPdfFile.create_from_file_initialized(
+            file_path=file_path,
+            center_name=center_name,
+            delete_source=delete_source,
+        )
+        
+        center = ensure_center(pdf, ctx.center_name)
+        
+        logger.info(f"Successfully set up report file from {center.name}")
+
+
+
+    # 3) Check if we already have a successful history entry for this object+file_type
+    has_success_history = ProcessingHistory.has_history_for_object(
+        obj=pdf,
+        success=True,
+    )
+
+    if has_success_history:
+        logger.info(
+            "RawPdfFile %s already has successful processing history (file_type=%s) - short-circuiting",
+            getattr(pdf, str(pdf.file_path)),
+            file_type,
+        )
+        # No need to run the pipeline again
+        return pdf, False
+
+    # 4) No successful history yet → ensure there is a history entry marking it as "in progress"/failed
+    ProcessingHistory.get_or_create_for_object(
+        obj=pdf,
+        # At this point we haven't finished anonymization; treat as not-success yet.
+        success=False,
+    )
+
+    logger.info(
+        "Report instance ready for processing: pk=%s, file_type=%s (needs_processing=True)",
+        pdf.pk,
+        file_type,
+    )
+
+    # Signal to the caller that the anonymization pipeline should run
+    return pdf, True

endoreg_db/import_files/file_storage/create_video_file.py

@@ -0,0 +1,75 @@
+# endoreg_db/import_files/storage/create_video_file.py
+import logging
+from typing import Tuple
+
+from django.db import IntegrityError
+
+from endoreg_db.import_files.context.import_context import ImportContext
+from endoreg_db.import_files.context.ensure_center import ensure_center
+from endoreg_db.models.media import VideoFile
+from endoreg_db.models.media.processing_history.processing_history import ProcessingHistory
+from endoreg_db.utils.hashs import get_video_hash
+
+logger = logging.getLogger(__name__)
+
+
+def create_or_retrieve_video_file(
+    ctx: ImportContext,
+) -> Tuple[VideoFile, bool]:
+    """
+    Create a new or retrieve an existing VideoFile for the given context.
+
+    Returns:
+        video     : VideoFile instance
+        retry     : whether we are re-processing an existing file
+    """
+    file_path = ctx.file_path
+    center_name = ctx.center_name
+    processor_name = ctx.processor_name
+    delete_source = ctx.delete_source
+    file_type = ctx.file_type
+
+    # 1) Determine the VideoFile instance to work with
+    if ctx.current_video is not None:
+        video = ctx.current_video
+        logger.info("Using existing VideoFIle from context: pk =%s", video.pk)
+    else:
+        logger.info("Creating new VideoFIle from %s for center %s", file_path, center_name)
+        video = VideoFile.create_from_file_initialized(
+            file_path=file_path,
+            center_name=center_name,
+            processor_name=processor_name,
+            delete_source=delete_source,
+        )
+        center = ensure_center(video, ctx.center_name)
+        
+        logger.info(f"Successfully set up report file from {center.name}")
+    # 3) Check if we already have a successful history entry for this object+file_type        
+    has_success_history = ProcessingHistory.has_history_for_object(
+        obj=video,
+        success=True,
+    )
+
+    if has_success_history:
+        logger.info(
+            "VideoFile %s already has successful processing history. (file_type:%s) - short-circuiting",
+            getattr(video, str(video.active_file_path)),
+            file_type,
+    )
+    
+    # 4) No successful history yet → ensure there is a history entry marking it as "in progress"/failed
+    ProcessingHistory.get_or_create_for_object(
+        obj=video,
+        success=False
+    )
+    
+    
+    logger.info(
+        "Video instance ready for processing: pk=%s, file_type=%s (needs_processing=True)",
+        video.pk,
+        file_type,
+    )
+    
+    # Signal to the caller that the anonymization pipeline should run
+    return video, True
+

endoreg_db/import_files/file_storage/sensitive_meta_storage.py

@@ -0,0 +1,39 @@
+# endoreg_db/import_files/storage/sensitive_meta_storage.py
+from typing import Union, Dict, Any
+
+from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.models.metadata import SensitiveMeta
+from endoreg_db.import_files.processing.sensitive_meta_adapter import (
+    normalize_lx_sensitive_meta,
+)
+from endoreg_db.import_files.context.default_sensitive_meta import default_sensitive_meta
+from logging import getLogger
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
+#
+
+logger = getLogger(__name__)
+
+def sensitive_meta_storage(
+    sensitive_meta: LxSM,
+    instance: Union[RawPdfFile, VideoFile],
+) -> bool:
+    """
+    Merge lx_anonymizer.SensitiveMeta into instance.sensitive_meta in the DB.
+
+    - Normalizes the dataclass into the dict format expected by the model logic
+    - Delegates to SensitiveMeta.update_from_dict() (which already calls logic.update_*)
+    """
+    local_meta = instance.sensitive_meta  # Django SensitiveMeta model instance
+    if not isinstance(local_meta, SensitiveMeta):
+        # If sensitice meta doesnt exist yet, ensure it
+        local_meta = default_sensitive_meta(instance)
+    assert isinstance(local_meta, SensitiveMeta)
+
+    try:
+        payload = normalize_lx_sensitive_meta(sensitive_meta)
+        local_meta.update_from_dict(payload)  # this calls your big logic.update_*
+    except Exception as e:
+        logger.error(f"{e}")
+        return False
+
+    return True

endoreg_db/import_files/file_storage/state_management.py

@@ -0,0 +1,400 @@
+from endoreg_db.models.media.processing_history.processing_history import ProcessingHistory
+from endoreg_db.utils.paths import IMPORT_REPORT_DIR, IMPORT_VIDEO_DIR, ANONYM_REPORT_DIR, ANONYM_VIDEO_DIR
+
+
+import logging
+import shutil
+from pathlib import Path
+from typing import Optional, Union
+
+from django.db import transaction
+
+from endoreg_db.import_files.context.import_context import ImportContext
+from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.models.state import RawPdfState, VideoState
+from endoreg_db.utils import paths as path_utils
+
+logger = logging.getLogger(__name__)
+
+
+def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[Union[RawPdfState, VideoState]]:
+    """
+    Helper: ensure instance.state exists and return it.
+    Mirrors PdfImportService._ensure_state.
+    """
+    if isinstance(instance, RawPdfFile):
+        state = getattr(instance, "state", None)
+    else:
+        state = getattr(instance, "state", None)
+        
+    if state is not None:
+        return state
+
+    if hasattr(instance, "get_or_create_state"):
+        state = instance.get_or_create_state()
+        instance.save()
+        return state
+
+    return None
+
+def mark_instance_processing_started(    
+    instance: Union[RawPdfFile, VideoFile],
+    ctx: ImportContext,):
+    state = _ensure_instance_state(instance)
+
+    with transaction.atomic():
+        if state is not None:
+
+            # In the old code, processing_started was set earlier; we guard here
+            if not getattr(state, "processing_started", False) and hasattr(
+                state, "mark_processing_started"
+            ):
+                state.mark_processing_started()
+
+
+def finalize_report_success(
+    ctx: ImportContext,
+) -> None:
+    """
+    Finalize a successful instance import/anonymization.
+
+    - Move anonymized Report from temp to canonical anonymized dir
+    - Update RawPdfFile.processed_file and .anonymized flag
+    - Mark RawPdfState as anonymized + sensitive_meta_processed
+    - Mark ProcessingHistory.success = True
+    """
+    instance = ctx.current_report
+    if not isinstance(instance, RawPdfFile):
+        logger.warning("finalize_success called with unsaved instance")
+        return
+    if not instance.pk:
+        logger.warning("finalize_success called with unsaved instance")
+        return
+
+    # --- Move anonymized path into final storage (if we have one) ---
+    final_path: Optional[Path] = None
+    if ctx.anonymized_path is None:
+        logger.warning(
+            "No anonymized_path for instance %s (hash=%s); skipping file move.",
+            instance.pk,
+            getattr(instance, "pdf_hash", None),
+        )
+        final_path = None
+    else:
+        pdf_hash = getattr(instance, "pdf_hash", None) or instance.pk
+        expected_final_path = ANONYM_REPORT_DIR / f"{pdf_hash}.pdf"
+
+        src = Path(ctx.anonymized_path)
+
+        logger.debug(
+            "finalize_report_success: src=%s (exists=%s, resolved=%s), expected_final=%s",
+            src,
+            src.exists(),
+            src.resolve(),
+            expected_final_path,
+        )
+
+        # If anonymizer already wrote to the final path, don't move
+        if src.resolve() == expected_final_path.resolve():
+            logger.info(
+                "Anonymizer output already at final path %s; skipping move.",
+                expected_final_path,
+            )
+            final_path = expected_final_path
+        else:
+            # Only move if the source actually exists
+            if not src.exists():
+                logger.error(
+                    "Anonymized file %s does not exist; cannot move to %s",
+                    src,
+                    expected_final_path,
+                )
+                final_path = None
+            else:
+                ANONYM_REPORT_DIR.mkdir(parents=True, exist_ok=True)
+                if expected_final_path.exists():
+                    expected_final_path.unlink()
+                shutil.move(str(src), str(expected_final_path))
+                final_path = expected_final_path
+                logger.info("Moved anonymized report to %s", final_path)
+
+        # Update FileField if we have a final path
+        if final_path is not None:
+            relative_name = path_utils.to_storage_relative(final_path)
+            current_name = getattr(instance.processed_file, "name", None)
+            if current_name != relative_name:
+                instance.processed_file.name = relative_name
+                logger.info("Updated processed_file to %s", relative_name)
+        try:
+            relative_name = str(ctx.anonymized_path)
+        except ValueError:
+            # Fallback: absolute path if outside STORAGE_DIR
+            relative_name = str(final_path)
+
+        current_name = getattr(instance.processed_file, "name", None)
+        if current_name != relative_name:
+            instance.processed_file.name = relative_name
+            logger.info(
+                "Updated processed_file reference to: %s",
+                instance.processed_file.name,
+            )
+
+
+    # --- Update RawPdfState flags (mirrors _finalize_processing) ---
+    state = _ensure_instance_state(instance)
+
+    with transaction.atomic():
+        if state is not None:
+
+            # In the old code, processing_started was set earlier; we guard here
+            if not getattr(state, "processing_started", False) and hasattr(
+                state, "mark_processing_started"
+            ):
+                state.mark_processing_started()
+
+            # We consider text/meta extraction + anonymization done at this point
+            if hasattr(state, "mark_anonymized"):
+                state.mark_anonymized()
+            if hasattr(state, "mark_sensitive_meta_processed"):
+                state.mark_sensitive_meta_processed()
+
+            state.save()
+
+        instance.save()
+
+    # --- ProcessingHistory entry ---
+    try:
+        with transaction.atomic():
+            ProcessingHistory.get_or_create_for_object(
+                obj=instance,
+                success=True,
+            )
+    except Exception as e:
+        logger.debug(
+            "Saving not possible; %s"
+            f"skipping ProcessingHistory.{e}",
+            instance.pk,
+        )
+
+def finalize_video_success(
+    ctx: ImportContext,
+) -> None:
+    """
+    Finalize a successful video import/anonymization.
+
+    - Move anonymized video from temp to canonical anonymized dir
+    - Update VideoFile.processed_file
+    - Mark VideoState as anonymized + sensitive_meta_processed
+    - Mark ProcessingHistory.success = True
+    """
+    instance = ctx.current_video
+    if not isinstance(instance, VideoFile):
+        logger.warning("finalize_video_success called with non-VideoFile instance")
+        return
+    if not instance.pk:
+        logger.warning("finalize_video_success called with unsaved instance")
+        return
+
+    # --- Move anonymized path into final storage (if we have one) ---
+    final_path: Optional[Path] = None
+
+    if ctx.anonymized_path is None:
+        logger.warning(
+            "No anonymized_path for video instance %s (hash=%s); skipping file move.",
+            instance.pk,
+            getattr(instance, "video_hash", None),
+        )
+    else:
+        # Use a stable naming convention: <video_hash>.mp4
+        video_hash = getattr(instance, "video_hash", None) or instance.pk
+        expected_final_path = ANONYM_VIDEO_DIR / f"{video_hash}.mp4"
+
+        src = Path(ctx.anonymized_path)
+
+        logger.debug(
+            "finalize_video_success: src=%s (exists=%s, resolved=%s), expected_final=%s",
+            src,
+            src.exists(),
+            src.resolve(),
+            expected_final_path,
+        )
+
+        # If anonymizer already wrote to the final path, don't move
+        try:
+            same_target = src.resolve() == expected_final_path.resolve()
+        except FileNotFoundError:
+            # src might not exist anymore
+            same_target = False
+
+        if same_target:
+            logger.info(
+                "Anonymizer output already at final video path %s; skipping move.",
+                expected_final_path,
+            )
+            final_path = expected_final_path
+        else:
+            if not src.exists():
+                logger.error(
+                    "Anonymized video %s does not exist; cannot move to %s",
+                    src,
+                    expected_final_path,
+                )
+                final_path = None
+            else:
+                ANONYM_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
+                if expected_final_path.exists():
+                    try:
+                        expected_final_path.unlink()
+                    except Exception as e:
+                        logger.warning(
+                            "Could not remove existing anonymized video %s: %s",
+                            expected_final_path,
+                            e,
+                        )
+                shutil.move(str(src), str(expected_final_path))
+                final_path = expected_final_path
+                logger.info("Moved anonymized video to %s", final_path)
+
+        # Update FileField if we have a final path
+        if final_path is not None:
+            relative_name = path_utils.to_storage_relative(final_path)
+            current_name = getattr(instance.processed_file, "name", None)
+            if current_name != relative_name:
+                instance.processed_file.name = relative_name
+                logger.info("Updated video processed_file to %s", relative_name)
+
+    # --- Update VideoState flags (mirrors report) ---
+    state = _ensure_instance_state(instance)
+
+    with transaction.atomic():
+        if state is not None:
+            if not getattr(state, "processing_started", False) and hasattr(
+                state, "mark_processing_started"
+            ):
+                state.mark_processing_started()
+
+            if hasattr(state, "mark_anonymized"):
+                state.mark_anonymized()
+            if hasattr(state, "mark_sensitive_meta_processed"):
+                state.mark_sensitive_meta_processed()
+
+            state.save()
+
+        instance.save()
+
+    # --- ProcessingHistory entry ---
+    try:
+        with transaction.atomic():
+            ProcessingHistory.get_or_create_for_object(
+                obj=instance,
+                success=True,
+            )
+    except Exception as e:
+        logger.debug(
+            "Saving not possible for video %s; skipping ProcessingHistory. Error: %s",
+            instance.pk,
+            e,
+        )
+
+
+def finalize_failure(
+    ctx: ImportContext,
+) -> None:
+    """
+    Finalize a failed instance import/anonymization.
+
+    - Reset RawPdfState flags to "not processed"
+    - Mark ProcessingHistory.success = False
+    """
+    if ctx.instance is None:
+        if isinstance(ctx.current_report, RawPdfFile):
+            ctx.instance = ctx.current_report
+        elif isinstance(ctx.current_video, VideoFile):
+            ctx.instance = ctx.current_video
+        else:
+            raise Exception
+    # Reset state flags similar to _mark_processing_incomplete / _cleanup_on_error
+    state = _ensure_instance_state(ctx.instance)
+
+    if state is not None:
+        try:
+            state.mark_processing_not_started()
+
+            state.save()
+            logger.info(
+                "Reset instance state for failed processing (instance pk=%s)",
+                ctx.instance.pk,
+            )
+        except Exception as e:
+            logger.warning(
+                "Failed to reset State for instance %s: %s",
+                ctx.instance.pk,
+                e,
+            )
+
+        try:
+            delete_associated_files(ctx)
+        except Exception as e:
+            logger.warning(f"There might be files remaining. {e}")
+
+    # History entry with success=False
+    if ctx.file_hash:
+        ProcessingHistory.get_or_create_for_object(
+            obj=ctx.instance,
+            success=False,
+        )
+    else:
+        logger.debug(
+            "No file_hash in context for instance %s when finalizing failure; "
+            "skipping ProcessingHistory.",
+            ctx.instance.pk,
+        )
+
+    logger.error(
+        "Report processing failed for %s",
+        ctx.file_path,
+    )
+
+def delete_associated_files(ctx:ImportContext):
+    try:
+        assert isinstance(ctx.original_path, Path)
+    except AssertionError as e:
+        logger.warning(f"Original file restored from sensitive copy. This is because the original file is gone. {e}")
+        if ctx.file_type =="video":
+            if isinstance(ctx.sensitive_path, Path):
+                try:
+                    ctx.original_path = Path(shutil.copy2(ctx.sensitive_path, IMPORT_VIDEO_DIR))
+                except Exception as e:
+                    logger.error(f"Error during safety copy: {e} Original file could not be restored!")
+                    raise
+        elif ctx.file_type == "report":
+            if isinstance(ctx.sensitive_path, Path):
+                try:
+                    ctx.original_path = Path(shutil.copy2(ctx.sensitive_path, IMPORT_REPORT_DIR))
+                except Exception as e:
+                    logger.error(f"Error during safety copy: {e} Original file could not be restored!")
+                    raise
+        
+    if isinstance(ctx.anonymized_path, Path):
+        try:
+            Path.unlink(ctx.anonymized_path)
+            assert(ctx.anonymized_path is not Path)
+
+        except Exception as e:
+            logger.error(f"Error when unlinking anonymized path. {e}")
+            raise
+        
+    ctx.anonymized_path = None
+    
+    if isinstance(ctx.sensitive_path, Path):
+        try:
+            Path.unlink(ctx.sensitive_path)
+            assert(ctx.sensitive_path is not Path)
+            
+        except Exception as e:
+            logger.error(f"Error when unlinking anonymized path. {e}")
+
+            raise
+        
+    ctx.sensitive_path = None
+    

endoreg_db/import_files/file_storage/storage.py

@@ -0,0 +1,36 @@
+import logging
+from pathlib import Path
+from endoreg_db.models.media.video.create_from_file import atomic_copy_with_fallback, atomic_move_with_fallback
+logger = logging.getLogger(__name__)
+
+
+def ensure_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def create_sensitive_copy(src: Path, sensitive_root: Path) -> Path:
+    """
+    Create a sensitive copy of `src` in `sensitive_root`.
+
+    Returns:
+        Path to the sensitive copy.
+    """
+    ensure_dir(sensitive_root)
+    dest = sensitive_root / src.name
+    logger.info("Creating sensitive copy: %s -> %s", src, dest)
+    atomic_copy_with_fallback(src, dest)
+    return dest
+
+
+def move_to_anonymized(temp_path: Path, anonymized_root: Path) -> Path:
+    """
+    Move a (temporary) anonymized file into the canonical anonymized root.
+
+    Returns:
+        Final path inside anonymized_root.
+    """
+    ensure_dir(anonymized_root)
+    dest = anonymized_root / temp_path.name
+    logger.info("Moving anonymized file: %s -> %s", temp_path, dest)
+    atomic_move_with_fallback(temp_path, dest)
+    return dest