endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/import_files/context/ensure_center.py

@@ -0,0 +1,17 @@
+from typing import Union, Optional
+
+from endoreg_db.models import Center
+from endoreg_db.models.media import RawPdfFile, VideoFile
+
+
+def ensure_center(
+    instance: Union[RawPdfFile, VideoFile], center: Optional[str]
+) -> Center:
+    if not isinstance(instance.center, Center):
+        raise AssertionError
+    if not isinstance(instance.center.name, str):
+        raise AssertionError
+    assert isinstance(instance.center.get_by_name(center), Center)
+    if not instance.center.get_by_name(center).name == instance.center.name:
+        raise AssertionError
+    return instance.center

endoreg_db/import_files/context/file_lock.py

@@ -0,0 +1,66 @@
+from contextlib import contextmanager
+from pathlib import Path
+import os
+import time
+from logging import getLogger
+from typing import Generator, Any
+
+logger = getLogger(__name__)
+
+STALE_LOCK_SECONDS = 6000
+MAX_LOCK_WAIT_SECONDS = 90
+
+
+@contextmanager
+def file_lock(path: Path) -> Generator[None, Any, None]:
+    """
+    Create a file lock to prevent duplicate processing of the same file.
+
+    Lock is created *next to* the source file: "<path>.lock".
+    """
+    lock_path = Path(str(path) + ".lock")
+    fd = None
+    try:
+        deadline = time.time() + MAX_LOCK_WAIT_SECONDS
+        while True:
+            try:
+                fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
+                break
+            except FileExistsError:
+                age = None
+                try:
+                    st = os.stat(lock_path)
+                    age = time.time() - st.st_mtime
+                except FileNotFoundError:
+                    age = None
+
+                if age is not None and age > STALE_LOCK_SECONDS:
+                    try:
+                        logger.warning(
+                            "Stale lock detected for %s (age %.0fs). Reclaiming lock...",
+                            path,
+                            age,
+                        )
+                        lock_path.unlink()
+                    except Exception as e:
+                        logger.warning(
+                            "Failed to remove stale lock %s: %s", lock_path, e
+                        )
+                    continue
+
+                if time.time() >= deadline:
+                    raise ValueError(f"File already being processed: {path}")
+                time.sleep(1.0)
+
+        os.write(fd, b"lock")
+        os.close(fd)
+        fd = None
+        yield
+    finally:
+        try:
+            if fd is not None:
+                os.close(fd)
+            if lock_path.exists():
+                lock_path.unlink()
+        except OSError:
+            pass

endoreg_db/import_files/context/import_context.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional, Dict, Any, Union
+
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta
+from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.utils.file_operations import sha256_file
+
+
+@dataclass
+class ImportContext:
+    file_path: Path
+    center_name: str
+    processor_name: str = "olympus-cv-500"
+    delete_source: bool = True
+
+    retry: bool = False
+    import_completed: bool = False
+    error_reason: str = ""
+
+    original_path: Optional[Path] = None
+    quarantine_path: Optional[Path] = None
+    sensitive_path: Optional[Path] = None
+    anonymized_path: Optional[Path] = None
+
+    current_report: Optional[RawPdfFile] = None
+    current_video: Optional[VideoFile] = None
+    current_meta: Optional[SensitiveMeta] = None
+
+    instance: Optional[Union[RawPdfFile, VideoFile]] = None
+    file_type: str = "undefined"
+
+    # will be populated in __post_init__
+    file_hash: Optional[str] = field(init=False)
+
+    original_text: Optional[str] = None
+    anonymized_text: Optional[str] = None
+    extracted_metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Compute raw file hash after dataclass is constructed."""
+        self.file_hash = sha256_file(self.file_path)

endoreg_db/import_files/context/validate_directories.py

@@ -0,0 +1,57 @@
+import logging
+from pathlib import Path
+from typing import Iterable
+from endoreg_db.utils.paths import (
+    ANONYM_REPORT_DIR,
+    ANONYM_VIDEO_DIR,
+    IMPORT_REPORT_DIR,
+    IMPORT_VIDEO_DIR,
+    SENSITIVE_REPORT_DIR,
+    SENSITIVE_VIDEO_DIR,
+)
+
+dirs = [
+    ANONYM_REPORT_DIR,
+    ANONYM_VIDEO_DIR,
+    IMPORT_REPORT_DIR,
+    IMPORT_VIDEO_DIR,
+    SENSITIVE_REPORT_DIR,
+    SENSITIVE_VIDEO_DIR,
+]
+
+
+logger = logging.getLogger(__name__)
+
+
+def validate_directories(dirs: Iterable[Path] = dirs) -> bool:
+    """
+    Ensure all directories in `dirs` exist.
+    Missing directories are created automatically.
+
+    Args:
+        dirs: Iterable of Path objects representing directories.
+
+    Returns:
+        bool: True if all directories exist or were created successfully,
+              False if any directory could not be created.
+    """
+    ok = True
+
+    for d in dirs:
+        try:
+            if not d.exists():
+                logger.info(f"Directory missing, creating: {d}")
+                d.mkdir(parents=True, exist_ok=True)
+
+            if not d.is_dir():
+                logger.error(f"Path exists but is not a directory: {d}")
+                ok = False
+
+        except Exception as e:
+            logger.error(f"Failed to create or validate directory '{d}': {e}")
+            ok = False
+
+    return ok
+
+
+validate_directories(dirs)

endoreg_db/import_files/file_storage/__init__.py

@@ -0,0 +1,15 @@
+# storage/__init__.py
+
+from . import create_report_file
+from . import create_video_file
+from . import sensitive_meta_storage
+from . import state_management
+from . import storage
+
+__all__ = [
+    "create_report_file",
+    "create_video_file",
+    "sensitive_meta_storage",
+    "state_management",
+    "storage",
+]

endoreg_db/import_files/file_storage/create_report_file.py

@@ -0,0 +1,99 @@
+# endoreg_db/import_files/storage/create_report_file.py
+import logging
+from typing import Tuple
+
+from endoreg_db.import_files.context.ensure_center import ensure_center
+from endoreg_db.import_files.context.import_context import ImportContext  #
+from endoreg_db.utils.file_operations import sha256_file
+from endoreg_db.models.media import RawPdfFile
+from endoreg_db.models.state.processing_history.processing_history import (
+    ProcessingHistory,
+)
+from endoreg_db.import_files.file_storage.state_management import finalize_failure
+
+logger = logging.getLogger(__name__)
+
+
+def create_or_retrieve_report_file(
+    ctx: ImportContext,
+) -> Tuple[RawPdfFile, bool, bool]:
+    """
+    Create a new or retrieve an existing RawPdfFile for the given context.
+
+    Returns:
+        pdf             : RawPdfFile instance
+        processed       : True if there is already a successful ProcessingHistory for this file
+        needs_processing: True if the pipeline should run for this file in this call
+    """
+    file_path = ctx.file_path
+    center_name = ctx.center_name
+    delete_source = ctx.delete_source
+    file_type = ctx.file_type  # logical key for history; can be None
+
+    # default assumptions
+    processed = False
+    needs_processing = True
+
+    if not isinstance(ctx.file_hash, str):
+        ctx.file_hash = sha256_file(ctx.file_path)
+
+    # Check if we already have a successful history entry for this object
+    has_success_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=True,
+    )
+    has_failure_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
+    )
+    if has_success_history:
+        logger.info(
+            "RawPdfFile pk= already has successful processing history (file_hash=%s) - short-circuiting",
+            ctx.file_hash,
+        )
+        processed = True
+        needs_processing = False
+        if not isinstance(ctx.current_report, RawPdfFile):
+            ctx.current_report = RawPdfFile.get_report_by_hash(ctx.file_hash)
+        return ctx.current_report, processed, needs_processing
+    elif has_failure_history:
+        if not isinstance(ctx.current_report, RawPdfFile):
+            ctx.current_report = RawPdfFile.get_report_by_hash(ctx.file_hash)
+        finalize_failure(ctx)
+        processed = True
+        needs_processing = True
+
+    # Determine the RawPdfFile instance to work with
+    if ctx.current_report is not None:
+        pdf = ctx.current_report
+        logger.info("Using existing RawPdfFile from context: pk=%s", pdf.pk)
+    else:
+        logger.info(
+            "Creating new RawPdfFile from %s for center %s",
+            file_path,
+            center_name,
+        )
+
+        pdf = RawPdfFile.create_from_file_initialized(
+            file_path=file_path,
+            center_name=center_name,
+            delete_source=delete_source,
+        )
+
+        center = ensure_center(pdf, ctx.center_name)
+        logger.info("Successfully set up report file from %s", center.name)
+
+    # No successful history yet → ensure there is a history entry marking it as "in progress"/failed
+    ProcessingHistory.get_or_create_for_hash(
+        obj=pdf,
+        file_hash=ctx.file_hash,
+        success=False,
+    )
+
+    logger.info(
+        "Report instance ready for processing: pk=%s, file_type=%s (needs_processing=True)",
+        pdf.pk,
+        file_type,
+    )
+
+    return pdf, processed, needs_processing

endoreg_db/import_files/file_storage/create_video_file.py

@@ -0,0 +1,104 @@
+# endoreg_db/import_files/storage/create_video_file.py
+import logging
+from typing import Tuple
+from endoreg_db.import_files.context.import_context import ImportContext
+from endoreg_db.import_files.context.ensure_center import ensure_center
+from endoreg_db.utils.file_operations import sha256_file
+from endoreg_db.models.media import VideoFile
+from endoreg_db.models.state.processing_history.processing_history import (
+    ProcessingHistory,
+)
+from endoreg_db.import_files.file_storage.state_management import finalize_failure
+
+logger = logging.getLogger(__name__)
+
+
+def create_or_retrieve_video_file(
+    ctx: ImportContext,
+) -> Tuple[VideoFile, bool, bool]:
+    """
+    Create a new or retrieve an existing VideoFile for the given context.
+
+    Returns:
+        video           : VideoFile instance
+        processed       : True if there is already a successful ProcessingHistory for this file
+        needs_processing: True if the pipeline should run for this file in this call
+    """
+    file_path = ctx.file_path
+
+    center_name = ctx.center_name
+    processor_name = ctx.processor_name
+    delete_source = ctx.delete_source
+    file_type = ctx.file_type  # logical key for history; can be None
+
+    # default assumptions (same semantics as report)
+    processed = False
+    needs_processing = True
+
+    if not isinstance(ctx.file_hash, str):
+        ctx.file_hash = sha256_file(ctx.file_path)
+
+        # 2) Check if we already have a successful history entry for this object
+    has_success_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=True,
+    )
+    has_failure_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
+    )
+
+    if has_success_history:
+        logger.info(
+            "VideoFile already has successful processing history (file_hash=%s) "
+            "- short-circuiting",
+            ctx.file_hash,
+        )
+        processed = True
+        needs_processing = False
+        if not isinstance(ctx.current_video, VideoFile):
+            ctx.current_video = VideoFile.get_video_by_content_hash(ctx.file_hash)
+        return ctx.current_video, processed, needs_processing
+    elif has_failure_history:
+        if not isinstance(ctx.current_video, VideoFile):
+            ctx.current_video = VideoFile.get_video_by_content_hash(ctx.file_hash)
+        finalize_failure(ctx)
+
+        processed = True
+        needs_processing = True
+
+    # Determine the VideoFile instance to work with
+    if ctx.current_video is not None:
+        video = ctx.current_video
+        logger.info("Using existing VideoFile from context: pk=%s", video.pk)
+    else:
+        logger.info(
+            "Creating new VideoFile from %s for center %s",
+            file_path,
+            center_name,
+        )
+        video = VideoFile.create_from_file_initialized(
+            file_path=file_path,
+            center_name=center_name,
+            processor_name=processor_name,
+            delete_source=delete_source,
+            video_hash=ctx.file_hash,
+        )
+        needs_processing = True
+
+        center = ensure_center(video, ctx.center_name)
+        logger.info("Successfully set up video file from %s", center.name)
+
+    # No successful history yet → ensure there is a history entry marking it as "in progress"/failed
+    ProcessingHistory.get_or_create_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
+    )
+
+    logger.info(
+        "Video instance ready for processing: pk=%s, file_type=%s (needs_processing=True)",
+        video.pk,
+        file_type,
+    )
+
+    return video, processed, needs_processing

endoreg_db/import_files/file_storage/sensitive_meta_storage.py

@@ -0,0 +1,42 @@
+# endoreg_db/import_files/storage/sensitive_meta_storage.py
+from typing import Union
+
+from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.models.metadata import SensitiveMeta
+from endoreg_db.import_files.processing.sensitive_meta_adapter import (
+    normalize_lx_sensitive_meta,
+)
+from endoreg_db.import_files.context.default_sensitive_meta import (
+    default_sensitive_meta,
+)
+from logging import getLogger
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
+#
+
+logger = getLogger(__name__)
+
+
+def sensitive_meta_storage(
+    sensitive_meta: LxSM,
+    instance: Union[RawPdfFile, VideoFile],
+) -> bool:
+    """
+    Merge lx_anonymizer.SensitiveMeta into instance.sensitive_meta in the DB.
+
+    - Normalizes the dataclass into the dict format expected by the model logic
+    - Delegates to SensitiveMeta.update_from_dict() (which already calls logic.update_*)
+    """
+    local_meta = instance.sensitive_meta  # Django SensitiveMeta model instance
+    if not isinstance(local_meta, SensitiveMeta):
+        # If sensitice meta doesnt exist yet, ensure it
+        local_meta = default_sensitive_meta(instance)
+    assert isinstance(local_meta, SensitiveMeta)
+
+    try:
+        payload = normalize_lx_sensitive_meta(sensitive_meta)
+        local_meta.update_from_dict(payload)  # this calls your big logic.update_*
+    except Exception as e:
+        logger.error(f"{e}")
+        return False
+
+    return True