endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

endoreg_db/helpers/download_segmentation_model.py

@@ -1,10 +1,11 @@
 import huggingface_hub
 from typing import Optional
 
+
 def download_segmentation_model(
     repo_id: str = "wg-lux/colo_segmentation_RegNetX800MF_base",
     filename: str = "model.safetensors",
-    cache_dir: Optional[str] = None
+    cache_dir: Optional[str] = None,
 ) -> str:
     """
     Downloads a segmentation model from Hugging Face and caches it locally.
@@ -16,7 +17,7 @@ def download_segmentation_model(
 
     Returns:
         str: The local path to the downloaded model.
-        
+
     Example:
         >>> model_path = download_segmentation_model()
         >>> # Downloads from wg-lux/colo_segmentation_RegNetX800MF_base
@@ -28,4 +29,4 @@ def download_segmentation_model(
         force_download=False,
         resume_download=True,
     )
-    return local_path
+    return local_path

endoreg_db/helpers/interact.py

@@ -1,6 +1 @@
-from endoreg_db.models import EndoscopyProcessor
-from django.core.management.base import BaseCommand
-from typing import TYPE_CHECKING
-from django.db.models import QuerySet
-
 

endoreg_db/helpers/test_video_helper.py

@@ -4,17 +4,16 @@ import re
 from typing import Optional
 
 from logging import getLogger
+
 logger = getLogger(__name__)
 
 
-from endoreg_db.utils.video.names import (
-    get_video_key_regex_by_examination_alias
-)
+from endoreg_db.utils.video.names import get_video_key_regex_by_examination_alias
 
 from django.conf import settings
 
 
-ASSET_DIR:Path = settings.ASSET_DIR
+ASSET_DIR: Path = settings.ASSET_DIR
 assert ASSET_DIR.exists(), f"ASSET_DIR does not exist: {ASSET_DIR}"
 
 _TEST_VIDEOS = {
@@ -29,10 +28,10 @@ _TEST_VIDEOS = {
 TEST_VIDEOS = {key: value for key, value in _TEST_VIDEOS.items() if value.exists()}
 
 
-def get_video_path(video_key:str) -> Path:
+def get_video_path(video_key: str) -> Path:
     """
     Returns the file path associated with the given video key.
-    
+
     Raises:
         ValueError: If the video key does not exist in the available test videos.
     """
@@ -41,19 +40,22 @@ def get_video_path(video_key:str) -> Path:
     else:
         raise ValueError(f"Video key '{video_key}' not found in TEST_VIDEOS.")
 
+
 def get_video_keys(
-    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+    examination_alias: Optional[str] = None,
+    content: Optional[str] = None,
+    is_anonymous: Optional[bool] = None,
 ):
     """
     Returns a list of video keys matching the specified examination alias, content type, and anonymity status.
-    
+
     If no direct matches are found, falls back to suffix-based filtering for anonymity. Logs warnings and errors when fallback logic is used or no matches are found.
-    
+
     Args:
         examination_alias: The examination alias to filter by, or None for any.
         content: The content type to filter by, or None for any.
         is_anonymous: Whether to filter for anonymous, non-anonymous, or both.
-    
+
     Returns:
         A list of matching video keys.
     """
@@ -62,49 +64,57 @@ def get_video_keys(
         pattern_parts = ["^"]
         if examination_alias:
             pattern_parts.append(re.escape(examination_alias))
-            pattern_parts.append("-.*-") # Match any content
+            pattern_parts.append("-.*-")  # Match any content
         else:
-            pattern_parts.append(".*-") # Match any examination alias and content
+            pattern_parts.append(".*-")  # Match any examination alias and content
 
         if is_anonymous is True:
             pattern_parts.append("anonymous$")
         elif is_anonymous is False:
             pattern_parts.append("non_anonymous$")
-        else: # is_anonymous is None
-            pattern_parts.append("(non_)?anonymous$") # Match either
+        else:  # is_anonymous is None
+            pattern_parts.append("(non_)?anonymous$")  # Match either
 
         pattern = "".join(pattern_parts)
     else:
         # Use the imported function if content is specified
-        pattern = get_video_key_regex_by_examination_alias(examination_alias, content, is_anonymous)
+        pattern = get_video_key_regex_by_examination_alias(
+            examination_alias, content, is_anonymous
+        )
         logger.warning(f"Generated pattern (from imported function): {pattern}")
 
-
     # Only consider keys for which the file actually exists
     keys_to_check = list(TEST_VIDEOS.keys())
     matched_keys = [key for key in keys_to_check if re.match(pattern, key)]
 
     # Fallback logic remains as a safety net, but ideally shouldn't be needed now for this case
     if not matched_keys and is_anonymous is False:
-        logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=False. Falling back to suffix check '-non_anonymous'.")
+        logger.warning(
+            f"Pattern '{pattern}' yielded no results for is_anonymous=False. Falling back to suffix check '-non_anonymous'."
+        )
         matched_keys = [key for key in keys_to_check if key.endswith("-non_anonymous")]
     elif not matched_keys and is_anonymous is True:
-         logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=True. Falling back to suffix check '-anonymous'.")
-         matched_keys = [key for key in keys_to_check if key.endswith("-anonymous")]
+        logger.warning(
+            f"Pattern '{pattern}' yielded no results for is_anonymous=True. Falling back to suffix check '-anonymous'."
+        )
+        matched_keys = [key for key in keys_to_check if key.endswith("-anonymous")]
 
     if not matched_keys:
-        logger.error(f"No keys found matching pattern '{pattern}' or fallback logic for keys: {keys_to_check}")
-
+        logger.error(
+            f"No keys found matching pattern '{pattern}' or fallback logic for keys: {keys_to_check}"
+        )
 
     return matched_keys
 
 
 def get_random_video_path_by_examination_alias(
-    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+    examination_alias: Optional[str] = None,
+    content: Optional[str] = None,
+    is_anonymous: Optional[bool] = None,
 ):
     """
     Returns the file path of a randomly selected video matching the specified examination alias, content type, and anonymity status.
-    
+
     Raises:
         ValueError: If no matching video keys are found for the given criteria.
     """
@@ -115,5 +125,3 @@ def get_random_video_path_by_examination_alias(
         return video_path  # Return the first match for simplicity
     else:
         raise ValueError("No matching video keys found for the given criteria.")
-
-

endoreg_db/import_files/__init__.py

@@ -23,5 +23,5 @@ __all__ = [
     "VideoImportService",
     "ImportContext",
     "validate_directories",
-    "default_sensitive_meta"
+    "default_sensitive_meta",
 ]

endoreg_db/import_files/context/__init__.py

@@ -4,4 +4,4 @@ from .import_context import ImportContext
 __all__ = [
     "file_lock",
     "ImportContext",
-]
+]

endoreg_db/import_files/context/default_sensitive_meta.py

@@ -3,7 +3,7 @@
 import os
 import logging
 from datetime import date
-from typing import Optional, Union
+from typing import Union
 
 from endoreg_db.models.media import RawPdfFile, VideoFile
 from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta  # adjust path
@@ -17,18 +17,17 @@ DEFAULT_CENTER_NAME = "endoreg_db_demo"
 DEFAULT_PATIENT_DOB = date(1970, 1, 1)
 
 
-def default_sensitive_meta(instance: Union[RawPdfFile, VideoFile]) -> SensitiveMeta | None:
+def default_sensitive_meta(
+    instance: Union[RawPdfFile, VideoFile],
+) -> SensitiveMeta | None:
     """
     Ensure the given instance has a minimal SensitiveMeta attached.
 
     Called after text extraction + merging; only creates meta if none exists.
     """
     if instance is None:
-        logger.warning(
-            "No instance available for ensuring default patient data"
-        )
+        logger.warning("No instance available for ensuring default patient data")
         return
-    
 
     if instance.sensitive_meta:
         # Already has meta; nothing to do
@@ -44,7 +43,9 @@ def default_sensitive_meta(instance: Union[RawPdfFile, VideoFile]) -> SensitiveM
             assert center_name is not None
             instance.center.name = center_name
         except AssertionError as e:
-            logger.debug(f"{e}Center name is not set! You can set it in .env under DEFAULT_CENTER_NAME using default from default_sensitive_meta")
+            logger.debug(
+                f"{e}Center name is not set! You can set it in .env under DEFAULT_CENTER_NAME using default from default_sensitive_meta"
+            )
             instance.center.name = DEFAULT_CENTER_NAME
             instance.center.get_by_name(DEFAULT_CENTER_NAME)
 
@@ -59,7 +60,9 @@ def default_sensitive_meta(instance: Union[RawPdfFile, VideoFile]) -> SensitiveM
             else DEFAULT_CENTER_NAME
         ),
         # optional: link file_path for debugging/tracing
-        "file_path": str(instance.file_path) if getattr(instance, "file_path", None) else None,
+        "file_path": str(instance.file_path)
+        if getattr(instance, "file_path", None)
+        else None,
     }
 
     try:
@@ -78,4 +81,3 @@ def default_sensitive_meta(instance: Union[RawPdfFile, VideoFile]) -> SensitiveM
             e,
         )
         return None
-

endoreg_db/import_files/context/ensure_center.py

@@ -1,17 +1,17 @@
 from typing import Union, Optional
-import os
 
 from endoreg_db.models import Center
 from endoreg_db.models.media import RawPdfFile, VideoFile
 
 
-def ensure_center(instance: Union[RawPdfFile, VideoFile], center: Optional[str]) -> Center:
+def ensure_center(
+    instance: Union[RawPdfFile, VideoFile], center: Optional[str]
+) -> Center:
     if not isinstance(instance.center, Center):
         raise AssertionError
     if not isinstance(instance.center.name, str):
         raise AssertionError
     assert isinstance(instance.center.get_by_name(center), Center)
-    if not instance.center.get_by_name(center).name ==instance.center.name:
+    if not instance.center.get_by_name(center).name == instance.center.name:
         raise AssertionError
     return instance.center
-

endoreg_db/import_files/context/file_lock.py

@@ -3,8 +3,6 @@ from pathlib import Path
 import os
 import time
 from logging import getLogger
-import errno
-import shutil
 from typing import Generator, Any
 
 logger = getLogger(__name__)
@@ -45,7 +43,9 @@ def file_lock(path: Path) -> Generator[None, Any, None]:
                         )
                         lock_path.unlink()
                     except Exception as e:
-                        logger.warning("Failed to remove stale lock %s: %s", lock_path, e)
+                        logger.warning(
+                            "Failed to remove stale lock %s: %s", lock_path, e
+                        )
                     continue
 
                 if time.time() >= deadline:

endoreg_db/import_files/context/import_context.py

@@ -4,40 +4,39 @@ from typing import Optional, Dict, Any, Union
 
 from lx_anonymizer.sensitive_meta_interface import SensitiveMeta
 from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.utils.file_operations import sha256_file
 
 
 @dataclass
 class ImportContext:
-    """
-    Tracking the import success and reasons of failure.
-    """
-
-    # core import parameters
     file_path: Path
     center_name: str
     processor_name: str = "olympus-cv-500"
     delete_source: bool = True
+
     retry: bool = False
     import_completed: bool = False
     error_reason: str = ""
 
-    # paths
     original_path: Optional[Path] = None
     quarantine_path: Optional[Path] = None
     sensitive_path: Optional[Path] = None
     anonymized_path: Optional[Path] = None
 
-    # associated objects
     current_report: Optional[RawPdfFile] = None
     current_video: Optional[VideoFile] = None
     current_meta: Optional[SensitiveMeta] = None
+
     instance: Optional[Union[RawPdfFile, VideoFile]] = None
-    
     file_type: str = "undefined"
-    
-    
-    # processing metadata
-    file_hash: str = ""
+
+    # will be populated in __post_init__
+    file_hash: Optional[str] = field(init=False)
+
     original_text: Optional[str] = None
     anonymized_text: Optional[str] = None
     extracted_metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Compute raw file hash after dataclass is constructed."""
+        self.file_hash = sha256_file(self.file_path)

endoreg_db/import_files/context/validate_directories.py

@@ -53,4 +53,5 @@ def validate_directories(dirs: Iterable[Path] = dirs) -> bool:
 
     return ok
 
+
 validate_directories(dirs)

endoreg_db/import_files/file_storage/create_report_file.py

@@ -2,67 +2,91 @@
 import logging
 from typing import Tuple
 
-from endoreg_db.import_files.context.import_context import ImportContext
-from endoreg_db.models.media import RawPdfFile
-from endoreg_db.models.media.processing_history.processing_history import ProcessingHistory
 from endoreg_db.import_files.context.ensure_center import ensure_center
+from endoreg_db.import_files.context.import_context import ImportContext  #
+from endoreg_db.utils.file_operations import sha256_file
+from endoreg_db.models.media import RawPdfFile
+from endoreg_db.models.state.processing_history.processing_history import (
+    ProcessingHistory,
+)
+from endoreg_db.import_files.file_storage.state_management import finalize_failure
+
 logger = logging.getLogger(__name__)
 
 
 def create_or_retrieve_report_file(
     ctx: ImportContext,
-) -> Tuple[RawPdfFile, bool]:
+) -> Tuple[RawPdfFile, bool, bool]:
     """
     Create a new or retrieve an existing RawPdfFile for the given context.
 
     Returns:
         pdf             : RawPdfFile instance
-        needs_processing: True if the pipeline should run for this file
-                          (no successful history yet for this object/file_type key)
+        processed       : True if there is already a successful ProcessingHistory for this file
+        needs_processing: True if the pipeline should run for this file in this call
     """
     file_path = ctx.file_path
     center_name = ctx.center_name
     delete_source = ctx.delete_source
     file_type = ctx.file_type  # logical key for history; can be None
 
-    # 1) Determine the RawPdfFile instance to work with
+    # default assumptions
+    processed = False
+    needs_processing = True
+
+    if not isinstance(ctx.file_hash, str):
+        ctx.file_hash = sha256_file(ctx.file_path)
+
+    # Check if we already have a successful history entry for this object
+    has_success_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=True,
+    )
+    has_failure_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
+    )
+    if has_success_history:
+        logger.info(
+            "RawPdfFile pk= already has successful processing history (file_hash=%s) - short-circuiting",
+            ctx.file_hash,
+        )
+        processed = True
+        needs_processing = False
+        if not isinstance(ctx.current_report, RawPdfFile):
+            ctx.current_report = RawPdfFile.get_report_by_hash(ctx.file_hash)
+        return ctx.current_report, processed, needs_processing
+    elif has_failure_history:
+        if not isinstance(ctx.current_report, RawPdfFile):
+            ctx.current_report = RawPdfFile.get_report_by_hash(ctx.file_hash)
+        finalize_failure(ctx)
+        processed = True
+        needs_processing = True
+
+    # Determine the RawPdfFile instance to work with
     if ctx.current_report is not None:
         pdf = ctx.current_report
         logger.info("Using existing RawPdfFile from context: pk=%s", pdf.pk)
     else:
-        logger.info("Creating new RawPdfFile from %s for center %s", file_path, center_name)
-        
+        logger.info(
+            "Creating new RawPdfFile from %s for center %s",
+            file_path,
+            center_name,
+        )
+
         pdf = RawPdfFile.create_from_file_initialized(
             file_path=file_path,
             center_name=center_name,
             delete_source=delete_source,
         )
-        
-        center = ensure_center(pdf, ctx.center_name)
-        
-        logger.info(f"Successfully set up report file from {center.name}")
-
 
+        center = ensure_center(pdf, ctx.center_name)
+        logger.info("Successfully set up report file from %s", center.name)
 
-    # 3) Check if we already have a successful history entry for this object+file_type
-    has_success_history = ProcessingHistory.has_history_for_object(
-        obj=pdf,
-        success=True,
-    )
-
-    if has_success_history:
-        logger.info(
-            "RawPdfFile %s already has successful processing history (file_type=%s) - short-circuiting",
-            getattr(pdf, str(pdf.file_path)),
-            file_type,
-        )
-        # No need to run the pipeline again
-        return pdf, False
-
-    # 4) No successful history yet → ensure there is a history entry marking it as "in progress"/failed
-    ProcessingHistory.get_or_create_for_object(
+    # No successful history yet → ensure there is a history entry marking it as "in progress"/failed
+    ProcessingHistory.get_or_create_for_hash(
         obj=pdf,
-        # At this point we haven't finished anonymization; treat as not-success yet.
+        file_hash=ctx.file_hash,
         success=False,
     )
 
@@ -72,5 +96,4 @@ def create_or_retrieve_report_file(
         file_type,
     )
 
-    # Signal to the caller that the anonymization pipeline should run
-    return pdf, True
+    return pdf, processed, needs_processing

endoreg_db/import_files/file_storage/create_video_file.py

@@ -1,75 +1,104 @@
 # endoreg_db/import_files/storage/create_video_file.py
 import logging
 from typing import Tuple
-
-from django.db import IntegrityError
-
 from endoreg_db.import_files.context.import_context import ImportContext
 from endoreg_db.import_files.context.ensure_center import ensure_center
+from endoreg_db.utils.file_operations import sha256_file
 from endoreg_db.models.media import VideoFile
-from endoreg_db.models.media.processing_history.processing_history import ProcessingHistory
-from endoreg_db.utils.hashs import get_video_hash
+from endoreg_db.models.state.processing_history.processing_history import (
+    ProcessingHistory,
+)
+from endoreg_db.import_files.file_storage.state_management import finalize_failure
 
 logger = logging.getLogger(__name__)
 
 
 def create_or_retrieve_video_file(
     ctx: ImportContext,
-) -> Tuple[VideoFile, bool]:
+) -> Tuple[VideoFile, bool, bool]:
     """
     Create a new or retrieve an existing VideoFile for the given context.
 
     Returns:
-        video     : VideoFile instance
-        retry     : whether we are re-processing an existing file
+        video           : VideoFile instance
+        processed       : True if there is already a successful ProcessingHistory for this file
+        needs_processing: True if the pipeline should run for this file in this call
     """
     file_path = ctx.file_path
+
     center_name = ctx.center_name
     processor_name = ctx.processor_name
     delete_source = ctx.delete_source
-    file_type = ctx.file_type
+    file_type = ctx.file_type  # logical key for history; can be None
+
+    # default assumptions (same semantics as report)
+    processed = False
+    needs_processing = True
+
+    if not isinstance(ctx.file_hash, str):
+        ctx.file_hash = sha256_file(ctx.file_path)
+
+        # 2) Check if we already have a successful history entry for this object
+    has_success_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=True,
+    )
+    has_failure_history = ProcessingHistory.has_history_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
+    )
 
-    # 1) Determine the VideoFile instance to work with
+    if has_success_history:
+        logger.info(
+            "VideoFile already has successful processing history (file_hash=%s) "
+            "- short-circuiting",
+            ctx.file_hash,
+        )
+        processed = True
+        needs_processing = False
+        if not isinstance(ctx.current_video, VideoFile):
+            ctx.current_video = VideoFile.get_video_by_content_hash(ctx.file_hash)
+        return ctx.current_video, processed, needs_processing
+    elif has_failure_history:
+        if not isinstance(ctx.current_video, VideoFile):
+            ctx.current_video = VideoFile.get_video_by_content_hash(ctx.file_hash)
+        finalize_failure(ctx)
+
+        processed = True
+        needs_processing = True
+
+    # Determine the VideoFile instance to work with
     if ctx.current_video is not None:
         video = ctx.current_video
-        logger.info("Using existing VideoFIle from context: pk =%s", video.pk)
+        logger.info("Using existing VideoFile from context: pk=%s", video.pk)
     else:
-        logger.info("Creating new VideoFIle from %s for center %s", file_path, center_name)
+        logger.info(
+            "Creating new VideoFile from %s for center %s",
+            file_path,
+            center_name,
+        )
         video = VideoFile.create_from_file_initialized(
             file_path=file_path,
             center_name=center_name,
             processor_name=processor_name,
             delete_source=delete_source,
+            video_hash=ctx.file_hash,
         )
+        needs_processing = True
+
         center = ensure_center(video, ctx.center_name)
-        
-        logger.info(f"Successfully set up report file from {center.name}")
-    # 3) Check if we already have a successful history entry for this object+file_type        
-    has_success_history = ProcessingHistory.has_history_for_object(
-        obj=video,
-        success=True,
-    )
+        logger.info("Successfully set up video file from %s", center.name)
 
-    if has_success_history:
-        logger.info(
-            "VideoFile %s already has successful processing history. (file_type:%s) - short-circuiting",
-            getattr(video, str(video.active_file_path)),
-            file_type,
+    # No successful history yet → ensure there is a history entry marking it as "in progress"/failed
+    ProcessingHistory.get_or_create_for_hash(
+        file_hash=ctx.file_hash,
+        success=False,
     )
-    
-    # 4) No successful history yet → ensure there is a history entry marking it as "in progress"/failed
-    ProcessingHistory.get_or_create_for_object(
-        obj=video,
-        success=False
-    )
-    
-    
+
     logger.info(
         "Video instance ready for processing: pk=%s, file_type=%s (needs_processing=True)",
         video.pk,
         file_type,
     )
-    
-    # Signal to the caller that the anonymization pipeline should run
-    return video, True
 
+    return video, processed, needs_processing

endoreg_db/import_files/file_storage/sensitive_meta_storage.py

@@ -1,18 +1,21 @@
 # endoreg_db/import_files/storage/sensitive_meta_storage.py
-from typing import Union, Dict, Any
+from typing import Union
 
 from endoreg_db.models.media import RawPdfFile, VideoFile
 from endoreg_db.models.metadata import SensitiveMeta
 from endoreg_db.import_files.processing.sensitive_meta_adapter import (
     normalize_lx_sensitive_meta,
 )
-from endoreg_db.import_files.context.default_sensitive_meta import default_sensitive_meta
+from endoreg_db.import_files.context.default_sensitive_meta import (
+    default_sensitive_meta,
+)
 from logging import getLogger
 from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
 #
 
 logger = getLogger(__name__)
 
+
 def sensitive_meta_storage(
     sensitive_meta: LxSM,
     instance: Union[RawPdfFile, VideoFile],