endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.9.2__py3-none-any.whl

endoreg_db/models/media/video/video_file.py

@@ -11,22 +11,25 @@ from django.core.validators import FileExtensionValidator
 from django.db import models
 from django.db.models import F
 from django.db.models.fields.files import FieldFile
-from librosa import frames_to_samples
-from pandas.core import frame
 
 from endoreg_db.utils.calc_duration_seconds import _calc_duration_vf
 from endoreg_db.utils.video.ffmpeg_wrapper import assemble_video_from_frames
 
 from ...label import Label, LabelVideoSegment
 from ...state import VideoState
-from ...utils import ANONYM_VIDEO_DIR, VIDEO_DIR
+from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, SENSITIVE_VIDEO_DIR
 
 # --- Import model-specific function modules ---
 from .create_from_file import _create_from_file
 from .pipe_1 import _pipe_1, _test_after_pipe_1
 from .pipe_2 import _pipe_2
 from .video_file_ai import _extract_text_from_video_frames, _predict_video_pipeline
-from .video_file_anonymize import _anonymize, _censor_outside_frames, _cleanup_raw_assets, _create_anonymized_frame_files
+from .video_file_anonymize import (
+    _anonymize,
+    _censor_outside_frames,
+    _cleanup_raw_assets,
+    _create_anonymized_frame_files,
+)
 from .video_file_frames import (
     _bulk_create_frames,
     _create_frame_object,
@@ -85,7 +88,6 @@ if TYPE_CHECKING:
         VideoImportMeta,
         VideoMeta,
         VideoState,
-        SensitiveMeta
     )
 
 
@@ -115,7 +117,7 @@ class VideoFile(models.Model):
     objects = VideoQuerySet.as_manager()
 
     raw_file = models.FileField(
-        upload_to=VIDEO_DIR.name,  # Use .name for relative path
+        upload_to=SENSITIVE_VIDEO_DIR.name,  # Use .name for relative path
         validators=[FileExtensionValidator(allowed_extensions=["mp4"])],
         null=True,
         blank=True,
@@ -127,7 +129,9 @@ class VideoFile(models.Model):
         blank=True,
     )
 
-    video_hash = models.CharField(max_length=255, unique=True, help_text="Hash of the raw video file.")
+    video_hash = models.CharField(
+        max_length=255, unique=True, help_text="Hash of the raw video file."
+    )
     processed_video_hash = models.CharField(
         max_length=255,
         unique=True,
@@ -144,7 +148,9 @@ class VideoFile(models.Model):
         related_name="video_file",
     )
     center = models.ForeignKey("Center", on_delete=models.PROTECT)
-    processor = models.ForeignKey("EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True)
+    processor = models.ForeignKey(
+        "EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True
+    )
     video_meta = models.OneToOneField(
         "VideoMeta",
         on_delete=models.SET_NULL,
@@ -166,7 +172,9 @@ class VideoFile(models.Model):
         null=True,
         related_name="video_files",
     )
-    ai_model_meta = models.ForeignKey("ModelMeta", on_delete=models.SET_NULL, blank=True, null=True)
+    ai_model_meta = models.ForeignKey(
+        "ModelMeta", on_delete=models.SET_NULL, blank=True, null=True
+    )
     state = models.OneToOneField(
         "VideoState",
         on_delete=models.SET_NULL,
@@ -174,7 +182,9 @@ class VideoFile(models.Model):
         blank=True,
         related_name="video_file",
     )
-    import_meta = models.OneToOneField("VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True)
+    import_meta = models.OneToOneField(
+        "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
+    )
 
     original_file_name = models.CharField(max_length=255, blank=True, null=True)
     uploaded_at = models.DateTimeField(auto_now_add=True)
@@ -291,7 +301,9 @@ class VideoFile(models.Model):
     bulk_create_frames = _bulk_create_frames
 
     # Define new methods that call the helper functions
-    def extract_specific_frame_range(self, start_frame: int, end_frame: int, overwrite: bool = False, **kwargs) -> bool:
+    def extract_specific_frame_range(
+        self, start_frame: int, end_frame: int, overwrite: bool = False, **kwargs
+    ) -> bool:
         """
         Extract frames from the video within the specified frame range.
 
@@ -314,9 +326,13 @@ class VideoFile(models.Model):
 
         # Log if unexpected kwargs are passed, beyond those used by the helper
         expected_helper_kwargs = {"quality", "ext", "verbose"}
-        unexpected_kwargs = {k: v for k, v in kwargs.items() if k not in expected_helper_kwargs}
+        unexpected_kwargs = {
+            k: v for k, v in kwargs.items() if k not in expected_helper_kwargs
+        }
         if unexpected_kwargs:
-            logger.warning(f"Unexpected keyword arguments for extract_specific_frame_range, will be ignored by helper: {unexpected_kwargs}")
+            logger.warning(
+                f"Unexpected keyword arguments for extract_specific_frame_range, will be ignored by helper: {unexpected_kwargs}"
+            )
 
         return _extract_frame_range_helper(
             video=self,
@@ -332,7 +348,9 @@ class VideoFile(models.Model):
         """
         Deletes frame files for a specific range [start_frame, end_frame).
         """
-        _delete_frame_range_helper(video=self, start_frame=start_frame, end_frame=end_frame)
+        _delete_frame_range_helper(
+            video=self, start_frame=start_frame, end_frame=end_frame
+        )
 
     delete_with_file = _delete_with_file
     get_base_frame_dir = _get_base_frame_dir
@@ -387,7 +405,9 @@ class VideoFile(models.Model):
         if isinstance(raw, FieldFile) and raw.name:
             return raw
 
-        raise ValueError("No active file available. VideoFile has neither raw nor processed file.")
+        raise ValueError(
+            "No active file available. VideoFile has neither raw nor processed file."
+        )
 
     @property
     def active_file_path(self) -> Path:
@@ -406,10 +426,14 @@ class VideoFile(models.Model):
         elif active is self.raw_file:
             path = _get_raw_file_path(self)
         else:
-            raise ValueError("No active file path available. VideoFile has neither raw nor processed file.")
+            raise ValueError(
+                "No active file path available. VideoFile has neither raw nor processed file."
+            )
 
         if path is None:
-            raise ValueError("Active file path could not be resolved. VideoFile raw file is missing.")
+            raise ValueError(
+                "Active file path could not be resolved. VideoFile raw file is missing."
+            )
         return path
 
     @property
@@ -426,7 +450,9 @@ class VideoFile(models.Model):
                 self.uuid,
                 exc,
             )
-            raise ValueError("Active file URL could not be resolved for this VideoFile.") from exc
+            raise ValueError(
+                "Active file URL could not be resolved for this VideoFile."
+            ) from exc
 
         if not url:
             raise ValueError("Active file URL is empty for this VideoFile.")
@@ -434,7 +460,9 @@ class VideoFile(models.Model):
         return str(url)
 
     @classmethod
-    def create_from_file(cls, file_path: Union[str, Path], center_name: str, **kwargs) -> Optional["VideoFile"]:
+    def create_from_file(
+        cls, file_path: Union[str, Path], center_name: str, **kwargs
+    ) -> Optional["VideoFile"]:
         # Ensure file_path is a Path object
         if isinstance(file_path, str):
             file_path = Path(file_path)
@@ -443,7 +471,9 @@ class VideoFile(models.Model):
             try:
                 center_name = os.environ["CENTER_NAME"]
             except KeyError:
-                logger.error("Center name must be provided to create VideoFile from file. You can set CENTER_NAME in environment variables.")
+                logger.error(
+                    "Center name must be provided to create VideoFile from file. You can set CENTER_NAME in environment variables."
+                )
                 return None
         return _create_from_file(cls, file_path, center_name=center_name, **kwargs)
 
@@ -503,7 +533,9 @@ class VideoFile(models.Model):
         # Delete file storage
         if self.raw_file and self.raw_file.storage.exists(self.raw_file.name):
             self.raw_file.storage.delete(self.raw_file.name)
-        if self.processed_file and self.processed_file.storage.exists(self.processed_file.name):
+        if self.processed_file and self.processed_file.storage.exists(
+            self.processed_file.name
+        ):
             self.processed_file.storage.delete(self.processed_file.name)
 
         # Use proper database connection
@@ -530,7 +562,9 @@ class VideoFile(models.Model):
             logger.error(f"Error deleting VideoFile {self.uuid}: {e}")
             raise
 
-    def validate_metadata_annotation(self, extracted_data_dict: Optional[dict] = None) -> bool:
+    def validate_metadata_annotation(
+        self, extracted_data_dict: Optional[dict] = None
+    ) -> bool:
         """
         Validate the metadata of the VideoFile instance.
 
@@ -543,12 +577,9 @@ class VideoFile(models.Model):
         video is preserved as the final validated output.
         """
 
-        if not self.sensitive_meta:
-            # Ensure a SensitiveMeta exists so validation can proceed.
-            self.sensitive_meta = self.get_or_create_sensitive_meta()
         # CRITICAL FIX: Delete RAW video file, not the processed (anonymized) one
         # CRITICAL: Update metadata BEFORE deleting raw video
-        if extracted_data_dict:
+        if extracted_data_dict and self.sensitive_meta:
             self.sensitive_meta.update_from_dict(extracted_data_dict)
         else:
             return False
@@ -564,7 +595,9 @@ class VideoFile(models.Model):
             # Clear the raw_file field in database (use delete() to avoid save issues)
             if self.raw_file:
                 self.raw_file.delete(save=False)
-            logger.info(f"Raw video deleted for {self.uuid}. Anonymized video preserved.")
+            logger.info(
+                f"Raw video deleted for {self.uuid}. Anonymized video preserved."
+            )
         else:
             logger.warning(
                 "Raw video file not found for deletion during validation %s.",
@@ -576,10 +609,14 @@ class VideoFile(models.Model):
             self.get_or_create_state().mark_anonymization_validated(save=True)
             # Save the VideoFile instance to persist changes
             self.save()
-            logger.info(f"Metadata annotation validated and saved for video {self.uuid}.")
+            logger.info(
+                f"Metadata annotation validated and saved for video {self.uuid}."
+            )
             return True
         else:
-            logger.error(f"Failed to validate metadata annotation for video {self.uuid}.")
+            logger.error(
+                f"Failed to validate metadata annotation for video {self.uuid}."
+            )
             return False
 
     def initialize(self):
@@ -600,7 +637,6 @@ class VideoFile(models.Model):
         # Create a new state if it doesn't exist
         self.state = self.get_or_create_state()
 
-        self.sensitive_meta = self.get_or_create_sensitive_meta()
         self.save()
         # Initialize frames based on the video specs
         self.initialize_frames()
@@ -613,7 +649,9 @@ class VideoFile(models.Model):
         """
         active_path = self.active_file_path
         file_name = active_path.name if active_path else "No file"
-        state = "Processed" if self.is_processed else ("Raw" if self.has_raw else "No File")
+        state = (
+            "Processed" if self.is_processed else ("Raw" if self.has_raw else "No File")
+        )
         return f"VideoFile ({state}): {file_name} (UUID: {self.uuid})"
 
     # --- Convenience state/meta helpers used in tests and admin workflows ---
@@ -622,6 +660,11 @@ class VideoFile(models.Model):
         Mark this video's processing state as having its sensitive meta fully processed.
         This proxies to the related VideoState and persists by default.
         """
+        sm = self.sensitive_meta
+        from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
+
+        if not isinstance(sm, SensitiveMeta):
+            raise AttributeError()
         state = self.get_or_create_state()
         state.mark_sensitive_meta_processed(save=save)
         return self
@@ -631,8 +674,13 @@ class VideoFile(models.Model):
         Mark the associated SensitiveMeta as verified by setting both DOB and names as verified.
         Ensures the SensitiveMeta and its state exist.
         """
-        sm = self.get_or_create_sensitive_meta()
+        sm = self.sensitive_meta
         # Use SensitiveMeta methods to update underlying SensitiveMetaState
+        from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
+
+        if not isinstance(sm, SensitiveMeta):
+            raise AttributeError()
+
         sm.mark_dob_verified()
         sm.mark_names_verified()
         return self
@@ -671,66 +719,9 @@ class VideoFile(models.Model):
 
         return state
 
-    def get_or_create_sensitive_meta(self) -> "SensitiveMeta":
-        """
-        Retrieve the associated SensitiveMeta instance for this video, creating and assigning one if it does not exist.
-
-        **Two-Phase Patient Data Pattern:**
-        This method implements a two-phase approach to handle incomplete patient data:
-
-        **Phase 1: Initial Creation (with defaults)**
-        - Creates SensitiveMeta with default patient data to prevent hash calculation errors
-        - Default values: patient_first_name="Patient", patient_last_name="Unknown", patient_dob=1990-01-01
-        - Allows video import to proceed even without extracted patient data
-        - Temporary hash and pseudo-entities are created
-
-        **Phase 2: Update (with extracted data)**
-        - Real patient data is extracted later (e.g., from video OCR via lx_anonymizer)
-        - update_from_dict() is called with actual patient information
-        - Hash is recalculated automatically using real data
-        - Correct pseudo-entities are created/linked based on new hash
-
-        **Example workflow:**
-        ```python
-        # Phase 1: Video creation
-        video = VideoFile.create_from_file_initialized(...)
-        video.initialize()  # Calls this method
-        # → SensitiveMeta created with defaults
-        # → Hash: sha256("Patient Unknown 1990-01-01...")
-
-        # Phase 2: Frame cleaning extracts real data
-        extracted = {"patient_first_name": "Max", "patient_last_name": "Mustermann", ...}
-        video.sensitive_meta.update_from_dict(extracted)
-        # → Hash: sha256("Max Mustermann 1985-03-15...") (RECALCULATED)
-        ```
-
-        Returns:
-            SensitiveMeta: The related SensitiveMeta instance.
-
-        See Also:
-            - sensitive_meta_logic.perform_save_logic() for hash calculation details
-            - sensitive_meta_logic.update_sensitive_meta_from_dict() for update mechanism
-        """
-        from datetime import date as dt_date
-
-        from endoreg_db.models import SensitiveMeta
-
-        if self.sensitive_meta is None:
-            # Use create_from_dict with default patient data
-            # to prevent "First name is required to calculate patient hash" error
-            default_data = {
-                "patient_first_name": "Patient",
-                "patient_last_name": "Unknown",
-                "patient_dob": dt_date(1990, 1, 1),
-                "examination_date": dt_date.today(),
-                "center": self.center,
-            }
-            self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
-            self.save(update_fields=["sensitive_meta"])
-            # Do not mark state as processed here; it will be set after extraction/validation steps
-        return self.sensitive_meta
-
-    def get_outside_segments(self, only_validated: bool = False) -> models.QuerySet["LabelVideoSegment"]:
+    def get_outside_segments(
+        self, only_validated: bool = False
+    ) -> models.QuerySet["LabelVideoSegment"]:
         """
         Return all video segments labeled as "outside" for this video.
 
@@ -762,7 +753,9 @@ class VideoFile(models.Model):
             return self.label_video_segments.none()
 
     @classmethod
-    def create_video_without_outside_frames(cls, instance: "VideoFile", only_validated: bool = False) -> bool:
+    def create_video_without_outside_frames(
+        cls, instance: "VideoFile", only_validated: bool = False
+    ) -> bool:
         """
         Creates a new video by excluding frames that belong to 'outside' segments.
 
@@ -775,40 +768,64 @@ class VideoFile(models.Model):
         video = instance
 
         if not video:
-            logger.warning("No processed video file available for VideoFile %s.", cls.uuid)
+            logger.warning(
+                "No processed video file available for VideoFile %s.", cls.uuid
+            )
             return False
         try:
-            extracted = video.extract_frames(quality=2, overwrite=False, ext="jpg", verbose=False, from_processed=True)
+            extracted = video.extract_frames(
+                quality=2,
+                overwrite=False,
+                ext="jpg",
+                verbose=False,
+                from_processed=True,
+            )
             assert extracted is True
         except AssertionError:
-            # Use default anonymization here
-            video.anonymize
-            extracted = video.extract_frames(quality=2, overwrite=False, ext="jpg", verbose=False, from_processed=True)
+            extracted = video.extract_frames(
+                quality=2,
+                overwrite=False,
+                ext="jpg",
+                verbose=False,
+                from_processed=True,
+            )
             assert extracted is True
         try:
             # Step 1: Get the "outside" labeled frames
             censored = _censor_outside_frames(video)
             frames = [instance.get_frame_dir_path()]
             assert len(frames) != 0
-            fps = video.fps if video.fps else 120.0  # Default to 30 FPS if fps is not set
+            fps = (
+                video.fps if video.fps else 120.0
+            )  # Default to 30 FPS if fps is not set
+            assert censored is True
             assert fps is not None
             assert video.width is not None
             assert video.height is not None
+            # assert isinstance(frames, list[Path]) #TODO improve TypeCheck
 
             # Step 2: Reassemble the video with frames excluding the 'outside' labeled frames
             output_video_path = Path(f"/path/to/output/{cls.uuid}_filtered.mp4")
             fps = cls.fps if cls.fps else 30.0  # Default to 30 FPS if fps is not set
-            new_video_file = assemble_video_from_frames(frames, output_video_path, fps, width=video.width, height=video.height)
+            new_video_file = assemble_video_from_frames(
+                frames, output_video_path, fps, width=video.width, height=video.height
+            )
             video.processed_file = new_video_file
             return True
         except AssertionError as ae:
-            logger.error(f"Assertion error while creating video without 'outside' frames for VideoFile {cls.uuid}: {ae}", exc_info=True)
+            logger.error(
+                f"Assertion error while creating video without 'outside' frames for VideoFile {cls.uuid}: {ae}",
+                exc_info=True,
+            )
             return False
         except Label.DoesNotExist:
             logger.warning("Outside label not found in the database.")
             return False
         except Exception as e:
-            logger.error(f"Error creating video without 'outside' frames for VideoFile {cls.uuid}: {e}", exc_info=True)
+            logger.error(
+                f"Error creating video without 'outside' frames for VideoFile {cls.uuid}: {e}",
+                exc_info=True,
+            )
             return False
 
     @classmethod
@@ -828,7 +845,9 @@ class VideoFile(models.Model):
             int: The count of VideoFile records, excluding this instance, where the modification timestamp matches the creation timestamp.
         """
         return (
-            VideoFile.objects.filter(date_modified=F("date_created"))  # compare the two fields in SQL
+            VideoFile.objects.filter(
+                date_modified=F("date_created")
+            )  # compare the two fields in SQL
             .exclude(pk=self.pk)  # exclude this instance
             .count()  # run a fast COUNT(*) on the filtered set
         )
@@ -851,7 +870,8 @@ class VideoFile(models.Model):
             raise ValueError("FPS must be set and greater than zero.")
         return frame_number / fps
 
-    def get_video_by_id(self, video_id: int) -> "VideoFile":
+    @staticmethod
+    def get_video_by_pk(pk: int) -> "VideoFile":
         """
         Retrieve a VideoFile instance by its primary key (ID).
 
@@ -864,4 +884,4 @@ class VideoFile(models.Model):
         Raises:
             VideoFile.DoesNotExist: If no VideoFile with the given ID exists.
         """
-        return self.objects.get(pk=video_id)
+        return VideoFile.objects.get(pk=video_id)

endoreg_db/models/media/video/video_file_io.py

@@ -6,9 +6,9 @@ from typing import TYPE_CHECKING, Iterator, Optional
 from django.db import transaction
 
 from ....utils import delete_field_file, ensure_local_file, storage_file_exists
-from ...utils import (
+from endoreg_db.utils.paths import (
     ANONYM_VIDEO_DIR,
-    VIDEO_DIR,
+    SENSITIVE_VIDEO_DIR,
     data_paths,
 )
 
@@ -23,36 +23,41 @@ def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
     if not (video.has_raw and video.raw_file.name):
         return None
 
-    # raw_file.name is a relative storage path like 'videos/<filename>'
-    raw_rel = Path(video.raw_file.name)
+    # 1) Canonical: use Django's storage path
+    try:
+        direct_path = Path(video.raw_file.path)
+        if direct_path.is_file():
+            return direct_path.resolve()
+        else:
+            logger.debug(
+                "raw_file.path for video %s is not a regular file: %s",
+                video.uuid,
+                direct_path,
+            )
+    except Exception as exc:
+        logger.debug(
+            "Could not access raw_file.path for video %s: %s",
+            video.uuid,
+            exc,
+        )
 
-    # If it already contains the video directory name, keep the tail
-    rel_name = raw_rel.name if raw_rel.parent.name == VIDEO_DIR.name else raw_rel
-    full_path = data_paths["video"] / rel_name
+    # 2) Fallback: use just the filename and search in known dirs
+    raw_rel = Path(video.raw_file.name)
+    filename = raw_rel.name  # strip any (possibly wrong) prefix
 
-    if full_path.exists():
-        return full_path.resolve()
+    candidates = [
+        data_paths["import_video"] / filename,
+        data_paths["sensitive_video"] / filename,
+    ]
 
-    sensitive_path = data_paths["video"] / "sensitive" / rel_name
-    if sensitive_path.exists():
-        return sensitive_path.resolve()
 
-    if storage_file_exists(video.raw_file):
-        try:
-            direct_path = Path(video.raw_file.path)
-            if direct_path.exists():
-                return direct_path.resolve()
-        except Exception as exc:
-            logger.debug(
-                "Could not access direct raw_file.path for video %s: %s",
-                video.uuid,
-                exc,
-            )
+    for candidate in candidates:
+        if candidate.is_file():
+            return candidate.resolve()
 
     logger.warning(
-        "Raw video file '%s' not found under %s or via stored FileField path for video %s.",
-        rel_name,
-        data_paths["video"],
+        "Raw video file '%s' not found in import/sensitive paths or via stored FileField path for video %s.",
+        video.raw_file.name,
         video.uuid,
     )
     return None

endoreg_db/models/medical/contraindication/README.md

@@ -0,0 +1 @@
+not yet properly implemented

endoreg_db/models/medical/examination/examination.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, cast
 
 from django.db import models
 
@@ -28,20 +28,40 @@ class Examination(models.Model):
     name = models.CharField(max_length=100, unique=True)
     examination_types = models.ManyToManyField("ExaminationType", blank=True)
     description = models.TextField(blank=True, null=True)
+    indications = models.ManyToManyField(
+        "ExaminationIndication",
+        related_name="examinations",
+        blank=True,
+    )
+    examination_times = models.ManyToManyField(
+        "ExaminationTime",
+        related_name="examinations",
+        blank=True,
+    )
+
+    findings = models.ManyToManyField(
+        "Finding",
+        blank=True,
+        related_name="examinations",
+    )
+    information_sources = models.ManyToManyField(
+        "InformationSource",
+        related_name="examinations",
+        blank=True,
+    )
 
     objects = ExaminationManager()
 
     if TYPE_CHECKING:
-        from endoreg_db.models import ExaminationIndication, Finding, FindingClassification
+        from endoreg_db.models import ExaminationIndication, ExaminationTime, Finding, FindingClassification, InformationSource
 
-        @property
-        def finding_classifications(self) -> "models.manager.RelatedManager[FindingClassification]": ...
+        indications = cast("models.manager.RelatedManager[ExaminationIndication]", indications)
+        examination_times = cast("models.manager.RelatedManager[ExaminationTime]", examination_times)
+        findings = cast("models.manager.RelatedManager[Finding]", findings)
+        information_sources = cast("models.manager.RelatedManager[InformationSource]", information_sources)
 
         @property
-        def indications(self) -> "models.manager.RelatedManager[ExaminationIndication]": ...
-
-        @property
-        def findings(self) -> "models.manager.RelatedManager[Finding]": ...
+        def finding_classifications(self) -> "models.manager.RelatedManager[FindingClassification]": ...
 
         @property
         def exam_reqset_links(self) -> "models.manager.RelatedManager[ExaminationRequirementSet]": ...