endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

endoreg_db/models/media/video/video_file.py

@@ -4,15 +4,18 @@ import logging
 import os
 import uuid
 from pathlib import Path
-from typing import TYPE_CHECKING, Optional, Union, cast
+from typing import TYPE_CHECKING, Optional, Self, Union, cast
 
 from django.core.files import File
 from django.core.validators import FileExtensionValidator
 from django.db import models
 from django.db.models import F
 from django.db.models.fields.files import FieldFile
+from librosa import frames_to_samples
+from pandas.core import frame
 
 from endoreg_db.utils.calc_duration_seconds import _calc_duration_vf
+from endoreg_db.utils.video.ffmpeg_wrapper import assemble_video_from_frames
 
 from ...label import Label, LabelVideoSegment
 from ...state import VideoState
@@ -23,11 +26,7 @@ from .create_from_file import _create_from_file
 from .pipe_1 import _pipe_1, _test_after_pipe_1
 from .pipe_2 import _pipe_2
 from .video_file_ai import _extract_text_from_video_frames, _predict_video_pipeline
-from .video_file_anonymize import (
-    _anonymize,
-    _cleanup_raw_assets,
-    _create_anonymized_frame_files,
-)
+from .video_file_anonymize import _anonymize, _censor_outside_frames, _cleanup_raw_assets, _create_anonymized_frame_files
 from .video_file_frames import (
     _bulk_create_frames,
     _create_frame_object,
@@ -72,6 +71,8 @@ from .video_file_meta import (
 logger = logging.getLogger(__name__)  # Changed from "video_file"
 
 if TYPE_CHECKING:
+    from django.db.models.fields.files import FieldFile
+
     from endoreg_db.models import (
         Center,
         EndoscopyProcessor,
@@ -84,6 +85,7 @@ if TYPE_CHECKING:
         VideoImportMeta,
         VideoMeta,
         VideoState,
+        SensitiveMeta
     )
 
 
@@ -125,9 +127,7 @@ class VideoFile(models.Model):
         blank=True,
     )
 
-    video_hash = models.CharField(
-        max_length=255, unique=True, help_text="Hash of the raw video file."
-    )
+    video_hash = models.CharField(max_length=255, unique=True, help_text="Hash of the raw video file.")
     processed_video_hash = models.CharField(
         max_length=255,
         unique=True,
@@ -142,45 +142,39 @@ class VideoFile(models.Model):
         null=True,
         blank=True,
         related_name="video_file",
-    )  # type: ignore
-    center = models.ForeignKey("Center", on_delete=models.PROTECT)  # type: ignore
-    processor = models.ForeignKey(
-        "EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True
-    )  # type: ignore
+    )
+    center = models.ForeignKey("Center", on_delete=models.PROTECT)
+    processor = models.ForeignKey("EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True)
     video_meta = models.OneToOneField(
         "VideoMeta",
         on_delete=models.SET_NULL,
         null=True,
         blank=True,
         related_name="video_file",
-    )  # type: ignore
+    )
     examination = models.ForeignKey(
         "PatientExamination",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
         related_name="video_files",
-    )  # type: ignore
+    )
     patient = models.ForeignKey(
         "Patient",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
         related_name="video_files",
-    )  # type: ignore
-    ai_model_meta = models.ForeignKey(
-        "ModelMeta", on_delete=models.SET_NULL, blank=True, null=True
-    )  # type: ignore
+    )
+    ai_model_meta = models.ForeignKey("ModelMeta", on_delete=models.SET_NULL, blank=True, null=True)
     state = models.OneToOneField(
         "VideoState",
         on_delete=models.SET_NULL,
         null=True,
         blank=True,
         related_name="video_file",
-    )  # type: ignore
-    import_meta = models.OneToOneField(
-        "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
-    )  # type: ignore
+    )
+    import_meta = models.OneToOneField("VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True)
 
     original_file_name = models.CharField(max_length=255, blank=True, null=True)
     uploaded_at = models.DateTimeField(auto_now_add=True)
@@ -206,17 +200,25 @@ class VideoFile(models.Model):
     date_modified = models.DateTimeField(auto_now=True)
 
     if TYPE_CHECKING:
-        label_video_segments: "models.QuerySet[LabelVideoSegment]"
-        frames: "models.QuerySet[Frame]"
-        center: "Center"
-        processor: "EndoscopyProcessor"
-        video_meta: "VideoMeta"
-        examination: "PatientExamination"
-        patient: "Patient"
-        sensitive_meta: "SensitiveMeta"
-        state: "VideoState"
-        ai_model_meta: "ModelMeta"
-        import_meta: "VideoImportMeta"
+        from django.db.models.manager import RelatedManager
+
+        @property
+        def label_video_segments(self) -> RelatedManager[LabelVideoSegment]: ...
+
+        @property
+        def frames(self) -> RelatedManager[Frame]: ...
+
+        center: models.ForeignKey["Center"]
+        processor: models.ForeignKey["EndoscopyProcessor | None"]
+        video_meta: models.OneToOneField["VideoMeta | None"]
+        examination: models.ForeignKey["PatientExamination | None"]
+        patient: models.ForeignKey["Patient | None"]
+        sensitive_meta: models.OneToOneField["SensitiveMeta | None"]
+        state: models.OneToOneField["VideoState | None"]
+        ai_model_meta: models.ForeignKey["ModelMeta | None"]
+        import_meta: models.OneToOneField["VideoImportMeta | None"]
+        raw_file = cast(FieldFile, raw_file)
+        processed_file = cast(FieldFile, processed_file)
 
     @property
     def ffmpeg_meta(self) -> "FFMpegMeta":
@@ -257,7 +259,10 @@ class VideoFile(models.Model):
         assert _file is not None, self.NO_ACTIVE_FILE
         if not _file or not _file.name:
             raise ValueError(self.NO_FILE_ASSOCIATED)
-        return _file.url
+        url = getattr(_file, "url", None)
+        if not url:
+            raise ValueError("Active raw file URL could not be resolved.")
+        return str(url)
 
     # Pipeline Functions
     pipe_1 = _pipe_1
@@ -286,9 +291,7 @@ class VideoFile(models.Model):
     bulk_create_frames = _bulk_create_frames
 
     # Define new methods that call the helper functions
-    def extract_specific_frame_range(
-        self, start_frame: int, end_frame: int, overwrite: bool = False, **kwargs
-    ) -> bool:
+    def extract_specific_frame_range(self, start_frame: int, end_frame: int, overwrite: bool = False, **kwargs) -> bool:
         """
         Extract frames from the video within the specified frame range.
 
@@ -311,13 +314,9 @@ class VideoFile(models.Model):
 
         # Log if unexpected kwargs are passed, beyond those used by the helper
         expected_helper_kwargs = {"quality", "ext", "verbose"}
-        unexpected_kwargs = {
-            k: v for k, v in kwargs.items() if k not in expected_helper_kwargs
-        }
+        unexpected_kwargs = {k: v for k, v in kwargs.items() if k not in expected_helper_kwargs}
         if unexpected_kwargs:
-            logger.warning(
-                f"Unexpected keyword arguments for extract_specific_frame_range, will be ignored by helper: {unexpected_kwargs}"
-            )
+            logger.warning(f"Unexpected keyword arguments for extract_specific_frame_range, will be ignored by helper: {unexpected_kwargs}")
 
         return _extract_frame_range_helper(
             video=self,
@@ -333,9 +332,7 @@ class VideoFile(models.Model):
         """
         Deletes frame files for a specific range [start_frame, end_frame).
         """
-        _delete_frame_range_helper(
-            video=self, start_frame=start_frame, end_frame=end_frame
-        )
+        _delete_frame_range_helper(video=self, start_frame=start_frame, end_frame=end_frame)
 
     delete_with_file = _delete_with_file
     get_base_frame_dir = _get_base_frame_dir
@@ -390,9 +387,7 @@ class VideoFile(models.Model):
         if isinstance(raw, FieldFile) and raw.name:
             return raw
 
-        raise ValueError(
-            "No active file available. VideoFile has neither raw nor processed file."
-        )
+        raise ValueError("No active file available. VideoFile has neither raw nor processed file.")
 
     @property
     def active_file_path(self) -> Path:
@@ -411,18 +406,35 @@ class VideoFile(models.Model):
         elif active is self.raw_file:
             path = _get_raw_file_path(self)
         else:
-            raise ValueError(
-                "No active file path available. VideoFile has neither raw nor processed file."
-            )
+            raise ValueError("No active file path available. VideoFile has neither raw nor processed file.")
 
         if path is None:
-            raise ValueError("Active file path could not be resolved.")
+            raise ValueError("Active file path could not be resolved. VideoFile raw file is missing.")
         return path
 
+    @property
+    def active_file_url(self) -> str:
+        """Return the URL of the active video file, if available."""
+        file_obj = self.active_file
+        if not isinstance(file_obj, FieldFile):
+            raise ValueError("Active file is not a valid Django FieldFile instance.")
+        try:
+            url = getattr(file_obj, "url", None)
+        except Exception as exc:  # storage backends may raise when missing
+            logger.warning(
+                "Active file URL unavailable for video %s: %s",
+                self.uuid,
+                exc,
+            )
+            raise ValueError("Active file URL could not be resolved for this VideoFile.") from exc
+
+        if not url:
+            raise ValueError("Active file URL is empty for this VideoFile.")
+
+        return str(url)
+
     @classmethod
-    def create_from_file(
-        cls, file_path: Union[str, Path], center_name: str, **kwargs
-    ) -> Optional["VideoFile"]:
+    def create_from_file(cls, file_path: Union[str, Path], center_name: str, **kwargs) -> Optional["VideoFile"]:
         # Ensure file_path is a Path object
         if isinstance(file_path, str):
             file_path = Path(file_path)
@@ -431,9 +443,7 @@ class VideoFile(models.Model):
             try:
                 center_name = os.environ["CENTER_NAME"]
             except KeyError:
-                logger.error(
-                    "Center name must be provided to create VideoFile from file. You can set CENTER_NAME in environment variables."
-                )
+                logger.error("Center name must be provided to create VideoFile from file. You can set CENTER_NAME in environment variables.")
                 return None
         return _create_from_file(cls, file_path, center_name=center_name, **kwargs)
 
@@ -478,19 +488,22 @@ class VideoFile(models.Model):
         _delete_frames(self)
 
         # Call the original delete method to remove the instance from the database
-        active_path = self.active_file_path
-        logger.info(f"Deleting VideoFile: {self.uuid} - {active_path}")
+        try:
+            active_path = self.active_file_path
+            logger.info(f"Deleting VideoFile: {self.uuid} - {active_path}")
+
+        except ValueError:
+            logger.info(f"Deleting VideoFile: {self.uuid} - No active file path found.")
+            active_path = None
 
         # Delete associated files if they exist
-        if active_path.exists():
+        if active_path and active_path.exists():
             active_path.unlink(missing_ok=True)
 
         # Delete file storage
         if self.raw_file and self.raw_file.storage.exists(self.raw_file.name):
             self.raw_file.storage.delete(self.raw_file.name)
-        if self.processed_file and self.processed_file.storage.exists(
-            self.processed_file.name
-        ):
+        if self.processed_file and self.processed_file.storage.exists(self.processed_file.name):
             self.processed_file.storage.delete(self.processed_file.name)
 
         # Use proper database connection
@@ -517,9 +530,7 @@ class VideoFile(models.Model):
             logger.error(f"Error deleting VideoFile {self.uuid}: {e}")
             raise
 
-    def validate_metadata_annotation(
-        self, extracted_data_dict: Optional[dict] = None
-    ) -> bool:
+    def validate_metadata_annotation(self, extracted_data_dict: Optional[dict] = None) -> bool:
         """
         Validate the metadata of the VideoFile instance.
 
@@ -531,57 +542,44 @@ class VideoFile(models.Model):
         **IMPORTANT:** Only the raw video is deleted. The processed (anonymized)
         video is preserved as the final validated output.
         """
-        from datetime import date as dt_date
-
-        from endoreg_db.models import SensitiveMeta
 
         if not self.sensitive_meta:
-            # CRITICAL FIX: Use create_from_dict with default patient data
-            default_data = {
-                "patient_first_name": "Patient",
-                "patient_last_name": "Unknown",
-                "patient_dob": dt_date(1990, 1, 1),
-                "examination_date": dt_date.today(),
-                "center": self.center,
-            }
-            self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
-
+            # Ensure a SensitiveMeta exists so validation can proceed.
+            self.sensitive_meta = self.get_or_create_sensitive_meta()
         # CRITICAL FIX: Delete RAW video file, not the processed (anonymized) one
         # CRITICAL: Update metadata BEFORE deleting raw video
-        # Metadata update may trigger frame extraction, which needs raw video
-        sensitive_meta = _update_text_metadata(
-            self, extracted_data_dict, overwrite=True
-        )
+        if extracted_data_dict:
+            self.sensitive_meta.update_from_dict(extracted_data_dict)
+        else:
+            return False
 
         # After validation and metadata update, only the anonymized video should remain
         from .video_file_io import _get_raw_file_path
 
         raw_path = _get_raw_file_path(self)
+
         if raw_path and raw_path.exists():
             logger.info(f"Deleting raw video file after validation: {raw_path}")
             raw_path.unlink(missing_ok=True)
             # Clear the raw_file field in database (use delete() to avoid save issues)
             if self.raw_file:
                 self.raw_file.delete(save=False)
-            logger.info(
-                f"Raw video deleted for {self.uuid}. Anonymized video preserved."
-            )
+            logger.info(f"Raw video deleted for {self.uuid}. Anonymized video preserved.")
         else:
-            logger.warning(f"Raw video file not found for deletion: {self.uuid}")
+            logger.warning(
+                "Raw video file not found for deletion during validation %s.",
+                self.uuid,
+            )
 
-        if sensitive_meta:
+        if self.sensitive_meta:
             # Mark as processed after validation
-            self.get_or_create_state().mark_sensitive_meta_processed(save=True)
+            self.get_or_create_state().mark_anonymization_validated(save=True)
             # Save the VideoFile instance to persist changes
             self.save()
-            logger.info(
-                f"Metadata annotation validated and saved for video {self.uuid}."
-            )
+            logger.info(f"Metadata annotation validated and saved for video {self.uuid}.")
             return True
         else:
-            logger.error(
-                f"Failed to validate metadata annotation for video {self.uuid}."
-            )
+            logger.error(f"Failed to validate metadata annotation for video {self.uuid}.")
             return False
 
     def initialize(self):
@@ -615,9 +613,7 @@ class VideoFile(models.Model):
         """
         active_path = self.active_file_path
         file_name = active_path.name if active_path else "No file"
-        state = (
-            "Processed" if self.is_processed else ("Raw" if self.has_raw else "No File")
-        )
+        state = "Processed" if self.is_processed else ("Raw" if self.has_raw else "No File")
         return f"VideoFile ({state}): {file_name} (UUID: {self.uuid})"
 
     # --- Convenience state/meta helpers used in tests and admin workflows ---
@@ -734,9 +730,7 @@ class VideoFile(models.Model):
             # Do not mark state as processed here; it will be set after extraction/validation steps
         return self.sensitive_meta
 
-    def get_outside_segments(
-        self, only_validated: bool = False
-    ) -> models.QuerySet["LabelVideoSegment"]:
+    def get_outside_segments(self, only_validated: bool = False) -> models.QuerySet["LabelVideoSegment"]:
         """
         Return all video segments labeled as "outside" for this video.
 
@@ -767,6 +761,56 @@ class VideoFile(models.Model):
             )
             return self.label_video_segments.none()
 
+    @classmethod
+    def create_video_without_outside_frames(cls, instance: "VideoFile", only_validated: bool = False) -> bool:
+        """
+        Creates a new video by excluding frames that belong to 'outside' segments.
+
+        Parameters:
+            only_validated (bool): If True, only validated segments are considered for frame exclusion.
+
+        Returns:
+            VideoFile: A new VideoFile instance with the frames excluding those labeled as 'outside'.
+        """
+        video = instance
+
+        if not video:
+            logger.warning("No processed video file available for VideoFile %s.", cls.uuid)
+            return False
+        try:
+            extracted = video.extract_frames(quality=2, overwrite=False, ext="jpg", verbose=False, from_processed=True)
+            assert extracted is True
+        except AssertionError:
+            # Use default anonymization here
+            video.anonymize
+            extracted = video.extract_frames(quality=2, overwrite=False, ext="jpg", verbose=False, from_processed=True)
+            assert extracted is True
+        try:
+            # Step 1: Get the "outside" labeled frames
+            censored = _censor_outside_frames(video)
+            frames = [instance.get_frame_dir_path()]
+            assert len(frames) != 0
+            fps = video.fps if video.fps else 120.0  # Default to 30 FPS if fps is not set
+            assert fps is not None
+            assert video.width is not None
+            assert video.height is not None
+
+            # Step 2: Reassemble the video with frames excluding the 'outside' labeled frames
+            output_video_path = Path(f"/path/to/output/{cls.uuid}_filtered.mp4")
+            fps = cls.fps if cls.fps else 30.0  # Default to 30 FPS if fps is not set
+            new_video_file = assemble_video_from_frames(frames, output_video_path, fps, width=video.width, height=video.height)
+            video.processed_file = new_video_file
+            return True
+        except AssertionError as ae:
+            logger.error(f"Assertion error while creating video without 'outside' frames for VideoFile {cls.uuid}: {ae}", exc_info=True)
+            return False
+        except Label.DoesNotExist:
+            logger.warning("Outside label not found in the database.")
+            return False
+        except Exception as e:
+            logger.error(f"Error creating video without 'outside' frames for VideoFile {cls.uuid}: {e}", exc_info=True)
+            return False
+
     @classmethod
     def get_all_videos(cls) -> models.QuerySet["VideoFile"]:
         """
@@ -784,9 +828,7 @@ class VideoFile(models.Model):
             int: The count of VideoFile records, excluding this instance, where the modification timestamp matches the creation timestamp.
         """
         return (
-            VideoFile.objects.filter(
-                date_modified=F("date_created")
-            )  # compare the two fields in SQL
+            VideoFile.objects.filter(date_modified=F("date_created"))  # compare the two fields in SQL
             .exclude(pk=self.pk)  # exclude this instance
             .count()  # run a fast COUNT(*) on the filtered set
         )