endoreg-db 0.8.6.4__py3-none-any.whl

endoreg_db/models/media/video/video_file_meta/text_meta.py

@@ -0,0 +1,134 @@
+import logging
+from typing import TYPE_CHECKING, Optional
+# --- Fix Imports ---
+from ....metadata import SensitiveMeta
+from ....metadata.sensitive_meta_logic import update_or_create_sensitive_meta_from_dict
+# --- End Fix ---
+from django.db import transaction
+
+if TYPE_CHECKING:
+    from ..video_file import VideoFile
+    # SensitiveMeta is already imported above
+
+logger = logging.getLogger(__name__)
+
+
+def _update_text_metadata(
+    video: "VideoFile",
+    extracted_data_dict: Optional[dict] = None,
+    ocr_frame_fraction: float = 0.01,
+    cap: int = 10,
+    overwrite: bool = False,
+) -> Optional["SensitiveMeta"]:
+    """
+    Extracts text from a fraction of video frames, updates or creates SensitiveMeta,
+    and potentially updates the VideoFile's date field. Requires frames to be extracted.
+    Raises ValueError if pre-conditions not met, RuntimeError on processing failure.
+
+    State Transitions:
+        - Pre-condition: Requires state.frames_extracted=True.
+        - Post-condition: Sets state.text_meta_extracted=True (even if no text found).
+    """
+    logger.debug(f"Updating text metadata for video {video.uuid}")
+    state = video.get_or_create_state()
+
+    # --- Pre-condition Checks ---
+    if not state.frames_extracted:
+        # Attempt to extract frames automatically if they're not available
+        logger.warning(f"Frames not extracted for video {video.uuid}. Attempting automatic frame extraction...")
+        try:
+            success = video.extract_frames(overwrite=False)
+            if success:
+                # Refresh state after frame extraction
+                state.refresh_from_db()
+                if state.frames_extracted:
+                    logger.info(f"Successfully extracted frames for video {video.uuid}")
+                else:
+                    # Force update the state if extraction was successful but state wasn't updated
+                    state.frames_extracted = True
+                    state.save(update_fields=['frames_extracted'])
+                    logger.info(f"Corrected frames_extracted state for video {video.uuid}")
+            else:
+                raise ValueError(f"Cannot update text metadata for video {video.uuid}: Frame extraction failed.")
+        except Exception as e:
+            logger.error(f"Failed to extract frames for video {video.uuid}: {e}")
+            raise ValueError(f"Cannot update text metadata for video {video.uuid}: Frames not extracted and automatic extraction failed: {e}")
+
+    if state.text_meta_extracted and not overwrite:
+        logger.info("Text already extracted for video %s and overwrite=False. Skipping.", video.uuid) # Changed to info
+        return video.sensitive_meta # Return existing meta if available
+    # --- End Pre-condition Checks ---
+
+    # Extract text using the AI helper function
+    # _extract_text_from_video_frames raises ValueError on pre-condition failure
+    try:
+        if not extracted_data_dict:
+            extracted_data_dict = video.extract_text_from_frames(
+                frame_fraction=ocr_frame_fraction, cap=cap
+            )
+    except Exception as text_extract_e:
+         logger.error("Failed during text extraction step for video %s: %s", video.uuid, text_extract_e, exc_info=True)
+         raise RuntimeError(f"Text extraction failed for video {video.uuid}") from text_extract_e
+
+    # --- Atomic Update Block ---
+    try:
+        with transaction.atomic():
+            # Refresh state in case it changed
+            state.refresh_from_db()
+            sensitive_meta_instance = video.sensitive_meta # Get current instance
+
+            if not extracted_data_dict:
+                logger.warning("No text extracted for video %s; skipping SensitiveMeta update.", video.uuid)
+                # Mark state as retrieved even if no data found, to avoid re-running unless overwrite=True
+                if not state.text_meta_extracted:
+                    state.text_meta_extracted = True
+                    state.save(update_fields=['text_meta_extracted'])
+                return sensitive_meta_instance # Return existing meta if available
+
+            # Add center info if not already present in extracted data
+            if 'center_name' not in extracted_data_dict and video.center:
+                extracted_data_dict['center_name'] = video.center.name
+            logger.debug("Data for SensitiveMeta update for video %s: %s", video.uuid, extracted_data_dict)
+
+            # Pass the Class, the data dict, and the current instance (or None)
+            # This function might raise exceptions if data is invalid
+            sensitive_meta, created = update_or_create_sensitive_meta_from_dict(
+                SensitiveMeta, # Pass the class
+                extracted_data_dict,
+                instance=sensitive_meta_instance # Pass current instance via keyword
+            )
+
+            # Update VideoFile fields if necessary
+            update_fields_video = []
+            if created or sensitive_meta != sensitive_meta_instance: # Check if relation needs update
+                video.sensitive_meta = sensitive_meta
+                update_fields_video.append('sensitive_meta')
+
+            if not video.date and sensitive_meta and extracted_data_dict.get('date'):
+                extracted_date = extracted_data_dict.get('date')
+                if extracted_date: # Ensure date is not None or empty
+                    video.date = extracted_date
+                    update_fields_video.append("date")
+
+            # Save VideoFile if fields changed
+            if update_fields_video:
+                video.save(update_fields=update_fields_video)
+
+            # Update state
+            if not state.text_meta_extracted:
+                state.text_meta_extracted = True
+                state.save(update_fields=['text_meta_extracted'])
+
+            # Mark sensitive meta as processed when updated via text metadata
+            if sensitive_meta:
+                state.mark_sensitive_meta_processed(save=True)
+                logger.info(f"Marked sensitive_meta_processed=True for video {video.uuid} after text metadata update")
+
+            logger.info("Successfully updated/created SensitiveMeta and state for video %s.", video.uuid) # Changed to info
+            return sensitive_meta
+
+    except Exception as e:
+        logger.error("Failed to update/create SensitiveMeta or state for video %s: %s", video.uuid, e, exc_info=True)
+        # Re-raise exception for the pipeline to catch
+        raise RuntimeError(f"Failed to update/create SensitiveMeta or state for video {video.uuid}") from e
+    # --- End Atomic Update Block ---

endoreg_db/models/media/video/video_file_meta/video_meta.py

@@ -0,0 +1,70 @@
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..video_file import VideoFile
+
+logger = logging.getLogger(__name__)
+
+def _update_video_meta(video: "VideoFile", save_instance: bool = True):
+    """
+    Updates or creates the technical VideoMeta from the raw video file.
+    Raises FileNotFoundError or ValueError on pre-condition failure, RuntimeError on processing failure.
+    """
+    from ....metadata import VideoMeta # Local import
+
+    logger.debug("Updating technical VideoMeta for video %s (from raw file).", video.uuid)
+
+    if not video.has_raw:
+        # DEFENSIVE: Log warning and skip instead of crashing
+        logger.warning(f"Raw video file path not available for {video.uuid}. Skipping VideoMeta update - this may indicate the video was processed and raw file moved.")
+        return  # Graceful skip instead of FileNotFoundError
+
+    raw_video_path = video.get_raw_file_path() # Use helper
+    if not raw_video_path or not raw_video_path.exists():
+        # DEFENSIVE: Log warning and skip instead of crashing production pipeline
+        logger.warning(f"Raw video file path {raw_video_path} does not exist for video {video.uuid}. Skipping VideoMeta update - this typically happens after video processing when raw files are moved to processed location.")
+        return  # Graceful skip instead of FileNotFoundError that crashes production
+
+    try:
+        vm = video.video_meta
+        if vm:
+            logger.info("Updating existing VideoMeta (PK: %s) for video %s.", vm.pk, video.uuid)
+            vm.update_meta(raw_video_path) # Assuming this method exists and raises on error
+            vm.save()
+        else:
+            if not video.center or not video.processor:
+                # Raise exception
+                raise ValueError(f"Cannot create VideoMeta for {video.uuid}: Center or Processor is missing.")
+
+            logger.info("Creating new VideoMeta for video %s.", video.uuid)
+            # Assuming create_from_file exists and raises on error
+            video.video_meta = VideoMeta.create_from_file(
+                video_path=raw_video_path,
+                center=video.center,
+                processor=video.processor,
+                save_instance=True # Let create_from_file handle saving
+            )
+            logger.info("Created and linked VideoMeta (PK: %s) for video %s.", video.video_meta.pk, video.uuid)
+
+        # Save the VideoFile instance itself if requested and if video_meta was linked/updated
+        if save_instance:
+            update_fields = ["video_meta"]
+            # Check if derived fields also need updating
+            if video.video_meta:
+                meta_fields = ["fps", "duration", "frame_count", "width", "height"]
+                for field in meta_fields:
+                    # Check if field is None on video but has value on meta
+                    if getattr(video, field) is None and getattr(video.video_meta, field, None) is not None:
+                        # No need to set attribute here, save method handles it
+                        update_fields.append(field)
+            # Ensure update_fields has unique values before saving
+            unique_update_fields = list(set(update_fields))
+            if unique_update_fields:
+                video.save(update_fields=unique_update_fields)
+                logger.info("Saved video %s after VideoMeta update (Fields: %s).", video.uuid, unique_update_fields)
+
+    except Exception as e:
+        logger.error("Failed to update/create VideoMeta for video %s: %s", video.uuid, e, exc_info=True)
+        # Re-raise exception
+        raise RuntimeError(f"Failed to update/create VideoMeta for video {video.uuid}") from e

endoreg_db/models/media/video/video_file_segments.py

@@ -0,0 +1,209 @@
+import logging
+from typing import TYPE_CHECKING, List, Dict, Tuple, Set
+from icecream import ic
+from pathlib import Path
+from django.db.models import Q  # Import Q for complex queries
+
+if TYPE_CHECKING:
+    from .video_file import VideoFile
+    from ...label import LabelVideoSegment
+    from ...metadata import VideoPredictionMeta
+    from ..frame import Frame
+    from django.db.models import QuerySet
+
+logger = logging.getLogger(__name__)
+
+def _convert_sequences_to_db_segments(
+    video: "VideoFile",
+    sequences: Dict[str, List[Tuple[int, int]]],
+    video_prediction_meta: "VideoPredictionMeta",
+):
+    """
+    Converts predicted sequences into LabelVideoSegment database objects
+    and ensures their corresponding state objects are created.
+    """
+    from ...label import Label, LabelVideoSegment  # Local import for models
+
+    logger.info("Converting sequences to LabelVideoSegments for video %s, prediction meta %s", video.uuid, video_prediction_meta.pk)
+    created_count = 0
+    skipped_count = 0
+    error_count = 0
+    state_created_count = 0
+    state_error_count = 0
+
+    processed_labels = set()
+
+    for label_name, sequence_list in sequences.items():
+        if not sequence_list:
+            continue
+
+        processed_labels.add(label_name)
+
+        try:
+            label = Label.objects.get(name=label_name)  # require pre-existing label
+        except Exception as e:
+            logger.error("Could not get or create Label '%s': %s", label_name, e, exc_info=True)
+            error_count += len(sequence_list)
+            continue
+
+        segments_to_create = []
+        for start_frame, end_frame in sequence_list:
+            if start_frame > end_frame or start_frame < 0:
+                logger.warning("Skipping invalid sequence for label '%s': start=%d, end=%d", label_name, start_frame, end_frame)
+                skipped_count += 1
+                continue
+
+            segments_to_create.append(
+                LabelVideoSegment(
+                    video_file=video,
+                    label=label,
+                    start_frame_number=start_frame,
+                    end_frame_number=end_frame,
+                    prediction_meta=video_prediction_meta,
+                )
+            )
+
+        if segments_to_create:
+            try:
+                LabelVideoSegment.objects.bulk_create(segments_to_create, ignore_conflicts=True)
+                created_count += len(segments_to_create)
+                logger.debug("Bulk created %d segments for label '%s'", len(segments_to_create), label_name)
+            except Exception as e:
+                logger.error("Error bulk creating segments for label '%s': %s", label_name, e, exc_info=True)
+                error_count += len(segments_to_create)
+
+    newly_created_segments = LabelVideoSegment.objects.filter(
+        video_file=video,
+        prediction_meta=video_prediction_meta,
+        label__name__in=processed_labels
+    )
+
+    logger.info("Attempting to create state objects for %d potentially new segments (Video: %s, PredictionMeta: %s)",
+                newly_created_segments.count(), video.uuid, video_prediction_meta.pk)
+
+    for segment in newly_created_segments:
+        try:
+            _state, created = segment.get_or_create_state()
+            if created:
+                state_created_count += 1
+        except Exception as e:
+            logger.error("Failed to get or create state for segment %s (Video: %s): %s", segment.pk, video.uuid, e, exc_info=True)
+            state_error_count += 1
+
+    logger.info(
+        "LabelVideoSegment conversion finished for video %s. Segments Created: %d, Skipped: %d, Errors: %d. States Created: %d, State Errors: %d",
+        video.uuid, created_count, skipped_count, error_count, state_created_count, state_error_count
+    )
+
+
+def _sequences_to_label_video_segments(
+    video: "VideoFile",
+    video_prediction_meta: "VideoPredictionMeta",
+):
+    """Converts stored sequences on the video object to LabelVideoSegments."""
+    if not video.sequences:
+        return
+
+    if not video_prediction_meta:
+        return
+
+    _convert_sequences_to_db_segments(
+        video=video,
+        sequences=video.sequences,
+        video_prediction_meta=video_prediction_meta,
+    )
+
+
+def _get_outside_segments(video: "VideoFile", outside_label_name: str = "outside") -> "QuerySet[LabelVideoSegment]":
+    """Gets LabelVideoSegments marked with the 'outside' label."""
+    from ...label import Label, LabelVideoSegment  # Local import for models
+
+    try:
+        outside_label = Label.objects.get(name__iexact=outside_label_name)
+        return video.label_video_segments.filter(label=outside_label)
+    except Label.DoesNotExist:
+        logger.warning("Label '%s' not found in the database.", outside_label_name)
+        return LabelVideoSegment.objects.none()
+    except AttributeError:
+        logger.error("VideoFile instance does not have 'label_video_segments' related manager.")
+        return LabelVideoSegment.objects.none()
+    except Exception as e:
+        logger.error("Error getting '%s' segments for video %s: %s", outside_label_name, video.uuid, e, exc_info=True)
+        return LabelVideoSegment.objects.none()
+
+
+def _get_outside_frame_numbers(video: "VideoFile", outside_label_name: str = "outside") -> Set[int]:
+    """
+    Gets a set of frame numbers corresponding to segments labeled as 'outside'.
+    """
+    outside_segments = _get_outside_segments(video, outside_label_name)
+    frame_numbers = set()
+    for segment in outside_segments:
+        frame_numbers.update(range(segment.start_frame_number, segment.end_frame_number + 1))
+    if frame_numbers:
+        logger.info("Found %d frame numbers marked as '%s' for video %s.", len(frame_numbers), outside_label_name, video.uuid)
+    else:
+        logger.info("No frame numbers marked as '%s' found for video %s.", outside_label_name, video.uuid)
+    return frame_numbers
+
+
+def _get_outside_frames(video: "VideoFile", outside_label_name: str = "outside") -> "QuerySet[Frame]":
+    """
+    Gets a QuerySet of all unique Frame objects that fall within any segment
+    labeled with the specified 'outside_label_name'.
+    """
+    from ..frame import Frame  # Local import
+
+    outside_segments = _get_outside_segments(video, outside_label_name)
+    if not outside_segments.exists():
+        return Frame.objects.none()
+
+    q_objects: Q | None = None
+    for segment in outside_segments:
+        clause = Q(frame_number__gte=segment.start_frame_number,
+                   frame_number__lt=segment.end_frame_number)
+        q_objects = clause if q_objects is None else q_objects | clause
+
+    if q_objects is None:
+        return Frame.objects.none()
+
+    try:
+        return video.frames.filter(q_objects).distinct().order_by('frame_number')
+    except Exception as e:
+        logger.error("Error filtering outside frames for video %s: %s", video.uuid, e, exc_info=True)
+        return Frame.objects.none()
+
+
+def _get_outside_frame_paths(video: "VideoFile", outside_label_name: str = "outside") -> List["Path"]:
+    """Gets the file paths of frames that fall within 'outside' segments."""
+    from pathlib import Path  # Local import
+    frames = _get_outside_frames(video, outside_label_name=outside_label_name)
+    frame_paths = []
+    for frame in frames:
+        try:
+            frame_paths.append(Path(frame.image.path))
+        except Exception as e:
+            logger.warning("Could not get path for frame %s (Number: %d): %s", frame.pk, frame.frame_number, e)
+            ic(f"Could not get path for frame {frame.pk}: {e}")
+
+    logger.info("Found %d frame paths within '%s' segments for video %s", len(frame_paths), outside_label_name, video.uuid)
+    return frame_paths
+
+
+def _label_segments_to_frame_annotations(video: "VideoFile"):
+    """Generates frame annotations based on existing LabelVideoSegments."""
+    logger.info("Generating frame annotations from segments for video %s", video.uuid)
+    processed_count = 0
+    try:
+        for lvs in video.label_video_segments.all():
+            lvs_duration = lvs.get_segment_len_in_s()
+            if lvs_duration >= 3:
+                try:
+                    lvs.generate_annotations()
+                    processed_count += 1
+                except Exception as e:
+                    logger.error("Error generating annotations for segment %s (Video %s): %s", lvs.pk, video.uuid, e)
+        logger.info("Processed %d segments for frame annotations for video %s", processed_count, video.uuid)
+    except AttributeError:
+        logger.error("Could not generate frame annotations for video %s. 'label_video_segments' related manager not found.", video.uuid)
+

endoreg_db/models/media/video/video_metadata.py

@@ -0,0 +1,65 @@
+"""
+Video Metadata Model
+
+Stores analysis results for videos (sensitive frames, detection statistics).
+Created as part of Phase 1.1: Video Correction API Endpoints.
+"""
+from django.db import models
+from .video_file import VideoFile
+
+class VideoMetadata(models.Model):
+    """
+    Stores analysis results for videos after sensitive frame detection.
+    
+    This model holds the output of frame analysis operations (MiniCPM, OCR+LLM)
+    and provides metrics for the correction UI.
+    """
+    video = models.OneToOneField(
+        VideoFile, 
+        on_delete=models.CASCADE, 
+        related_name='metadata',
+        help_text="Video file this metadata belongs to"
+    )
+    
+    # Analysis Results
+    sensitive_frame_count = models.IntegerField(
+        null=True, 
+        blank=True,
+        help_text="Number of frames detected as containing sensitive information"
+    )
+    sensitive_ratio = models.FloatField(
+        null=True, 
+        blank=True,
+        help_text="Ratio of sensitive frames to total frames (0.0-1.0)"
+    )
+    sensitive_frame_ids = models.TextField(
+        null=True, 
+        blank=True,
+        help_text="JSON array of sensitive frame indices (0-based)"
+    )
+    
+    # Metadata
+    analyzed_at = models.DateTimeField(
+        auto_now=True,
+        help_text="Timestamp of last analysis"
+    )
+    
+    class Meta:
+        db_table = 'video_metadata'
+        verbose_name = 'Video Metadata'
+        verbose_name_plural = 'Video Metadata'
+    
+    def __str__(self):
+        return f"Metadata for {self.video.uuid} ({self.sensitive_frame_count or 0} sensitive frames)"
+    
+    @property
+    def has_analysis(self) -> bool:
+        """Check if this video has been analyzed."""
+        return self.sensitive_frame_count is not None
+    
+    @property
+    def sensitive_percentage(self) -> float:
+        """Get sensitivity as percentage (0-100)."""
+        if self.sensitive_ratio is not None:
+            return self.sensitive_ratio * 100
+        return 0.0

endoreg_db/models/media/video/video_processing.py

@@ -0,0 +1,152 @@
+"""
+Video Processing History Model
+
+Tracks all video correction operations (masking, frame removal, reprocessing).
+Created as part of Phase 1.1: Video Correction API Endpoints.
+"""
+from django.db import models
+from django.utils import timezone
+from .video_file import VideoFile
+
+class VideoProcessingHistory(models.Model):
+    """
+    History of all video processing operations.
+    
+    Stores configuration and results of masking, frame removal, and reprocessing
+    operations for audit trail and download access.
+    """
+    
+    # Operation Types
+    OPERATION_MASKING = 'mask_overlay'
+    OPERATION_FRAME_REMOVAL = 'frame_removal'
+    OPERATION_ANALYSIS = 'analysis'
+    OPERATION_REPROCESSING = 'reprocessing'
+    
+    OPERATION_CHOICES = [
+        (OPERATION_MASKING, 'Mask Overlay'),
+        (OPERATION_FRAME_REMOVAL, 'Frame Removal'),
+        (OPERATION_ANALYSIS, 'Sensitivity Analysis'),
+        (OPERATION_REPROCESSING, 'Full Reprocessing'),
+    ]
+    
+    # Status Types
+    STATUS_PENDING = 'pending'
+    STATUS_RUNNING = 'running'
+    STATUS_SUCCESS = 'success'
+    STATUS_FAILURE = 'failure'
+    STATUS_CANCELLED = 'cancelled'
+    
+    STATUS_CHOICES = [
+        (STATUS_PENDING, 'Pending'),
+        (STATUS_RUNNING, 'Running'),
+        (STATUS_SUCCESS, 'Success'),
+        (STATUS_FAILURE, 'Failure'),
+        (STATUS_CANCELLED, 'Cancelled'),
+    ]
+    
+    video = models.ForeignKey(
+        VideoFile, 
+        on_delete=models.CASCADE, 
+        related_name='processing_history',
+        help_text="Video file this operation was performed on"
+    )
+    
+    operation = models.CharField(
+        max_length=50,
+        choices=OPERATION_CHOICES,
+        help_text="Type of processing operation"
+    )
+    
+    status = models.CharField(
+        max_length=20,
+        choices=STATUS_CHOICES,
+        default=STATUS_PENDING,
+        help_text="Current status of the operation"
+    )
+    
+    # Configuration & Results
+    config = models.JSONField(
+        default=dict,
+        help_text="Operation configuration (mask settings, frame list, etc.)"
+    )
+    
+    output_file = models.CharField(
+        max_length=500,
+        blank=True,
+        help_text="Path to output file (relative to MEDIA_ROOT)"
+    )
+    
+    details = models.TextField(
+        blank=True,
+        help_text="Additional details or error messages"
+    )
+    
+    # Celery Integration
+    task_id = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text="Celery task ID for progress tracking"
+    )
+    
+    # Timestamps
+    created_at = models.DateTimeField(
+        auto_now_add=True,
+        help_text="When the operation was started"
+    )
+    
+    completed_at = models.DateTimeField(
+        null=True,
+        blank=True,
+        help_text="When the operation completed (success or failure)"
+    )
+    
+    class Meta:
+        db_table = 'video_processing_history'
+        verbose_name = 'Video Processing History'
+        verbose_name_plural = 'Video Processing Histories'
+        ordering = ['-created_at']
+        indexes = [
+            models.Index(fields=['video', '-created_at']),
+            models.Index(fields=['task_id']),
+            models.Index(fields=['status']),
+        ]
+    
+    def __str__(self):
+        return f"{self.get_operation_display()} on {self.video.uuid} - {self.get_status_display()}"
+    
+    def mark_running(self, save=True):
+        """Mark operation as running."""
+        self.status = self.STATUS_RUNNING
+        if save:
+            self.save(update_fields=['status'])
+    
+    def mark_success(self, output_file=None, details=None, save=True):
+        """Mark operation as successful."""
+        self.status = self.STATUS_SUCCESS
+        self.completed_at = timezone.now()
+        if output_file:
+            self.output_file = output_file
+        if details:
+            self.details = details
+        if save:
+            self.save(update_fields=['status', 'completed_at', 'output_file', 'details'])
+    
+    def mark_failure(self, error_message, save=True):
+        """Mark operation as failed."""
+        self.status = self.STATUS_FAILURE
+        self.completed_at = timezone.now()
+        self.details = error_message
+        if save:
+            self.save(update_fields=['status', 'completed_at', 'details'])
+    
+    @property
+    def duration(self):
+        """Calculate operation duration if completed."""
+        if self.completed_at and self.created_at:
+            return (self.completed_at - self.created_at).total_seconds()
+        return None
+    
+    @property
+    def is_complete(self) -> bool:
+        """Check if operation is in a terminal state."""
+        return self.status in [self.STATUS_SUCCESS, self.STATUS_FAILURE, self.STATUS_CANCELLED]