endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/models/media/video/video_file_frames/_initialize_frames.py

@@ -0,0 +1,205 @@
+import logging
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Optional
+
+from django.db import OperationalError
+from tqdm import tqdm
+
+if TYPE_CHECKING:
+    from endoreg_db.models import VideoFile
+
+logger = logging.getLogger(__name__)
+
+
+def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = None):
+    """
+    Initializes Frame objects in the database based on either provided existing
+    frame paths or the expected frame count from the video metadata.
+
+    If `frame_paths` is provided, Frame objects are created/updated and marked
+    as `is_extracted=True`.
+    If `frame_paths` is None, Frame objects are created based on `video.frame_count`
+    and marked as `is_extracted=False`.
+
+    Updates state.frames_initialized and state.frame_count.
+    Uses bulk_create with ignore_conflicts=True, so it won't fail if frames already exist.
+
+    Raises RuntimeError on failure to create/update frames or update state.
+
+    State Transitions:
+        - On Success: Sets state.frames_initialized=True, state.frame_count=N.
+        - On Failure: Does not change state (error is raised).
+    """
+    from endoreg_db.models import Frame
+    from endoreg_db.models.media.video.video_file_frames._bulk_create_frames import (
+        _bulk_create_frames,
+    )
+    from endoreg_db.models.media.video.video_file_frames._create_frame_object import (
+        _create_frame_object,
+    )
+
+    frames_to_create = []
+    num_expected_or_provided = 0
+    mark_as_extracted = False
+
+    if frame_paths:
+        logger.info(
+            "Initializing Frame objects based on %d provided paths for video %s.",
+            len(frame_paths),
+            video.video_hash,
+        )
+        mark_as_extracted = True
+        num_expected_or_provided = len(frame_paths)
+        for frame_path in tqdm(
+            frame_paths,
+            desc=f"Initializing Frames from Paths {video.video_hash}",
+            unit="frame",
+        ):
+            try:
+                frame_number = int(frame_path.stem.split("_")[-1])
+                relative_path_str = frame_path.name
+                frames_to_create.append(
+                    _create_frame_object(
+                        video,
+                        frame_number,
+                        relative_path_str,
+                        extracted=mark_as_extracted,
+                    )
+                )
+            except (ValueError, IndexError) as e:
+                logger.warning(
+                    "Could not parse frame number from %s: %s", frame_path.name, e
+                )
+                continue
+    else:
+        expected_frame_count = video.frame_count
+        if expected_frame_count is None or expected_frame_count <= 0:
+            logger.warning(
+                "Cannot initialize frames for video %s: Frame count is %s.",
+                video.video_hash,
+                expected_frame_count,
+            )
+            try:
+                state = video.get_or_create_state()
+                if state.frames_initialized or state.frame_count is not None:
+                    state.frames_initialized = False
+                    state.frame_count = None
+                    state.save(update_fields=["frames_initialized", "frame_count"])
+            except Exception as state_e:
+                logger.error(
+                    "Failed to reset state during empty initialization for video %s: %s",
+                    video.video_hash,
+                    state_e,
+                    exc_info=True,
+                )
+            return
+
+        logger.info(
+            "Initializing %d expected Frame objects for video %s (is_extracted=False).",
+            expected_frame_count,
+            video.video_hash,
+        )
+        mark_as_extracted = False
+        num_expected_or_provided = expected_frame_count
+        for frame_number in tqdm(
+            range(expected_frame_count),
+            desc=f"Initializing Expected Frames {video.video_hash}",
+            unit="frame",
+        ):
+            relative_path_str = f"frame_{frame_number:07d}.jpg"
+            frames_to_create.append(
+                _create_frame_object(
+                    video, frame_number, relative_path_str, extracted=mark_as_extracted
+                )
+            )
+
+    if frames_to_create:
+        for attempt in range(5):
+            try:
+                _bulk_create_frames(video, frames_to_create)
+                num_created_or_ignored = len(frames_to_create)
+                logger.info(
+                    "Bulk create attempted for %d Frame objects for video %s (ignore_conflicts=True).",
+                    num_created_or_ignored,
+                    video.video_hash,
+                )
+
+                if mark_as_extracted:
+                    frame_numbers_to_update = [f.frame_number for f in frames_to_create]
+                    if frame_numbers_to_update:
+                        update_count = Frame.objects.filter(
+                            video=video,
+                            frame_number__in=frame_numbers_to_update,
+                            is_extracted=False,
+                        ).update(is_extracted=True)
+                        if update_count > 0:
+                            logger.info(
+                                "Marked %d existing Frame objects as is_extracted=True for video %s.",
+                                update_count,
+                                video.video_hash,
+                            )
+
+                try:
+                    state = video.get_or_create_state()
+                    state.frames_initialized = True
+                    state.frame_count = num_expected_or_provided
+                    state.save(update_fields=["frames_initialized", "frame_count"])
+                    logger.info(
+                        "Set frames_initialized=True and frame_count=%d for video %s.",
+                        num_expected_or_provided,
+                        video.video_hash,
+                    )
+                except Exception as state_e:
+                    logger.error(
+                        "Failed to update state after frame initialization for video %s: %s",
+                        video.video_hash,
+                        state_e,
+                        exc_info=True,
+                    )
+                    raise RuntimeError(
+                        f"Failed to update state after frame initialization for video {video.video_hash}"
+                    ) from state_e
+
+            except OperationalError as e:
+                if "database is locked" in str(e):
+                    logger.warning(
+                        "Database is locked, retrying frame initialization for video %s (attempt %d/5).",
+                        video.video_hash,
+                        attempt + 1,
+                    )
+                    time.sleep(2**attempt)
+                    if attempt < 4:
+                        continue
+                    else:
+                        raise RuntimeError(
+                            f"Failed to initialize frames for video {video.video_hash}."
+                        ) from e
+                logger.error(
+                    "Error initializing frames for video %s: %s",
+                    video.video_hash,
+                    e,
+                    exc_info=True,
+                )
+                raise RuntimeError(
+                    f"Failed to initialize frames for video {video.video_hash}."
+                ) from e
+
+    else:
+        logger.warning(
+            "No valid frames found/generated to initialize for video %s.",
+            video.video_hash,
+        )
+        try:
+            state = video.get_or_create_state()
+            if state.frames_initialized or state.frame_count is not None:
+                state.frames_initialized = False
+                state.frame_count = None
+                state.save(update_fields=["frames_initialized", "frame_count"])
+        except Exception as state_e:
+            logger.error(
+                "Failed to reset state during empty initialization for video %s: %s",
+                video.video_hash,
+                state_e,
+                exc_info=True,
+            )

endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py

@@ -0,0 +1,228 @@
+import logging
+import os
+from typing import TYPE_CHECKING
+
+from django.db import transaction
+
+from endoreg_db.models.media.video.video_file_io import _get_frame_dir_path
+
+# Assuming ffmpeg_wrapper has or will have this function
+from endoreg_db.utils.video.ffmpeg_wrapper import (
+    extract_frame_range as ffmpeg_extract_frame_range,
+)
+
+if TYPE_CHECKING:
+    from endoreg_db.models import VideoFile
+
+logger = logging.getLogger(__name__)
+
+
+def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
+    """
+    Deletes frame image files within the specified range [start_frame, end_frame)
+    and updates their is_extracted status to False. Runs within the caller's transaction.
+    """
+
+    logger.info(
+        "Deleting frame files for video %s in range [%d, %d)",
+        video.video_hash,
+        start_frame,
+        end_frame,
+    )
+    frames_to_delete = video.frames.filter(
+        frame_number__gte=start_frame, frame_number__lt=end_frame, is_extracted=True
+    )
+
+    deleted_count = 0
+    paths_to_delete = [
+        frame.file_path for frame in frames_to_delete
+    ]  # Get paths before potential DB changes
+
+    # Update DB first
+    update_count = frames_to_delete.update(is_extracted=False)
+    logger.info(
+        "Marked %d Frame objects as is_extracted=False for video %s range [%d, %d).",
+        update_count,
+        video.video_hash,
+        start_frame,
+        end_frame,
+    )
+
+    # Then delete files
+    for frame_path in paths_to_delete:
+        try:
+            if frame_path.exists():
+                os.remove(frame_path)
+                deleted_count += 1
+        except Exception as e:
+            # Log warning but continue; DB state is already updated.
+            logger.warning(
+                "Could not delete frame file %s for video %s: %s",
+                frame_path,
+                video.video_hash,
+                e,
+            )
+
+    logger.info(
+        "Attempted deletion of %d files for video %s range [%d, %d). Actual deleted: %d.",
+        len(paths_to_delete),
+        video.video_hash,
+        start_frame,
+        end_frame,
+        deleted_count,
+    )
+
+
+@transaction.atomic
+def _extract_frame_range(
+    video: "VideoFile",
+    start_frame: int,
+    end_frame: int,
+    quality: int = 2,
+    overwrite: bool = False,
+    ext="jpg",
+    verbose=False,
+) -> bool:
+    """
+    Extracts frames within the range [start_frame, end_frame) using ffmpeg.
+    Updates corresponding Frame objects' is_extracted flag.
+    Runs within the caller's transaction.
+    Does NOT update VideoState.frames_extracted.
+    """
+
+    if not video.has_raw:
+        raise FileNotFoundError(
+            f"Raw video file not available for {video.video_hash}. Cannot extract frame range."
+        )
+
+    raw_file_path = video.get_raw_file_path()
+    if not raw_file_path or not raw_file_path.exists():
+        raise FileNotFoundError(
+            f"Raw video file not found at {raw_file_path} for video {video.video_hash}. Cannot extract frame range."
+        )
+
+    frame_dir = _get_frame_dir_path(video)
+    if not frame_dir:
+        raise ValueError(
+            f"Cannot determine frame directory path for video {video.video_hash}."
+        )
+
+    # Check frames within the range that might already exist
+    frames_in_range = video.frames.filter(
+        frame_number__gte=start_frame, frame_number__lt=end_frame
+    )
+    existing_extracted_in_range = frames_in_range.filter(is_extracted=True)
+
+    if existing_extracted_in_range.exists():
+        if overwrite:
+            logger.info(
+                "Overwrite=True, deleting existing frame files in range [%d, %d) for video %s before extraction.",
+                start_frame,
+                end_frame,
+                video.video_hash,
+            )
+            _delete_frame_range(video, start_frame, end_frame)
+        else:
+            logger.info(
+                "Frames already exist in range [%d, %d) for video %s and overwrite=False. Skipping extraction for this range.",
+                start_frame,
+                end_frame,
+                video.video_hash,
+            )
+            updated_count = frames_in_range.filter(is_extracted=False).update(
+                is_extracted=True
+            )
+            if updated_count > 0:
+                logger.info(
+                    "Marked %d existing Frame objects in range [%d, %d) as extracted for video %s.",
+                    updated_count,
+                    start_frame,
+                    end_frame,
+                    video.video_hash,
+                )
+            return True  # Indicate success as frames are considered present
+
+    frame_dir.mkdir(parents=True, exist_ok=True)
+    extracted_paths = []
+
+    try:
+        logger.info(
+            "Starting frame range extraction [%d, %d) for video %s to %s",
+            start_frame,
+            end_frame,
+            video.video_hash,
+            frame_dir,
+        )
+        extracted_paths = ffmpeg_extract_frame_range(
+            raw_file_path, frame_dir, start_frame, end_frame, quality=quality, ext=ext
+        )
+
+        logger.info(
+            "ffmpeg extraction process completed for video %s range [%d, %d). Found %d files.",
+            video.video_hash,
+            start_frame,
+            end_frame,
+            len(extracted_paths),
+        )
+
+        update_count = frames_in_range.update(is_extracted=True)
+        logger.info(
+            "Marked %d Frame objects in range [%d, %d) as is_extracted=True for video %s.",
+            update_count,
+            start_frame,
+            end_frame,
+            video.video_hash,
+        )
+
+        return True
+
+    except FileNotFoundError as err:
+        logger.error(
+            "Frame range extraction [%d, %d) failed for video %s: %s",
+            start_frame,
+            end_frame,
+            video.video_hash,
+            err,
+            exc_info=True,
+        )
+        raise
+
+    except Exception as e:
+        logger.error(
+            "Frame range extraction [%d, %d) or DB update failed for video %s: %s",
+            start_frame,
+            end_frame,
+            video.video_hash,
+            e,
+            exc_info=True,
+        )
+
+        logger.warning(
+            "Attempting file cleanup in range [%d, %d) for video %s due to extraction error.",
+            start_frame,
+            end_frame,
+            video.video_hash,
+        )
+        files_to_check = (
+            extracted_paths if "extracted_paths" in locals() and extracted_paths else []
+        )
+        if not files_to_check:
+            files_to_check = [
+                frame_dir / f"frame_{i:07d}.{ext}"
+                for i in range(start_frame, end_frame)
+            ]
+
+        for potential_file in files_to_check:
+            if potential_file.exists():
+                try:
+                    os.remove(potential_file)
+                except OSError as unlink_err:
+                    logger.error(
+                        "Failed to delete potential frame %s during cleanup: %s",
+                        potential_file,
+                        unlink_err,
+                    )
+
+        raise RuntimeError(
+            f"Frame range extraction or update failed for video {video.video_hash} range [{start_frame}, {end_frame})."
+        ) from e

endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py

@@ -0,0 +1,107 @@
+import logging
+from typing import TYPE_CHECKING, Set
+
+from django.db import transaction
+
+if TYPE_CHECKING:
+    from endoreg_db.models import VideoFile
+
+logger = logging.getLogger(__name__)
+
+
+@transaction.atomic
+def _mark_frames_extracted_status(
+    video: "VideoFile", extracted_frame_numbers: Set[int], status: bool
+):
+    """
+    Bulk updates the is_extracted status for a set of frame numbers.
+    """
+    from endoreg_db.models.media.frame import Frame
+
+    if not extracted_frame_numbers:
+        logger.warning(
+            "No frame numbers provided to update status for video %s.", video.video_hash
+        )
+        return 0
+
+    # --- Enhanced Logging ---
+    min_frame = min(extracted_frame_numbers) if extracted_frame_numbers else "N/A"
+    max_frame = max(extracted_frame_numbers) if extracted_frame_numbers else "N/A"
+    contains_zero = 0 in extracted_frame_numbers
+    logger.info(
+        "Attempting to mark %d Frame objects as is_extracted=%s for video %s. Frame numbers range: [%s-%s]. Contains frame 0: %s",
+        len(extracted_frame_numbers),
+        status,
+        video.video_hash,
+        min_frame,
+        max_frame,
+        contains_zero,
+    )
+    # --- End Enhanced Logging ---
+
+    try:
+        # Update Frame objects based on frame_number
+        # Convert set to list for potentially better compatibility with some DB backends
+        updated_count = Frame.objects.filter(
+            video=video, frame_number__in=list(extracted_frame_numbers)
+        ).update(is_extracted=status)
+
+        logger.info(
+            "Database reported updating %d Frame objects to is_extracted=%s for video %s.",
+            updated_count,
+            status,
+            video.video_hash,
+        )
+
+        # Verification step
+        if updated_count != len(extracted_frame_numbers):
+            logger.warning(
+                "Mismatch during status update for video %s. Expected to update %d frames, but DB reported updating %d.",
+                video.video_hash,
+                len(extracted_frame_numbers),
+                updated_count,
+            )
+            # --- Add detailed check for frame 0 if status is True and it should have been updated ---
+            if (
+                status is True
+                and contains_zero
+                and updated_count < len(extracted_frame_numbers)
+            ):
+                try:
+                    # Check the status of frame 0 directly after the update attempt
+                    frame_zero = Frame.objects.get(video=video, frame_number=0)
+                    if not frame_zero.is_extracted:
+                        logger.error(
+                            "Verification check: Frame 0 (PK: %s) was NOT updated to is_extracted=True for video %s.",
+                            frame_zero.pk,
+                            video.video_hash,
+                        )
+                    else:
+                        # This case should ideally not happen if updated_count < expected count, but log just in case
+                        logger.info(
+                            "Verification check: Frame 0 (PK: %s) IS is_extracted=True for video %s, despite count mismatch.",
+                            frame_zero.pk,
+                            video.video_hash,
+                        )
+                except Frame.DoesNotExist:
+                    logger.error(
+                        "Verification check: Frame 0 does not exist for video %s during status check.",
+                        video.video_hash,
+                    )
+                except Exception as verify_e:
+                    logger.error(
+                        "Verification check: Error checking frame 0 status for video %s: %s",
+                        video.video_hash,
+                        verify_e,
+                    )
+            # --- End detailed check ---
+
+        return updated_count
+    except Exception as e:
+        logger.error(
+            "Failed to bulk update is_extracted status for video %s: %s",
+            video.video_hash,
+            e,
+            exc_info=True,
+        )
+        raise  # Re-raise to ensure transaction rollback if needed