endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/models/media/video/create_from_file.py

@@ -0,0 +1,402 @@
+import logging
+import shutil
+import uuid
+from importlib import import_module
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Type
+
+# Import the new exceptions from the correct path
+from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
+from endoreg_db.utils.paths import (
+    IMPORT_VIDEO_DIR,
+    SENSITIVE_VIDEO_DIR,
+    TRANSCODING_DIR,
+)
+
+if TYPE_CHECKING:
+    from endoreg_db.models import VideoFile
+
+import endoreg_db.utils.paths as path_utils
+
+from ....utils.video.ffmpeg_wrapper import transcode_videofile_if_required
+
+logger = logging.getLogger(__name__)
+
+
+def check_storage_capacity(
+    src_path: Path, dst_root: Path, safety_margin: float = 1.2
+) -> None:
+    """
+    Check if there's enough storage space before starting operations.
+
+    Args:
+        src_path: Source file path
+        dst_root: Destination root directory
+        safety_margin: Safety factor (1.2 = 20% extra space required)
+
+    Raises:
+        InsufficientStorageError: If insufficient storage space
+    """
+    try:
+        src_size = src_path.stat().st_size
+        required_space = int(src_size * safety_margin)
+
+        # Check free space on destination
+        free_space = shutil.disk_usage(dst_root).free
+
+        if free_space < required_space:
+            raise InsufficientStorageError(
+                f"Insufficient storage space. Required: {required_space / 1e9:.1f} GB, Available: {free_space / 1e9:.1f} GB on {dst_root}",
+                required_space=required_space,
+                available_space=free_space,
+            )
+
+        logger.info(
+            f"Storage check passed: {free_space / 1e9:.1f} GB available, {required_space / 1e9:.1f} GB required"
+        )
+
+    except OSError as e:
+        logger.warning(f"Could not check storage capacity: {e}")
+        # Don't fail the operation, just log the warning
+
+
+def atomic_copy_with_fallback(
+    src_path: Path = IMPORT_VIDEO_DIR, dst_path: Path = SENSITIVE_VIDEO_DIR
+) -> bool:
+    """
+    Atomically copy file from src to dst, preserving the source file.
+
+    Args:
+        src_path: Source file path
+        dst_path: Destination file path
+
+    Returns:
+        True if successful
+
+    Raises:
+        InsufficientStorageError: If not enough space for the operation
+        OSError: For other file system errors
+    """
+    try:
+        # Check space before copy
+        src_size = src_path.stat().st_size
+        free_space = shutil.disk_usage(dst_path.parent).free
+
+        if free_space < src_size * 1.1:  # 10% safety margin
+            raise InsufficientStorageError(
+                f"Insufficient space for copy operation. Required: {src_size / 1e9:.1f} GB, Available: {free_space / 1e9:.1f} GB",
+                required_space=src_size,
+                available_space=free_space,
+            )
+
+        # Use a temporary name during copy for atomicity
+        temp_dst = dst_path.with_suffix(dst_path.suffix + ".tmp")
+
+        try:
+            shutil.copy2(str(src_path), str(temp_dst))
+            temp_dst.rename(dst_path)
+            logger.debug(f"Copy successful: {src_path} -> {dst_path}")
+            return True
+        except Exception:
+            # Clean up temp file if copy failed
+            if temp_dst.exists():
+                temp_dst.unlink(missing_ok=True)
+            raise
+
+    except Exception as e:
+        logger.error(f"Copy operation failed: {src_path} -> {dst_path}: {e}")
+        raise
+
+
+def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
+    """
+    Atomically move file from src to dst, with fallback to copy+remove.
+
+    Args:
+        src_path: Source file path
+        dst_path: Destination file path
+
+    Returns:
+        True if successful
+
+    Raises:
+        InsufficientStorageError: If not enough space for the operation
+        OSError: For other file system errors
+    """
+    try:
+        # First try atomic move (same filesystem)
+        try:
+            src_path.rename(dst_path)
+            logger.debug(f"Atomic move successful: {src_path} -> {dst_path}")
+            return True
+        except OSError as e:
+            if e.errno == 18:  # Cross-device link
+                logger.debug("Cross-device move detected, falling back to copy+remove")
+            else:
+                raise
+
+        # Check space before cross-filesystem copy
+        src_size = src_path.stat().st_size
+        free_space = shutil.disk_usage(dst_path.parent).free
+
+        if free_space < src_size * 1.1:  # 10% safety margin
+            raise InsufficientStorageError(
+                f"Insufficient space for copy operation. Required: {src_size / 1e9:.1f} GB, Available: {free_space / 1e9:.1f} GB",
+                required_space=src_size,
+                available_space=free_space,
+            )
+
+        # Fallback to copy+remove for cross-filesystem moves
+        logger.info(f"Copying file (cross-filesystem): {src_path} -> {dst_path}")
+
+        # Use a temporary name during copy for atomicity
+        temp_dst = dst_path.with_suffix(dst_path.suffix + ".tmp")
+
+        try:
+            shutil.copy2(str(src_path), str(temp_dst))
+            temp_dst.rename(dst_path)
+            src_path.unlink()  # Remove source only after successful copy
+            logger.debug(f"Copy+remove successful: {src_path} -> {dst_path}")
+            return True
+
+        except OSError as e:
+            # Clean up temp file on failure
+            if temp_dst.exists():
+                temp_dst.unlink(missing_ok=True)
+            # Re-raise with better context
+            if e.errno == 28:  # No space left on device
+                raise InsufficientStorageError(
+                    f"No space left on device during copy: {e}",
+                    required_space=src_path.stat().st_size,
+                    available_space=shutil.disk_usage(dst_path.parent).free,
+                )
+            raise
+
+    except Exception as e:
+        logger.error(f"Failed to move {src_path} -> {dst_path}: {e}")
+        raise
+
+
+def _get_data_paths():
+    """Return the current data_paths mapping (supports patched instances in tests)."""
+    utils_module = import_module("endoreg_db.utils")
+    return getattr(utils_module, "data_paths")
+
+
+def _get_path(mapping, key, default):
+    """Access mapping by key using __getitem__ so MagicMocks with side effects work."""
+    if mapping is None:
+        return default
+    try:
+        return mapping[key]
+    except (KeyError, TypeError):
+        return default
+
+
+def _create_from_file(
+    cls_model: Type["VideoFile"],
+    file_path: Path,
+    center_name: str,
+    processor_name: Optional[str],
+    video_hash: str,
+    video_dir: Path = IMPORT_VIDEO_DIR,
+    save: bool = True,
+    delete_source: bool = False,
+    **kwargs,
+) -> "VideoFile":
+    """
+    Creates a VideoFile instance from a given video file path with improved error handling.
+
+    Raises:
+        InsufficientStorageError: When not enough disk space
+        TranscodingError: When video transcoding fails
+        ValueError: When required objects (Center, Processor) not found
+        RuntimeError: For other processing errors
+    """
+    from endoreg_db.models.administration.center.center import Center
+    from endoreg_db.models.medical.hardware import EndoscopyProcessor
+
+    original_file_name = file_path.name
+    original_suffix = file_path.suffix
+    final_storage_path = None
+    transcoded_file_path = None
+
+    try:
+        # Ensure we operate under the canonical video path root
+        data_paths = _get_data_paths()
+        resolved_video_dir = _get_path(data_paths, "sensitive_video", video_dir)
+        video_dir = Path(resolved_video_dir)
+        storage_root_default = Path(video_dir).parent
+        resolved_storage_root = _get_path(data_paths, "storage", storage_root_default)
+        storage_root = Path(resolved_storage_root)
+        storage_root.mkdir(parents=True, exist_ok=True)
+
+        # Check storage capacity before starting any work
+        check_storage_capacity(file_path, storage_root)
+
+        # 1. Transcode if necessary
+        logger.debug("Checking transcoding requirement for %s", file_path)
+        temp_transcode_dir = TRANSCODING_DIR
+        temp_transcode_dir.mkdir(parents=True, exist_ok=True)
+
+        # Use a unique name for the potential transcoded file
+        temp_transcoded_output_path = (
+            temp_transcode_dir / f"{uuid.uuid4()}{original_suffix}"
+        )
+
+        try:
+            transcoded_file_path = transcode_videofile_if_required(
+                input_path=file_path, output_path=temp_transcoded_output_path
+            )
+            if transcoded_file_path is None:
+                raise TranscodingError(
+                    f"Transcoding check/process failed for {file_path}"
+                )
+        except Exception as e:
+            raise TranscodingError(f"Video transcoding failed: {e}") from e
+
+        logger.debug("Using file for hashing: %s", transcoded_file_path)
+
+        # 3. Check if hash already exists
+        if cls_model.check_hash_exists(video_hash=video_hash):
+            existing_video = cls_model.objects.get(video_hash=video_hash)
+            logger.warning(
+                "Video with hash %s already exists (UUID: %s)",
+                video_hash,
+                existing_video.video_hash,
+            )
+
+            # Check if the existing video has a valid file
+            existing_raw_path = existing_video.get_raw_file_path()
+            if (
+                existing_video.has_raw
+                and existing_raw_path
+                and existing_raw_path.exists()
+            ):
+                logger.warning(
+                    "Video with hash %s already exists and file is present. Returning existing instance.",
+                    video_hash,
+                )
+                # Clean up transcoded file if it was created temporarily
+                if transcoded_file_path != file_path and transcoded_file_path.exists():
+                    transcoded_file_path.unlink(missing_ok=True)
+                return existing_video
+
+            logger.warning(
+                "Video with hash %s exists but file is missing. Deleting orphaned record.",
+                video_hash,
+            )
+            existing_video.delete()
+
+        filename = f"{video_hash}{transcoded_file_path.suffix}"
+        final_storage_path = video_dir / filename
+        
+        # Ensure the DIRECTORY exists (video_dir), not the parent of a nested file
+        final_storage_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # 5. Move or Copy the file to final storage using improved method
+        try:
+            if delete_source and transcoded_file_path == file_path:
+                logger.debug(
+                    "Moving original file %s to %s", file_path, final_storage_path
+                )
+                atomic_move_with_fallback(file_path, final_storage_path)
+            elif delete_source and transcoded_file_path != file_path:
+                logger.debug(
+                    "Moving transcoded file %s to %s",
+                    transcoded_file_path,
+                    final_storage_path,
+                )
+                atomic_move_with_fallback(transcoded_file_path, final_storage_path)
+            else:
+                logger.debug(
+                    "Copying file %s to %s", transcoded_file_path, final_storage_path
+                )
+                atomic_copy_with_fallback(transcoded_file_path, final_storage_path)
+                if transcoded_file_path != file_path and transcoded_file_path.exists():
+                    logger.debug(
+                        "Cleaning up temporary transcoded file %s", transcoded_file_path
+                    )
+                    transcoded_file_path.unlink(missing_ok=True)
+        except InsufficientStorageError:
+            # Re-raise storage errors as-is
+            raise
+        except Exception as e:
+            raise RuntimeError(f"Failed to move file to final storage: {e}") from e
+
+        # 7. Get related objects
+        try:
+            center = Center.objects.get(name=center_name)
+            processor = (
+                EndoscopyProcessor.objects.get(name=processor_name)
+                if processor_name
+                else None
+            )
+            logger.debug(
+                "Found Center: %s, Processor: %s",
+                center.name,
+                processor.name if processor else "None",
+            )
+        except Center.DoesNotExist as e:
+            logger.error("Center '%s' not found", center_name)
+            if final_storage_path and final_storage_path.exists():
+                final_storage_path.unlink(missing_ok=True)
+            raise ValueError(f"Center '{center_name}' not found.") from e
+        except EndoscopyProcessor.DoesNotExist as e:
+            logger.error("Processor '%s' not found", processor_name)
+            if final_storage_path and final_storage_path.exists():
+                final_storage_path.unlink(missing_ok=True)
+            raise ValueError(f"Processor '{processor_name}' not found.") from e
+
+        # 8. Create the VideoFile instance
+        logger.info("Creating new VideoFile instance with hash: %s", video_hash)
+        # Store FileField path relative to storage root including the videos prefix
+
+        relative_name = path_utils.to_storage_relative(final_storage_path)
+        video = cls_model(
+            raw_file=relative_name,
+            processed_file=None,
+            center=center,
+            processor=processor,
+            original_file_name=original_file_name,
+            video_hash=video_hash,
+            processed_video_hash=None,
+            suffix=original_suffix,
+        )
+
+        # 9. Save the instance if requested
+        if save:
+            logger.info("Saving new VideoFile instance (Hash:%s", video_hash)
+            video.save()
+            logger.info(
+                "Successfully created VideoFile PK %s ",
+                video.pk,
+            )
+
+        return video
+
+    except (InsufficientStorageError, TranscodingError, ValueError):
+        # Re-raise these specific errors as-is
+        raise
+    except Exception as e:
+        logger.error(
+            "Failed to create VideoFile from %s: (%s) %s",
+            file_path,
+            type(e).__name__,
+            e,
+            exc_info=True,
+        )
+        # Clean up any created files
+        if final_storage_path and final_storage_path.exists():
+            logger.warning("Cleaning up orphaned file: %s", final_storage_path)
+            final_storage_path.unlink(missing_ok=True)
+        if (
+            transcoded_file_path
+            and transcoded_file_path != file_path
+            and transcoded_file_path.exists()
+        ):
+            logger.warning(
+                "Cleaning up orphaned transcoded file: %s", transcoded_file_path
+            )
+            transcoded_file_path.unlink(missing_ok=True)
+        raise RuntimeError(f"Video processing failed: {e}") from e

endoreg_db/models/media/video/pipe_1.py

@@ -0,0 +1,258 @@
+import logging
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+
+from django.db import transaction
+
+from endoreg_db.helpers.download_segmentation_model import download_segmentation_model
+
+# Added imports
+
+# Configure logging
+logger = logging.getLogger(__name__)  # Changed from "video_file"
+
+if TYPE_CHECKING:
+    from endoreg_db.models import VideoFile
+
+    # --- Pipeline 1 ---
+
+
+def _pipe_1(
+    video_file: "VideoFile",
+    model_name: str,
+    model_meta_version: Optional[int] = None,
+    delete_frames_after: bool = False,
+    ocr_frame_fraction: float = 0.001,
+    ocr_cap: int = 10,
+    smooth_window_size_s: int = 1,
+    binarize_threshold: float = 0.5,
+    test_run: bool = False,
+    n_test_frames: int = 10,
+) -> bool:
+    """
+    Pipeline 1: Extract frames, text, predict, create segments, optionally delete frames.
+    """
+    success = False  # Initialize success flag
+    from endoreg_db.models import AiModel, LabelVideoSegment
+
+    from ...metadata import ModelMeta, VideoPredictionMeta
+    from .video_file_segments import _convert_sequences_to_db_segments  # Added import
+
+    video_file.refresh_from_db()
+    video_file.update_video_meta()
+
+    logger.info(f"Starting Pipe 1 for video {video_file.video_hash}")
+    try:
+        # 1. Heavy I/O operations outside the transaction block
+        logger.info("Pipe 1: Extracting frames...")
+        video_file.extract_frames(
+            overwrite=False
+        )  # Avoid overwriting if already extracted
+
+        logger.info("Pipe 1: Extracting text metadata...")
+        video_file.update_text_metadata(
+            ocr_frame_fraction=ocr_frame_fraction, cap=ocr_cap, overwrite=False
+        )
+        with transaction.atomic():
+            state = video_file.get_or_create_state()
+            if not state.frames_extracted:
+                logger.error(
+                    "Pipe 1 failed: Frame extraction did not complete successfully."
+                )
+                return False
+
+            # 3. Perform Initial Prediction
+            logger.info(f"Pipe 1: Performing prediction with model '{model_name}'...")
+            try:
+                ai_model_obj = AiModel.objects.get(name=model_name)
+                if model_meta_version is not None:
+                    model_meta = ai_model_obj.metadata_versions.get(
+                        version=model_meta_version
+                    )
+                else:
+                    model_meta = ai_model_obj.get_latest_version()
+            except AiModel.DoesNotExist:
+                logger.error(f"Pipe 1 failed: Model '{model_name}' not found.")
+                try:
+                    model_name = download_segmentation_model()
+                    ai_model_obj = AiModel.objects.get(name=model_name)
+                    if model_meta_version is not None:
+                        model_meta = ai_model_obj.metadata_versions.get(
+                            version=model_meta_version
+                        )
+                    else:
+                        model_meta = ai_model_obj.get_latest_version()
+                except AiModel.DoesNotExist:
+                    logger.error(f"Pipe 1 failed: Model '{model_name}' not found.")
+                    return False
+
+            except ModelMeta.DoesNotExist:
+                try:
+                    model_name = download_segmentation_model()
+                    ai_model_obj = AiModel.objects.get(name=model_name)
+                    if model_meta_version is not None:
+                        model_meta = ai_model_obj.metadata_versions.get(
+                            version=model_meta_version
+                        )
+                    else:
+                        model_meta = ai_model_obj.get_latest_version()
+                except ModelMeta.DoesNotExist:
+                    logger.error(
+                        f"Pipe 1 failed: ModelMeta version {model_meta_version} for model '{model_name}' not found."
+                    )
+                    return False
+            try:
+                sequences: Optional[Dict[str, List[Tuple[int, int]]]] = (
+                    video_file.predict_video(
+                        model_meta=model_meta,
+                        smooth_window_size_s=smooth_window_size_s,
+                        binarize_threshold=binarize_threshold,
+                        test_run=test_run,
+                        n_test_frames=n_test_frames,
+                    )
+                )
+            except Exception as e:
+                logger.error(f"Pipe 1 failed during prediction: {e}", exc_info=True)
+                return False
+
+            if sequences is None:
+                logger.error("Pipe 1 failed: Prediction pipeline returned None.")
+                return False
+            logger.info("Pipe 1: Prediction complete.")
+
+            # --- Set and Save State ---
+            state.initial_prediction_completed = True
+            state.save(update_fields=["initial_prediction_completed"])
+            logger.info("Pipe 1: Set initial_prediction_completed state to True.")
+
+            logger.info(f"Pipe 1: Sequences returned from prediction: {sequences}")
+            if not sequences:
+                logger.warning(
+                    "Pipe 1: Prediction returned empty sequences dictionary. No LabelVideoSegments will be created."
+                )
+
+            # 4. Create LabelVideoSegments
+            logger.info("Pipe 1: Creating LabelVideoSegments from predictions...")
+            try:
+                video_prediction_meta = VideoPredictionMeta.objects.get(
+                    video_file=video_file, model_meta=model_meta
+                )
+                logger.info(
+                    f"Pipe 1: Calling _convert_sequences_to_db_segments for video {video_file.video_hash} with prediction meta {video_prediction_meta.pk}"
+                )
+                _convert_sequences_to_db_segments(
+                    video=video_file,
+                    sequences=sequences,
+                    video_prediction_meta=video_prediction_meta,
+                )
+                video_file.sequences = sequences
+                video_file.save(update_fields=["sequences"])
+                state.lvs_created = True
+                state.save(update_fields=["lvs_created"])
+                logger.info("Pipe 1: Set lvs_created state to True.")
+                logger.info("Pipe 1: LabelVideoSegment creation complete.")
+                lvs_count_after = LabelVideoSegment.objects.filter(
+                    video_file=video_file
+                ).count()
+                logger.info(
+                    f"Pipe 1: Found {lvs_count_after} LabelVideoSegments after conversion attempt."
+                )
+            except VideoPredictionMeta.DoesNotExist:
+                logger.error(
+                    "Pipe 1 failed: Could not find VideoPredictionMeta after prediction."
+                )
+                raise
+
+        logger.info(f"Pipe 1 completed successfully for video {video_file.video_hash}")
+        success = True  # Set success flag
+        return True
+
+    except Exception as e:
+        logger.error(
+            f"Pipe 1 failed for video {video_file.video_hash}: {e}", exc_info=True
+        )
+        return False
+    finally:
+        # 5. Optionally delete frames
+        if delete_frames_after and success:  # Check success flag
+            logger.info("Pipe 1: Deleting frames after processing...")
+            try:
+                video_file.delete_frames()
+                logger.info("Pipe 1: Frame deletion complete.")
+            except Exception as e:
+                logger.error(f"Pipe 1 failed during frame deletion: {e}", exc_info=True)
+        else:
+            logger.info("Pipe 1: Frame deletion skipped.")
+
+
+# --- Test after Pipe 1 ---
+def _test_after_pipe_1(
+    video_file: "VideoFile", start_frame: int = 0, end_frame: int = 100
+) -> bool:
+    """
+    Simulates human annotation validation after Pipe 1.
+    Creates 'outside' segments and marks sensitive meta as verified.
+    """
+    from ...label import Label, LabelVideoSegment
+
+    logger.info(f"Starting _test_after_pipe_1 for video {video_file.video_hash}")
+    try:
+        # 1. Create 'outside' LabelVideoSegments
+        try:
+            outside_label = Label.objects.get(name__iexact="outside")
+            logger.info(
+                f"Creating 'outside' annotation segment [{start_frame}-{end_frame}]"
+            )
+            # Create a segment - assuming custom_create handles saving
+            outside_segment = LabelVideoSegment.objects.create(  # Assign to variable
+                video_file=video_file,
+                label=outside_label,
+                start_frame_number=start_frame,
+                end_frame_number=end_frame,
+                prediction_meta=None,
+            )
+            # Ensure the segment has a state and mark it as validated
+            segment_state, created = (
+                outside_segment.get_or_create_state()
+            )  # Unpack the tuple
+            segment_state.is_validated = True
+            segment_state.save()
+            logger.info(
+                f"Marked 'outside' segment {outside_segment.pk} as validated. Created: {created}"
+            )
+
+        except Label.DoesNotExist:
+            logger.error("_test_after_pipe_1 failed: 'outside' Label not found.")
+            return False
+        except Exception as e:
+            logger.error(
+                f"_test_after_pipe_1 failed during segment creation: {e}", exc_info=True
+            )
+            return False
+
+        # 2. Set Sensitive Metadata state to verified
+        if video_file.sensitive_meta:
+            sm = video_file.sensitive_meta
+            st_state = sm.state
+            assert st_state, "SensitiveMeta state is None. Cannot set to verified."
+            logger.info("Setting sensitive meta state to verified.")
+
+            # Example: using a boolean field
+            video_file.sensitive_meta.state.dob_verified = True
+            video_file.sensitive_meta.state.names_verified = True
+            video_file.sensitive_meta.state.save()  # Save the SensitiveMeta instance
+            logger.info("Sensitive meta state updated.")
+
+        else:
+            logger.warning("_test_after_pipe_1: No sensitive meta found to verify.")
+
+        logger.info(
+            f"_test_after_pipe_1 completed successfully for video {video_file.video_hash}"
+        )
+        return True
+
+    except Exception as e:
+        logger.error(
+            f"_test_after_pipe_1 failed for video {video_file.video_hash}: {e}",
+            exc_info=True,
+        )
+        return False