endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

endoreg_db/models/media/video/create_from_file.py

@@ -1,70 +1,73 @@
-import shutil
 import logging
+import shutil
 import uuid
+from importlib import import_module
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Type
 
 # Import the new exceptions from the correct path
 from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
-from ...utils import VIDEO_DIR, TMP_VIDEO_DIR
-from importlib import import_module
+from endoreg_db.utils.paths import IMPORT_VIDEO_DIR, SENSITIVE_VIDEO_DIR, TRANSCODING_DIR
 
 if TYPE_CHECKING:
     from endoreg_db.models import VideoFile
 
-from ....utils.video.ffmpeg_wrapper import transcode_videofile_if_required
-from ....utils.hashs import get_video_hash
+import endoreg_db.utils.paths as path_utils
 from ....utils.file_operations import get_uuid_filename
+from ....utils.hashs import get_video_hash
+from ....utils.video.ffmpeg_wrapper import transcode_videofile_if_required
 
 logger = logging.getLogger(__name__)
 
 
-def check_storage_capacity(src_path: Path, dst_root: Path, safety_margin: float = 1.2) -> None:
+def check_storage_capacity(
+    src_path: Path, dst_root: Path, safety_margin: float = 1.2
+) -> None:
     """
     Check if there's enough storage space before starting operations.
-    
+
     Args:
         src_path: Source file path
         dst_root: Destination root directory
         safety_margin: Safety factor (1.2 = 20% extra space required)
-        
+
     Raises:
         InsufficientStorageError: If insufficient storage space
     """
     try:
         src_size = src_path.stat().st_size
         required_space = int(src_size * safety_margin)
-        
+
         # Check free space on destination
         free_space = shutil.disk_usage(dst_root).free
-        
+
         if free_space < required_space:
             raise InsufficientStorageError(
-                f"Insufficient storage space. Required: {required_space/1e9:.1f} GB, "
-                f"Available: {free_space/1e9:.1f} GB on {dst_root}",
+                f"Insufficient storage space. Required: {required_space / 1e9:.1f} GB, Available: {free_space / 1e9:.1f} GB on {dst_root}",
                 required_space=required_space,
-                available_space=free_space
+                available_space=free_space,
             )
-            
-        logger.info(f"Storage check passed: {free_space/1e9:.1f} GB available, "
-                   f"{required_space/1e9:.1f} GB required")
-                   
+
+        logger.info(
+            f"Storage check passed: {free_space / 1e9:.1f} GB available, {required_space / 1e9:.1f} GB required"
+        )
+
     except OSError as e:
         logger.warning(f"Could not check storage capacity: {e}")
         # Don't fail the operation, just log the warning
 
 
-def atomic_copy_with_fallback(src_path: Path, dst_path: Path) -> bool:
+def atomic_copy_with_fallback(src_path: Path = IMPORT_VIDEO_DIR, dst_path: Path = SENSITIVE_VIDEO_DIR) -> bool:
     """
     Atomically copy file from src to dst, preserving the source file.
-    
+
     Args:
         src_path: Source file path
         dst_path: Destination file path
-        
+
     Returns:
         True if successful
-        
+
     Raises:
         InsufficientStorageError: If not enough space for the operation
         OSError: For other file system errors
@@ -73,18 +76,17 @@ def atomic_copy_with_fallback(src_path: Path, dst_path: Path) -> bool:
         # Check space before copy
         src_size = src_path.stat().st_size
         free_space = shutil.disk_usage(dst_path.parent).free
-        
+
         if free_space < src_size * 1.1:  # 10% safety margin
             raise InsufficientStorageError(
-                f"Insufficient space for copy operation. Required: {src_size/1e9:.1f} GB, "
-                f"Available: {free_space/1e9:.1f} GB",
+                f"Insufficient space for copy operation. Required: {src_size / 1e9:.1f} GB, Available: {free_space / 1e9:.1f} GB",
                 required_space=src_size,
-                available_space=free_space
+                available_space=free_space,
             )
-        
+
         # Use a temporary name during copy for atomicity
-        temp_dst = dst_path.with_suffix(dst_path.suffix + '.tmp')
-        
+        temp_dst = dst_path.with_suffix(dst_path.suffix + ".tmp")
+
         try:
             shutil.copy2(str(src_path), str(temp_dst))
             temp_dst.rename(dst_path)
@@ -95,7 +97,7 @@ def atomic_copy_with_fallback(src_path: Path, dst_path: Path) -> bool:
             if temp_dst.exists():
                 temp_dst.unlink(missing_ok=True)
             raise
-    
+
     except Exception as e:
         logger.error(f"Copy operation failed: {src_path} -> {dst_path}: {e}")
         raise
@@ -104,14 +106,14 @@ def atomic_copy_with_fallback(src_path: Path, dst_path: Path) -> bool:
 def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
     """
     Atomically move file from src to dst, with fallback to copy+remove.
-    
+
     Args:
         src_path: Source file path
         dst_path: Destination file path
-        
+
     Returns:
         True if successful
-        
+
     Raises:
         InsufficientStorageError: If not enough space for the operation
         OSError: For other file system errors
@@ -127,32 +129,31 @@ def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
                 logger.debug("Cross-device move detected, falling back to copy+remove")
             else:
                 raise
-        
+
         # Check space before cross-filesystem copy
         src_size = src_path.stat().st_size
         free_space = shutil.disk_usage(dst_path.parent).free
-        
+
         if free_space < src_size * 1.1:  # 10% safety margin
             raise InsufficientStorageError(
-                f"Insufficient space for copy operation. Required: {src_size/1e9:.1f} GB, "
-                f"Available: {free_space/1e9:.1f} GB",
+                f"Insufficient space for copy operation. Required: {src_size / 1e9:.1f} GB, Available: {free_space / 1e9:.1f} GB",
                 required_space=src_size,
-                available_space=free_space
+                available_space=free_space,
             )
-        
+
         # Fallback to copy+remove for cross-filesystem moves
         logger.info(f"Copying file (cross-filesystem): {src_path} -> {dst_path}")
-        
+
         # Use a temporary name during copy for atomicity
-        temp_dst = dst_path.with_suffix(dst_path.suffix + '.tmp')
-        
+        temp_dst = dst_path.with_suffix(dst_path.suffix + ".tmp")
+
         try:
             shutil.copy2(str(src_path), str(temp_dst))
             temp_dst.rename(dst_path)
             src_path.unlink()  # Remove source only after successful copy
             logger.debug(f"Copy+remove successful: {src_path} -> {dst_path}")
             return True
-            
+
         except OSError as e:
             # Clean up temp file on failure
             if temp_dst.exists():
@@ -162,10 +163,10 @@ def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
                 raise InsufficientStorageError(
                     f"No space left on device during copy: {e}",
                     required_space=src_path.stat().st_size,
-                    available_space=shutil.disk_usage(dst_path.parent).free
+                    available_space=shutil.disk_usage(dst_path.parent).free,
                 )
             raise
-            
+
     except Exception as e:
         logger.error(f"Failed to move {src_path} -> {dst_path}: {e}")
         raise
@@ -192,14 +193,14 @@ def _create_from_file(
     file_path: Path,
     center_name: str,
     processor_name: Optional[str] = None,
-    video_dir: Path = VIDEO_DIR,
+    video_dir: Path = IMPORT_VIDEO_DIR,
     save: bool = True,
     delete_source: bool = False,
-    **kwargs
+    **kwargs,
 ) -> "VideoFile":
     """
     Creates a VideoFile instance from a given video file path with improved error handling.
-    
+
     Raises:
         InsufficientStorageError: When not enough disk space
         TranscodingError: When video transcoding fails
@@ -208,7 +209,7 @@ def _create_from_file(
     """
     from endoreg_db.models.administration.center.center import Center
     from endoreg_db.models.medical.hardware import EndoscopyProcessor
-    
+
     original_file_name = file_path.name
     original_suffix = file_path.suffix
     final_storage_path = None
@@ -223,25 +224,28 @@ def _create_from_file(
         resolved_storage_root = _get_path(data_paths, "storage", storage_root_default)
         storage_root = Path(resolved_storage_root)
         storage_root.mkdir(parents=True, exist_ok=True)
-        
+
         # Check storage capacity before starting any work
         check_storage_capacity(file_path, storage_root)
 
         # 1. Transcode if necessary
         logger.debug("Checking transcoding requirement for %s", file_path)
-        temp_transcode_dir = TMP_VIDEO_DIR / 'transcoding'
+        temp_transcode_dir = TRANSCODING_DIR
         temp_transcode_dir.mkdir(parents=True, exist_ok=True)
-        
+
         # Use a unique name for the potential transcoded file
-        temp_transcoded_output_path = temp_transcode_dir / f"{uuid.uuid4()}{original_suffix}"
+        temp_transcoded_output_path = (
+            temp_transcode_dir / f"{uuid.uuid4()}{original_suffix}"
+        )
 
         try:
             transcoded_file_path = transcode_videofile_if_required(
-                input_path=file_path,
-                output_path=temp_transcoded_output_path
+                input_path=file_path, output_path=temp_transcoded_output_path
             )
             if transcoded_file_path is None:
-                raise TranscodingError(f"Transcoding check/process failed for {file_path}")
+                raise TranscodingError(
+                    f"Transcoding check/process failed for {file_path}"
+                )
         except Exception as e:
             raise TranscodingError(f"Video transcoding failed: {e}") from e
 
@@ -250,24 +254,42 @@ def _create_from_file(
         # 2. Calculate hash (this will be the raw_video_hash)
         video_hash = get_video_hash(transcoded_file_path)
         if not video_hash:
-            raise ValueError(f"Could not calculate video hash for {transcoded_file_path}")
-        logger.info("Calculated raw video hash: %s for %s", video_hash, original_file_name)
+            raise ValueError(
+                f"Could not calculate video hash for {transcoded_file_path}"
+            )
+        logger.info(
+            "Calculated raw video hash: %s for %s", video_hash, original_file_name
+        )
 
         # 3. Check if hash already exists
         if cls_model.check_hash_exists(video_hash=video_hash):
             existing_video = cls_model.objects.get(video_hash=video_hash)
-            logger.warning("Video with hash %s already exists (UUID: %s)", video_hash, existing_video.uuid)
-            
+            logger.warning(
+                "Video with hash %s already exists (UUID: %s)",
+                video_hash,
+                existing_video.uuid,
+            )
+
             # Check if the existing video has a valid file
             existing_raw_path = existing_video.get_raw_file_path()
-            if existing_video.has_raw and existing_raw_path and existing_raw_path.exists():
-                logger.warning("Video with hash %s already exists and file is present. Returning existing instance.", video_hash)
+            if (
+                existing_video.has_raw
+                and existing_raw_path
+                and existing_raw_path.exists()
+            ):
+                logger.warning(
+                    "Video with hash %s already exists and file is present. Returning existing instance.",
+                    video_hash,
+                )
                 # Clean up transcoded file if it was created temporarily
                 if transcoded_file_path != file_path and transcoded_file_path.exists():
                     transcoded_file_path.unlink(missing_ok=True)
                 return existing_video
 
-            logger.warning("Video with hash %s exists but file is missing. Deleting orphaned record.", video_hash)
+            logger.warning(
+                "Video with hash %s exists but file is missing. Deleting orphaned record.",
+                video_hash,
+            )
             existing_video.delete()
 
         # 4. Generate UUID and final storage path
@@ -278,16 +300,26 @@ def _create_from_file(
         # 5. Move or Copy the file to final storage using improved method
         try:
             if delete_source and transcoded_file_path == file_path:
-                logger.debug("Moving original file %s to %s", file_path, final_storage_path)
+                logger.debug(
+                    "Moving original file %s to %s", file_path, final_storage_path
+                )
                 atomic_move_with_fallback(file_path, final_storage_path)
             elif delete_source and transcoded_file_path != file_path:
-                logger.debug("Moving transcoded file %s to %s", transcoded_file_path, final_storage_path)
+                logger.debug(
+                    "Moving transcoded file %s to %s",
+                    transcoded_file_path,
+                    final_storage_path,
+                )
                 atomic_move_with_fallback(transcoded_file_path, final_storage_path)
             else:
-                logger.debug("Copying file %s to %s", transcoded_file_path, final_storage_path)
+                logger.debug(
+                    "Copying file %s to %s", transcoded_file_path, final_storage_path
+                )
                 atomic_copy_with_fallback(transcoded_file_path, final_storage_path)
                 if transcoded_file_path != file_path and transcoded_file_path.exists():
-                    logger.debug("Cleaning up temporary transcoded file %s", transcoded_file_path)
+                    logger.debug(
+                        "Cleaning up temporary transcoded file %s", transcoded_file_path
+                    )
                     transcoded_file_path.unlink(missing_ok=True)
         except InsufficientStorageError:
             # Re-raise storage errors as-is
@@ -298,15 +330,29 @@ def _create_from_file(
         # 6. Verify hash after move/copy
         final_hash = get_video_hash(final_storage_path)
         if final_hash != video_hash:
-            logger.error("Hash mismatch after file operation! Expected %s, got %s", video_hash, final_hash)
+            logger.error(
+                "Hash mismatch after file operation! Expected %s, got %s",
+                video_hash,
+                final_hash,
+            )
             final_storage_path.unlink(missing_ok=True)
-            raise RuntimeError(f"Hash mismatch after file operation for {final_storage_path}")
+            raise RuntimeError(
+                f"Hash mismatch after file operation for {final_storage_path}"
+            )
 
         # 7. Get related objects
         try:
             center = Center.objects.get(name=center_name)
-            processor = EndoscopyProcessor.objects.get(name=processor_name) if processor_name else None
-            logger.debug("Found Center: %s, Processor: %s", center.name, processor.name if processor else "None")
+            processor = (
+                EndoscopyProcessor.objects.get(name=processor_name)
+                if processor_name
+                else None
+            )
+            logger.debug(
+                "Found Center: %s, Processor: %s",
+                center.name,
+                processor.name if processor else "None",
+            )
         except Center.DoesNotExist as e:
             logger.error("Center '%s' not found", center_name)
             if final_storage_path and final_storage_path.exists():
@@ -321,8 +367,10 @@ def _create_from_file(
         # 8. Create the VideoFile instance
         logger.info("Creating new VideoFile instance with UUID: %s", uuid_val)
         # Store FileField path relative to storage root including the videos prefix
-        storage_base = Path(_get_path(data_paths, "storage", final_storage_path.parent))
-        relative_name = (final_storage_path.relative_to(storage_base)).as_posix()
+        # TODO Review removal, since this is now newly handled by path_utils
+        #storage_base = Path(_get_path(data_paths, "storage", final_storage_path.parent))
+        
+        relative_name = path_utils.to_storage_relative(final_storage_path)        
         video = cls_model(
             uuid=uuid_val,
             raw_file=relative_name,
@@ -339,7 +387,9 @@ def _create_from_file(
         if save:
             logger.info("Saving new VideoFile instance (UUID: %s)", uuid_val)
             video.save()
-            logger.info("Successfully created VideoFile PK %s (UUID: %s)", video.pk, video.uuid)
+            logger.info(
+                "Successfully created VideoFile PK %s (UUID: %s)", video.pk, video.uuid
+            )
 
         return video
 
@@ -347,12 +397,24 @@ def _create_from_file(
         # Re-raise these specific errors as-is
         raise
     except Exception as e:
-        logger.error("Failed to create VideoFile from %s: (%s) %s", file_path, type(e).__name__, e, exc_info=True)
+        logger.error(
+            "Failed to create VideoFile from %s: (%s) %s",
+            file_path,
+            type(e).__name__,
+            e,
+            exc_info=True,
+        )
         # Clean up any created files
         if final_storage_path and final_storage_path.exists():
             logger.warning("Cleaning up orphaned file: %s", final_storage_path)
             final_storage_path.unlink(missing_ok=True)
-        if transcoded_file_path and transcoded_file_path != file_path and transcoded_file_path.exists():
-            logger.warning("Cleaning up orphaned transcoded file: %s", transcoded_file_path)
+        if (
+            transcoded_file_path
+            and transcoded_file_path != file_path
+            and transcoded_file_path.exists()
+        ):
+            logger.warning(
+                "Cleaning up orphaned transcoded file: %s", transcoded_file_path
+            )
             transcoded_file_path.unlink(missing_ok=True)
-        raise RuntimeError(f"Video processing failed: {e}") from e
+        raise RuntimeError(f"Video processing failed: {e}") from e

endoreg_db/models/media/video/pipe_2.py

@@ -1,16 +1,16 @@
-from itertools import filterfalse
 import logging
 from typing import TYPE_CHECKING
-from django.db import transaction
 
+from django.db import transaction
 
 # Configure logging
-logger = logging.getLogger(__name__) # Changed from "video_file"
+logger = logging.getLogger(__name__)  # Changed from "video_file"
 
 if TYPE_CHECKING:
     from endoreg_db.models import VideoFile, VideoState
 
-def _pipe_2(video_file:"VideoFile") -> bool:
+
+def _pipe_2(video_file: "VideoFile") -> bool:
     """
     Process the given video file through pipeline 2 operations which include frame extraction,
     anonymization of the video, and deletion of sensitive meta data.
@@ -70,11 +70,10 @@ def _pipe_2(video_file:"VideoFile") -> bool:
                 logger.info("Pipe 2: Anonymization complete.")
         else:
             logger.info("Pipe 2: Video already anonymized.")
-        
 
         # --- Part 3: Final DB operations (now in its own atomic transaction) ---
         with transaction.atomic():
-            video_file.refresh_from_db() # Ensure we have the latest video_file state for these ops
+            video_file.refresh_from_db()  # Ensure we have the latest video_file state for these ops
 
             # Set sensitive_meta_processed True atomically
             state: "VideoState" = video_file.get_or_create_state()
@@ -86,8 +85,8 @@ def _pipe_2(video_file:"VideoFile") -> bool:
                 try:
                     sm_pk = video_file.sensitive_meta.pk
                     video_file.sensitive_meta.delete()
-                    video_file.sensitive_meta = None # Important after SET_NULL
-                    video_file.save(update_fields=['sensitive_meta']) # Persist the null relation
+                    video_file.sensitive_meta = None  # Important after SET_NULL
+                    video_file.save(update_fields=["sensitive_meta"])  # Persist the null relation
                     logger.info("Pipe 2: Deleted sensitive meta object (PK: %s).", sm_pk)
                 except Exception as e:
                     logger.error("Pipe 2: Failed to delete sensitive meta object: %s", e, exc_info=True)
@@ -102,4 +101,4 @@ def _pipe_2(video_file:"VideoFile") -> bool:
         # This will catch exceptions from I/O operations if they raise,
         # or from the final transaction block, or any other unhandled error.
         logger.error(f"Pipe 2 failed for video {video_file.uuid} with unhandled exception: {e}", exc_info=True)
-        return False
+        return False