endoreg-db 0.8.1__py3-none-any.whl → 0.8.2.1__py3-none-any.whl

endoreg_db/helpers/download_segmentation_model.py

@@ -0,0 +1,31 @@
+import huggingface_hub
+from typing import Optional
+
+def download_segmentation_model(
+    repo_id: str = "wg-lux/colo_segmentation_RegNetX800MF_base",
+    filename: str = "model.safetensors",
+    cache_dir: Optional[str] = None
+) -> str:
+    """
+    Downloads a segmentation model from Hugging Face and caches it locally.
+
+    Args:
+        repo_id (str): The Hugging Face repository ID (default: wg-lux model).
+        filename (str): The specific file to download from the repo (default: model.safetensors).
+        cache_dir (str): The directory to cache the downloaded model. If None, uses HF default cache.
+
+    Returns:
+        str: The local path to the downloaded model.
+        
+    Example:
+        >>> model_path = download_segmentation_model()
+        >>> # Downloads from wg-lux/colo_segmentation_RegNetX800MF_base
+    """
+    local_path = huggingface_hub.hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        cache_dir=cache_dir,
+        force_download=False,
+        resume_download=True,
+    )
+    return local_path

endoreg_db/migrations/0003_add_center_display_name.py

@@ -0,0 +1,30 @@
+from django.db import migrations, models
+
+
+def populate_display_name(apps, schema_editor):
+    Center = apps.get_model('endoreg_db', 'Center')
+    for center in Center.objects.all():
+        if not center.display_name:
+            center.display_name = center.name
+            center.save(update_fields=['display_name'])
+
+
+def reset_display_name(apps, schema_editor):
+    Center = apps.get_model('endoreg_db', 'Center')
+    Center.objects.update(display_name='')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0002_add_video_correction_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='center',
+            name='display_name',
+            field=models.CharField(blank=True, default='', max_length=255),
+        ),
+        migrations.RunPython(populate_display_name, reset_display_name),
+    ]

endoreg_db/models/administration/center/center.py

@@ -19,6 +19,7 @@ class Center(models.Model):
 
     # import_id = models.IntegerField(primary_key=True)
     name = models.CharField(max_length=255)
+    display_name = models.CharField(max_length=255, blank=True, default="")
 
     first_names = models.ManyToManyField(
         to="FirstName",
@@ -45,8 +46,13 @@ class Center(models.Model):
     def natural_key(self) -> tuple[str]:
         return (self.name,)
 
+    def save(self, *args, **kwargs):
+        if not self.display_name:
+            self.display_name = self.name
+        super().save(*args, **kwargs)
+
     def __str__(self) -> str:
-        return str(object=self.name)
+        return str(object=self.display_name or self.name)
 
     def get_first_names(self):
         return self.first_names.all()

endoreg_db/models/media/pdf/raw_pdf.py

@@ -383,37 +383,42 @@ class RawPdfFile(models.Model):
         new_file_name, _uuid = get_uuid_filename(file_path)
         logger.info(f"Generated new filename: {new_file_name}")
 
-        # Create model instance (without file initially)
-        raw_pdf = cls(
-            pdf_hash=pdf_hash,
-            center=center,
-        )
-
-        # Assign file using Django's File wrapper and save
+        # Create model instance via manager so creation can be intercepted/mocked during tests
         try:
             with file_path.open("rb") as f:
                 django_file = File(f, name=new_file_name)
-                raw_pdf.file = django_file  # type: ignore # Assign the file object
-                # Save the instance - Django storage handles the file copy/move
-                raw_pdf.save()
-                _file = raw_pdf.file
-                assert _file is not None
-                logger.info(f"Created and saved new RawPdfFile {raw_pdf.pk} with file {_file.name}")
-
-                # Verify file exists in storage after save
-                if not _file.storage.exists(_file.name):
-                    logger.error(f"File was not saved correctly to storage path {_file.name} after model save.")
-                    raise IOError(f"File not found at expected storage path after save: {_file.name}")
-                # Log the absolute path for debugging if possible (depends on storage)
-                try:
-                    logger.info(f"File saved to absolute path: {_file.path}")
-                except NotImplementedError:
-                    logger.info(f"File saved to storage path: {_file.name} (Absolute path not available from storage)")
+                raw_pdf = cls.objects.create(
+                    pdf_hash=pdf_hash,
+                    center=center,
+                    file=django_file,
+                )
+
+            _file = raw_pdf.file
+            assert _file is not None
+            logger.info(
+                "Created and saved new RawPdfFile %s with file %s", raw_pdf.pk, _file.name
+            )
+
+            if not _file.storage.exists(_file.name):
+                logger.error(
+                    "File was not saved correctly to storage path %s after model save.",
+                    _file.name,
+                )
+                raise IOError(
+                    f"File not found at expected storage path after save: {_file.name}"
+                )
+
+            try:
+                logger.info("File saved to absolute path: %s", _file.path)
+            except NotImplementedError:
+                logger.info(
+                    "File saved to storage path: %s (Absolute path not available from storage)",
+                    _file.name,
+                )
 
         except Exception as e:
-            logger.error(f"Error processing or saving file {file_path} for new record: {e}")
-            # If save failed, the instance might be partially created but not fully saved.
-            raise  # Re-raise the exception
+            logger.error("Error processing or saving file %s for new record: %s", file_path, e)
+            raise
 
         # Delete source file *after* successful save and verification
         if delete_source:

endoreg_db/models/media/video/create_from_file.py

@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Optional, Type
 
 # Import the new exceptions from the correct path
 from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
-from ...utils import VIDEO_DIR, TMP_VIDEO_DIR, data_paths
+from ...utils import VIDEO_DIR, TMP_VIDEO_DIR
+from importlib import import_module
 
 if TYPE_CHECKING:
     from endoreg_db.models import VideoFile
@@ -170,6 +171,22 @@ def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
         raise
 
 
+def _get_data_paths():
+    """Return the current data_paths mapping (supports patched instances in tests)."""
+    utils_module = import_module("endoreg_db.utils")
+    return getattr(utils_module, "data_paths")
+
+
+def _get_path(mapping, key, default):
+    """Access mapping by key using __getitem__ so MagicMocks with side effects work."""
+    if mapping is None:
+        return default
+    try:
+        return mapping[key]
+    except (KeyError, TypeError):
+        return default
+
+
 def _create_from_file(
     cls_model: Type["VideoFile"],
     file_path: Path,
@@ -199,8 +216,12 @@ def _create_from_file(
 
     try:
         # Ensure we operate under the canonical video path root
-        video_dir = data_paths.get("video", video_dir)
-        storage_root = Path(video_dir).parent
+        data_paths = _get_data_paths()
+        resolved_video_dir = _get_path(data_paths, "video", video_dir)
+        video_dir = Path(resolved_video_dir)
+        storage_root_default = Path(video_dir).parent
+        resolved_storage_root = _get_path(data_paths, "storage", storage_root_default)
+        storage_root = Path(resolved_storage_root)
         storage_root.mkdir(parents=True, exist_ok=True)
         
         # Check storage capacity before starting any work
@@ -300,7 +321,8 @@ def _create_from_file(
         # 8. Create the VideoFile instance
         logger.info("Creating new VideoFile instance with UUID: %s", uuid_val)
         # Store FileField path relative to storage root including the videos prefix
-        relative_name = (final_storage_path.relative_to(data_paths['storage'])).as_posix()
+        storage_base = Path(_get_path(data_paths, "storage", final_storage_path.parent))
+        relative_name = (final_storage_path.relative_to(storage_base)).as_posix()
         video = cls_model(
             uuid=uuid_val,
             raw_file=relative_name,

endoreg_db/models/media/video/pipe_1.py

@@ -1,6 +1,7 @@
 import logging
 from typing import TYPE_CHECKING, Optional, Dict, List, Tuple
 from django.db import transaction
+from endoreg_db.helpers.download_segmentation_model import download_segmentation_model
 
 # Added imports
 
@@ -50,6 +51,8 @@ def _pipe_1(
             if not state.frames_extracted:
                 logger.error("Pipe 1 failed: Frame extraction did not complete successfully.")
                 return False
+            
+            
 
             # 3. Perform Initial Prediction
             logger.info(f"Pipe 1: Performing prediction with model '{model_name}'...")
@@ -61,7 +64,16 @@ def _pipe_1(
                     model_meta = ai_model_obj.get_latest_version()
             except AiModel.DoesNotExist:
                 logger.error(f"Pipe 1 failed: Model '{model_name}' not found.")
-                return False
+                try:
+                    model_name = download_segmentation_model()
+                    ai_model_obj = AiModel.objects.get(name=model_name)
+                    if model_meta_version is not None:
+                        model_meta = ai_model_obj.metadata_versions.get(version=model_meta_version)
+                    else:
+                        model_meta = ai_model_obj.get_latest_version()
+                except AiModel.DoesNotExist:
+                    logger.error(f"Pipe 1 failed: Model '{model_name}' not found.")
+                    return False
             except ModelMeta.DoesNotExist:
                 logger.error(
                     f"Pipe 1 failed: ModelMeta version {model_meta_version} for model '{model_name}' not found."

endoreg_db/models/media/video/video_file.py

@@ -126,7 +126,7 @@ class VideoFile(models.Model):
 
     sensitive_meta = models.OneToOneField(
         "SensitiveMeta", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
-    )
+    ) # type: ignore
     center = models.ForeignKey("Center", on_delete=models.PROTECT)
     processor = models.ForeignKey(
         "EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True
@@ -465,7 +465,18 @@ class VideoFile(models.Model):
         # Use proper database connection
         if using is None:
             using = 'default'
-        
+
+        raw_file_path = self.get_raw_file_path()
+        if raw_file_path:
+            raw_file_path = Path(raw_file_path)
+            lock_path = raw_file_path.with_suffix(raw_file_path.suffix + ".lock")
+            if lock_path.exists():
+                try:
+                    lock_path.unlink()
+                    logger.info(f"Removed processing lock: {lock_path}")
+                except Exception as e:
+                    logger.warning(f"Could not remove processing lock {lock_path}: {e}")
+                
         try:
             # Call parent delete with proper parameters
             super().delete(using=using, keep_parents=keep_parents)
@@ -572,15 +583,28 @@ class VideoFile(models.Model):
         super().save(*args, **kwargs)
 
     def get_or_create_state(self) -> "VideoState":
-        """
-        Return the related VideoState instance for this video, creating and assigning a new one if none exists.
-        
-        Returns:
-            VideoState: The associated VideoState instance.
-        """
-        if self.state is None:
-            self.state = VideoState.objects.create()
-        return self.state
+        """Ensure this video has a persisted ``VideoState`` and return it."""
+
+        state = self.state
+
+        # When tests reuse cached instances across database flushes, ``state`` may reference
+        # a row that no longer exists. Guard against that by validating persistence.
+        state_pk = getattr(state, "pk", None)
+        if state is not None and state_pk is not None:
+            if not VideoState.objects.filter(pk=state_pk).exists():
+                state = None
+
+        if state is None:
+            # Create a fresh state to avoid refresh_from_db() failures on unsaved instances.
+            state = VideoState.objects.create()
+            self.state = state
+
+            # Persist the relation immediately if the VideoFile already exists in the DB so
+            # later refreshes see the association without requiring additional saves.
+            if self.pk:
+                self.save(update_fields=["state"])
+
+        return state
 
     def get_or_create_sensitive_meta(self) -> "SensitiveMeta":
         """
@@ -592,8 +616,7 @@ class VideoFile(models.Model):
         from endoreg_db.models import SensitiveMeta
         if self.sensitive_meta is None:
             self.sensitive_meta = SensitiveMeta.objects.create(center = self.center)
-            # Mark as processed when creating new SensitiveMeta
-            self.get_or_create_state().mark_sensitive_meta_processed(save=True)
+            # Do not mark processed here; it will be set after extraction/validation steps
         return self.sensitive_meta
 
     def get_outside_segments(self, only_validated: bool = False) -> models.QuerySet["LabelVideoSegment"]:

endoreg_db/models/media/video/video_file_anonymize.py

@@ -12,6 +12,7 @@ from django.conf import settings
 
 from endoreg_db.utils.hashs import get_video_hash
 from endoreg_db.utils.validate_endo_roi import validate_endo_roi
+from endoreg_db.utils.paths import STORAGE_DIR
 from ....utils.video.ffmpeg_wrapper import assemble_video_from_frames
 from ...utils import anonymize_frame  # Import from models.utils
 from .video_file_segments import _get_outside_frames, _get_outside_frame_numbers
@@ -268,7 +269,7 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
             raise ValueError(f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid}).")
 
         video.processed_video_hash = new_processed_hash
-        video.processed_file.name = video.get_target_anonymized_video_path().relative_to(settings.MEDIA_ROOT).as_posix()
+        video.processed_file.name = video.get_target_anonymized_video_path().relative_to(STORAGE_DIR).as_posix()
 
         update_fields = [
             "processed_video_hash",

endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py

@@ -97,6 +97,7 @@ def _extract_frame_range(
             return True  # Indicate success as frames are considered present
 
     frame_dir.mkdir(parents=True, exist_ok=True)
+    extracted_paths = []
 
     try:
         logger.info("Starting frame range extraction [%d, %d) for video %s to %s", start_frame, end_frame, video.uuid, frame_dir)
@@ -111,6 +112,17 @@ def _extract_frame_range(
 
         return True
 
+    except FileNotFoundError as err:
+        logger.error(
+            "Frame range extraction [%d, %d) failed for video %s: %s",
+            start_frame,
+            end_frame,
+            video.uuid,
+            err,
+            exc_info=True,
+        )
+        raise
+
     except Exception as e:
         logger.error("Frame range extraction [%d, %d) or DB update failed for video %s: %s", start_frame, end_frame, video.uuid, e, exc_info=True)
 

endoreg_db/models/media/video/video_file_io.py

@@ -32,13 +32,15 @@ def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
                 if sensitive_path.exists():
                     return sensitive_path.resolve()
                 
+                # Check direct raw_file.path if available
                 # Check direct raw_file.path if available
                 try:
                     direct_path = Path(video.raw_file.path)
                     if direct_path.exists():
                         return direct_path.resolve()
-                except Exception:
-                    pass  # Fallback to original behavior
+                except Exception as e:
+                    logger.debug("Could not access direct raw_file.path for video %s: %s", video.uuid, e)
+                    # Fallback to checking alternative paths
                 
                 # Check common alternative paths
                 alternative_paths = [

endoreg_db/models/metadata/video_meta.py

@@ -13,7 +13,7 @@ else:
     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
 
 # Import the new utility function
-from ...utils.video.ffmpeg_wrapper import get_stream_info
+from ...utils.video import ffmpeg_wrapper
 
 logger = logging.getLogger(__name__)
 
@@ -214,7 +214,7 @@ class FFMpegMeta(models.Model):
         """
         logger.info("Running ffprobe on %s", file_path)
         try:
-            probe_data = get_stream_info(file_path)  # Use the new utility
+            probe_data = ffmpeg_wrapper.get_stream_info(file_path)  # Use the new utility
         except Exception as probe_err:
             logger.error("ffprobe execution failed for %s: %s", file_path, probe_err, exc_info=True)
             raise RuntimeError(f"ffprobe execution failed for {file_path}") from probe_err

endoreg_db/serializers/anonymization.py

@@ -28,6 +28,9 @@ class SensitiveMetaValidateSerializer(serializers.Serializer):
     patient_gender = serializers.CharField(required=False, allow_blank=True)
     center_name = serializers.CharField(required=False, allow_blank=True)
     is_verified = serializers.BooleanField(required=False, default=True)
+    file_type = serializers.ChoiceField(
+        choices=['video', 'pdf'], required=False
+    )  # Optional: "video" oder "pdf"
 
     def validate_patient_dob(self, value):
         """