endoreg-db 0.8.2__py3-none-any.whl → 0.8.2.1__py3-none-any.whl

endoreg_db/migrations/0003_add_center_display_name.py

@@ -0,0 +1,30 @@
+from django.db import migrations, models
+
+
+def populate_display_name(apps, schema_editor):
+    Center = apps.get_model('endoreg_db', 'Center')
+    for center in Center.objects.all():
+        if not center.display_name:
+            center.display_name = center.name
+            center.save(update_fields=['display_name'])
+
+
+def reset_display_name(apps, schema_editor):
+    Center = apps.get_model('endoreg_db', 'Center')
+    Center.objects.update(display_name='')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0002_add_video_correction_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='center',
+            name='display_name',
+            field=models.CharField(blank=True, default='', max_length=255),
+        ),
+        migrations.RunPython(populate_display_name, reset_display_name),
+    ]

endoreg_db/models/administration/center/center.py

@@ -19,6 +19,7 @@ class Center(models.Model):
 
     # import_id = models.IntegerField(primary_key=True)
     name = models.CharField(max_length=255)
+    display_name = models.CharField(max_length=255, blank=True, default="")
 
     first_names = models.ManyToManyField(
         to="FirstName",
@@ -45,8 +46,13 @@ class Center(models.Model):
     def natural_key(self) -> tuple[str]:
         return (self.name,)
 
+    def save(self, *args, **kwargs):
+        if not self.display_name:
+            self.display_name = self.name
+        super().save(*args, **kwargs)
+
     def __str__(self) -> str:
-        return str(object=self.name)
+        return str(object=self.display_name or self.name)
 
     def get_first_names(self):
         return self.first_names.all()

endoreg_db/models/media/pdf/raw_pdf.py

@@ -383,37 +383,42 @@ class RawPdfFile(models.Model):
         new_file_name, _uuid = get_uuid_filename(file_path)
         logger.info(f"Generated new filename: {new_file_name}")
 
-        # Create model instance (without file initially)
-        raw_pdf = cls(
-            pdf_hash=pdf_hash,
-            center=center,
-        )
-
-        # Assign file using Django's File wrapper and save
+        # Create model instance via manager so creation can be intercepted/mocked during tests
         try:
             with file_path.open("rb") as f:
                 django_file = File(f, name=new_file_name)
-                raw_pdf.file = django_file  # type: ignore # Assign the file object
-                # Save the instance - Django storage handles the file copy/move
-                raw_pdf.save()
-                _file = raw_pdf.file
-                assert _file is not None
-                logger.info(f"Created and saved new RawPdfFile {raw_pdf.pk} with file {_file.name}")
-
-                # Verify file exists in storage after save
-                if not _file.storage.exists(_file.name):
-                    logger.error(f"File was not saved correctly to storage path {_file.name} after model save.")
-                    raise IOError(f"File not found at expected storage path after save: {_file.name}")
-                # Log the absolute path for debugging if possible (depends on storage)
-                try:
-                    logger.info(f"File saved to absolute path: {_file.path}")
-                except NotImplementedError:
-                    logger.info(f"File saved to storage path: {_file.name} (Absolute path not available from storage)")
+                raw_pdf = cls.objects.create(
+                    pdf_hash=pdf_hash,
+                    center=center,
+                    file=django_file,
+                )
+
+            _file = raw_pdf.file
+            assert _file is not None
+            logger.info(
+                "Created and saved new RawPdfFile %s with file %s", raw_pdf.pk, _file.name
+            )
+
+            if not _file.storage.exists(_file.name):
+                logger.error(
+                    "File was not saved correctly to storage path %s after model save.",
+                    _file.name,
+                )
+                raise IOError(
+                    f"File not found at expected storage path after save: {_file.name}"
+                )
+
+            try:
+                logger.info("File saved to absolute path: %s", _file.path)
+            except NotImplementedError:
+                logger.info(
+                    "File saved to storage path: %s (Absolute path not available from storage)",
+                    _file.name,
+                )
 
         except Exception as e:
-            logger.error(f"Error processing or saving file {file_path} for new record: {e}")
-            # If save failed, the instance might be partially created but not fully saved.
-            raise  # Re-raise the exception
+            logger.error("Error processing or saving file %s for new record: %s", file_path, e)
+            raise
 
         # Delete source file *after* successful save and verification
         if delete_source:

endoreg_db/models/media/video/create_from_file.py

@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Optional, Type
 
 # Import the new exceptions from the correct path
 from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
-from ...utils import VIDEO_DIR, TMP_VIDEO_DIR, data_paths
+from ...utils import VIDEO_DIR, TMP_VIDEO_DIR
+from importlib import import_module
 
 if TYPE_CHECKING:
     from endoreg_db.models import VideoFile
@@ -170,6 +171,22 @@ def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
         raise
 
 
+def _get_data_paths():
+    """Return the current data_paths mapping (supports patched instances in tests)."""
+    utils_module = import_module("endoreg_db.utils")
+    return getattr(utils_module, "data_paths")
+
+
+def _get_path(mapping, key, default):
+    """Access mapping by key using __getitem__ so MagicMocks with side effects work."""
+    if mapping is None:
+        return default
+    try:
+        return mapping[key]
+    except (KeyError, TypeError):
+        return default
+
+
 def _create_from_file(
     cls_model: Type["VideoFile"],
     file_path: Path,
@@ -199,8 +216,12 @@ def _create_from_file(
 
     try:
         # Ensure we operate under the canonical video path root
-        video_dir = data_paths.get("video", video_dir)
-        storage_root = Path(video_dir).parent
+        data_paths = _get_data_paths()
+        resolved_video_dir = _get_path(data_paths, "video", video_dir)
+        video_dir = Path(resolved_video_dir)
+        storage_root_default = Path(video_dir).parent
+        resolved_storage_root = _get_path(data_paths, "storage", storage_root_default)
+        storage_root = Path(resolved_storage_root)
         storage_root.mkdir(parents=True, exist_ok=True)
         
         # Check storage capacity before starting any work
@@ -300,7 +321,8 @@ def _create_from_file(
         # 8. Create the VideoFile instance
         logger.info("Creating new VideoFile instance with UUID: %s", uuid_val)
         # Store FileField path relative to storage root including the videos prefix
-        relative_name = (final_storage_path.relative_to(data_paths['storage'])).as_posix()
+        storage_base = Path(_get_path(data_paths, "storage", final_storage_path.parent))
+        relative_name = (final_storage_path.relative_to(storage_base)).as_posix()
         video = cls_model(
             uuid=uuid_val,
             raw_file=relative_name,

endoreg_db/models/media/video/video_file.py

@@ -126,7 +126,7 @@ class VideoFile(models.Model):
 
     sensitive_meta = models.OneToOneField(
         "SensitiveMeta", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
-    )
+    ) # type: ignore
     center = models.ForeignKey("Center", on_delete=models.PROTECT)
     processor = models.ForeignKey(
         "EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True
@@ -465,7 +465,18 @@ class VideoFile(models.Model):
         # Use proper database connection
         if using is None:
             using = 'default'
-        
+
+        raw_file_path = self.get_raw_file_path()
+        if raw_file_path:
+            raw_file_path = Path(raw_file_path)
+            lock_path = raw_file_path.with_suffix(raw_file_path.suffix + ".lock")
+            if lock_path.exists():
+                try:
+                    lock_path.unlink()
+                    logger.info(f"Removed processing lock: {lock_path}")
+                except Exception as e:
+                    logger.warning(f"Could not remove processing lock {lock_path}: {e}")
+                
         try:
             # Call parent delete with proper parameters
             super().delete(using=using, keep_parents=keep_parents)
@@ -572,15 +583,28 @@ class VideoFile(models.Model):
         super().save(*args, **kwargs)
 
     def get_or_create_state(self) -> "VideoState":
-        """
-        Return the related VideoState instance for this video, creating and assigning a new one if none exists.
-        
-        Returns:
-            VideoState: The associated VideoState instance.
-        """
-        if self.state is None:
-            self.state = VideoState.objects.create()
-        return self.state
+        """Ensure this video has a persisted ``VideoState`` and return it."""
+
+        state = self.state
+
+        # When tests reuse cached instances across database flushes, ``state`` may reference
+        # a row that no longer exists. Guard against that by validating persistence.
+        state_pk = getattr(state, "pk", None)
+        if state is not None and state_pk is not None:
+            if not VideoState.objects.filter(pk=state_pk).exists():
+                state = None
+
+        if state is None:
+            # Create a fresh state to avoid refresh_from_db() failures on unsaved instances.
+            state = VideoState.objects.create()
+            self.state = state
+
+            # Persist the relation immediately if the VideoFile already exists in the DB so
+            # later refreshes see the association without requiring additional saves.
+            if self.pk:
+                self.save(update_fields=["state"])
+
+        return state
 
     def get_or_create_sensitive_meta(self) -> "SensitiveMeta":
         """
@@ -592,8 +616,7 @@ class VideoFile(models.Model):
         from endoreg_db.models import SensitiveMeta
         if self.sensitive_meta is None:
             self.sensitive_meta = SensitiveMeta.objects.create(center = self.center)
-            # Mark as processed when creating new SensitiveMeta
-            self.get_or_create_state().mark_sensitive_meta_processed(save=True)
+            # Do not mark processed here; it will be set after extraction/validation steps
         return self.sensitive_meta
 
     def get_outside_segments(self, only_validated: bool = False) -> models.QuerySet["LabelVideoSegment"]:

endoreg_db/models/media/video/video_file_anonymize.py

@@ -12,6 +12,7 @@ from django.conf import settings
 
 from endoreg_db.utils.hashs import get_video_hash
 from endoreg_db.utils.validate_endo_roi import validate_endo_roi
+from endoreg_db.utils.paths import STORAGE_DIR
 from ....utils.video.ffmpeg_wrapper import assemble_video_from_frames
 from ...utils import anonymize_frame  # Import from models.utils
 from .video_file_segments import _get_outside_frames, _get_outside_frame_numbers
@@ -268,7 +269,7 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
             raise ValueError(f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid}).")
 
         video.processed_video_hash = new_processed_hash
-        video.processed_file.name = video.get_target_anonymized_video_path().relative_to(settings.MEDIA_ROOT).as_posix()
+        video.processed_file.name = video.get_target_anonymized_video_path().relative_to(STORAGE_DIR).as_posix()
 
         update_fields = [
             "processed_video_hash",

endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py

@@ -97,6 +97,7 @@ def _extract_frame_range(
             return True  # Indicate success as frames are considered present
 
     frame_dir.mkdir(parents=True, exist_ok=True)
+    extracted_paths = []
 
     try:
         logger.info("Starting frame range extraction [%d, %d) for video %s to %s", start_frame, end_frame, video.uuid, frame_dir)
@@ -111,6 +112,17 @@ def _extract_frame_range(
 
         return True
 
+    except FileNotFoundError as err:
+        logger.error(
+            "Frame range extraction [%d, %d) failed for video %s: %s",
+            start_frame,
+            end_frame,
+            video.uuid,
+            err,
+            exc_info=True,
+        )
+        raise
+
     except Exception as e:
         logger.error("Frame range extraction [%d, %d) or DB update failed for video %s: %s", start_frame, end_frame, video.uuid, e, exc_info=True)
 

endoreg_db/models/media/video/video_file_io.py

@@ -32,13 +32,15 @@ def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
                 if sensitive_path.exists():
                     return sensitive_path.resolve()
                 
+                # Check direct raw_file.path if available
                 # Check direct raw_file.path if available
                 try:
                     direct_path = Path(video.raw_file.path)
                     if direct_path.exists():
                         return direct_path.resolve()
-                except Exception:
-                    pass  # Fallback to original behavior
+                except Exception as e:
+                    logger.debug("Could not access direct raw_file.path for video %s: %s", video.uuid, e)
+                    # Fallback to checking alternative paths
                 
                 # Check common alternative paths
                 alternative_paths = [

endoreg_db/models/metadata/video_meta.py

@@ -13,7 +13,7 @@ else:
     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
 
 # Import the new utility function
-from ...utils.video.ffmpeg_wrapper import get_stream_info
+from ...utils.video import ffmpeg_wrapper
 
 logger = logging.getLogger(__name__)
 
@@ -214,7 +214,7 @@ class FFMpegMeta(models.Model):
         """
         logger.info("Running ffprobe on %s", file_path)
         try:
-            probe_data = get_stream_info(file_path)  # Use the new utility
+            probe_data = ffmpeg_wrapper.get_stream_info(file_path)  # Use the new utility
         except Exception as probe_err:
             logger.error("ffprobe execution failed for %s: %s", file_path, probe_err, exc_info=True)
             raise RuntimeError(f"ffprobe execution failed for {file_path}") from probe_err

endoreg_db/services/pdf_import.py

@@ -5,6 +5,7 @@ Provides high-level functions for importing and anonymizing PDF files,
 combining RawPdfFile creation with text extraction and anonymization.
 """
 from datetime import date, datetime
+import errno
 import logging
 import shutil
 import sys
@@ -13,12 +14,11 @@ import hashlib
 from pathlib import Path
 from typing import TYPE_CHECKING, Union
 from contextlib import contextmanager
-from django.conf.locale import tr
 from django.db import transaction
 from endoreg_db.models.media.pdf.raw_pdf import RawPdfFile
 from endoreg_db.models.state.raw_pdf import RawPdfState
 from endoreg_db.models import SensitiveMeta
-from endoreg_db.utils.paths import PDF_DIR, STORAGE_DIR
+from endoreg_db.utils import paths as path_utils
 import time
 
 logger = logging.getLogger(__name__)
@@ -111,14 +111,44 @@ class PdfImportService:
                     break
                 h.update(b)
         return h.hexdigest()
+
+    def _get_pdf_dir(self) -> Path | None:
+        """Resolve the configured PDF directory to a concrete Path."""
+        candidate = getattr(path_utils, "PDF_DIR", None)
+        if isinstance(candidate, Path):
+            return candidate
+        if candidate is None:
+            return None
+        try:
+            derived = candidate / "."
+        except Exception:
+            derived = None
+
+        if derived is not None:
+            try:
+                return Path(derived)
+            except Exception:
+                return None
+
+        try:
+            return Path(str(candidate))
+        except Exception:
+            return None
     
     def _quarantine(self, source: Path) -> Path:
         """Move file to quarantine directory to prevent re-processing."""
-        qdir = PDF_DIR / "_processing"
+        qdir = path_utils.PDF_DIR / "_processing"
         qdir.mkdir(parents=True, exist_ok=True)
         target = qdir / source.name
-        # atomic rename on same filesystem
-        source.rename(target)
+        try:
+            # Try atomic rename first (fastest when on same filesystem)
+            source.rename(target)
+        except OSError as exc:
+            if exc.errno == errno.EXDEV:
+                # Cross-device move, fall back to shutil.move which copies+removes
+                shutil.move(str(source), str(target))
+            else:
+                raise
         return target
     
     def _ensure_state(self, pdf_file: "RawPdfFile"):
@@ -287,6 +317,7 @@ class PdfImportService:
         """Initialize the processing context for the current PDF."""
         self.processing_context = {
             'file_path': Path(file_path),
+            'original_file_path': Path(file_path),
             'center_name': center_name,
             'delete_source': delete_source,
             'retry': retry,
@@ -379,11 +410,18 @@ class PdfImportService:
 
     def _setup_processing_environment(self):
         """Setup processing environment and state."""
+        original_path = self.processing_context.get('file_path')
+
         # Create sensitive file copy
-        self.create_sensitive_file(self.current_pdf, self.processing_context['file_path'])
+        self.create_sensitive_file(self.current_pdf, original_path)
         
         # Update file path to point to sensitive copy
         self.processing_context['file_path'] = self.current_pdf.file.path
+        self.processing_context['sensitive_copy_created'] = True
+        try:
+            self.processing_context['sensitive_file_path'] = Path(self.current_pdf.file.path)
+        except Exception:
+            self.processing_context['sensitive_file_path'] = None
         
         # Ensure state exists
         state = self.current_pdf.get_or_create_state()
@@ -415,14 +453,14 @@ class PdfImportService:
             logger.info("Starting text extraction and metadata processing with ReportReader...")
             
             # Setup output directories
-            crops_dir = PDF_DIR / 'cropped_regions'
-            anonymized_dir = PDF_DIR / 'anonymized'
+            crops_dir = path_utils.PDF_DIR / 'cropped_regions'
+            anonymized_dir = path_utils.PDF_DIR / 'anonymized'
             crops_dir.mkdir(parents=True, exist_ok=True)
             anonymized_dir.mkdir(parents=True, exist_ok=True)
 
             # Initialize ReportReader
             report_reader = ReportReader(
-                report_root_path=STORAGE_DIR,
+                report_root_path=str(path_utils.STORAGE_DIR),
                 locale="de_DE",
                 text_date_format="%d.%m.%Y"
             )
@@ -603,7 +641,7 @@ class PdfImportService:
         try:
             # Prefer storing a path relative to STORAGE_DIR so Django serves it correctly
             try:
-                relative_name = str(anonymized_path.relative_to(STORAGE_DIR))
+                relative_name = str(anonymized_path.relative_to(path_utils.STORAGE_DIR))
             except ValueError:
                 # Fallback to absolute path if the file lives outside STORAGE_DIR
                 relative_name = str(anonymized_path)
@@ -717,18 +755,96 @@ class PdfImportService:
         except Exception as e:
             logger.warning(f"Error during cleanup: {e}")
         finally:
+            # Remove any sensitive copy created during this processing run
+            sensitive_created = self.processing_context.get('sensitive_copy_created')
+            if sensitive_created:
+                pdf_obj = self.current_pdf
+                try:
+                    if pdf_obj:
+                        file_field = getattr(pdf_obj, "file", None)
+                        if file_field and getattr(file_field, "name", None):
+                            storage_name = file_field.name
+                            file_field.delete(save=False)
+                            logger.debug("Deleted sensitive copy %s during error cleanup", storage_name)
+                except Exception as cleanup_exc:
+                    logger.warning("Failed to remove sensitive copy during error cleanup: %s", cleanup_exc)
+
             # Always clean up processed files set to prevent blocks
             file_path = self.processing_context.get('file_path')
             if file_path and str(file_path) in self.processed_files:
                 self.processed_files.remove(str(file_path))
                 logger.debug(f"Removed {file_path} from processed files during error cleanup")
 
+            try:
+                original_path = self.processing_context.get('original_file_path')
+                logger.debug("PDF cleanup original path: %s (%s)", original_path, type(original_path))
+                raw_dir = original_path.parent if isinstance(original_path, Path) else None
+                if (
+                    isinstance(original_path, Path)
+                    and original_path.exists()
+                    and not self.processing_context.get('sensitive_copy_created')
+                ):
+                    try:
+                        original_path.unlink()
+                        logger.info("Removed original file %s during error cleanup", original_path)
+                    except Exception as remove_exc:
+                        logger.warning("Could not remove original file %s during error cleanup: %s", original_path, remove_exc)
+                pdf_dir = self._get_pdf_dir()
+                if not pdf_dir and raw_dir:
+                    base_dir = raw_dir.parent
+                    dir_name = getattr(path_utils, "PDF_DIR_NAME", "pdfs")
+                    fallback_pdf_dir = base_dir / dir_name
+                    logger.debug(
+                        "PDF cleanup fallback resolution - base: %s, dir_name: %s, exists: %s",
+                        base_dir,
+                        dir_name,
+                        fallback_pdf_dir.exists(),
+                    )
+                    if fallback_pdf_dir.exists():
+                        pdf_dir = fallback_pdf_dir
+
+                # Remove empty PDF subdirectories that might have been created during setup
+                if pdf_dir and pdf_dir.exists():
+                    for subdir_name in ("sensitive", "cropped_regions", "anonymized", "_processing"):
+                        subdir_path = pdf_dir / subdir_name
+                        if subdir_path.exists() and subdir_path.is_dir():
+                            try:
+                                next(subdir_path.iterdir())
+                            except StopIteration:
+                                try:
+                                    subdir_path.rmdir()
+                                    logger.debug("Removed empty directory %s during error cleanup", subdir_path)
+                                except OSError as rm_err:
+                                    logger.debug("Could not remove directory %s: %s", subdir_path, rm_err)
+                            except Exception as iter_err:
+                                logger.debug("Could not inspect directory %s: %s", subdir_path, iter_err)
+
+                raw_count = len(list(raw_dir.glob("*"))) if raw_dir and raw_dir.exists() else None
+                pdf_count = len(list(pdf_dir.glob("*"))) if pdf_dir and pdf_dir.exists() else None
+
+                sensitive_path = self.processing_context.get('sensitive_file_path')
+                if sensitive_path:
+                    sensitive_parent = Path(sensitive_path).parent
+                    sensitive_count = len(list(sensitive_parent.glob("*"))) if sensitive_parent.exists() else None
+                else:
+                    sensitive_dir = pdf_dir / "sensitive" if pdf_dir else None
+                    sensitive_count = len(list(sensitive_dir.glob("*"))) if sensitive_dir and sensitive_dir.exists() else None
+
+                logger.info(
+                    "PDF import error cleanup counts - raw: %s, pdf: %s, sensitive: %s",
+                    raw_count,
+                    pdf_count,
+                    sensitive_count,
+                )
+            except Exception:
+                pass
+
     def _cleanup_processing_context(self):
         """Cleanup processing context."""
         try:
             # Clean up temporary directories
             if self.processing_context.get('text_extracted'):
-                crops_dir = PDF_DIR / 'cropped_regions'
+                crops_dir = path_utils.PDF_DIR / 'cropped_regions'
                 if crops_dir.exists() and not any(crops_dir.iterdir()):
                     crops_dir.rmdir()
                     
@@ -857,7 +973,7 @@ class PdfImportService:
         if not source_path:
             raise ValueError("No file path available for creating sensitive file")
 
-        SENSITIVE_DIR = PDF_DIR / "sensitive"
+        SENSITIVE_DIR = path_utils.PDF_DIR / "sensitive"
         target = SENSITIVE_DIR / f"{pdf_file.pdf_hash}.pdf"
 
         try:
@@ -880,7 +996,7 @@ class PdfImportService:
             # Update FileField to reference the file under STORAGE_DIR
             # We avoid re-saving file content (the file is already at target); set .name relative to STORAGE_DIR
             try:
-                relative_name = str(target.relative_to(STORAGE_DIR)) #just point the Django FileField to the file that the anonymizer already created in data/pdfs/anonymized/.
+                relative_name = str(target.relative_to(path_utils.STORAGE_DIR))  # Point Django FileField to sensitive storage
             except ValueError:
                 # Fallback: if target is not under STORAGE_DIR, store absolute path (not ideal)
                 relative_name = str(target)
@@ -934,7 +1050,7 @@ class PdfImportService:
         if pdf_problematic:
             # Quarantine the file
             logger.warning(f"Quarantining problematic PDF: {pdf_file.pdf_hash}, reason: {quarantine_reason}")
-            quarantine_dir = PDF_DIR / "quarantine"
+            quarantine_dir = path_utils.PDF_DIR / "quarantine"
             os.makedirs(quarantine_dir, exist_ok=True)
             
             quarantine_path = quarantine_dir / f"{pdf_file.pdf_hash}.pdf"
@@ -950,7 +1066,7 @@ class PdfImportService:
         else:
             # Archive the file normally
             logger.info(f"Archiving successfully processed PDF: {pdf_file.pdf_hash}")
-            archive_dir = PDF_DIR / "processed"
+            archive_dir = path_utils.PDF_DIR / "processed"
             os.makedirs(archive_dir, exist_ok=True)
             
             archive_path = archive_dir / f"{pdf_file.pdf_hash}.pdf"