endoreg-db 0.6.4__py3-none-any.whl → 0.8.2__py3-none-any.whl

endoreg_db/models/media/pdf/report_file.py

@@ -0,0 +1,162 @@
+from ...utils import DOCUMENT_DIR, STORAGE_DIR
+from django.db import models
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ...administration import (
+        Center,
+        Patient,
+    )
+    from ...medical import (
+        PatientExamination,
+    )
+    from ...metadata import SensitiveMeta
+
+class DocumentTypeManager(models.Manager):
+    """
+    Custom manager for DocumentType.
+    """
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class DocumentType(models.Model):
+    """
+    Represents the type of a document.
+    """
+    name = models.CharField(max_length=255, unique=True)
+    description = models.TextField(blank=True, null=True)
+
+    objects = DocumentTypeManager()
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return str(self.name)
+    
+    class Meta:
+        verbose_name = "Document Type"
+        verbose_name_plural = "Document Types"
+
+class AbstractDocument(models.Model):
+    """
+    Abstract base class for documents.
+    """
+    meta = models.JSONField(blank=True, null=True)
+    text = models.TextField(blank=True, null=True)
+    date = models.DateField(blank=True, null=True)
+    time = models.TimeField(blank=True, null=True)
+    file = models.FileField(
+        upload_to=DOCUMENT_DIR.relative_to(STORAGE_DIR),
+        blank=True,
+        null=True,
+    )
+
+    center = models.ForeignKey(
+        "Center",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+    )
+
+    type = models.ForeignKey(
+        DocumentType,
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+    )
+
+    if TYPE_CHECKING:
+        center: "Center"
+        type: "DocumentType"
+
+    class Meta:
+        abstract = True
+
+
+    
+
+class AbstractExaminationReport(AbstractDocument):
+    """
+    Abstract base class for examination reports.
+    """
+    patient = models.ForeignKey(
+        "Patient", on_delete=models.DO_NOTHING, blank=True, null=True
+    )
+
+    patient_examination = models.ForeignKey(
+        "PatientExamination",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+    )
+
+    examiners = models.ManyToManyField(
+        "Examiner",
+        blank=True,
+    )
+
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True
+    )
+
+    if TYPE_CHECKING:
+        center: "Center"
+        type: "DocumentType"
+        patient: "Patient"
+        patient_examination: "PatientExamination"
+        sensitive_meta: "SensitiveMeta"
+
+
+    class Meta:
+        abstract = True
+
+    
+    def get_or_create_examiner(self, examiner_first_name, examiner_last_name):
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    def set_examination_date_and_time(self, report_meta=None):
+        raise NotImplementedError("Subclasses must implement this method.")
+
+
+
+class AnonymExaminationReport(AbstractExaminationReport):
+
+    def get_or_create_examiner(self, examiner_first_name:str, examiner_last_name:str):
+        from ...administration.person import Examiner
+
+        examiner_center = self.center
+
+        examiner, created = Examiner.objects.get_or_create(
+            first_name=examiner_first_name,
+            last_name=examiner_last_name,
+            center=examiner_center,
+        )
+
+        return examiner, created
+
+    def set_examination_date_and_time(self, report_meta=None):
+        #TODO
+        if not report_meta:
+            report_meta = self.meta
+        # examination_date_str = report_meta["examination_date"]
+        # examination_time_str = report_meta["examination_time"]
+
+        # if examination_date_str:
+        #     # TODO: get django DateField compatible date from string (e.g. "2021-01-01")
+        #     self.date = date.fromisoformat(examination_date_str)
+        # if examination_time_str:
+        #     # TODO: get django TimeField compatible time from string (e.g. "12:00")
+        #     self.time = time.fromisoformat(examination_time_str)
+
+class AnonymHistologyReport(AbstractExaminationReport):
+    """
+    Represents a histology report.
+    """
+
+
+    def get_or_create_examiner(self, examiner_first_name, examiner_last_name):
+        raise NotImplementedError("Subclasses must implement this method.")

endoreg_db/models/media/pdf/report_reader/report_reader_config.py

@@ -0,0 +1,77 @@
+# ReportReaderConfig Class
+# Description: This class is used to store the configuration of the ReportReader
+
+# PATIENT_INFO_LINE_FLAG = "Patient: "
+# ENDOSCOPE_INFO_LINE_FLAG = "Gerät: "
+# EXAMINER_INFO_LINE_FLAG = "1. Unters.:"
+# CUT_OFF_BELOW_LINE_FLAG = "________________"
+
+
+# CUT_OFF_ABOVE_LINE_FLAGS = [
+#     ENDOSCOPE_INFO_LINE_FLAG,
+#     EXAMINER_INFO_LINE_FLAG,
+# ]
+
+# CUT_OFF_BELOW_LINE_FLAGS = [
+#         CUT_OFF_BELOW_LINE_FLAG
+#     ]
+
+from django.db import models
+
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from .report_reader_flag import ReportReaderFlag
+    from ....administration.person import FirstName, LastName
+    from ....administration.center import Center
+    from ....metadata import PdfType
+
+class ReportReaderConfig(models.Model):
+    """
+    Configuration settings for parsing PDF reports (ReportReader).
+
+    Stores locale, name lists, date format, and flags used to identify key information lines
+    and text sections to ignore.
+    """
+    locale = models.CharField(default="de_DE", max_length=10)
+    first_names = models.ManyToManyField('FirstName', related_name='report_reader_configs')
+    last_names = models.ManyToManyField('LastName', related_name='report_reader_configs')
+    text_date_format = models.CharField(default = "%d.%m.%Y", max_length=10)
+    patient_info_line_flag = models.ForeignKey("ReportReaderFlag", related_name='report_reader_configs_patient_info_line', on_delete=models.CASCADE)
+    endoscope_info_line_flag = models.ForeignKey("ReportReaderFlag", related_name='report_reader_configs_endoscope_info_line', on_delete=models.CASCADE)
+    examiner_info_line_flag = models.ForeignKey("ReportReaderFlag", related_name='report_reader_configs_examiner_info_line', on_delete=models.CASCADE)
+    cut_off_below = models.ManyToManyField("ReportReaderFlag", related_name='report_reader_configs_cut_off_below')
+    cut_off_above = models.ManyToManyField("ReportReaderFlag", related_name='report_reader_configs_cut_off_above')
+    
+    if TYPE_CHECKING:
+        first_names: models.QuerySet["FirstName"]
+        last_names: models.QuerySet["LastName"]
+        patient_info_line_flag: "ReportReaderFlag"
+        endoscope_info_line_flag: "ReportReaderFlag"
+        examiner_info_line_flag: "ReportReaderFlag"
+        cut_off_below: models.QuerySet["ReportReaderFlag"]
+        cut_off_above: models.QuerySet["ReportReaderFlag"]
+    
+
+    def __str__(self):
+        """Returns a string representation including the locale and primary key."""
+        _str = f"ReportReaderConfig: {self.locale} (id: {self.pk}\n"
+        return _str
+    
+    def update_names_by_center(self, center:"Center", save = True):
+        """Updates the first and last name lists based on the names associated with a Center."""
+        self.first_names.set(center.first_names.all())
+        self.last_names.set(center.last_names.all())
+        if save:
+            self.save()
+
+    def update_flags_by_pdf_type(self, pdf_type:"PdfType", save = True):
+        """Updates the line identification flags based on a specific PdfType."""
+        self.patient_info_line_flag = pdf_type.patient_info_line_flag
+        self.endoscope_info_line_flag = pdf_type.endoscope_info_line_flag
+        self.examiner_info_line_flag = pdf_type.examiner_info_line_flag
+        self.cut_off_below.set(pdf_type.cut_off_below.all())
+        self.cut_off_above.set(pdf_type.cut_off_above.all())
+        if save:
+            self.save()
+

endoreg_db/models/media/video/__init__.py

@@ -0,0 +1,4 @@
+from .video_file import VideoFile
+__all__ = [
+    "VideoFile",
+]

endoreg_db/models/media/video/create_from_file.py

@@ -0,0 +1,336 @@
+import shutil
+import logging
+import uuid
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Type
+
+# Import the new exceptions from the correct path
+from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
+from ...utils import VIDEO_DIR, TMP_VIDEO_DIR, data_paths
+
+if TYPE_CHECKING:
+    from endoreg_db.models import VideoFile
+
+from ....utils.video.ffmpeg_wrapper import transcode_videofile_if_required
+from ....utils.hashs import get_video_hash
+from ....utils.file_operations import get_uuid_filename
+
+logger = logging.getLogger(__name__)
+
+
+def check_storage_capacity(src_path: Path, dst_root: Path, safety_margin: float = 1.2) -> None:
+    """
+    Check if there's enough storage space before starting operations.
+    
+    Args:
+        src_path: Source file path
+        dst_root: Destination root directory
+        safety_margin: Safety factor (1.2 = 20% extra space required)
+        
+    Raises:
+        InsufficientStorageError: If insufficient storage space
+    """
+    try:
+        src_size = src_path.stat().st_size
+        required_space = int(src_size * safety_margin)
+        
+        # Check free space on destination
+        free_space = shutil.disk_usage(dst_root).free
+        
+        if free_space < required_space:
+            raise InsufficientStorageError(
+                f"Insufficient storage space. Required: {required_space/1e9:.1f} GB, "
+                f"Available: {free_space/1e9:.1f} GB on {dst_root}",
+                required_space=required_space,
+                available_space=free_space
+            )
+            
+        logger.info(f"Storage check passed: {free_space/1e9:.1f} GB available, "
+                   f"{required_space/1e9:.1f} GB required")
+                   
+    except OSError as e:
+        logger.warning(f"Could not check storage capacity: {e}")
+        # Don't fail the operation, just log the warning
+
+
+def atomic_copy_with_fallback(src_path: Path, dst_path: Path) -> bool:
+    """
+    Atomically copy file from src to dst, preserving the source file.
+    
+    Args:
+        src_path: Source file path
+        dst_path: Destination file path
+        
+    Returns:
+        True if successful
+        
+    Raises:
+        InsufficientStorageError: If not enough space for the operation
+        OSError: For other file system errors
+    """
+    try:
+        # Check space before copy
+        src_size = src_path.stat().st_size
+        free_space = shutil.disk_usage(dst_path.parent).free
+        
+        if free_space < src_size * 1.1:  # 10% safety margin
+            raise InsufficientStorageError(
+                f"Insufficient space for copy operation. Required: {src_size/1e9:.1f} GB, "
+                f"Available: {free_space/1e9:.1f} GB",
+                required_space=src_size,
+                available_space=free_space
+            )
+        
+        # Use a temporary name during copy for atomicity
+        temp_dst = dst_path.with_suffix(dst_path.suffix + '.tmp')
+        
+        try:
+            shutil.copy2(str(src_path), str(temp_dst))
+            temp_dst.rename(dst_path)
+            logger.debug(f"Copy successful: {src_path} -> {dst_path}")
+            return True
+        except Exception:
+            # Clean up temp file if copy failed
+            if temp_dst.exists():
+                temp_dst.unlink(missing_ok=True)
+            raise
+    
+    except Exception as e:
+        logger.error(f"Copy operation failed: {src_path} -> {dst_path}: {e}")
+        raise
+
+
+def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
+    """
+    Atomically move file from src to dst, with fallback to copy+remove.
+    
+    Args:
+        src_path: Source file path
+        dst_path: Destination file path
+        
+    Returns:
+        True if successful
+        
+    Raises:
+        InsufficientStorageError: If not enough space for the operation
+        OSError: For other file system errors
+    """
+    try:
+        # First try atomic move (same filesystem)
+        try:
+            src_path.rename(dst_path)
+            logger.debug(f"Atomic move successful: {src_path} -> {dst_path}")
+            return True
+        except OSError as e:
+            if e.errno == 18:  # Cross-device link
+                logger.debug("Cross-device move detected, falling back to copy+remove")
+            else:
+                raise
+        
+        # Check space before cross-filesystem copy
+        src_size = src_path.stat().st_size
+        free_space = shutil.disk_usage(dst_path.parent).free
+        
+        if free_space < src_size * 1.1:  # 10% safety margin
+            raise InsufficientStorageError(
+                f"Insufficient space for copy operation. Required: {src_size/1e9:.1f} GB, "
+                f"Available: {free_space/1e9:.1f} GB",
+                required_space=src_size,
+                available_space=free_space
+            )
+        
+        # Fallback to copy+remove for cross-filesystem moves
+        logger.info(f"Copying file (cross-filesystem): {src_path} -> {dst_path}")
+        
+        # Use a temporary name during copy for atomicity
+        temp_dst = dst_path.with_suffix(dst_path.suffix + '.tmp')
+        
+        try:
+            shutil.copy2(str(src_path), str(temp_dst))
+            temp_dst.rename(dst_path)
+            src_path.unlink()  # Remove source only after successful copy
+            logger.debug(f"Copy+remove successful: {src_path} -> {dst_path}")
+            return True
+            
+        except OSError as e:
+            # Clean up temp file on failure
+            if temp_dst.exists():
+                temp_dst.unlink(missing_ok=True)
+            # Re-raise with better context
+            if e.errno == 28:  # No space left on device
+                raise InsufficientStorageError(
+                    f"No space left on device during copy: {e}",
+                    required_space=src_path.stat().st_size,
+                    available_space=shutil.disk_usage(dst_path.parent).free
+                )
+            raise
+            
+    except Exception as e:
+        logger.error(f"Failed to move {src_path} -> {dst_path}: {e}")
+        raise
+
+
+def _create_from_file(
+    cls_model: Type["VideoFile"],
+    file_path: Path,
+    center_name: str,
+    processor_name: Optional[str] = None,
+    video_dir: Path = VIDEO_DIR,
+    save: bool = True,
+    delete_source: bool = False,
+    **kwargs
+) -> "VideoFile":
+    """
+    Creates a VideoFile instance from a given video file path with improved error handling.
+    
+    Raises:
+        InsufficientStorageError: When not enough disk space
+        TranscodingError: When video transcoding fails
+        ValueError: When required objects (Center, Processor) not found
+        RuntimeError: For other processing errors
+    """
+    from endoreg_db.models.administration.center.center import Center
+    from endoreg_db.models.medical.hardware import EndoscopyProcessor
+    
+    original_file_name = file_path.name
+    original_suffix = file_path.suffix
+    final_storage_path = None
+    transcoded_file_path = None
+
+    try:
+        # Ensure we operate under the canonical video path root
+        video_dir = data_paths.get("video", video_dir)
+        storage_root = Path(video_dir).parent
+        storage_root.mkdir(parents=True, exist_ok=True)
+        
+        # Check storage capacity before starting any work
+        check_storage_capacity(file_path, storage_root)
+
+        # 1. Transcode if necessary
+        logger.debug("Checking transcoding requirement for %s", file_path)
+        temp_transcode_dir = TMP_VIDEO_DIR / 'transcoding'
+        temp_transcode_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Use a unique name for the potential transcoded file
+        temp_transcoded_output_path = temp_transcode_dir / f"{uuid.uuid4()}{original_suffix}"
+
+        try:
+            transcoded_file_path = transcode_videofile_if_required(
+                input_path=file_path,
+                output_path=temp_transcoded_output_path
+            )
+            if transcoded_file_path is None:
+                raise TranscodingError(f"Transcoding check/process failed for {file_path}")
+        except Exception as e:
+            raise TranscodingError(f"Video transcoding failed: {e}") from e
+
+        logger.debug("Using file for hashing: %s", transcoded_file_path)
+
+        # 2. Calculate hash (this will be the raw_video_hash)
+        video_hash = get_video_hash(transcoded_file_path)
+        if not video_hash:
+            raise ValueError(f"Could not calculate video hash for {transcoded_file_path}")
+        logger.info("Calculated raw video hash: %s for %s", video_hash, original_file_name)
+
+        # 3. Check if hash already exists
+        if cls_model.check_hash_exists(video_hash=video_hash):
+            existing_video = cls_model.objects.get(video_hash=video_hash)
+            logger.warning("Video with hash %s already exists (UUID: %s)", video_hash, existing_video.uuid)
+            
+            # Check if the existing video has a valid file
+            existing_raw_path = existing_video.get_raw_file_path()
+            if existing_video.has_raw and existing_raw_path and existing_raw_path.exists():
+                logger.warning("Video with hash %s already exists and file is present. Returning existing instance.", video_hash)
+                # Clean up transcoded file if it was created temporarily
+                if transcoded_file_path != file_path and transcoded_file_path.exists():
+                    transcoded_file_path.unlink(missing_ok=True)
+                return existing_video
+
+            logger.warning("Video with hash %s exists but file is missing. Deleting orphaned record.", video_hash)
+            existing_video.delete()
+
+        # 4. Generate UUID and final storage path
+        new_file_name, uuid_val = get_uuid_filename(transcoded_file_path)
+        final_storage_path = video_dir / new_file_name
+        final_storage_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # 5. Move or Copy the file to final storage using improved method
+        try:
+            if delete_source and transcoded_file_path == file_path:
+                logger.debug("Moving original file %s to %s", file_path, final_storage_path)
+                atomic_move_with_fallback(file_path, final_storage_path)
+            elif delete_source and transcoded_file_path != file_path:
+                logger.debug("Moving transcoded file %s to %s", transcoded_file_path, final_storage_path)
+                atomic_move_with_fallback(transcoded_file_path, final_storage_path)
+            else:
+                logger.debug("Copying file %s to %s", transcoded_file_path, final_storage_path)
+                atomic_copy_with_fallback(transcoded_file_path, final_storage_path)
+                if transcoded_file_path != file_path and transcoded_file_path.exists():
+                    logger.debug("Cleaning up temporary transcoded file %s", transcoded_file_path)
+                    transcoded_file_path.unlink(missing_ok=True)
+        except InsufficientStorageError:
+            # Re-raise storage errors as-is
+            raise
+        except Exception as e:
+            raise RuntimeError(f"Failed to move file to final storage: {e}") from e
+
+        # 6. Verify hash after move/copy
+        final_hash = get_video_hash(final_storage_path)
+        if final_hash != video_hash:
+            logger.error("Hash mismatch after file operation! Expected %s, got %s", video_hash, final_hash)
+            final_storage_path.unlink(missing_ok=True)
+            raise RuntimeError(f"Hash mismatch after file operation for {final_storage_path}")
+
+        # 7. Get related objects
+        try:
+            center = Center.objects.get(name=center_name)
+            processor = EndoscopyProcessor.objects.get(name=processor_name) if processor_name else None
+            logger.debug("Found Center: %s, Processor: %s", center.name, processor.name if processor else "None")
+        except Center.DoesNotExist as e:
+            logger.error("Center '%s' not found", center_name)
+            if final_storage_path and final_storage_path.exists():
+                final_storage_path.unlink(missing_ok=True)
+            raise ValueError(f"Center '{center_name}' not found.") from e
+        except EndoscopyProcessor.DoesNotExist as e:
+            logger.error("Processor '%s' not found", processor_name)
+            if final_storage_path and final_storage_path.exists():
+                final_storage_path.unlink(missing_ok=True)
+            raise ValueError(f"Processor '{processor_name}' not found.") from e
+
+        # 8. Create the VideoFile instance
+        logger.info("Creating new VideoFile instance with UUID: %s", uuid_val)
+        # Store FileField path relative to storage root including the videos prefix
+        relative_name = (final_storage_path.relative_to(data_paths['storage'])).as_posix()
+        video = cls_model(
+            uuid=uuid_val,
+            raw_file=relative_name,
+            processed_file=None,
+            center=center,
+            processor=processor,
+            original_file_name=original_file_name,
+            video_hash=video_hash,
+            processed_video_hash=None,
+            suffix=original_suffix,
+        )
+
+        # 9. Save the instance if requested
+        if save:
+            logger.info("Saving new VideoFile instance (UUID: %s)", uuid_val)
+            video.save()
+            logger.info("Successfully created VideoFile PK %s (UUID: %s)", video.pk, video.uuid)
+
+        return video
+
+    except (InsufficientStorageError, TranscodingError, ValueError):
+        # Re-raise these specific errors as-is
+        raise
+    except Exception as e:
+        logger.error("Failed to create VideoFile from %s: (%s) %s", file_path, type(e).__name__, e, exc_info=True)
+        # Clean up any created files
+        if final_storage_path and final_storage_path.exists():
+            logger.warning("Cleaning up orphaned file: %s", final_storage_path)
+            final_storage_path.unlink(missing_ok=True)
+        if transcoded_file_path and transcoded_file_path != file_path and transcoded_file_path.exists():
+            logger.warning("Cleaning up orphaned transcoded file: %s", transcoded_file_path)
+            transcoded_file_path.unlink(missing_ok=True)
+        raise RuntimeError(f"Video processing failed: {e}") from e