endoreg-db 0.5.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

endoreg_db/models/data_file/base_classes/frame_helpers.py

@@ -0,0 +1,17 @@
+from django.core.files import File
+import io
+from pathlib import Path
+from typing import List
+
+
+def prepare_bulk_frames(frame_paths: List[Path]):
+    """
+    Reads the frame paths into memory as Django File objects.
+    This avoids 'seek of closed file' errors by using BytesIO for each frame.
+    """
+    for path in frame_paths:
+        frame_number = int(path.stem.split("_")[1])
+        with open(path, "rb") as f:
+            content = f.read()
+        file_obj = File(io.BytesIO(content), name=path.name)
+        yield frame_number, file_obj

endoreg_db/models/data_file/base_classes/prepare_bulk_frames.py

@@ -0,0 +1,19 @@
+from django.core.files import File
+
+
+import io
+from pathlib import Path
+from typing import List
+
+
+def prepare_bulk_frames(frame_paths: List[Path]):
+    """
+    Reads the frame paths into memory as Django File objects.
+    This avoids 'seek of closed file' errors by using BytesIO for each frame.
+    """
+    for path in frame_paths:
+        frame_number = int(path.stem.split("_")[1])
+        with open(path, "rb") as f:
+            content = f.read()
+        file_obj = File(io.BytesIO(content), name=path.name)
+        yield frame_number, file_obj

endoreg_db/models/data_file/base_classes/utils.py

@@ -0,0 +1,80 @@
+"""
+Utility functions for data file classes.
+"""
+
+import os
+from pathlib import Path
+import cv2
+import numpy as np
+
+DJANGO_NAME_SALT = os.environ.get("DJANGO_NAME_SALT", "default_salt")
+
+# Directory stuff
+PSEUDO_DIR = Path(os.environ.get("DJANGO_PSEUDO_DIR", Path("./erc_data")))
+STORAGE_LOCATION = PSEUDO_DIR
+FRAME_DIR_NAME = os.environ.get("DJANGO_FRAME_DIR_NAME", "db_frames")
+RAW_FRAME_DIR_NAME = os.environ.get("DJANGO_RAW_FRAME_DIR_NAME", "db_raw_frames")
+VIDEO_DIR_NAME = os.environ.get("DJANGO_VIDEO_DIR_NAME", "db_videos")
+RAW_VIDEO_DIR_NAME = os.environ.get("DJANGO_RAW_VIDEO_DIR_NAME", "db_raw_videos")
+
+FRAME_DIR = STORAGE_LOCATION / FRAME_DIR_NAME
+VIDEO_DIR = STORAGE_LOCATION / VIDEO_DIR_NAME
+RAW_VIDEO_DIR = STORAGE_LOCATION / RAW_VIDEO_DIR_NAME
+
+TEST_RUN = os.environ.get("TEST_RUN", False)
+TEST_RUN_FRAME_NUMBER = os.environ.get("TEST_RUN_FRAME_NUMBER", 1000)
+
+VIDEO_DIR.mkdir(parents=True, exist_ok=True)
+RAW_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
+
+# AI Stuff
+FRAME_PROCESSING_BATCH_SIZE = os.environ.get("DJANGO_FRAME_PROCESSING_BATCH_SIZE", 10)
+
+
+def anonymize_frame(
+    raw_frame_path: Path, target_frame_path: Path, endo_roi, all_black: bool = False
+):
+    """
+    Anonymize the frame by blacking out all pixels that are not in the endoscope ROI.
+    """
+
+    frame = cv2.imread(raw_frame_path.as_posix())  # pylint: disable=no-member
+
+    # make black frame with same size as original frame
+    new_frame = np.zeros_like(frame)
+
+    if not all_black:
+        # endo_roi is dict with keys "x", "y", "width", "heigth"
+        x = endo_roi["x"]
+        y = endo_roi["y"]
+        width = endo_roi["width"]
+        height = endo_roi["height"]
+
+        # copy endoscope roi to black frame
+        new_frame[y : y + height, x : x + width] = frame[y : y + height, x : x + width]
+    cv2.imwrite(target_frame_path.as_posix(), new_frame)  # pylint: disable=no-member
+
+    return frame
+
+
+def copy_with_progress(src: str, dst: str, buffer_size=1024 * 1024):
+    """
+    Make a copy of a file with progress bar.
+
+    Args:
+        src (str): Source file path.
+        dst (str): Destination file path.
+        buffer_size (int): Buffer size for copying.
+    """
+    total_size = os.path.getsize(src)
+    copied_size = 0
+
+    with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
+        while True:
+            buf = fsrc.read(buffer_size)
+            if not buf:
+                break
+            fdst.write(buf)
+            copied_size += len(buf)
+            progress = copied_size / total_size * 100
+            print(f"\rProgress: {progress:.2f}%", end="")

endoreg_db/models/data_file/frame.py

@@ -1,45 +1,29 @@
-from endoreg_db.models.annotation.image_classification import ImageClassificationAnnotation
+from typing import TYPE_CHECKING
+
 from endoreg_db.models.label.label import Label
 from .base_classes import AbstractFrame
 from django.db import models
 
+from .base_classes.utils import FRAME_DIR_NAME, RAW_FRAME_DIR_NAME
+
+if TYPE_CHECKING:
+    from endoreg_db.models import RawVideoFile, Video
+
+
+class RawFrame(AbstractFrame):
+    image = models.ImageField(upload_to=RAW_FRAME_DIR_NAME, blank=True, null=True)
+    video = models.ForeignKey(
+        "RawVideoFile", on_delete=models.CASCADE, related_name="frames"
+    )
+
+    if TYPE_CHECKING:
+        video: "RawVideoFile"
+        label: "Label"
+
+
 class Frame(AbstractFrame):
+    image = models.ImageField(upload_to=FRAME_DIR_NAME, blank=True, null=True)
     video = models.ForeignKey("Video", on_delete=models.CASCADE, related_name="frames")
 
-class LegacyFrame(AbstractFrame):
-    video = models.ForeignKey("LegacyVideo", on_delete=models.CASCADE, related_name='frames')
-    image = models.ImageField(upload_to="legacy_frames", blank=True, null=True)
-    suffix = models.CharField(max_length=255)
-    # ImageClassificationAnnotation has a foreign key to this model (related name: image_classification_annotations)
-
-    class Meta:
-        unique_together = ('video', 'frame_number')
-        indexes = [
-            models.Index(fields=['video', 'frame_number']),
-        ]
-
-    def get_classification_annotations(self):
-        """
-        Get all image classification annotations for this frame.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_frame=self)
-
-    def get_classification_annotations_by_label(self, label:Label):
-        """
-        Get all image classification annotations for this frame with the given label.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_frame=self, label=label)
-
-    def get_classification_annotations_by_value(self, value:bool):
-        """
-        Get all image classification annotations for this frame with the given value.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_frame=self, value=value)
-
-    def get_classification_annotations_by_label_and_value(self, label:Label, value:bool):
-        """
-        Get all image classification annotations for this frame with the given label and value.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_frame=self, label=label, value=value)
-
-    
+    if TYPE_CHECKING:
+        video: "Video"

endoreg_db/models/data_file/import_classes/__init__.py

@@ -1,6 +1,3 @@
-import os
-from pathlib import Path
-
 from .raw_video import RawVideoFile
 from .raw_pdf import RawPdfFile
 
@@ -14,19 +11,8 @@ from .raw_pdf import RawPdfFile
 # main method is import_files which expects a path to a directory containing files to import.
 # creates correct import file object depending on file type by checking the file extension
 
-class FileImporter:
-    def __init__(self, directory):
-        self.directory = directory
 
-    def import_files(self):
-        directory_path = Path(self.directory)
-        for file in directory_path.iterdir():
-            if file.is_file():
-                if file.suffix.lower() in ['.mov', '.mp4']:
-                    RawVideoFile.create_from_file(file)
-                else:
-                    raise ValueError(f"File type {file.suffix} not supported")
-            else:
-                raise ValueError(f"{file} is not a file")
-            
-        
+__all__ = [
+    "RawPdfFile",
+    "RawVideoFile",
+]

endoreg_db/models/data_file/import_classes/raw_pdf.py

@@ -7,182 +7,254 @@
 
 from django.db import models
 from django.core.files.storage import FileSystemStorage
-from django.core.files import File
 from django.conf import settings
-from django.utils import timezone
 from django.core.exceptions import ValidationError
 from django.core.validators import FileExtensionValidator
 from endoreg_db.utils.file_operations import get_uuid_filename
+from icecream import ic
 
 from agl_report_reader.report_reader import ReportReader
 
 from endoreg_db.utils.hashs import get_pdf_hash
 from ..metadata import SensitiveMeta
+from ..base_classes.abstract_pdf import AbstractPdfFile
 
 # setup logging to pdf_import.log
 import logging
-logger = logging.getLogger('pdf_import')
 
 import shutil
+from pathlib import Path
 
-# get pdf location from settings, default to ~/erc_data/raw_pdf and create if not exists
-PSEUDO_DIR_RAW_PDF = getattr(settings, 'PSEUDO_DIR_RAW_PDF', settings.BASE_DIR / 'erc_data/raw_pdf')
+from ..base_classes.utils import (
+    STORAGE_LOCATION,
+)
 
-class RawPdfFile(models.Model):
+logger = logging.getLogger("pdf_import")
+
+RAW_PDF_DIR_NAME = "raw_pdf"
+RAW_PDF_DIR = STORAGE_LOCATION / RAW_PDF_DIR_NAME
+
+if not RAW_PDF_DIR.exists():
+    RAW_PDF_DIR.mkdir(parents=True)
+
+
+class RawPdfFile(AbstractPdfFile):
     file = models.FileField(
-        upload_to='raw_pdf/',
-        validators=[FileExtensionValidator(allowed_extensions=['pdf'])],
-        storage=FileSystemStorage(location=PSEUDO_DIR_RAW_PDF.resolve().as_posix()),
+        upload_to=f"{RAW_PDF_DIR_NAME}/",
+        validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
+        storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
     )
 
-    pdf_hash = models.CharField(max_length=255, unique=True)
-    pdf_type = models.ForeignKey('PdfType', on_delete=models.CASCADE)
-    center = models.ForeignKey('Center', on_delete=models.CASCADE)
+    patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="raw_pdf_files",
+    )
 
-    state_report_processing_required = models.BooleanField(default = True)
+    state_report_processing_required = models.BooleanField(default=True)
     state_report_processed = models.BooleanField(default=False)
-
+    raw_meta = models.JSONField(blank=True, null=True)
     # report_file = models.OneToOneField("ReportFile", on_delete=models.CASCADE, null=True, blank=True)
-    sensitive_meta = models.OneToOneField(
-        'SensitiveMeta',
-        on_delete=models.CASCADE,
-        related_name='raw_pdf_file',
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.SET_NULL,
+        related_name="raw_pdf_files",
         null=True,
         blank=True,
     )
 
-    text = models.TextField(blank=True, null=True)
-    anonymized_text = models.TextField(blank=True, null=True)
-
-    raw_meta = models.JSONField(blank=True, null=True)
-
-    created_at = models.DateTimeField(auto_now_add=True)
+    report_file = models.ForeignKey(
+        "ReportFile",
+        on_delete=models.SET_NULL,
+        related_name="raw_pdf_files",
+        null=True,
+        blank=True,
+    )
 
-    def __str__(self):
-        str_repr = f"RawPdfFile: {self.file.name}"
-        return str_repr
+    anonymized_text = models.TextField(blank=True, null=True)
 
     @classmethod
     def create_from_file(
         cls,
-        file_path,
+        file_path: Path,
         center_name,
-        pdf_type_name, # to be depreceated / changed since we now import all pdfs from same directory
-        destination_dir,
         save=True,
+        delete_source=True,
     ):
-        from endoreg_db.models import PdfType, Center
+        from endoreg_db.models import Center
+
         logger.info(f"Creating RawPdfFile object from file: {file_path}")
-        original_file_name = file_path.name
+        ic(f"Creating RawPdfFile object from file: {file_path}")
 
         new_file_name, uuid = get_uuid_filename(file_path)
 
-        if not destination_dir.exists():
-            destination_dir.mkdir(parents=True)
-
         pdf_hash = get_pdf_hash(file_path)
-
+        ic(pdf_hash)
+        new_file_path = RAW_PDF_DIR / new_file_name
         # check if pdf file already exists
+
         if cls.objects.filter(pdf_hash=pdf_hash).exists():
+            existing_pdf_file = cls.objects.filter(pdf_hash=pdf_hash).get()
             logger.warning(f"RawPdfFile with hash {pdf_hash} already exists")
-            return None
-        
-        assert pdf_type_name is not None, "pdf_type_name is required"
+            ic(f"RawPdfFile with hash {pdf_hash} already exists")
+
+            existing_pdf_file.verify_existing_file(fallback_file=file_path)
+
+            return existing_pdf_file
+
+        else:
+            ic(f"No existing pdf file found for hash {pdf_hash}")
+
+        # assert pdf_type_name is not None, "pdf_type_name is required"
         assert center_name is not None, "center_name is required"
 
-        pdf_type = PdfType.objects.get(name=pdf_type_name)
+        # pdf_type = PdfType.objects.get(name=pdf_type_name)
         center = Center.objects.get(name=center_name)
 
-        new_file_path = destination_dir / new_file_name
-
         logger.info(f"Copying file to {new_file_path}")
-        success = shutil.copy(file_path, new_file_path)
-         
+        ic(f"Copying file to {new_file_path}")
+        _success = shutil.copy(file_path, new_file_path)
+
         # validate copy operation by comparing hashs
         assert get_pdf_hash(new_file_path) == pdf_hash, "Copy operation failed"
 
         raw_pdf = cls(
             file=new_file_path.resolve().as_posix(),
             pdf_hash=pdf_hash,
-            pdf_type=pdf_type,
+            # pdf_type=pdf_type,
             center=center,
         )
+        raw_pdf.save()
         logger.info(f"RawPdfFile object created: {raw_pdf}")
+        ic(f"RawPdfFile object created: {raw_pdf}")
 
         # remove source file
-        file_path.unlink()
-        logger.info(f"Source file removed: {file_path}")
+        if delete_source:
+            file_path.unlink()
+            logger.info(f"Source file removed: {file_path}")
+            ic(f"Source file removed: {file_path}")
 
         if save:
             raw_pdf.save()
-        
 
         return raw_pdf
 
-    def process_file(self, verbose = False):
-        
+    def save(self, *args, **kwargs):
+        if not self.file.name.endswith(".pdf"):
+            raise ValidationError("Only PDF files are allowed")
+
+        if not self.pdf_hash:
+            self.pdf_hash = get_pdf_hash(self.file.path)
+
+        super().save(*args, **kwargs)
+
+    def verify_existing_file(self, fallback_file):
+        if not Path(self.file.path).exists():
+            logger.warning(f"File not found: {self.file.path}")
+            logger.warning(f"Using fallback file: {fallback_file}")
+            ic(f"File not found: {self.file.path}")
+            ic(f"Copy fallback file: {fallback_file} to existing filepath")
+
+            shutil.copy(fallback_file, self.file.path)
+
+            self.save()
+
+    def process_file(self, verbose=False):
         pdf_path = self.file.path
         rr_config = self.get_report_reader_config()
 
-        rr = ReportReader(**rr_config) #FIXME In future we need to pass a configuration file 
-        # This configuration file should be associated with pdf type 
+        rr = ReportReader(
+            **rr_config
+        )  # FIXME In future we need to pass a configuration file
+        # This configuration file should be associated with pdf type
+
+        text, anonymized_text, report_meta = rr.process_report(
+            pdf_path, verbose=verbose
+        )
 
-        text, anonymized_text, report_meta = rr.process_report(pdf_path, verbose=verbose)
+        self.text = text
+        self.anonymized_text = anonymized_text
+
+        report_meta["center_name"] = self.center.name
         if not self.sensitive_meta:
             sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
-            sensitive_meta.save()
             self.sensitive_meta = sensitive_meta
 
-        else: 
+        else:
             # update existing sensitive meta
             sensitive_meta = self.sensitive_meta
             sensitive_meta.update_from_dict(report_meta)
 
-        return text, anonymized_text, report_meta
-    
-    def update(self, save=True, verbose = True):
-        try:
-            self.text, self.anonymized_text, self.raw_meta = self.process_file(verbose = verbose)
-            self.state_report_processed = True
-            self.state_report_processing_required = False
-        
-            if save: 
-    
-                self.save()
-
-            return True
-
-        except:
-            logger.error(f"Error processing file: {self.file.path}")
-            return False
+        self.raw_meta = report_meta
 
-    def save(self, *args, **kwargs):
-        if not self.file.name.endswith('.pdf'):
-            raise ValidationError('Only PDF files are allowed')
-        
-        if not self.pdf_hash:
-            self.pdf_hash = get_pdf_hash(self.file.path)
-
-        super().save(*args, **kwargs)
+        sensitive_meta.save()
+        self.save()
 
+        return text, anonymized_text, report_meta
 
     def get_report_reader_config(self):
-        if self.pdf_type.endoscope_info_line:
-            endoscope_info_line = self.pdf_type.endoscope_info_line.value
+        from endoreg_db.models import PdfType, Center
+        from warnings import warn
+
+        if not self.pdf_type:
+            warn("PdfType not set, using default settings")
+            pdf_type = PdfType.default_pdf_type()
+        else:
+            pdf_type: PdfType = self.pdf_type
+        center: Center = self.center
+        if pdf_type.endoscope_info_line:
+            endoscope_info_line = pdf_type.endoscope_info_line.value
+
         else:
             endoscope_info_line = None
         settings_dict = {
             "locale": "de_DE",
-            "employee_first_names": [_.name for _ in self.center.first_names.all()],
-            "employee_last_names": [_.name for _ in self.center.last_names.all()],
-            "text_date_format":'%d.%m.%Y',
+            "employee_first_names": [_.name for _ in center.first_names.all()],
+            "employee_last_names": [_.name for _ in center.last_names.all()],
+            "text_date_format": "%d.%m.%Y",
             "flags": {
-                "patient_info_line": self.pdf_type.patient_info_line.value,
+                "patient_info_line": pdf_type.patient_info_line.value,
                 "endoscope_info_line": endoscope_info_line,
-                "examiner_info_line": self.pdf_type.examiner_info_line.value,
-                "cut_off_below": [_.value for _ in self.pdf_type.cut_off_below_lines.all()],
-                "cut_off_above": [_.value for _ in self.pdf_type.cut_off_above_lines.all()],
-            }
+                "examiner_info_line": pdf_type.examiner_info_line.value,
+                "cut_off_below": [_.value for _ in pdf_type.cut_off_below_lines.all()],
+                "cut_off_above": [_.value for _ in pdf_type.cut_off_above_lines.all()],
+            },
         }
 
         return settings_dict
+
+    def get_or_create_report_file(self):
+        from endoreg_db.models import ReportFile
+
+        if self.report_file:
+            report_file = self.report_file
+
+        elif ReportFile.objects.filter(pdf_hash=self.pdf_hash).exists():
+            report_file = ReportFile.objects.filter(pdf_hash=self.pdf_hash).get()
+            self.report_file = report_file
+            self.save()
+        else:
+            # TODO  Make sure all required states are set
+            patient = self.sensitive_meta.get_or_create_pseudo_patient()
+            examiner = self.sensitive_meta.get_or_create_pseudo_examiner()
+            patient_examination = (
+                self.sensitive_meta.get_or_create_pseudo_patient_examination()
+            )
+
+            report_file = ReportFile.objects.create(
+                pdf_hash=self.pdf_hash,
+                center=self.center,
+                sensitive_meta=self.sensitive_meta,
+                patient=patient,
+                examiner=examiner,
+                examination=patient_examination,
+                text=self.anonymized_text,
+            )
+
+            report_file.save()
+            self.report_file = report_file
+            self.save()
+
+        return report_file