endoreg-db 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

endoreg_db/models/data_file/import_classes/raw_pdf.py

@@ -0,0 +1,260 @@
+# models/data_file/import_classes/raw_pdf.py
+# django db model "RawPdf"
+# Class to store raw pdf file using django file field
+# Class contains classmethod to create object from pdf file
+# objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
+# ------------------------------------------------------------------------------
+
+from django.db import models
+from django.core.files.storage import FileSystemStorage
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.core.validators import FileExtensionValidator
+from endoreg_db.utils.file_operations import get_uuid_filename
+from icecream import ic
+
+from agl_report_reader.report_reader import ReportReader
+
+from endoreg_db.utils.hashs import get_pdf_hash
+from ..metadata import SensitiveMeta
+from ..base_classes.abstract_pdf import AbstractPdfFile
+
+# setup logging to pdf_import.log
+import logging
+
+import shutil
+from pathlib import Path
+
+from ..base_classes.utils import (
+    STORAGE_LOCATION,
+)
+
+logger = logging.getLogger("pdf_import")
+
+RAW_PDF_DIR_NAME = "raw_pdf"
+RAW_PDF_DIR = STORAGE_LOCATION / RAW_PDF_DIR_NAME
+
+if not RAW_PDF_DIR.exists():
+    RAW_PDF_DIR.mkdir(parents=True)
+
+
+class RawPdfFile(AbstractPdfFile):
+    file = models.FileField(
+        upload_to=f"{RAW_PDF_DIR_NAME}/",
+        validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
+        storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
+    )
+
+    patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="raw_pdf_files",
+    )
+
+    state_report_processing_required = models.BooleanField(default=True)
+    state_report_processed = models.BooleanField(default=False)
+    raw_meta = models.JSONField(blank=True, null=True)
+    # report_file = models.OneToOneField("ReportFile", on_delete=models.CASCADE, null=True, blank=True)
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.SET_NULL,
+        related_name="raw_pdf_files",
+        null=True,
+        blank=True,
+    )
+
+    report_file = models.ForeignKey(
+        "ReportFile",
+        on_delete=models.SET_NULL,
+        related_name="raw_pdf_files",
+        null=True,
+        blank=True,
+    )
+
+    anonymized_text = models.TextField(blank=True, null=True)
+
+    @classmethod
+    def create_from_file(
+        cls,
+        file_path: Path,
+        center_name,
+        save=True,
+        delete_source=True,
+    ):
+        from endoreg_db.models import Center
+
+        logger.info(f"Creating RawPdfFile object from file: {file_path}")
+        ic(f"Creating RawPdfFile object from file: {file_path}")
+
+        new_file_name, uuid = get_uuid_filename(file_path)
+
+        pdf_hash = get_pdf_hash(file_path)
+        ic(pdf_hash)
+        new_file_path = RAW_PDF_DIR / new_file_name
+        # check if pdf file already exists
+
+        if cls.objects.filter(pdf_hash=pdf_hash).exists():
+            existing_pdf_file = cls.objects.filter(pdf_hash=pdf_hash).get()
+            logger.warning(f"RawPdfFile with hash {pdf_hash} already exists")
+            ic(f"RawPdfFile with hash {pdf_hash} already exists")
+
+            existing_pdf_file.verify_existing_file(fallback_file=file_path)
+
+            return existing_pdf_file
+
+        else:
+            ic(f"No existing pdf file found for hash {pdf_hash}")
+
+        # assert pdf_type_name is not None, "pdf_type_name is required"
+        assert center_name is not None, "center_name is required"
+
+        # pdf_type = PdfType.objects.get(name=pdf_type_name)
+        center = Center.objects.get(name=center_name)
+
+        logger.info(f"Copying file to {new_file_path}")
+        ic(f"Copying file to {new_file_path}")
+        _success = shutil.copy(file_path, new_file_path)
+
+        # validate copy operation by comparing hashs
+        assert get_pdf_hash(new_file_path) == pdf_hash, "Copy operation failed"
+
+        raw_pdf = cls(
+            file=new_file_path.resolve().as_posix(),
+            pdf_hash=pdf_hash,
+            # pdf_type=pdf_type,
+            center=center,
+        )
+        raw_pdf.save()
+        logger.info(f"RawPdfFile object created: {raw_pdf}")
+        ic(f"RawPdfFile object created: {raw_pdf}")
+
+        # remove source file
+        if delete_source:
+            file_path.unlink()
+            logger.info(f"Source file removed: {file_path}")
+            ic(f"Source file removed: {file_path}")
+
+        if save:
+            raw_pdf.save()
+
+        return raw_pdf
+
+    def save(self, *args, **kwargs):
+        if not self.file.name.endswith(".pdf"):
+            raise ValidationError("Only PDF files are allowed")
+
+        if not self.pdf_hash:
+            self.pdf_hash = get_pdf_hash(self.file.path)
+
+        super().save(*args, **kwargs)
+
+    def verify_existing_file(self, fallback_file):
+        if not Path(self.file.path).exists():
+            logger.warning(f"File not found: {self.file.path}")
+            logger.warning(f"Using fallback file: {fallback_file}")
+            ic(f"File not found: {self.file.path}")
+            ic(f"Copy fallback file: {fallback_file} to existing filepath")
+
+            shutil.copy(fallback_file, self.file.path)
+
+            self.save()
+
+    def process_file(self, verbose=False):
+        pdf_path = self.file.path
+        rr_config = self.get_report_reader_config()
+
+        rr = ReportReader(
+            **rr_config
+        )  # FIXME In future we need to pass a configuration file
+        # This configuration file should be associated with pdf type
+
+        text, anonymized_text, report_meta = rr.process_report(
+            pdf_path, verbose=verbose
+        )
+
+        self.text = text
+        self.anonymized_text = anonymized_text
+
+        report_meta["center_name"] = self.center.name
+        if not self.sensitive_meta:
+            sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
+            self.sensitive_meta = sensitive_meta
+
+        else:
+            # update existing sensitive meta
+            sensitive_meta = self.sensitive_meta
+            sensitive_meta.update_from_dict(report_meta)
+
+        self.raw_meta = report_meta
+
+        sensitive_meta.save()
+        self.save()
+
+        return text, anonymized_text, report_meta
+
+    def get_report_reader_config(self):
+        from endoreg_db.models import PdfType, Center
+        from warnings import warn
+
+        if not self.pdf_type:
+            warn("PdfType not set, using default settings")
+            pdf_type = PdfType.default_pdf_type()
+        else:
+            pdf_type: PdfType = self.pdf_type
+        center: Center = self.center
+        if pdf_type.endoscope_info_line:
+            endoscope_info_line = pdf_type.endoscope_info_line.value
+
+        else:
+            endoscope_info_line = None
+        settings_dict = {
+            "locale": "de_DE",
+            "employee_first_names": [_.name for _ in center.first_names.all()],
+            "employee_last_names": [_.name for _ in center.last_names.all()],
+            "text_date_format": "%d.%m.%Y",
+            "flags": {
+                "patient_info_line": pdf_type.patient_info_line.value,
+                "endoscope_info_line": endoscope_info_line,
+                "examiner_info_line": pdf_type.examiner_info_line.value,
+                "cut_off_below": [_.value for _ in pdf_type.cut_off_below_lines.all()],
+                "cut_off_above": [_.value for _ in pdf_type.cut_off_above_lines.all()],
+            },
+        }
+
+        return settings_dict
+
+    def get_or_create_report_file(self):
+        from endoreg_db.models import ReportFile
+
+        if self.report_file:
+            report_file = self.report_file
+
+        elif ReportFile.objects.filter(pdf_hash=self.pdf_hash).exists():
+            report_file = ReportFile.objects.filter(pdf_hash=self.pdf_hash).get()
+            self.report_file = report_file
+            self.save()
+        else:
+            # TODO  Make sure all required states are set
+            patient = self.sensitive_meta.get_or_create_pseudo_patient()
+            examiner = self.sensitive_meta.get_or_create_pseudo_examiner()
+            patient_examination = (
+                self.sensitive_meta.get_or_create_pseudo_patient_examination()
+            )
+
+            report_file = ReportFile.objects.create(
+                pdf_hash=self.pdf_hash,
+                center=self.center,
+                sensitive_meta=self.sensitive_meta,
+                patient=patient,
+                examiner=examiner,
+                examination=patient_examination,
+                text=self.anonymized_text,
+            )
+
+            report_file.save()
+            self.report_file = report_file
+            self.save()
+
+        return report_file

endoreg_db/models/data_file/import_classes/raw_video.py

@@ -0,0 +1,288 @@
+import shutil
+import subprocess
+from pathlib import Path
+from django.db import models
+from typing import TYPE_CHECKING, List, Tuple
+
+from icecream import ic
+from tqdm import tqdm
+import cv2
+from django.core.validators import FileExtensionValidator
+from django.core.files.storage import FileSystemStorage
+
+from endoreg_db.utils.validate_endo_roi import validate_endo_roi
+from ..base_classes.utils import (
+    anonymize_frame,
+    RAW_VIDEO_DIR_NAME,
+    VIDEO_DIR,
+    STORAGE_LOCATION,
+)
+from ..base_classes.abstract_video import AbstractVideoFile
+
+if TYPE_CHECKING:
+    # import Queryset
+    from django.db.models import QuerySet
+    from endoreg_db.models import (
+        SensitiveMeta,
+        LabelVideoSegment,
+    )
+
+
+# pylint: disable=attribute-defined-outside-init,no-member
+class RawVideoFile(AbstractVideoFile):
+    """ """
+
+    file = models.FileField(
+        upload_to=RAW_VIDEO_DIR_NAME,
+        validators=[FileExtensionValidator(allowed_extensions=["mp4"])],  # FIXME
+        storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
+    )
+
+    patient = models.ForeignKey(
+        "Patient", on_delete=models.SET_NULL, blank=True, null=True
+    )
+
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.SET_NULL,
+        related_name="raw_videos",
+        null=True,
+        blank=True,
+    )
+
+    video = models.ForeignKey(
+        "Video",
+        on_delete=models.SET_NULL,
+        related_name="raw_videos",
+        null=True,
+        blank=True,
+    )
+
+    if TYPE_CHECKING:
+        sensitive_meta: "SensitiveMeta"
+        label_video_segments: "QuerySet[LabelVideoSegment]"
+
+    # Crop Frames
+    state_anonymized_frames_generated = models.BooleanField(default=False)
+
+    ## OCR
+    state_ocr_required = models.BooleanField(default=True)
+    state_ocr_completed = models.BooleanField(default=False)
+    ## Validation
+    state_outside_validated = models.BooleanField(default=False)
+    state_ocr_result_validated = models.BooleanField(default=False)
+
+    state_sensitive_data_retrieved = models.BooleanField(default=False)
+
+    # Censor Outside
+    state_censor_outside_required = models.BooleanField(default=True)
+    state_censor_outside_completed = models.BooleanField(default=False)
+    state_make_anonymized_video_required = models.BooleanField(default=True)
+    state_make_anonymized_video_completed = models.BooleanField(default=False)
+
+    def get_anonymized_video_path(self):
+        video_dir = VIDEO_DIR
+        video_suffix = Path(self.file.path).suffix
+        video_name = f"{self.uuid}{video_suffix}"
+        anonymized_video_name = f"TMP_anonymized_{video_name}"
+        anonymized_video_path = video_dir / anonymized_video_name
+
+        return anonymized_video_path
+
+    def censor_outside_frames(self):
+        assert self.state_frames_extracted, "Frames not extracted"
+        assert self.state_initial_prediction_completed, (
+            "Initial prediction not completed"
+        )
+        assert self.state_sensitive_data_retrieved, "Sensitive data not retrieved"
+
+        ic(
+            "WARNING: Outside validation is not yet implemented and automatically set to true in this function"
+        )
+
+        self.state_outside_validated = True
+        self.save()
+
+        assert self.state_outside_validated, "Outside validation not completed"
+
+        outside_frame_paths = self.get_outside_frame_paths()
+
+        if not outside_frame_paths:
+            ic("No outside frames found")
+
+        else:
+            ic(f"Found {len(outside_frame_paths)} outside frames")
+            # use cv2 to replace all outside frames with completely black frames
+
+            for frame_path in tqdm(outside_frame_paths):
+                frame = cv2.imread(frame_path.as_posix())
+                frame.fill(0)
+                cv2.imwrite(frame_path.as_posix(), frame)
+
+        self.state_censor_outside_required = False
+        self.state_censor_outside_completed = True
+        self.save()
+
+    def get_anonymized_frame_dir(self):
+        anonymized_frame_dir = Path(self.frame_dir).parent / f"tmp_{self.uuid}"
+        return anonymized_frame_dir
+
+    def make_temporary_anonymized_frames(self) -> Tuple[Path, List[Path]]:
+        anonymized_frame_dir = self.get_anonymized_frame_dir()
+
+        assert self.state_frames_extracted, "Frames not extracted"
+        assert self.processor, "Processor not set"
+
+        anonymized_frame_dir.mkdir(parents=True, exist_ok=True)
+        endo_roi = self.get_endo_roi()
+        assert validate_endo_roi(endo_roi), "Endoscope ROI is not valid"
+        generated_frame_paths = []
+
+        all_frames = self.frames.all()
+        outside_frames = self.get_outside_frames()  #
+        outside_frame_numbers = [frame.frame_number for frame in outside_frames]
+
+        # anonymize frames: copy endo-roi content while making other pixels black. (frames are Path objects to jpgs or pngs)
+        for frame in tqdm(all_frames):
+            frame_path = Path(frame.image.path)
+            frame_name = frame_path.name
+            frame_number = frame.frame_number
+
+            if frame_number in outside_frame_numbers:
+                all_black = True
+            else:
+                all_black = False
+
+            target_frame_path = anonymized_frame_dir / frame_name
+            anonymize_frame(
+                frame_path, target_frame_path, endo_roi, all_black=all_black
+            )
+            generated_frame_paths.append(target_frame_path)
+
+        return anonymized_frame_dir, generated_frame_paths
+
+    def make_anonymized_video(self):
+        """
+        Make an anonymized video from the anonymized frames.
+        """
+
+        assert self.state_initial_prediction_completed, (
+            "Initial prediction not completed"
+        )
+        assert self.state_sensitive_data_retrieved, "Sensitive data not retrieved"
+
+        ic(
+            "WARNING: Outside validation is not yet implemented and automatically set to true in this function"
+        )
+        self.state_outside_validated = True
+        self.save()
+
+        assert self.state_outside_validated, "Outside validation not completed"
+
+        _anonymized_frame_dir, generated_frame_paths = (
+            self.make_temporary_anonymized_frames()
+        )
+
+        anonymized_video_path = self.get_anonymized_video_path()
+        # if anonymized video already exists, delete it
+        if anonymized_video_path.exists():
+            anonymized_video_path.unlink()
+
+        # Use ffmpeg and the frame paths to create a video
+        fps = self.get_fps()
+        height, width = cv2.imread(generated_frame_paths[0].as_posix()).shape[:2]
+        ic("Assembling anonymized video")
+        ic(f"Frame width: {width}, height: {height}")
+        ic(f"FPS: {fps}")
+
+        command = [
+            "ffmpeg",
+            "-y",
+            "-pattern_type",
+            "glob",
+            "-f",
+            "image2",
+            "-framerate",
+            str(fps),
+            "-i",
+            f"{generated_frame_paths[0].parent.as_posix()}/frame_[0-9]*.jpg",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            "-vf",
+            f"scale={width}:{height}",
+            anonymized_video_path.as_posix(),
+        ]
+
+        subprocess.run(command, check=True)
+        ic(f"Anonymized video saved at {anonymized_video_path}")
+
+        self.state_make_anonymized_video_required = False
+        self.state_make_anonymized_video_completed = True
+        self.save()
+
+        return anonymized_video_path, generated_frame_paths
+
+    def delete_frames_anonymized(self):
+        """
+        Delete anonymized frames extracted from the video file.
+        """
+        frame_dir = Path(self.frame_dir)
+        anonymized_frame_dir = frame_dir.parent / f"anonymized_{self.uuid}"
+        if anonymized_frame_dir.exists():
+            shutil.rmtree(anonymized_frame_dir)
+            return f"Anonymized frames deleted from {anonymized_frame_dir}"
+        else:
+            return f"No anonymized frames to delete for {self.file.name}"
+
+    def get_or_create_video(self):
+        from endoreg_db.models import Video, Patient, PatientExamination
+
+        video = self.video
+        expected_path = self.get_anonymized_video_path()
+        if not video:
+            video_hash = self.video_hash
+            if Video.objects.filter(video_hash=video_hash).exists():
+                video = Video.objects.filter(video_hash=video_hash).first()
+
+            else:
+                if not expected_path.exists():
+                    ic(
+                        f"No anonymized video found at {expected_path}, Creating new one"
+                    )
+                    video_path, frame_paths = self.make_anonymized_video()
+
+                else:
+                    ic(f"Anonymized video found at {expected_path}")
+                    video_path = expected_path
+                    frame_dir = self.get_anonymized_frame_dir()
+                    ic(f"Frame dir: {frame_dir}")
+                    frame_paths = list(frame_dir.glob("*.jpg"))
+                    ic(f"Found {len(frame_paths)} frames")
+
+                video_object = Video.create_from_file(
+                    video_path,
+                    self.center,
+                    self.processor,
+                    video_dir=VIDEO_DIR,
+                    frame_paths=frame_paths,
+                )
+
+                ex: PatientExamination = self.sensitive_meta.pseudo_examination
+                pat: Patient = self.sensitive_meta.pseudo_patient
+                video_object.examination = ex
+                video_object.patient = pat
+
+                self.video = video_object
+                self.save()
+                video_object.sync_from_raw_video()
+
+                ic(f"Video object created: {video_object}")
+                return video_object
+
+            self.video = video
+            self.save()
+
+        # self.vi
+        return video

endoreg_db/models/data_file/metadata/__init__.py

@@ -0,0 +1,13 @@
+from .sensitive_meta import SensitiveMeta
+from .pdf_meta import PdfMeta, PdfType
+from .video_meta import VideoMeta, FFMpegMeta, VideoImportMeta
+
+
+__all__ = [
+    "SensitiveMeta",
+    "PdfMeta",
+    "PdfType",
+    "VideoMeta",
+    "FFMpegMeta",
+    "VideoImportMeta",
+]

endoreg_db/models/data_file/metadata/pdf_meta.py

@@ -0,0 +1,74 @@
+from django.db import models
+
+# import endoreg_center_id from django settings
+from django.conf import settings
+
+
+# import File class
+from django.core.files import File
+
+# # check if endoreg_center_id is set
+# if not hasattr(settings, 'ENDOREG_CENTER_ID'):
+#     ENDOREG_CENTER_ID = 9999
+# else:
+#     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
+
+class PdfType(models.Model):
+    name = models.CharField(max_length=255)
+
+    patient_info_line = models.ForeignKey(
+        "ReportReaderFlag",
+        related_name="pdf_type_patient_info_line", 
+        on_delete=models.CASCADE
+    )
+    endoscope_info_line = models.ForeignKey(
+        "ReportReaderFlag",
+        related_name="pdf_type_endoscopy_info_line", 
+        on_delete=models.CASCADE,
+    )
+    examiner_info_line = models.ForeignKey(
+        "ReportReaderFlag",
+        related_name="pdf_type_examiner_info_line", 
+        on_delete=models.CASCADE
+    )
+    cut_off_above_lines = models.ManyToManyField(
+        "ReportReaderFlag",
+        related_name="pdf_type_cut_off_above_lines", 
+    )
+    cut_off_below_lines = models.ManyToManyField(
+        "ReportReaderFlag",
+        related_name="pdf_type_cut_off_below_lines", 
+    )
+
+
+    def __str__(self):
+        summary = f"{self.name}"
+        # add lines to summary
+        summary += f"\nPatient Info Line: {self.patient_info_line.value}"
+        summary += f"\nEndoscope Info Line: {self.endoscope_info_line.value}"
+        summary += f"\nExaminer Info Line: {self.examiner_info_line.value}"
+        summary += f"\nCut Off Above Lines: {[_.value for _ in self.cut_off_above_lines.all()]}"
+        summary += f"\nCut Off Below Lines: {[_.value for _ in self.cut_off_below_lines.all()]}"
+    
+        return summary
+
+    @classmethod
+    def default_pdf_type(cls):
+        return PdfType.objects.get(name="ukw-endoscopy-examination-report-generic")
+
+class PdfMeta(models.Model):
+    pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
+    date = models.DateField()
+    time = models.TimeField()
+    pdf_hash = models.CharField(max_length=255, unique=True)
+
+    def __str__(self):
+        return self.pdf_hash
+
+    @classmethod
+    def create_from_file(cls, pdf_file):
+        pdf_file = File(pdf_file)
+        pdf_meta = cls(file=pdf_file)
+        pdf_meta.save()
+        return pdf_meta
+