endoreg-db 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

endoreg_db/models/data_file/metadata/sensitive_meta.py

@@ -0,0 +1,290 @@
+from django.db import models
+from endoreg_db.utils.hashs import (
+    get_patient_hash,
+    get_patient_examination_hash,
+    # get_hash_string,
+)
+from hashlib import sha256
+
+# from datetime import date
+from icecream import ic
+import os
+
+# get DJANGO_SALT from settings
+SECRET_SALT = os.getenv("DJANGO_SALT", "default_salt")
+
+
+class SensitiveMeta(models.Model):
+    examination_date = models.DateField(blank=True, null=True)
+    pseudo_patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
+    patient_first_name = models.CharField(max_length=255, blank=True, null=True)
+    patient_last_name = models.CharField(max_length=255, blank=True, null=True)
+    patient_dob = models.DateField(blank=True, null=True)
+    pseudo_examination = models.ForeignKey(
+        "PatientExamination",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
+    patient_gender = models.ForeignKey(
+        "Gender",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
+    examiners = models.ManyToManyField(
+        "Examiner",
+        blank=True,
+    )
+    center = models.ForeignKey(
+        "Center",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
+
+    examiner_first_name = models.CharField(max_length=255, blank=True, null=True)
+    examiner_last_name = models.CharField(max_length=255, blank=True, null=True)
+
+    examination_hash = models.CharField(max_length=255, blank=True, null=True)
+    patient_hash = models.CharField(max_length=255, blank=True, null=True)
+
+    endoscope_type = models.CharField(max_length=255, blank=True, null=True)
+    endoscope_sn = models.CharField(max_length=255, blank=True, null=True)
+    state_verified = models.BooleanField(default=False)
+    state_hash_generated = models.BooleanField(default=False)
+    state_names_substituted = models.BooleanField(default=False)
+    state_dob_substituted = models.BooleanField(default=False)
+    state_examination_date_substituted = models.BooleanField(default=False)
+    state_endoscope_sn_substituted = models.BooleanField(default=False)
+    state_examiners_substituted = models.BooleanField(default=False)
+
+    @classmethod
+    def create_from_dict(cls, data: dict):
+        from endoreg_db.models import Center, Examiner
+        from endoreg_db.utils import guess_name_gender
+
+        # data can contain more fields than the model has
+        field_names = [_.name for _ in cls._meta.fields]
+        selected_data = {k: v for k, v in data.items() if k in field_names}
+
+        first_name = selected_data.get("patient_first_name")
+        last_name = selected_data.get("patient_last_name")
+        center_name = data.get("center_name")
+
+        try:
+            center = Center.objects.get_by_natural_key(center_name)
+        except Exception as exc:
+            raise ValueError(f"Center with name {center_name} does not exist") from exc
+
+        selected_data["center"] = center
+
+        try:
+            # TODO Add to documentation and replace with better method
+            gender = guess_name_gender(first_name)
+        except:
+            raise ValueError(f"Gender for name {first_name} could not be guessed")
+
+        selected_data["patient_gender"] = gender
+
+        if first_name and last_name:
+            # TODO Add to documentation
+            cls._update_name_db(first_name, last_name)
+
+        sensitive_meta = cls.objects.create(**selected_data)
+        sensitive_meta.get_or_create_pseudo_examiner()
+        sensitive_meta.get_or_create_pseudo_patient()
+        sensitive_meta.get_or_create_pseudo_patient_examination()
+
+        ic("EXAMINER_FIRST_NAME", sensitive_meta.examiner_first_name)
+        ic("EXAMINER_LAST_NAME", sensitive_meta.examiner_last_name)
+
+        return sensitive_meta
+
+    def get_or_create_pseudo_examiner(self):
+        ic("GETTING OR CREATING EXAMINER")
+
+        if self.examiners.exists():
+            examiner = self.examiners.first()
+            ic(f"Exisiting examiner: {examiner}")
+
+        else:
+            examiner = self.create_pseudo_examiner()
+            ic(f"Created examiner: {examiner}")
+
+        return examiner
+
+    def create_pseudo_examiner(self):
+        from endoreg_db.models import Examiner, Center
+
+        first_name = self.examiner_first_name
+        last_name = self.examiner_last_name
+        center = self.center
+        ic("CREATING EXAMINER", first_name, last_name, center)
+        if not first_name or not last_name or not center:
+            default_center = Center.objects.get_by_natural_key("endoreg_db_demo")
+            examiner, _created = Examiner.custom_get_or_create(
+                first_name="Unknown", last_name="Unknown", center=default_center
+            )
+        else:
+            examiner, _created = Examiner.custom_get_or_create(
+                first_name=first_name, last_name=last_name, center=center
+            )
+        self.examiners.add(examiner)
+        self.save()
+
+        return examiner
+
+    def get_or_create_pseudo_patient(self):
+        if not self.pseudo_patient:
+            self.pseudo_patient = self.create_pseudo_patient()
+            self.save()
+        return self.pseudo_patient
+
+    def create_pseudo_patient(self):
+        from endoreg_db.models import Patient
+        from datetime import date
+
+        dob = self.patient_dob
+
+        if isinstance(dob, str):
+            dob = date.fromisoformat(dob)
+
+        month = dob.month
+        year = dob.year
+
+        patient_hash = self.get_patient_hash()
+        patient, _created = Patient.get_or_create_pseudo_patient_by_hash(
+            patient_hash=patient_hash,
+            center=self.center,
+            gender=self.patient_gender,
+            birth_year=year,
+            birth_month=month,
+        )
+
+        return patient
+
+    def get_or_create_pseudo_patient_examination(self):
+        from endoreg_db.models import PatientExamination
+
+        patient_hash = self.get_patient_hash()
+        examination_hash = self.get_patient_examination_hash()
+
+        patient_examination, _created = (
+            PatientExamination.get_or_create_pseudo_patient_examination_by_hash(
+                patient_hash, examination_hash
+            )
+        )
+
+        self.pseudo_examination = patient_examination
+
+        return patient_examination
+
+    def update_from_dict(self, data: dict):
+        # data can contain more fields than the model has
+        field_names = [_.name for _ in self._meta.fields]
+        selected_data = {k: v for k, v in data.items() if k in field_names}
+
+        for k, v in selected_data.items():
+            setattr(self, k, v)
+
+        self.save()
+        first_name = self.patient_first_name
+        last_name = self.patient_last_name
+
+        if first_name and last_name:
+            SensitiveMeta._update_name_db(first_name=first_name, last_name=last_name)
+
+        if not self.examination_hash:
+            self.examination_hash = self.get_patient_examination_hash()
+        if not self.patient_hash:
+            self.patient_hash = self.get_patient_hash()
+
+        examiner_first_name = data.get("examiner_first_name", "")
+        examiner_last_name = data.get("examiner_last_name", "")
+
+        if examiner_first_name and examiner_last_name:
+            self.examiner_first_name = examiner_first_name
+            self.examiner_last_name = examiner_last_name
+        _examiner = self.get_or_create_pseudo_examiner()
+
+        return self
+
+    def __str__(self):
+        result_str = "SensitiveMeta:"
+        result_str += f"\tExamination Date: {self.examination_date}"
+        result_str += f"\tFirst Name: {self.patient_first_name}"
+        result_str += f"\tLast Name: {self.patient_last_name}"
+        result_str += f"\tDate of Birth: (*{self.patient_dob})"
+        result_str += f"\tGender: {self.patient_gender}"
+        result_str += f"\tCenter: {self.center}"
+        result_str += f"\tExaminers: {self.examiners.all()}"
+        result_str += f"\tEndoscope Type: {self.endoscope_type}"
+        result_str += f"\tEndoscope SN: {self.endoscope_sn}"
+        result_str += f"\tState Verified: {self.state_verified}"
+        result_str += f"\tPatient Hash: {self.patient_hash}"
+        result_str += f"\tExamination Hash: {self.examination_hash}"
+
+        return result_str
+
+    def __repr__(self):
+        return self.__str__()
+
+    def get_patient_hash(self, salt=SECRET_SALT):
+        dob = self.patient_dob
+        first_name = self.patient_first_name
+        last_name = self.patient_last_name
+        center = self.center
+
+        assert dob, "Patient DOB is required"
+        assert center, "Center is required"
+
+        hash_str = get_patient_hash(
+            first_name=first_name,
+            last_name=last_name,
+            dob=dob,
+            center=self.center.name,
+            salt=salt,
+        )
+        return sha256(hash_str.encode()).hexdigest()
+
+    def get_patient_examination_hash(self, salt=SECRET_SALT):
+        dob = self.patient_dob
+        first_name = self.patient_first_name
+        last_name = self.patient_last_name
+        examination_date = self.examination_date
+        center = self.center
+
+        assert dob, "Patient DOB is required"
+        assert examination_date, "Examination date is required"
+
+        hash_str = get_patient_examination_hash(
+            first_name=first_name,
+            last_name=last_name,
+            dob=dob,
+            examination_date=examination_date,
+            center=center.name,
+            salt=salt,
+        )
+
+        return sha256(hash_str.encode()).hexdigest()
+
+    # override save method to update hashes
+    def save(self, *args, **kwargs):
+        self.examination_hash = self.get_patient_examination_hash()
+        self.patient_hash = self.get_patient_hash()
+        self.pseudo_patient = self.create_pseudo_patient()
+        self.pseudo_examination = self.get_or_create_pseudo_patient_examination()
+        super().save(*args, **kwargs)
+
+    @classmethod
+    def _update_name_db(cls, first_name, last_name):
+        from endoreg_db.models import FirstName, LastName
+
+        FirstName.objects.get_or_create(name=first_name)
+        LastName.objects.get_or_create(name=last_name)

endoreg_db/models/data_file/metadata/video_meta.py

@@ -0,0 +1,199 @@
+from django.db import models
+import subprocess
+import json
+from pathlib import Path
+from typing import Optional, TYPE_CHECKING
+
+# import endoreg_center_id from django settings
+from django.conf import settings
+
+# check if endoreg_center_id is set
+if not hasattr(settings, "ENDOREG_CENTER_ID"):
+    ENDOREG_CENTER_ID = 9999
+else:
+    ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
+
+if TYPE_CHECKING:
+    from endoreg_db.models import EndoscopyProcessor, Endoscope, Center
+
+
+# VideoMeta
+class VideoMeta(models.Model):
+    processor = models.ForeignKey(
+        "EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
+    )
+    endoscope = models.ForeignKey(
+        "Endoscope", on_delete=models.CASCADE, blank=True, null=True
+    )
+    center = models.ForeignKey("Center", on_delete=models.CASCADE)
+    import_meta = models.OneToOneField(
+        "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
+    )
+    ffmpeg_meta = models.OneToOneField(
+        "FFMpegMeta", on_delete=models.CASCADE, blank=True, null=True
+    )  ##
+
+    @classmethod
+    def create_from_file(
+        cls,
+        file_path: Path,
+        center: Optional["Center"],
+        processor: Optional["EndoscopyProcessor"] = None,
+        endoscope: Optional["Endoscope"] = None,
+    ):
+        """Create a new VideoMeta from a file."""
+        meta = cls.objects.create(center=center)
+        meta.update_meta(file_path)
+
+        if processor:
+            meta.processor = processor
+            meta.get_endo_roi()
+        if endoscope:
+            meta.endoscope = endoscope
+
+        meta.save()
+
+        return meta
+
+    def __str__(self):
+        processor_name = self.processor.name if self.processor is not None else "None"
+        endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
+        center_name = self.center.name if self.center is not None else "None"
+        ffmpeg_meta_str = self.ffmpeg_meta.__str__()
+        import_meta_str = self.import_meta.__str__()
+
+        result_html = ""
+
+        result_html += f"Processor: {processor_name}\n"
+        result_html += f"Endoscope: {endoscope_name}\n"
+        result_html += f"Center: {center_name}\n"
+        result_html += f"FFMpeg Meta: {ffmpeg_meta_str}\n"
+        result_html += f"Import Meta: {import_meta_str}\n"
+
+        return result_html
+
+    # import meta should be created when video meta is created
+    def save(self, *args, **kwargs):
+        if self.import_meta is None:
+            self.import_meta = VideoImportMeta.objects.create()
+        super(VideoMeta, self).save(*args, **kwargs)
+
+    def initialize_ffmpeg_meta(self, file_path):
+        """Initializes FFMpeg metadata for the video file if not already done."""
+        self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
+        self.save()
+
+    def update_meta(self, file_path):
+        """Updates the video metadata from the file."""
+        self.initialize_ffmpeg_meta(file_path)
+        self.save()
+
+    def get_endo_roi(self):
+        from endoreg_db.models import EndoscopyProcessor
+
+        processor: EndoscopyProcessor = self.processor
+        endo_roi = processor.get_roi_endoscope_image()
+        return endo_roi
+
+    def get_fps(self):
+        if not self.ffmpeg_meta:
+            return None
+
+        return self.ffmpeg_meta.frame_rate
+
+
+class FFMpegMeta(models.Model):
+    # Existing fields
+    duration = models.FloatField(blank=True, null=True)
+    width = models.IntegerField(blank=True, null=True)
+    height = models.IntegerField(blank=True, null=True)
+    frame_rate = models.FloatField(blank=True, null=True)
+
+    # New fields for comprehensive information
+    video_codec = models.CharField(max_length=50, blank=True, null=True)
+    audio_codec = models.CharField(max_length=50, blank=True, null=True)
+    audio_channels = models.IntegerField(blank=True, null=True)
+    audio_sample_rate = models.IntegerField(blank=True, null=True)
+
+    # Existing __str__ method can be updated to include new fields
+
+    @classmethod
+    def create_from_file(cls, file_path: Path):
+        cmd = [
+            "ffprobe",
+            "-v",
+            "quiet",
+            "-print_format",
+            "json",
+            "-show_streams",
+            str(file_path),
+        ]
+        proc = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        probe = json.loads(proc.stdout)
+
+        video_stream = next(
+            (stream for stream in probe["streams"] if stream["codec_type"] == "video"),
+            None,
+        )
+        audio_streams = [
+            stream for stream in probe["streams"] if stream["codec_type"] == "audio"
+        ]
+
+        if not video_stream:
+            return None
+
+        metadata = {
+            "duration": float(video_stream.get("duration", 0)),
+            "width": int(video_stream.get("width", 0)),
+            "height": int(video_stream.get("height", 0)),
+            "frame_rate": float(
+                next(iter(video_stream.get("avg_frame_rate", "").split("/")), 0)
+            ),
+            "video_codec": video_stream.get("codec_name", ""),
+        }
+
+        if audio_streams:
+            first_audio_stream = audio_streams[0]
+            metadata.update(
+                {
+                    "audio_codec": first_audio_stream.get("codec_name", ""),
+                    "audio_channels": int(first_audio_stream.get("channels", 0)),
+                    "audio_sample_rate": int(first_audio_stream.get("sample_rate", 0)),
+                }
+            )
+
+        return cls.objects.create(**metadata)
+
+    def __str__(self):
+        result_html = ""
+
+        result_html += f"Duration: {self.duration}\n"
+        result_html += f"Width: {self.width}\n"
+        result_html += f"Height: {self.height}\n"
+        result_html += f"Frame Rate: {self.frame_rate}\n"
+        result_html += f"Video Codec: {self.video_codec}\n"
+        result_html += f"Audio Codec: {self.audio_codec}\n"
+        result_html += f"Audio Channels: {self.audio_channels}\n"
+        result_html += f"Audio Sample Rate: {self.audio_sample_rate}\n"
+
+        return result_html
+
+
+class VideoImportMeta(models.Model):
+    video_anonymized = models.BooleanField(default=False)
+    video_patient_data_detected = models.BooleanField(default=False)
+    outside_detected = models.BooleanField(default=False)
+    patient_data_removed = models.BooleanField(default=False)
+    outside_removed = models.BooleanField(default=False)
+
+    def __str__(self):
+        result_html = ""
+
+        result_html += f"Video anonymized: {self.video_anonymized}\n"
+        result_html += (
+            f"Video patient data detected: {self.video_patient_data_detected}\n"
+        )
+        result_html += f"Outside detected: {self.outside_detected}\n"
+        result_html += f"Patient data removed: {self.patient_data_removed}\n"
+        result_html += f"Outside removed: {self.outside_removed}\n"
+        return result_html

endoreg_db/models/data_file/report_file.py

@@ -0,0 +1,56 @@
+from datetime import date, time
+from django.db import models
+from .base_classes.abstract_pdf import AbstractPdfFile
+
+
+class ReportFile(AbstractPdfFile):
+    meta = models.JSONField(blank=True, null=True)
+    text = models.TextField(blank=True, null=True)
+    # sensitive_meta = models.ForeignKey(
+    #     "SensitiveMeta",
+    #     on_delete=models.SET_NULL,
+    #     related_name="report_files",
+    #     null=True,
+    #     blank=True,
+    # )
+    # examination = models.ForeignKey(
+    #     "PatientExamination",
+    #     on_delete=models.SET_NULL,
+    #     blank=True,
+    #     null=True,
+    #     related_name="report_files",
+    # )
+    # patient = models.ForeignKey(
+    #     "Patient", on_delete=models.DO_NOTHING, blank=True, null=True
+    # )
+    # examiner = models.ForeignKey(
+    #     "Examiner", on_delete=models.DO_NOTHING, blank=True, null=True
+    # )
+    date = models.DateField(blank=True, null=True)
+    time = models.TimeField(blank=True, null=True)
+
+    def get_or_create_examiner(self, examiner_first_name, examiner_last_name):
+        from ..persons import Examiner
+
+        examiner_center = self.center
+
+        examiner, created = Examiner.objects.get_or_create(
+            first_name=examiner_first_name,
+            last_name=examiner_last_name,
+            center=examiner_center,
+        )
+
+        return examiner, created
+
+    def set_examination_date_and_time(self, report_meta=None):
+        if not report_meta:
+            report_meta = self.meta
+        examination_date_str = report_meta["examination_date"]
+        examination_time_str = report_meta["examination_time"]
+
+        if examination_date_str:
+            # TODO: get django DateField compatible date from string (e.g. "2021-01-01")
+            self.date = date.fromisoformat(examination_date_str)
+        if examination_time_str:
+            # TODO: get django TimeField compatible time from string (e.g. "12:00")
+            self.time = time.fromisoformat(examination_time_str)

endoreg_db/models/data_file/video/__init__.py

@@ -0,0 +1,11 @@
+from .video import (
+    Video,
+)
+from ..metadata import (
+    VideoImportMeta,
+)
+
+__all__ = [
+    "Video",
+    "VideoImportMeta",
+]

endoreg_db/models/data_file/video/import_meta.py

@@ -0,0 +1,25 @@
+# from django.db import models
+
+# class VideoImportMeta(models.Model):
+#     processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE)
+#     endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
+#     center = models.ForeignKey('Center', on_delete=models.CASCADE)
+#     video_anonymized = models.BooleanField(default=False)
+#     video_patient_data_detected = models.BooleanField(default=False)
+#     outside_detected = models.BooleanField(default=False)
+#     patient_data_removed = models.BooleanField(default=False)
+#     outside_removed = models.BooleanField(default=False)
+
+#     def __str__(self):
+#         result_html = ""
+
+#         result_html += f"Processor: {self.processor.name}<br>"
+#         result_html += f"Endoscope: {self.endoscope.name}<br>"
+#         result_html += f"Center: {self.center.name}<br>"
+#         result_html += f"Video anonymized: {self.video_anonymized}<br>"
+#         result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
+#         result_html += f"Outside detected: {self.outside_detected}<br>"
+#         result_html += f"Patient data removed: {self.patient_data_removed}<br>"
+#         result_html += f"Outside removed: {self.outside_removed}<br>"
+#         return result_html
+