endoreg-db 0.5.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

endoreg_db/models/data_file/metadata/sensitive_meta.py

@@ -1,21 +1,190 @@
 from django.db import models
+from endoreg_db.utils.hashs import (
+    get_patient_hash,
+    get_patient_examination_hash,
+    # get_hash_string,
+)
+from hashlib import sha256
+
+# from datetime import date
+from icecream import ic
+import os
+
+# get DJANGO_SALT from settings
+SECRET_SALT = os.getenv("DJANGO_SALT", "default_salt")
+
 
 class SensitiveMeta(models.Model):
     examination_date = models.DateField(blank=True, null=True)
+    pseudo_patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
     patient_first_name = models.CharField(max_length=255, blank=True, null=True)
     patient_last_name = models.CharField(max_length=255, blank=True, null=True)
     patient_dob = models.DateField(blank=True, null=True)
+    pseudo_examination = models.ForeignKey(
+        "PatientExamination",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
+    patient_gender = models.ForeignKey(
+        "Gender",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
+    examiners = models.ManyToManyField(
+        "Examiner",
+        blank=True,
+    )
+    center = models.ForeignKey(
+        "Center",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+    )
+
+    examiner_first_name = models.CharField(max_length=255, blank=True, null=True)
+    examiner_last_name = models.CharField(max_length=255, blank=True, null=True)
+
+    examination_hash = models.CharField(max_length=255, blank=True, null=True)
+    patient_hash = models.CharField(max_length=255, blank=True, null=True)
+
     endoscope_type = models.CharField(max_length=255, blank=True, null=True)
     endoscope_sn = models.CharField(max_length=255, blank=True, null=True)
+    state_verified = models.BooleanField(default=False)
+    state_hash_generated = models.BooleanField(default=False)
+    state_names_substituted = models.BooleanField(default=False)
+    state_dob_substituted = models.BooleanField(default=False)
+    state_examination_date_substituted = models.BooleanField(default=False)
+    state_endoscope_sn_substituted = models.BooleanField(default=False)
+    state_examiners_substituted = models.BooleanField(default=False)
 
     @classmethod
     def create_from_dict(cls, data: dict):
+        from endoreg_db.models import Center, Examiner
+        from endoreg_db.utils import guess_name_gender
+
         # data can contain more fields than the model has
         field_names = [_.name for _ in cls._meta.fields]
         selected_data = {k: v for k, v in data.items() if k in field_names}
 
-        return cls.objects.create(**selected_data)
-    
+        first_name = selected_data.get("patient_first_name")
+        last_name = selected_data.get("patient_last_name")
+        center_name = data.get("center_name")
+
+        try:
+            center = Center.objects.get_by_natural_key(center_name)
+        except Exception as exc:
+            raise ValueError(f"Center with name {center_name} does not exist") from exc
+
+        selected_data["center"] = center
+
+        try:
+            # TODO Add to documentation and replace with better method
+            gender = guess_name_gender(first_name)
+        except:
+            raise ValueError(f"Gender for name {first_name} could not be guessed")
+
+        selected_data["patient_gender"] = gender
+
+        if first_name and last_name:
+            # TODO Add to documentation
+            cls._update_name_db(first_name, last_name)
+
+        sensitive_meta = cls.objects.create(**selected_data)
+        sensitive_meta.get_or_create_pseudo_examiner()
+        sensitive_meta.get_or_create_pseudo_patient()
+        sensitive_meta.get_or_create_pseudo_patient_examination()
+
+        ic("EXAMINER_FIRST_NAME", sensitive_meta.examiner_first_name)
+        ic("EXAMINER_LAST_NAME", sensitive_meta.examiner_last_name)
+
+        return sensitive_meta
+
+    def get_or_create_pseudo_examiner(self):
+        ic("GETTING OR CREATING EXAMINER")
+
+        if self.examiners.exists():
+            examiner = self.examiners.first()
+            ic(f"Exisiting examiner: {examiner}")
+
+        else:
+            examiner = self.create_pseudo_examiner()
+            ic(f"Created examiner: {examiner}")
+
+        return examiner
+
+    def create_pseudo_examiner(self):
+        from endoreg_db.models import Examiner, Center
+
+        first_name = self.examiner_first_name
+        last_name = self.examiner_last_name
+        center = self.center
+        ic("CREATING EXAMINER", first_name, last_name, center)
+        if not first_name or not last_name or not center:
+            default_center = Center.objects.get_by_natural_key("endoreg_db_demo")
+            examiner, _created = Examiner.custom_get_or_create(
+                first_name="Unknown", last_name="Unknown", center=default_center
+            )
+        else:
+            examiner, _created = Examiner.custom_get_or_create(
+                first_name=first_name, last_name=last_name, center=center
+            )
+        self.examiners.add(examiner)
+        self.save()
+
+        return examiner
+
+    def get_or_create_pseudo_patient(self):
+        if not self.pseudo_patient:
+            self.pseudo_patient = self.create_pseudo_patient()
+            self.save()
+        return self.pseudo_patient
+
+    def create_pseudo_patient(self):
+        from endoreg_db.models import Patient
+        from datetime import date
+
+        dob = self.patient_dob
+
+        if isinstance(dob, str):
+            dob = date.fromisoformat(dob)
+
+        month = dob.month
+        year = dob.year
+
+        patient_hash = self.get_patient_hash()
+        patient, _created = Patient.get_or_create_pseudo_patient_by_hash(
+            patient_hash=patient_hash,
+            center=self.center,
+            gender=self.patient_gender,
+            birth_year=year,
+            birth_month=month,
+        )
+
+        return patient
+
+    def get_or_create_pseudo_patient_examination(self):
+        from endoreg_db.models import PatientExamination
+
+        patient_hash = self.get_patient_hash()
+        examination_hash = self.get_patient_examination_hash()
+
+        patient_examination, _created = (
+            PatientExamination.get_or_create_pseudo_patient_examination_by_hash(
+                patient_hash, examination_hash
+            )
+        )
+
+        self.pseudo_examination = patient_examination
+
+        return patient_examination
+
     def update_from_dict(self, data: dict):
         # data can contain more fields than the model has
         field_names = [_.name for _ in self._meta.fields]
@@ -23,9 +192,99 @@ class SensitiveMeta(models.Model):
 
         for k, v in selected_data.items():
             setattr(self, k, v)
-        
+
         self.save()
-    
+        first_name = self.patient_first_name
+        last_name = self.patient_last_name
+
+        if first_name and last_name:
+            SensitiveMeta._update_name_db(first_name=first_name, last_name=last_name)
+
+        if not self.examination_hash:
+            self.examination_hash = self.get_patient_examination_hash()
+        if not self.patient_hash:
+            self.patient_hash = self.get_patient_hash()
+
+        examiner_first_name = data.get("examiner_first_name", "")
+        examiner_last_name = data.get("examiner_last_name", "")
+
+        if examiner_first_name and examiner_last_name:
+            self.examiner_first_name = examiner_first_name
+            self.examiner_last_name = examiner_last_name
+        _examiner = self.get_or_create_pseudo_examiner()
+
+        return self
+
     def __str__(self):
-        return f"SensitiveMeta: {self.examination_date} {self.patient_first_name} {self.patient_last_name} (*{self.patient_dob})"
-    
+        result_str = "SensitiveMeta:"
+        result_str += f"\tExamination Date: {self.examination_date}"
+        result_str += f"\tFirst Name: {self.patient_first_name}"
+        result_str += f"\tLast Name: {self.patient_last_name}"
+        result_str += f"\tDate of Birth: (*{self.patient_dob})"
+        result_str += f"\tGender: {self.patient_gender}"
+        result_str += f"\tCenter: {self.center}"
+        result_str += f"\tExaminers: {self.examiners.all()}"
+        result_str += f"\tEndoscope Type: {self.endoscope_type}"
+        result_str += f"\tEndoscope SN: {self.endoscope_sn}"
+        result_str += f"\tState Verified: {self.state_verified}"
+        result_str += f"\tPatient Hash: {self.patient_hash}"
+        result_str += f"\tExamination Hash: {self.examination_hash}"
+
+        return result_str
+
+    def __repr__(self):
+        return self.__str__()
+
+    def get_patient_hash(self, salt=SECRET_SALT):
+        dob = self.patient_dob
+        first_name = self.patient_first_name
+        last_name = self.patient_last_name
+        center = self.center
+
+        assert dob, "Patient DOB is required"
+        assert center, "Center is required"
+
+        hash_str = get_patient_hash(
+            first_name=first_name,
+            last_name=last_name,
+            dob=dob,
+            center=self.center.name,
+            salt=salt,
+        )
+        return sha256(hash_str.encode()).hexdigest()
+
+    def get_patient_examination_hash(self, salt=SECRET_SALT):
+        dob = self.patient_dob
+        first_name = self.patient_first_name
+        last_name = self.patient_last_name
+        examination_date = self.examination_date
+        center = self.center
+
+        assert dob, "Patient DOB is required"
+        assert examination_date, "Examination date is required"
+
+        hash_str = get_patient_examination_hash(
+            first_name=first_name,
+            last_name=last_name,
+            dob=dob,
+            examination_date=examination_date,
+            center=center.name,
+            salt=salt,
+        )
+
+        return sha256(hash_str.encode()).hexdigest()
+
+    # override save method to update hashes
+    def save(self, *args, **kwargs):
+        self.examination_hash = self.get_patient_examination_hash()
+        self.patient_hash = self.get_patient_hash()
+        self.pseudo_patient = self.create_pseudo_patient()
+        self.pseudo_examination = self.get_or_create_pseudo_patient_examination()
+        super().save(*args, **kwargs)
+
+    @classmethod
+    def _update_name_db(cls, first_name, last_name):
+        from endoreg_db.models import FirstName, LastName
+
+        FirstName.objects.get_or_create(name=first_name)
+        LastName.objects.get_or_create(name=last_name)

endoreg_db/models/data_file/metadata/video_meta.py

@@ -1,38 +1,77 @@
 from django.db import models
-import ffmpeg
+import subprocess
+import json
 from pathlib import Path
+from typing import Optional, TYPE_CHECKING
 
 # import endoreg_center_id from django settings
 from django.conf import settings
 
 # check if endoreg_center_id is set
-if not hasattr(settings, 'ENDOREG_CENTER_ID'):
+if not hasattr(settings, "ENDOREG_CENTER_ID"):
     ENDOREG_CENTER_ID = 9999
 else:
     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
 
+if TYPE_CHECKING:
+    from endoreg_db.models import EndoscopyProcessor, Endoscope, Center
+
+
 # VideoMeta
 class VideoMeta(models.Model):
-    processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
-    endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
-    center = models.ForeignKey('Center', on_delete=models.CASCADE)
-    import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
-    ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
+    processor = models.ForeignKey(
+        "EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
+    )
+    endoscope = models.ForeignKey(
+        "Endoscope", on_delete=models.CASCADE, blank=True, null=True
+    )
+    center = models.ForeignKey("Center", on_delete=models.CASCADE)
+    import_meta = models.OneToOneField(
+        "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
+    )
+    ffmpeg_meta = models.OneToOneField(
+        "FFMpegMeta", on_delete=models.CASCADE, blank=True, null=True
+    )  ##
 
-    def __str__(self):
+    @classmethod
+    def create_from_file(
+        cls,
+        file_path: Path,
+        center: Optional["Center"],
+        processor: Optional["EndoscopyProcessor"] = None,
+        endoscope: Optional["Endoscope"] = None,
+    ):
+        """Create a new VideoMeta from a file."""
+        meta = cls.objects.create(center=center)
+        meta.update_meta(file_path)
+
+        if processor:
+            meta.processor = processor
+            meta.get_endo_roi()
+        if endoscope:
+            meta.endoscope = endoscope
+
+        meta.save()
+
+        return meta
 
+    def __str__(self):
         processor_name = self.processor.name if self.processor is not None else "None"
         endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
         center_name = self.center.name if self.center is not None else "None"
+        ffmpeg_meta_str = self.ffmpeg_meta.__str__()
+        import_meta_str = self.import_meta.__str__()
 
         result_html = ""
 
-        result_html += f"Processor: {processor_name}<br>"
-        result_html += f"Endoscope: {endoscope_name}<br>"
-        result_html += f"Center: {center_name}<br>"
+        result_html += f"Processor: {processor_name}\n"
+        result_html += f"Endoscope: {endoscope_name}\n"
+        result_html += f"Center: {center_name}\n"
+        result_html += f"FFMpeg Meta: {ffmpeg_meta_str}\n"
+        result_html += f"Import Meta: {import_meta_str}\n"
 
         return result_html
-    
+
     # import meta should be created when video meta is created
     def save(self, *args, **kwargs):
         if self.import_meta is None:
@@ -50,15 +89,18 @@ class VideoMeta(models.Model):
         self.save()
 
     def get_endo_roi(self):
-        endo_roi = self.processor.get_roi_endoscope_image()
+        from endoreg_db.models import EndoscopyProcessor
+
+        processor: EndoscopyProcessor = self.processor
+        endo_roi = processor.get_roi_endoscope_image()
         return endo_roi
 
     def get_fps(self):
         if not self.ffmpeg_meta:
             return None
-    
+
         return self.ffmpeg_meta.frame_rate
- 
+
 
 class FFMpegMeta(models.Model):
     # Existing fields
@@ -66,7 +108,7 @@ class FFMpegMeta(models.Model):
     width = models.IntegerField(blank=True, null=True)
     height = models.IntegerField(blank=True, null=True)
     frame_rate = models.FloatField(blank=True, null=True)
-    
+
     # New fields for comprehensive information
     video_codec = models.CharField(max_length=50, blank=True, null=True)
     audio_codec = models.CharField(max_length=50, blank=True, null=True)
@@ -77,45 +119,67 @@ class FFMpegMeta(models.Model):
 
     @classmethod
     def create_from_file(cls, file_path: Path):
-        """Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
-        try:
-            probe = ffmpeg.probe(file_path.resolve().as_posix())
-        except: # ffmpeg.Error as e:
-            # print(e.stderr)
-            print(f"Error while probing {file_path}")
-            return None
-
-        video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
-        audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
-
-        # Check for the existence of a video stream
-        if video_stream is None:
-            print(f"No video stream found in {file_path}")
+        cmd = [
+            "ffprobe",
+            "-v",
+            "quiet",
+            "-print_format",
+            "json",
+            "-show_streams",
+            str(file_path),
+        ]
+        proc = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        probe = json.loads(proc.stdout)
+
+        video_stream = next(
+            (stream for stream in probe["streams"] if stream["codec_type"] == "video"),
+            None,
+        )
+        audio_streams = [
+            stream for stream in probe["streams"] if stream["codec_type"] == "audio"
+        ]
+
+        if not video_stream:
             return None
 
-        # Extract and store video metadata
         metadata = {
-            'duration': float(video_stream.get('duration', 0)),
-            'width': int(video_stream.get('width', 0)),
-            'height': int(video_stream.get('height', 0)),
-            'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
-            'video_codec': video_stream.get('codec_name', ''),
+            "duration": float(video_stream.get("duration", 0)),
+            "width": int(video_stream.get("width", 0)),
+            "height": int(video_stream.get("height", 0)),
+            "frame_rate": float(
+                next(iter(video_stream.get("avg_frame_rate", "").split("/")), 0)
+            ),
+            "video_codec": video_stream.get("codec_name", ""),
         }
 
-        # If there are audio streams, extract and store audio metadata from the first stream
         if audio_streams:
             first_audio_stream = audio_streams[0]
-            metadata.update({
-                'audio_codec': first_audio_stream.get('codec_name', ''),
-                'audio_channels': int(first_audio_stream.get('channels', 0)),
-                'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
-            })
+            metadata.update(
+                {
+                    "audio_codec": first_audio_stream.get("codec_name", ""),
+                    "audio_channels": int(first_audio_stream.get("channels", 0)),
+                    "audio_sample_rate": int(first_audio_stream.get("sample_rate", 0)),
+                }
+            )
 
-        # Create and return the FFMpegMeta instance
         return cls.objects.create(**metadata)
 
-class VideoImportMeta(models.Model):
+    def __str__(self):
+        result_html = ""
+
+        result_html += f"Duration: {self.duration}\n"
+        result_html += f"Width: {self.width}\n"
+        result_html += f"Height: {self.height}\n"
+        result_html += f"Frame Rate: {self.frame_rate}\n"
+        result_html += f"Video Codec: {self.video_codec}\n"
+        result_html += f"Audio Codec: {self.audio_codec}\n"
+        result_html += f"Audio Channels: {self.audio_channels}\n"
+        result_html += f"Audio Sample Rate: {self.audio_sample_rate}\n"
 
+        return result_html
+
+
+class VideoImportMeta(models.Model):
     video_anonymized = models.BooleanField(default=False)
     video_patient_data_detected = models.BooleanField(default=False)
     outside_detected = models.BooleanField(default=False)
@@ -125,9 +189,11 @@ class VideoImportMeta(models.Model):
     def __str__(self):
         result_html = ""
 
-        result_html += f"Video anonymized: {self.video_anonymized}<br>"
-        result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
-        result_html += f"Outside detected: {self.outside_detected}<br>"
-        result_html += f"Patient data removed: {self.patient_data_removed}<br>"
-        result_html += f"Outside removed: {self.outside_removed}<br>"
-        return result_html
+        result_html += f"Video anonymized: {self.video_anonymized}\n"
+        result_html += (
+            f"Video patient data detected: {self.video_patient_data_detected}\n"
+        )
+        result_html += f"Outside detected: {self.outside_detected}\n"
+        result_html += f"Patient data removed: {self.patient_data_removed}\n"
+        result_html += f"Outside removed: {self.outside_removed}\n"
+        return result_html

endoreg_db/models/data_file/report_file.py

@@ -1,82 +1,52 @@
-from django.db import models
-from ..center import Center
-import hashlib
 from datetime import date, time
+from django.db import models
+from .base_classes.abstract_pdf import AbstractPdfFile
+
 
-class ReportFile(models.Model):
-    pdf = models.FileField(upload_to="raw_report_pdfs", blank=True, null=True)
-    pdf_hash = models.CharField(max_length=255, unique=True)
-    center = models.ForeignKey(Center, on_delete=models.CASCADE)
+class ReportFile(AbstractPdfFile):
     meta = models.JSONField(blank=True, null=True)
     text = models.TextField(blank=True, null=True)
-    text_anonymized = models.TextField(blank=True, null=True)
-    patient = models.ForeignKey("Patient", on_delete=models.CASCADE, blank=True, null=True)
-    examiner = models.ForeignKey("Examiner", on_delete=models.CASCADE, blank=True, null=True)
+    # sensitive_meta = models.ForeignKey(
+    #     "SensitiveMeta",
+    #     on_delete=models.SET_NULL,
+    #     related_name="report_files",
+    #     null=True,
+    #     blank=True,
+    # )
+    # examination = models.ForeignKey(
+    #     "PatientExamination",
+    #     on_delete=models.SET_NULL,
+    #     blank=True,
+    #     null=True,
+    #     related_name="report_files",
+    # )
+    # patient = models.ForeignKey(
+    #     "Patient", on_delete=models.DO_NOTHING, blank=True, null=True
+    # )
+    # examiner = models.ForeignKey(
+    #     "Examiner", on_delete=models.DO_NOTHING, blank=True, null=True
+    # )
     date = models.DateField(blank=True, null=True)
     time = models.TimeField(blank=True, null=True)
 
-    def get_pdf_hash(self):
-        #FIXME should use endoreg_db.utils.get_pdf_hash in future
-        pdf = self.pdf
-        pdf_hash = None
-
-        if pdf:
-            # Open the file in binary mode and read its contents
-            with pdf.open(mode='rb') as f:
-                pdf_contents = f.read()
-                # Create a hash object using SHA-256 algorithm
-                hash_object = hashlib.sha256(pdf_contents, usedforsecurity=False)
-                # Get the hexadecimal representation of the hash
-                pdf_hash = hash_object.hexdigest()
-                assert len(pdf_hash) <= 255, "Hash length exceeds 255 characters"
-
-        return pdf_hash
-
-    def initialize_metadata_in_db(self, report_meta=None):
-        if not report_meta:
-            report_meta = self.meta
-        self.set_examination_date_and_time(report_meta)
-        self.patient, created = self.get_or_create_patient(report_meta)
-        self.examiner, created = self.get_or_create_examiner(report_meta)
-        self.save()    
-
-    def get_or_create_patient(self, report_meta=None):
-        from ..persons import Patient
-        if not report_meta:
-            report_meta = self.meta
-        patient_first_name = report_meta['patient_first_name']
-        patient_last_name = report_meta['patient_last_name']
-        patient_dob = report_meta['patient_dob']
-
-        patient, created = Patient.objects.get_or_create(
-            first_name=patient_first_name,
-            last_name=patient_last_name,
-            dob=patient_dob
-        )
-
-        return patient, created
-    
-    def get_or_create_examiner(self, report_meta= None):
+    def get_or_create_examiner(self, examiner_first_name, examiner_last_name):
         from ..persons import Examiner
-        if not report_meta:
-            report_meta = self.meta
-        examiner_first_name = report_meta['examiner_first_name']
-        examiner_last_name = report_meta['examiner_last_name']
+
         examiner_center = self.center
 
         examiner, created = Examiner.objects.get_or_create(
             first_name=examiner_first_name,
             last_name=examiner_last_name,
-            center=examiner_center
+            center=examiner_center,
         )
 
         return examiner, created
-        
+
     def set_examination_date_and_time(self, report_meta=None):
         if not report_meta:
             report_meta = self.meta
-        examination_date_str = report_meta['examination_date']
-        examination_time_str = report_meta['examination_time']
+        examination_date_str = report_meta["examination_date"]
+        examination_time_str = report_meta["examination_time"]
 
         if examination_date_str:
             # TODO: get django DateField compatible date from string (e.g. "2021-01-01")
@@ -84,6 +54,3 @@ class ReportFile(models.Model):
         if examination_time_str:
             # TODO: get django TimeField compatible time from string (e.g. "12:00")
             self.time = time.fromisoformat(examination_time_str)
-        
-
-

endoreg_db/models/data_file/video/__init__.py

@@ -1,7 +1,11 @@
 from .video import (
     Video,
-    LegacyVideo,
 )
 from ..metadata import (
     VideoImportMeta,
-)
+)
+
+__all__ = [
+    "Video",
+    "VideoImportMeta",
+]