endoreg-db 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

endoreg_db/models/data_file/import_classes/processing_functions/__init__.py

@@ -0,0 +1,35 @@
+from .video import (
+    get_videos_scheduled_for_frame_extraction, 
+    extract_frames_from_video,
+    extract_frames_from_videos,
+
+    get_videos_scheduled_for_ocr,
+    videos_scheduled_for_ocr_preflight,
+    perform_ocr_on_video,
+    perform_ocr_on_videos,
+
+    videos_scheduled_for_initial_prediction_preflight,
+    get_videos_scheduled_for_initial_prediction,
+    get_multilabel_model,
+    get_multilabel_classifier,
+    get_crops,
+    perform_initial_prediction_on_video,
+    perform_initial_prediction_on_videos,
+
+    videos_scheduled_for_prediction_import_preflight,
+    get_videos_scheduled_for_prediction_import,
+    import_predictions_for_video,
+    import_predictions_for_videos,
+    
+
+    delete_frames_preflight,
+    get_videos_scheduled_for_frame_deletion,
+    delete_frames_for_video,
+    delete_frames,
+)
+
+from .pdf import (
+    get_pdf_files_scheduled_for_processing,
+    process_pdf_file,
+    process_pdf_files,
+)

endoreg_db/models/data_file/import_classes/processing_functions/pdf.py

@@ -0,0 +1,28 @@
+from ..raw_pdf import RawPdfFile
+from logging import getLogger
+#
+# # setup logging to pdf_import.log
+logger = getLogger('examination_pdf_import')
+
+def get_pdf_files_scheduled_for_processing():
+    reports = RawPdfFile.objects.filter(
+        state_report_processing_required=True
+    )
+    return reports
+
+def process_pdf_file(report:RawPdfFile):
+    if report.update(save=True, verbose=True):
+        logger.info(f"Report {report} processed successfully")
+        return True
+    else:
+        logger.error(f"Report {report} processing failed")
+        return False
+
+
+def process_pdf_files():
+    reports = get_pdf_files_scheduled_for_processing()
+    for report in reports:
+        process_pdf_file(report)
+    
+
+

endoreg_db/models/data_file/import_classes/{processing_functions.py → processing_functions/video.py}

@@ -1,4 +1,5 @@
-from .raw_video import RawVideoFile
+from ..raw_video import RawVideoFile
+import json 
 
 # # Starting point
 # Automated tasks generate RawVideoFile objects in our db.
@@ -142,40 +143,31 @@ def perform_initial_prediction_on_video(
         fps = fps
     )
 
-    # pred_target_dir = video.get_pred_target_dir()
-    
-    result_targets = [
-        "predictions",
-        "smooth_predictions",
-        "binary_predictions",
-        "raw_sequences",
-        "filtered_sequences"
-    ]
     
     # Predictions
     _path = video.get_predictions_path()
     with open(_path, "w") as f:
-        json.dump(result_targets["predictions"])
+        json.dump(result_dict["predictions"], f, indent = 4)
 
     # smooth_predictions
     _path = video.get_smooth_predictions_path()
     with open(_path, "w") as f:
-        json.dump(result_targets["smooth_predictions"])
+        json.dump(result_dict["smooth_predictions"], f, indent = 4)
 
     # binary_predictions
     _path = video.get_binary_predictions_path()
     with open(_path, "w") as f:
-        json.dump(result_targets["binary_predictions"])
+        json.dump(result_dict["binary_predictions"], f, indent = 4)
 
     # Raw Sequences
     _path = video.get_raw_sequences_path()
     with open(_path, "w") as f:
-        json.dump(result_targets["raw_sequences"])
+        json.dump(result_dict["raw_sequences"], f, indent = 4)
 
     # filtered_sequences
     _path = video.get_filtered_sequences_path()
     with open(_path, "w") as f:
-        json.dump(result_targets["filtered_sequences"])
+        json.dump(result_dict["filtered_sequences"], f, indent = 4)
 
     
     # update state_initial_prediction_completed
@@ -209,8 +201,8 @@ def videos_scheduled_for_prediction_import_preflight():
     
 def get_videos_scheduled_for_prediction_import():
     return RawVideoFile.objects.filter(
-        state_prediction_import_required=True,
-        state_prediction_import_completed=False,
+        state_initial_prediction_import_required=True,
+        state_initial_prediction_import_completed=False,
         state_initial_prediction_completed=True
     )
 
@@ -219,9 +211,9 @@ def import_predictions_for_video(video:RawVideoFile):
     pass
 
     # update state_prediction_import_completed
-    video.state_prediction_import_required = False
-    video.state_prediction_import_completed = True
-    video.save()
+    # video.state_initial_prediction_import_required = False
+    # video.state_initial_prediction_import_completed = True
+    # video.save()
 
     return video
 
@@ -230,7 +222,6 @@ def import_predictions_for_videos():
     for video in videos:
         import_predictions_for_video(video)
 
-
 # # Step 4 - Delete Frames if not needed anymore
 # function to query for videos scheduled for frame deletion,
 # first we need to set state_frames_required = False for videos with:

endoreg_db/models/data_file/import_classes/raw_pdf.py

@@ -0,0 +1,185 @@
+# models/data_file/import_classes/raw_pdf.py
+# django db model "RawPdf"
+# Class to store raw pdf file using django file field
+# Class contains classmethod to create object from pdf file
+# objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
+# ------------------------------------------------------------------------------
+
+from django.db import models
+from django.core.files.storage import FileSystemStorage
+from django.core.files import File
+from django.conf import settings
+from django.utils import timezone
+from django.core.exceptions import ValidationError
+from django.core.validators import FileExtensionValidator
+from endoreg_db.utils.file_operations import get_uuid_filename
+
+from agl_report_reader.report_reader import ReportReader
+
+from endoreg_db.utils.hashs import get_pdf_hash
+from ..metadata import SensitiveMeta
+
+# setup logging to pdf_import.log
+import logging
+logger = logging.getLogger('pdf_import')
+
+import shutil
+
+class RawPdfFile(models.Model):
+    file = models.FileField(
+        upload_to='raw_pdf/',
+        validators=[FileExtensionValidator(allowed_extensions=['pdf'])],
+        storage=FileSystemStorage(location=settings.PSEUDO_DIR_RAW_PDF.resolve().as_posix()),
+    )
+
+    pdf_hash = models.CharField(max_length=255, unique=True)
+    pdf_type = models.ForeignKey('PdfType', on_delete=models.CASCADE)
+    center = models.ForeignKey('Center', on_delete=models.CASCADE)
+
+    state_report_processing_required = models.BooleanField(default = True)
+    state_report_processed = models.BooleanField(default=False)
+
+    # report_file = models.OneToOneField("ReportFile", on_delete=models.CASCADE, null=True, blank=True)
+    sensitive_meta = models.OneToOneField(
+        'SensitiveMeta',
+        on_delete=models.CASCADE,
+        related_name='raw_pdf_file',
+        null=True,
+        blank=True,
+    )
+
+    text = models.TextField(blank=True, null=True)
+    anonymized_text = models.TextField(blank=True, null=True)
+
+    raw_meta = models.JSONField(blank=True, null=True)
+
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        str_repr = f"RawPdfFile: {self.file.name}"
+        return str_repr
+
+    @classmethod
+    def create_from_file(
+        cls,
+        file_path,
+        center_name,
+        pdf_type_name,
+        destination_dir,
+        save=True,
+    ):
+        from endoreg_db.models import PdfType, Center
+        logger.info(f"Creating RawPdfFile object from file: {file_path}")
+        original_file_name = file_path.name
+
+        new_file_name, uuid = get_uuid_filename(file_path)
+
+        if not destination_dir.exists():
+            destination_dir.mkdir(parents=True)
+
+        pdf_hash = get_pdf_hash(file_path)
+
+        # check if pdf file already exists
+        if cls.objects.filter(pdf_hash=pdf_hash).exists():
+            logger.warning(f"RawPdfFile with hash {pdf_hash} already exists")
+            return None
+        
+        assert pdf_type_name is not None, "pdf_type_name is required"
+        assert center_name is not None, "center_name is required"
+
+        pdf_type = PdfType.objects.get(name=pdf_type_name)
+        center = Center.objects.get(name=center_name)
+
+        new_file_path = destination_dir / new_file_name
+
+        logger.info(f"Copying file to {new_file_path}")
+        success = shutil.copy(file_path, new_file_path)
+         
+        # validate copy operation by comparing hashs
+        assert get_pdf_hash(new_file_path) == pdf_hash, "Copy operation failed"
+
+        raw_pdf = cls(
+            file=new_file_path.resolve().as_posix(),
+            pdf_hash=pdf_hash,
+            pdf_type=pdf_type,
+            center=center,
+        )
+        logger.info(f"RawPdfFile object created: {raw_pdf}")
+
+        # remove source file
+        file_path.unlink()
+        logger.info(f"Source file removed: {file_path}")
+
+        if save:
+            raw_pdf.save()
+        
+
+        return raw_pdf
+
+    def process_file(self, verbose = False):
+        
+        pdf_path = self.file.path
+        rr_config = self.get_report_reader_config()
+
+        rr = ReportReader(**rr_config) #FIXME In future we need to pass a configuration file 
+        # This configuration file should be associated with pdf type 
+
+        text, anonymized_text, report_meta = rr.process_report(pdf_path, verbose=verbose)
+        if not self.sensitive_meta:
+            sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
+            sensitive_meta.save()
+            self.sensitive_meta = sensitive_meta
+
+        else: 
+            # update existing sensitive meta
+            sensitive_meta = self.sensitive_meta
+            sensitive_meta.update_from_dict(report_meta)
+
+        return text, anonymized_text, report_meta
+    
+    def update(self, save=True, verbose = True):
+        try:
+            self.text, self.anonymized_text, self.raw_meta = self.process_file(verbose = verbose)
+            self.state_report_processed = True
+            self.state_report_processing_required = False
+        
+            if save: 
+    
+                self.save()
+
+            return True
+
+        except:
+            logger.error(f"Error processing file: {self.file.path}")
+            return False
+
+    def save(self, *args, **kwargs):
+        if not self.file.name.endswith('.pdf'):
+            raise ValidationError('Only PDF files are allowed')
+        
+        if not self.pdf_hash:
+            self.pdf_hash = get_pdf_hash(self.file.path)
+
+        super().save(*args, **kwargs)
+
+
+    def get_report_reader_config(self):
+        if self.pdf_type.endoscope_info_line:
+            endoscope_info_line = self.pdf_type.endoscope_info_line.value
+        else:
+            endoscope_info_line = None
+        settings_dict = {
+            "locale": "de_DE",
+            "employee_first_names": [_.name for _ in self.center.first_names.all()],
+            "employee_last_names": [_.name for _ in self.center.last_names.all()],
+            "text_date_format":'%d.%m.%Y',
+            "flags": {
+                "patient_info_line": self.pdf_type.patient_info_line.value,
+                "endoscope_info_line": endoscope_info_line,
+                "examiner_info_line": self.pdf_type.examiner_info_line.value,
+                "cut_off_below": [_.value for _ in self.pdf_type.cut_off_below_lines.all()],
+                "cut_off_above": [_.value for _ in self.pdf_type.cut_off_above_lines.all()],
+            }
+        }
+
+        return settings_dict

endoreg_db/models/data_file/import_classes/raw_video.py

@@ -15,9 +15,11 @@ from ..metadata import VideoMeta, SensitiveMeta
 class RawVideoFile(models.Model):
     uuid = models.UUIDField()
     file = models.FileField(upload_to="raw_data/")
+    
     sensitive_meta = models.OneToOneField(
         "SensitiveMeta", on_delete=models.CASCADE, blank=True, null=True
-    )
+    ) 
+
     center = models.ForeignKey("Center", on_delete=models.CASCADE)
     processor = models.ForeignKey(
         "EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
@@ -69,7 +71,7 @@ class RawVideoFile(models.Model):
     def create_from_file(
         cls,
         file_path: Path,
-        video_dir_parent: Path,
+        video_dir: Path,
         center_name: str,
         processor_name: str,
         frame_dir_parent: Path,
@@ -87,8 +89,8 @@ class RawVideoFile(models.Model):
         if not framedir.exists():
             framedir.mkdir(parents=True, exist_ok=True)
 
-        if not video_dir_parent.exists():
-            video_dir_parent.mkdir(parents=True, exist_ok=True)
+        if not video_dir.exists():
+            video_dir.mkdir(parents=True, exist_ok=True)
 
         video_hash = get_video_hash(file_path)
 
@@ -98,7 +100,7 @@ class RawVideoFile(models.Model):
         processor = EndoscopyProcessor.objects.get(name=processor_name)
         assert processor is not None, "Processor must exist"
 
-        new_filepath = video_dir_parent / new_file_name
+        new_filepath = video_dir / new_file_name
 
         print(f"Moving {file_path} to {new_filepath}")
         shutil.move(file_path.resolve().as_posix(), new_filepath.resolve().as_posix())
@@ -245,7 +247,7 @@ class RawVideoFile(models.Model):
         return paths
 
     def get_prediction_dir(self):
-        return Path(elf.prediction_dir)
+        return Path(self.prediction_dir)
 
     def get_predictions_path(self, suffix = ".json"):
         pred_dir = self.get_prediction_dir()

endoreg_db/models/data_file/metadata/__init__.py

@@ -1,133 +1,3 @@
-from django.db import models
-import ffmpeg
-from pathlib import Path
 from .sensitive_meta import SensitiveMeta
-
-# import endoreg_center_id from django settings
-from django.conf import settings
-
-# check if endoreg_center_id is set
-if not hasattr(settings, 'ENDOREG_CENTER_ID'):
-    ENDOREG_CENTER_ID = 9999
-else:
-    ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
-
-# VideoMeta
-class VideoMeta(models.Model):
-    processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
-    endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
-    center = models.ForeignKey('Center', on_delete=models.CASCADE)
-    import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
-    ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
-
-    def __str__(self):
-
-        processor_name = self.processor.name if self.processor is not None else "None"
-        endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
-        center_name = self.center.name if self.center is not None else "None"
-
-        result_html = ""
-
-        result_html += f"Processor: {processor_name}<br>"
-        result_html += f"Endoscope: {endoscope_name}<br>"
-        result_html += f"Center: {center_name}<br>"
-
-        return result_html
-    
-    # import meta should be created when video meta is created
-    def save(self, *args, **kwargs):
-        if self.import_meta is None:
-            self.import_meta = VideoImportMeta.objects.create()
-        super(VideoMeta, self).save(*args, **kwargs)
-
-    def initialize_ffmpeg_meta(self, file_path):
-        """Initializes FFMpeg metadata for the video file if not already done."""
-        self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
-        self.save()
-
-    def update_meta(self, file_path):
-        """Updates the video metadata from the file."""
-        self.initialize_ffmpeg_meta(file_path)
-        self.save()
-
-    def get_endo_roi(self):
-        endo_roi = self.processor.get_roi_endoscope_image()
-        return endo_roi
-
-    def get_fps(self):
-        if not self.ffmpeg_meta:
-            return None
-    
-        return self.ffmpeg_meta.frame_rate
-            
-
-class FFMpegMeta(models.Model):
-    # Existing fields
-    duration = models.FloatField(blank=True, null=True)
-    width = models.IntegerField(blank=True, null=True)
-    height = models.IntegerField(blank=True, null=True)
-    frame_rate = models.FloatField(blank=True, null=True)
-    
-    # New fields for comprehensive information
-    video_codec = models.CharField(max_length=50, blank=True, null=True)
-    audio_codec = models.CharField(max_length=50, blank=True, null=True)
-    audio_channels = models.IntegerField(blank=True, null=True)
-    audio_sample_rate = models.IntegerField(blank=True, null=True)
-
-    # Existing __str__ method can be updated to include new fields
-
-    @classmethod
-    def create_from_file(cls, file_path: Path):
-        """Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
-        try:
-            probe = ffmpeg.probe(str(file_path))
-        except ffmpeg.Error as e:
-            print(e.stderr)
-            return None
-
-        video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
-        audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
-
-        # Check for the existence of a video stream
-        if video_stream is None:
-            print(f"No video stream found in {file_path}")
-            return None
-
-        # Extract and store video metadata
-        metadata = {
-            'duration': float(video_stream.get('duration', 0)),
-            'width': int(video_stream.get('width', 0)),
-            'height': int(video_stream.get('height', 0)),
-            'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
-            'video_codec': video_stream.get('codec_name', ''),
-        }
-
-        # If there are audio streams, extract and store audio metadata from the first stream
-        if audio_streams:
-            first_audio_stream = audio_streams[0]
-            metadata.update({
-                'audio_codec': first_audio_stream.get('codec_name', ''),
-                'audio_channels': int(first_audio_stream.get('channels', 0)),
-                'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
-            })
-
-        # Create and return the FFMpegMeta instance
-        return cls.objects.create(**metadata)
-
-class VideoImportMeta(models.Model):
-
-    video_anonymized = models.BooleanField(default=False)
-    video_patient_data_detected = models.BooleanField(default=False)
-    outside_detected = models.BooleanField(default=False)
-    patient_data_removed = models.BooleanField(default=False)
-    outside_removed = models.BooleanField(default=False)
-
-    def __str__(self):
-        result_html = ""
-
-        result_html += f"Video anonymized: {self.video_anonymized}<br>"
-        result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
-        result_html += f"Outside detected: {self.outside_detected}<br>"
-        result_html += f"Patient data removed: {self.patient_data_removed}<br>"
-        result_html += f"Outside removed: {self.outside_removed}<br>"
-        return result_html
+from .pdf_meta import PdfMeta, PdfType
+from .video_meta import VideoMeta, FFMpegMeta, VideoImportMeta

endoreg_db/models/data_file/metadata/pdf_meta.py

@@ -0,0 +1,70 @@
+from django.db import models
+
+# import endoreg_center_id from django settings
+from django.conf import settings
+
+
+# import File class
+from django.core.files import File
+
+# # check if endoreg_center_id is set
+# if not hasattr(settings, 'ENDOREG_CENTER_ID'):
+#     ENDOREG_CENTER_ID = 9999
+# else:
+#     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
+
+class PdfType(models.Model):
+    name = models.CharField(max_length=255)
+
+    patient_info_line = models.ForeignKey(
+        "ReportReaderFlag",
+        related_name="pdf_type_patient_info_line", 
+        on_delete=models.CASCADE
+    )
+    endoscope_info_line = models.ForeignKey(
+        "ReportReaderFlag",
+        related_name="pdf_type_endoscopy_info_line", 
+        on_delete=models.CASCADE,
+    )
+    examiner_info_line = models.ForeignKey(
+        "ReportReaderFlag",
+        related_name="pdf_type_examiner_info_line", 
+        on_delete=models.CASCADE
+    )
+    cut_off_above_lines = models.ManyToManyField(
+        "ReportReaderFlag",
+        related_name="pdf_type_cut_off_above_lines", 
+    )
+    cut_off_below_lines = models.ManyToManyField(
+        "ReportReaderFlag",
+        related_name="pdf_type_cut_off_below_lines", 
+    )
+
+
+    def __str__(self):
+        summary = f"{self.name}"
+        # add lines to summary
+        summary += f"\nPatient Info Line: {self.patient_info_line.value}"
+        summary += f"\nEndoscope Info Line: {self.endoscope_info_line.value}"
+        summary += f"\nExaminer Info Line: {self.examiner_info_line.value}"
+        summary += f"\nCut Off Above Lines: {[_.value for _ in self.cut_off_above_lines.all()]}"
+        summary += f"\nCut Off Below Lines: {[_.value for _ in self.cut_off_below_lines.all()]}"
+    
+        return summary
+
+class PdfMeta(models.Model):
+    pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
+    date = models.DateField()
+    time = models.TimeField()
+    pdf_hash = models.CharField(max_length=255, unique=True)
+
+    def __str__(self):
+        return self.pdf_hash
+
+    @classmethod
+    def create_from_file(cls, pdf_file):
+        pdf_file = File(pdf_file)
+        pdf_meta = cls(file=pdf_file)
+        pdf_meta.save()
+        return pdf_meta
+

endoreg_db/models/data_file/metadata/sensitive_meta.py

@@ -10,4 +10,22 @@ class SensitiveMeta(models.Model):
 
     @classmethod
     def create_from_dict(cls, data: dict):
-        return cls.objects.create(**data)
+        # data can contain more fields than the model has
+        field_names = [_.name for _ in cls._meta.fields]
+        selected_data = {k: v for k, v in data.items() if k in field_names}
+
+        return cls.objects.create(**selected_data)
+    
+    def update_from_dict(self, data: dict):
+        # data can contain more fields than the model has
+        field_names = [_.name for _ in self._meta.fields]
+        selected_data = {k: v for k, v in data.items() if k in field_names}
+
+        for k, v in selected_data.items():
+            setattr(self, k, v)
+        
+        self.save()
+    
+    def __str__(self):
+        return f"SensitiveMeta: {self.examination_date} {self.patient_first_name} {self.patient_last_name} (*{self.patient_dob})"
+