endoreg-db 0.5.2__py3-none-any.whl → 0.6.0__py3-none-any.whl

endoreg_db/models/data_file/import_classes/raw_pdf.py

@@ -1,188 +0,0 @@
-# models/data_file/import_classes/raw_pdf.py
-# django db model "RawPdf"
-# Class to store raw pdf file using django file field
-# Class contains classmethod to create object from pdf file
-# objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
-# ------------------------------------------------------------------------------
-
-from django.db import models
-from django.core.files.storage import FileSystemStorage
-from django.core.files import File
-from django.conf import settings
-from django.utils import timezone
-from django.core.exceptions import ValidationError
-from django.core.validators import FileExtensionValidator
-from endoreg_db.utils.file_operations import get_uuid_filename
-
-from agl_report_reader.report_reader import ReportReader
-
-from endoreg_db.utils.hashs import get_pdf_hash
-from ..metadata import SensitiveMeta
-
-# setup logging to pdf_import.log
-import logging
-logger = logging.getLogger('pdf_import')
-
-import shutil
-
-# get pdf location from settings, default to ~/erc_data/raw_pdf and create if not exists
-PSEUDO_DIR_RAW_PDF = getattr(settings, 'PSEUDO_DIR_RAW_PDF', settings.BASE_DIR / 'erc_data/raw_pdf')
-
-class RawPdfFile(models.Model):
-    file = models.FileField(
-        upload_to='raw_pdf/',
-        validators=[FileExtensionValidator(allowed_extensions=['pdf'])],
-        storage=FileSystemStorage(location=PSEUDO_DIR_RAW_PDF.resolve().as_posix()),
-    )
-
-    pdf_hash = models.CharField(max_length=255, unique=True)
-    pdf_type = models.ForeignKey('PdfType', on_delete=models.CASCADE)
-    center = models.ForeignKey('Center', on_delete=models.CASCADE)
-
-    state_report_processing_required = models.BooleanField(default = True)
-    state_report_processed = models.BooleanField(default=False)
-
-    # report_file = models.OneToOneField("ReportFile", on_delete=models.CASCADE, null=True, blank=True)
-    sensitive_meta = models.OneToOneField(
-        'SensitiveMeta',
-        on_delete=models.CASCADE,
-        related_name='raw_pdf_file',
-        null=True,
-        blank=True,
-    )
-
-    text = models.TextField(blank=True, null=True)
-    anonymized_text = models.TextField(blank=True, null=True)
-
-    raw_meta = models.JSONField(blank=True, null=True)
-
-    created_at = models.DateTimeField(auto_now_add=True)
-
-    def __str__(self):
-        str_repr = f"RawPdfFile: {self.file.name}"
-        return str_repr
-
-    @classmethod
-    def create_from_file(
-        cls,
-        file_path,
-        center_name,
-        pdf_type_name, # to be depreceated / changed since we now import all pdfs from same directory
-        destination_dir,
-        save=True,
-    ):
-        from endoreg_db.models import PdfType, Center
-        logger.info(f"Creating RawPdfFile object from file: {file_path}")
-        original_file_name = file_path.name
-
-        new_file_name, uuid = get_uuid_filename(file_path)
-
-        if not destination_dir.exists():
-            destination_dir.mkdir(parents=True)
-
-        pdf_hash = get_pdf_hash(file_path)
-
-        # check if pdf file already exists
-        if cls.objects.filter(pdf_hash=pdf_hash).exists():
-            logger.warning(f"RawPdfFile with hash {pdf_hash} already exists")
-            return None
-        
-        assert pdf_type_name is not None, "pdf_type_name is required"
-        assert center_name is not None, "center_name is required"
-
-        pdf_type = PdfType.objects.get(name=pdf_type_name)
-        center = Center.objects.get(name=center_name)
-
-        new_file_path = destination_dir / new_file_name
-
-        logger.info(f"Copying file to {new_file_path}")
-        success = shutil.copy(file_path, new_file_path)
-         
-        # validate copy operation by comparing hashs
-        assert get_pdf_hash(new_file_path) == pdf_hash, "Copy operation failed"
-
-        raw_pdf = cls(
-            file=new_file_path.resolve().as_posix(),
-            pdf_hash=pdf_hash,
-            pdf_type=pdf_type,
-            center=center,
-        )
-        logger.info(f"RawPdfFile object created: {raw_pdf}")
-
-        # remove source file
-        file_path.unlink()
-        logger.info(f"Source file removed: {file_path}")
-
-        if save:
-            raw_pdf.save()
-        
-
-        return raw_pdf
-
-    def process_file(self, verbose = False):
-        
-        pdf_path = self.file.path
-        rr_config = self.get_report_reader_config()
-
-        rr = ReportReader(**rr_config) #FIXME In future we need to pass a configuration file 
-        # This configuration file should be associated with pdf type 
-
-        text, anonymized_text, report_meta = rr.process_report(pdf_path, verbose=verbose)
-        if not self.sensitive_meta:
-            sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
-            sensitive_meta.save()
-            self.sensitive_meta = sensitive_meta
-
-        else: 
-            # update existing sensitive meta
-            sensitive_meta = self.sensitive_meta
-            sensitive_meta.update_from_dict(report_meta)
-
-        return text, anonymized_text, report_meta
-    
-    def update(self, save=True, verbose = True):
-        try:
-            self.text, self.anonymized_text, self.raw_meta = self.process_file(verbose = verbose)
-            self.state_report_processed = True
-            self.state_report_processing_required = False
-        
-            if save: 
-    
-                self.save()
-
-            return True
-
-        except:
-            logger.error(f"Error processing file: {self.file.path}")
-            return False
-
-    def save(self, *args, **kwargs):
-        if not self.file.name.endswith('.pdf'):
-            raise ValidationError('Only PDF files are allowed')
-        
-        if not self.pdf_hash:
-            self.pdf_hash = get_pdf_hash(self.file.path)
-
-        super().save(*args, **kwargs)
-
-
-    def get_report_reader_config(self):
-        if self.pdf_type.endoscope_info_line:
-            endoscope_info_line = self.pdf_type.endoscope_info_line.value
-        else:
-            endoscope_info_line = None
-        settings_dict = {
-            "locale": "de_DE",
-            "employee_first_names": [_.name for _ in self.center.first_names.all()],
-            "employee_last_names": [_.name for _ in self.center.last_names.all()],
-            "text_date_format":'%d.%m.%Y',
-            "flags": {
-                "patient_info_line": self.pdf_type.patient_info_line.value,
-                "endoscope_info_line": endoscope_info_line,
-                "examiner_info_line": self.pdf_type.examiner_info_line.value,
-                "cut_off_below": [_.value for _ in self.pdf_type.cut_off_below_lines.all()],
-                "cut_off_above": [_.value for _ in self.pdf_type.cut_off_above_lines.all()],
-            }
-        }
-
-        return settings_dict

endoreg_db/models/data_file/import_classes/raw_video.py

@@ -1,343 +0,0 @@
-from django.db import models
-from pathlib import Path
-from collections import defaultdict, Counter
-
-from endoreg_db.utils.hashs import get_video_hash
-from endoreg_db.utils.file_operations import get_uuid_filename
-from endoreg_db.utils.ocr import extract_text_from_rois
-
-import shutil
-import os
-import subprocess
-
-from ..metadata import VideoMeta, SensitiveMeta
-
-class RawVideoFile(models.Model):
-    uuid = models.UUIDField()
-    file = models.FileField(upload_to="raw_data/")
-    
-    sensitive_meta = models.OneToOneField(
-        "SensitiveMeta", on_delete=models.CASCADE, blank=True, null=True
-    ) 
-
-    center = models.ForeignKey("Center", on_delete=models.CASCADE)
-    processor = models.ForeignKey(
-        "EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
-    )
-    video_meta = models.OneToOneField(
-        "VideoMeta", on_delete=models.CASCADE, blank=True, null=True
-    )
-    original_file_name = models.CharField(max_length=255)
-    video_hash = models.CharField(max_length=255, unique=True)
-    uploaded_at = models.DateTimeField(auto_now_add=True)
-
-    # Frame Extraction States
-    state_frames_required = models.BooleanField(default=True)
-    state_frames_extracted = models.BooleanField(default=False)
-
-    # Video
-    ## Prediction
-    state_initial_prediction_required = models.BooleanField(default=True)
-    state_initial_prediction_completed = models.BooleanField(default=False)
-    state_initial_prediction_import_required = models.BooleanField(default=True)
-    state_initial_prediction_import_completed = models.BooleanField(default=False)
-    ## OCR
-    state_ocr_required = models.BooleanField(default=True)
-    state_ocr_completed = models.BooleanField(default=False)
-    ## Validation
-    state_outside_validated = models.BooleanField(default=False)
-    state_ocr_result_validated = models.BooleanField(default=False)
-
-    state_sensitive_data_retrieved = models.BooleanField(default=False)
-
-    # Dataset complete?
-    state_histology_required = models.BooleanField(blank=True, null=True)
-    state_histology_available = models.BooleanField(default=False)
-    state_follow_up_intervention_required = models.BooleanField(blank=True, null=True)
-    state_follow_up_intervention_available = models.BooleanField(default=False)
-    state_dataset_complete = models.BooleanField(default=False)
-
-    # Finalizing for Upload
-    state_anonym_video_required = models.BooleanField(default=True)
-    state_anonym_video_performed = models.BooleanField(default=False)
-    state_original_reports_deleted = models.BooleanField(default=False)
-    state_original_video_deleted = models.BooleanField(default=False)
-    state_finalized = models.BooleanField(default=False)
-
-    frame_dir = models.CharField(max_length=255)
-    prediction_dir = models.CharField(max_length=255)
-
-    @classmethod
-    def create_from_file(
-        cls,
-        file_path: Path,
-        video_dir: Path,
-        center_name: str,
-        processor_name: str,
-        frame_dir_parent: Path,
-        save: bool = True,
-    ):
-        from endoreg_db.models import Center, EndoscopyProcessor
-
-        print(f"Creating RawVideoFile from {file_path}")
-        original_file_name = file_path.name
-        # Rename and and move
-
-        new_file_name, uuid = get_uuid_filename(file_path)
-        framedir: Path = frame_dir_parent / str(uuid)
-
-        if not framedir.exists():
-            framedir.mkdir(parents=True, exist_ok=True)
-
-        if not video_dir.exists():
-            video_dir.mkdir(parents=True, exist_ok=True)
-
-        video_hash = get_video_hash(file_path)
-
-        center = Center.objects.get(name=center_name)
-        assert center is not None, "Center must exist"
-
-        processor = EndoscopyProcessor.objects.get(name=processor_name)
-        assert processor is not None, "Processor must exist"
-
-        new_filepath = video_dir / new_file_name
-
-        print(f"Moving {file_path} to {new_filepath}")
-        shutil.move(file_path.resolve().as_posix(), new_filepath.resolve().as_posix())
-        print(f"Moved to {new_filepath}")
-
-        # Make sure file was transferred correctly and hash is correct
-        if not new_filepath.exists():
-            print(f"File {file_path} was not transferred correctly to {new_filepath}")
-            return None
-
-        new_hash = get_video_hash(new_filepath)
-        if new_hash != video_hash:
-            print(f"Hash of file {file_path} is not correct")
-            return None
-
-        # make sure that no other file with the same hash exists
-        if cls.objects.filter(video_hash=video_hash).exists():
-            # log and print warnint
-            print(f"File with hash {video_hash} already exists")
-            return None
-
-        else:
-            print(center)
-            # Create a new instance of RawVideoFile
-            raw_video_file = cls(
-                uuid=uuid,
-                file=new_filepath.resolve().as_posix(),
-                center=center,
-                processor=processor,
-                original_file_name=original_file_name,
-                video_hash=video_hash,
-                frame_dir=framedir.as_posix(),
-            )
-
-            # Save the instance to the database
-            raw_video_file.save()
-
-            return raw_video_file
-
-    def __str__(self):
-        return self.file.name
-
-    def get_endo_roi(self):
-        endo_roi = self.video_meta.get_endo_roi()
-        return endo_roi
-
-    # video meta should be created when video file is created
-    def save(self, *args, **kwargs):
-        if self.video_meta is None:
-            center = self.center
-            processor = self.processor
-            self.video_meta = VideoMeta.objects.create(
-                center=center, processor=processor
-            )
-            self.video_meta.initialize_ffmpeg_meta(self.file.path)
-        super(RawVideoFile, self).save(*args, **kwargs)
-
-    def extract_frames(
-        self,
-        quality: int = 2,
-        frame_dir: Path = None,
-        overwrite: bool = False,
-        ext="jpg",
-    ):
-        """
-        Extract frames from the video file and save them to the frame_dir.
-        For this, ffmpeg must be available in in the current environment.
-        """
-        if frame_dir is None:
-            frame_dir = Path(self.frame_dir)
-        else:
-            frame_dir = Path(frame_dir)
-
-        if not frame_dir.exists():
-            frame_dir.mkdir(parents=True, exist_ok=True)
-
-        if not overwrite and len(list(frame_dir.glob("*.jpg"))) > 0:
-            print(f"Frames already extracted for {self.file.name}")
-            return
-
-        video_path = Path(self.file.path).resolve().as_posix()
-
-        frame_path_string = frame_dir.resolve().as_posix()
-        command = [
-            "ffmpeg",
-            "-i",
-            video_path,  #
-            "-q:v",
-            str(quality),
-            os.path.join(frame_path_string, f"frame_%07d.{ext}"),
-        ]
-
-        # Ensure FFmpeg is available
-        if not shutil.which("ffmpeg"):
-            raise EnvironmentError(
-                "FFmpeg could not be found. Ensure it is installed and in your PATH."
-            )
-
-        # Extract frames from the video file
-        # Execute the command
-        result = subprocess.run(command, capture_output=True, text=True)
-        if result.returncode != 0:
-            raise Exception(f"Error extracting frames: {result.stderr}")
-
-        self.state_frames_extracted = True
-
-        return f"Frames extracted to {frame_dir} ({frame_path_string}) with quality {quality}"
-
-    def delete_frames(self):
-        """
-        Delete frames extracted from the video file.
-        """
-        frame_dir = Path(self.frame_dir)
-        if frame_dir.exists():
-            shutil.rmtree(frame_dir)
-            self.state_frames_extracted = False
-            self.save()
-            return f"Frames deleted from {frame_dir}"
-        else:
-            return f"No frames to delete for {self.file.name}"
-
-    def get_frame_path(self, n: int = 0):
-        """
-        Get the path to the n-th frame extracted from the video file.
-        Note that the frame numbering starts at 1 in our naming convention.
-        """
-        # Adjust index
-        n = n + 1
-
-        frame_dir = Path(self.frame_dir)
-        return frame_dir / f"frame_{n:07d}.jpg"
-    
-    def get_frame_paths(self):
-        if not self.state_frames_extracted:
-            return None
-        frame_dir = Path(self.frame_dir)
-        paths = [p for p in frame_dir.glob('*')]
-        indices = [int(p.stem.split("_")[1]) for p in paths]
-        path_index_tuples = list(zip(paths, indices))
-        # sort ascending by index
-        path_index_tuples.sort(key=lambda x: x[1])
-        paths, indices = zip(*path_index_tuples)
-
-        return paths
-
-    def get_prediction_dir(self):
-        return Path(self.prediction_dir)
-
-    def get_predictions_path(self, suffix = ".json"):
-        pred_dir = self.get_prediction_dir()
-        return pred_dir.joinpath("predictions").with_suffix(suffix)
-    
-    def get_smooth_predictions_path(self, suffix = ".json"):
-        pred_dir = self.get_prediction_dir()
-        return pred_dir.joinpath("smooth_predictions").with_suffix(suffix)
-    
-    def get_binary_predictions_path(self, suffix = ".json"):
-        pred_dir = self.get_prediction_dir()
-        return pred_dir.joinpath("binary_predictions").with_suffix(suffix)
-    
-    def get_raw_sequences_path(self, suffix = ".json"):
-        pred_dir = self.get_prediction_dir()
-        return pred_dir.joinpath("raw_sequences").with_suffix(suffix)
-
-    def get_filtered_sequences_path(self, suffix=".json"):
-        pred_dir = self.get_prediction_dir()
-        return pred_dir.joinpath("filtered_sequences").with_suffix(suffix)
-
-    def extract_text_information(self, frame_fraction: float = 0.001):
-        """
-        Extract text information from the video file.
-        Makes sure that frames are extracted and then processes the frames.
-        gets all frames from frame_dir and selects a fraction of them to process (at least 1)
-        """
-        if not self.state_frames_extracted:
-            print(f"Frames not extracted for {self.file.name}")
-            return None
-
-        processor = self.processor
-
-        frame_dir = Path(self.frame_dir)
-        frames = list(frame_dir.glob("*"))
-        n_frames = len(frames)
-        n_frames_to_process = max(1, int(frame_fraction * n_frames))
-
-        # Select evenly spaced frames
-        frames = frames[:: n_frames // n_frames_to_process]
-
-        # extract text from each frame and store the value to
-        # defaultdict of lists.
-        # Then, extract the most frequent value from each list
-        # Finally, return the dictionary of most frequent values
-
-        # Create a defaultdict to store the extracted text from each ROI
-        rois_texts = defaultdict(list)
-
-        print(f"Processing {n_frames_to_process} frames from {self.file.name}")
-        # Process frames
-        for frame_path in frames[:n_frames_to_process]:
-            extracted_texts = extract_text_from_rois(frame_path, processor)
-            for roi, text in extracted_texts.items():
-                rois_texts[roi].append(text)
-
-        # Get the most frequent text values for each ROI using Counter
-        for key in rois_texts.keys():
-            counter = Counter([text for text in rois_texts[key] if text])
-            rois_texts[key] = counter.most_common(1)[0][0] if counter else None
-
-        return rois_texts
-
-    def update_text_metadata(self, ocr_frame_fraction=0.001):
-        print(f"Updating metadata for {self.file.name}")
-        texts = self.extract_text_information(ocr_frame_fraction)
-
-        self.sensitive_meta = SensitiveMeta.create_from_dict(texts)
-        self.state_sensitive_data_retrieved = True
-        self.save()
-
-        # Resulting dict depends on defined ROIs for this processor type!
-
-    def update_video_meta(self):
-        video_meta = self.video_meta
-        video_path = Path(self.file.path)
-
-        if video_meta is None:
-            video_meta = VideoMeta.create_from_video(video_path)
-            self.video_meta = video_meta
-            self.save()
-
-        else:
-            video_meta.update_meta(video_path)
-
-    def get_fps(self):
-        if self.video_meta is None:
-            self.update_video_meta()
-
-        if self.video_meta.ffmpeg_meta is None:
-            self.video_meta.initialize_ffmpeg_meta(self.file.path)
-
-        return self.video_meta.get_fps()

endoreg_db/models/data_file/metadata/__init__.py

@@ -1,3 +0,0 @@
-from .sensitive_meta import SensitiveMeta
-from .pdf_meta import PdfMeta, PdfType
-from .video_meta import VideoMeta, FFMpegMeta, VideoImportMeta

endoreg_db/models/data_file/metadata/pdf_meta.py

@@ -1,70 +0,0 @@
-from django.db import models
-
-# import endoreg_center_id from django settings
-from django.conf import settings
-
-
-# import File class
-from django.core.files import File
-
-# # check if endoreg_center_id is set
-# if not hasattr(settings, 'ENDOREG_CENTER_ID'):
-#     ENDOREG_CENTER_ID = 9999
-# else:
-#     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
-
-class PdfType(models.Model):
-    name = models.CharField(max_length=255)
-
-    patient_info_line = models.ForeignKey(
-        "ReportReaderFlag",
-        related_name="pdf_type_patient_info_line", 
-        on_delete=models.CASCADE
-    )
-    endoscope_info_line = models.ForeignKey(
-        "ReportReaderFlag",
-        related_name="pdf_type_endoscopy_info_line", 
-        on_delete=models.CASCADE,
-    )
-    examiner_info_line = models.ForeignKey(
-        "ReportReaderFlag",
-        related_name="pdf_type_examiner_info_line", 
-        on_delete=models.CASCADE
-    )
-    cut_off_above_lines = models.ManyToManyField(
-        "ReportReaderFlag",
-        related_name="pdf_type_cut_off_above_lines", 
-    )
-    cut_off_below_lines = models.ManyToManyField(
-        "ReportReaderFlag",
-        related_name="pdf_type_cut_off_below_lines", 
-    )
-
-
-    def __str__(self):
-        summary = f"{self.name}"
-        # add lines to summary
-        summary += f"\nPatient Info Line: {self.patient_info_line.value}"
-        summary += f"\nEndoscope Info Line: {self.endoscope_info_line.value}"
-        summary += f"\nExaminer Info Line: {self.examiner_info_line.value}"
-        summary += f"\nCut Off Above Lines: {[_.value for _ in self.cut_off_above_lines.all()]}"
-        summary += f"\nCut Off Below Lines: {[_.value for _ in self.cut_off_below_lines.all()]}"
-    
-        return summary
-
-class PdfMeta(models.Model):
-    pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
-    date = models.DateField()
-    time = models.TimeField()
-    pdf_hash = models.CharField(max_length=255, unique=True)
-
-    def __str__(self):
-        return self.pdf_hash
-
-    @classmethod
-    def create_from_file(cls, pdf_file):
-        pdf_file = File(pdf_file)
-        pdf_meta = cls(file=pdf_file)
-        pdf_meta.save()
-        return pdf_meta
-

endoreg_db/models/data_file/metadata/sensitive_meta.py

@@ -1,31 +0,0 @@
-from django.db import models
-
-class SensitiveMeta(models.Model):
-    examination_date = models.DateField(blank=True, null=True)
-    patient_first_name = models.CharField(max_length=255, blank=True, null=True)
-    patient_last_name = models.CharField(max_length=255, blank=True, null=True)
-    patient_dob = models.DateField(blank=True, null=True)
-    endoscope_type = models.CharField(max_length=255, blank=True, null=True)
-    endoscope_sn = models.CharField(max_length=255, blank=True, null=True)
-
-    @classmethod
-    def create_from_dict(cls, data: dict):
-        # data can contain more fields than the model has
-        field_names = [_.name for _ in cls._meta.fields]
-        selected_data = {k: v for k, v in data.items() if k in field_names}
-
-        return cls.objects.create(**selected_data)
-    
-    def update_from_dict(self, data: dict):
-        # data can contain more fields than the model has
-        field_names = [_.name for _ in self._meta.fields]
-        selected_data = {k: v for k, v in data.items() if k in field_names}
-
-        for k, v in selected_data.items():
-            setattr(self, k, v)
-        
-        self.save()
-    
-    def __str__(self):
-        return f"SensitiveMeta: {self.examination_date} {self.patient_first_name} {self.patient_last_name} (*{self.patient_dob})"
-