endoreg-db 0.3.6__py3-none-any.whl → 0.8.6.1__py3-none-any.whl

endoreg_db/models/data_file/import_classes/__init__.py

@@ -1,32 +0,0 @@
-import os
-from pathlib import Path
-
-from .raw_video import RawVideoFile
-from .raw_pdf import RawPdfFile
-
-# FileImporter class
-# This class is used to import data from a file into the database.
-# Expects a directory containing files to import.
-# creates correct import file object depending on file type
-
-# FileImporter class
-# This class is used to import data from a file into the database by creating objects for the files.
-# main method is import_files which expects a path to a directory containing files to import.
-# creates correct import file object depending on file type by checking the file extension
-
-class FileImporter:
-    def __init__(self, directory):
-        self.directory = directory
-
-    def import_files(self):
-        directory_path = Path(self.directory)
-        for file in directory_path.iterdir():
-            if file.is_file():
-                if file.suffix.lower() in ['.mov', '.mp4']:
-                    RawVideoFile.create_from_file(file)
-                else:
-                    raise ValueError(f"File type {file.suffix} not supported")
-            else:
-                raise ValueError(f"{file} is not a file")
-            
-        

endoreg_db/models/data_file/import_classes/processing_functions/__init__.py

@@ -1,35 +0,0 @@
-from .video import (
-    get_videos_scheduled_for_frame_extraction, 
-    extract_frames_from_video,
-    extract_frames_from_videos,
-
-    get_videos_scheduled_for_ocr,
-    videos_scheduled_for_ocr_preflight,
-    perform_ocr_on_video,
-    perform_ocr_on_videos,
-
-    videos_scheduled_for_initial_prediction_preflight,
-    get_videos_scheduled_for_initial_prediction,
-    get_multilabel_model,
-    get_multilabel_classifier,
-    get_crops,
-    perform_initial_prediction_on_video,
-    perform_initial_prediction_on_videos,
-
-    videos_scheduled_for_prediction_import_preflight,
-    get_videos_scheduled_for_prediction_import,
-    import_predictions_for_video,
-    import_predictions_for_videos,
-    
-
-    delete_frames_preflight,
-    get_videos_scheduled_for_frame_deletion,
-    delete_frames_for_video,
-    delete_frames,
-)
-
-from .pdf import (
-    get_pdf_files_scheduled_for_processing,
-    process_pdf_file,
-    process_pdf_files,
-)

endoreg_db/models/data_file/import_classes/processing_functions/pdf.py

@@ -1,28 +0,0 @@
-from ..raw_pdf import RawPdfFile
-from logging import getLogger
-#
-# # setup logging to pdf_import.log
-logger = getLogger('examination_pdf_import')
-
-def get_pdf_files_scheduled_for_processing():
-    reports = RawPdfFile.objects.filter(
-        state_report_processing_required=True
-    )
-    return reports
-
-def process_pdf_file(report:RawPdfFile):
-    if report.update(save=True, verbose=True):
-        logger.info(f"Report {report} processed successfully")
-        return True
-    else:
-        logger.error(f"Report {report} processing failed")
-        return False
-
-
-def process_pdf_files():
-    reports = get_pdf_files_scheduled_for_processing()
-    for report in reports:
-        process_pdf_file(report)
-    
-
-

endoreg_db/models/data_file/import_classes/processing_functions/video.py

@@ -1,260 +0,0 @@
-from ..raw_video import RawVideoFile
-import json 
-
-# # Starting point
-# Automated tasks generate RawVideoFile objects in our db.
-# Each object has state_{NAME} attributes.
-# We will create functions which query the db for RawVideoFile 
-# objects with specific state_{NAME} attributes.
-# Then, we perform the necessary operations on the RawVideoFile and
-# update the state_{NAME} attributes accordingly.
-
-# # Step 1 - Frame Extraction
-# function to query for videos scheduled for frame extraction,
-# these have state_frames_required and state_frames_extracted
-def get_videos_scheduled_for_frame_extraction():
-    return RawVideoFile.objects.filter(
-        state_frames_required=True,
-        state_frames_extracted=False
-    )
-
-def extract_frames_from_video(video:RawVideoFile):
-    # extract frames from video
-    video.extract_frames()
-
-    # update state_frames_extracted
-    video.state_frames_extracted = True
-    video.save()
-
-    return video
-
-def extract_frames_from_videos():
-    videos = get_videos_scheduled_for_frame_extraction()
-    for video in videos:
-        extract_frames_from_video(video)
-
-# # Step 2 - OCR
-# function to query for videos scheduled for OCR,
-# these have 
-# state_ocr_required = True and state_ocr_completed = False and state_frames_extracted = True
-def get_videos_scheduled_for_ocr():
-    return RawVideoFile.objects.filter(
-        state_ocr_required=True,
-        state_ocr_completed=False,
-        state_frames_extracted=True
-    )
-
-# function to set state_frames_required to True for videos 
-# which are scheduled for OCR but have not had frames extracted
-def videos_scheduled_for_ocr_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_ocr_required=True,
-        state_ocr_completed=False,
-        state_frames_extracted=False
-    )
-    for video in videos:
-        video.state_frames_required = True
-        video.save()
-
-def perform_ocr_on_video(video:RawVideoFile):
-    # perform OCR on video
-    video.update_text_metadata()
-
-    # update state_ocr_completed
-    video.state_ocr_completed = True
-    video.save()
-
-    return video
-
-def perform_ocr_on_videos():
-    videos = get_videos_scheduled_for_ocr()
-    for video in videos:
-        perform_ocr_on_video(video)
-
-
-# # Step 3 - initial Prediction
-# function to query for videos scheduled for initial prediction,
-# these have
-# state_initial_prediction_required = True and state_initial_prediction_completed = False and state_frames_extracted = True
-def videos_scheduled_for_initial_prediction_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_initial_prediction_required=True,
-        state_initial_prediction_completed=False,
-        state_frames_extracted=False
-    )
-    for video in videos:
-        video.state_frames_required = True
-        video.save()
-
-def get_videos_scheduled_for_initial_prediction():
-    return RawVideoFile.objects.filter(
-        state_initial_prediction_required=True,
-        state_initial_prediction_completed=False,
-        state_frames_extracted=True
-    )
-
-from pathlib import Path
-def get_multilabel_model(model_path:Path):
-    from agl_predict_endo_frame.model_loader import MultiLabelClassificationNet
-    model_path_str = model_path.resolve().as_posix()
-    model = MultiLabelClassificationNet.load_from_checkpoint(model_path_str)
-    model.cuda()
-    model.eval()
-    return model
-
-def get_multilabel_classifier(model, verbose:bool=False):
-    from agl_predict_endo_frame.predict import Classifier
-    classifier = Classifier(model, verbose = verbose)
-    return classifier
-
-def get_crops(video, paths):
-    endo_roi_dict = video.get_endo_roi()
-    # dict with x, y, width height
-    # crops is list of touples with (y_min, y_max, x_min, x_max)
-    crop_tuple = (
-        endo_roi_dict["y"],
-        endo_roi_dict["y"] + endo_roi_dict["height"],
-        endo_roi_dict["x"],
-        endo_roi_dict["x"] + endo_roi_dict["width"],  
-    )
-    crops = [crop_tuple for _ in paths]
-    return crops
-
-# model = MultiLabelClassificationNet.load_from_checkpoint("model/colo_segmentation_RegNetX800MF_6.ckpt")
-def perform_initial_prediction_on_video(
-    video:RawVideoFile, model_path,
-    window_size_s, min_seq_len_s
-):
-
-    model = get_multilabel_model(model_path)
-    classifier = get_multilabel_classifier(model, verbose = True)
-
-    paths = video.get_frame_paths()
-    string_paths = [p.resolve().as_posix() for p in paths]
-    crops = get_crops(video, string_paths)
-    fps = video.get_fps()
-
-    predictions = classifier.pipe(string_paths, crops)
-    readable_predictions = [classifier.readable(p) for p in predictions]
-    result_dict = classifier.post_process_predictions_serializable(
-        readable_predictions,
-        window_size_s = window_size_s,
-        min_seq_len_s = min_seq_len_s,
-        fps = fps
-    )
-
-    
-    # Predictions
-    _path = video.get_predictions_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["predictions"], f, indent = 4)
-
-    # smooth_predictions
-    _path = video.get_smooth_predictions_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["smooth_predictions"], f, indent = 4)
-
-    # binary_predictions
-    _path = video.get_binary_predictions_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["binary_predictions"], f, indent = 4)
-
-    # Raw Sequences
-    _path = video.get_raw_sequences_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["raw_sequences"], f, indent = 4)
-
-    # filtered_sequences
-    _path = video.get_filtered_sequences_path()
-    with open(_path, "w") as f:
-        json.dump(result_dict["filtered_sequences"], f, indent = 4)
-
-    
-    # update state_initial_prediction_completed
-    video.state_initial_prediction_required = False
-    video.state_initial_prediction_completed = True
-    video.state_initial_prediction_import_required = True
-    video.state_initial_prediction_import_completed = False
-    video.save()
-
-    return video
-
-def perform_initial_prediction_on_videos(
-    model_path,
-    window_size_s, min_seq_len_s
-):
-    videos = get_videos_scheduled_for_initial_prediction()
-    for video in videos:
-        perform_initial_prediction_on_video(
-        video,
-        model_path, window_size_s, min_seq_len_s
-    )
-
-def videos_scheduled_for_prediction_import_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_initial_prediction_completed=True,
-        state_initial_prediction_import_completed=False
-    )
-    for video in videos:
-        video.state_initial_prediction_required = True
-        video.save()
-    
-def get_videos_scheduled_for_prediction_import():
-    return RawVideoFile.objects.filter(
-        state_initial_prediction_import_required=True,
-        state_initial_prediction_import_completed=False,
-        state_initial_prediction_completed=True
-    )
-
-def import_predictions_for_video(video:RawVideoFile):
-    # import predictions for video
-    pass
-
-    # update state_prediction_import_completed
-    # video.state_initial_prediction_import_required = False
-    # video.state_initial_prediction_import_completed = True
-    # video.save()
-
-    return video
-
-def import_predictions_for_videos():
-    videos = get_videos_scheduled_for_prediction_import()
-    for video in videos:
-        import_predictions_for_video(video)
-
-# # Step 4 - Delete Frames if not needed anymore
-# function to query for videos scheduled for frame deletion,
-# first we need to set state_frames_required = False for videos with:
-# state_ocr_required = False and state_ocr_completed = True and
-# state_initial_prediction_required = False and state_initial_prediction_completed = True
-def delete_frames_preflight():
-    videos = RawVideoFile.objects.filter(
-        state_ocr_required=False,
-        state_ocr_completed=True,
-        state_initial_prediction_required=False,
-        state_initial_prediction_completed=True
-    )
-    for video in videos:
-        video.state_frames_required = False
-        video.save()
-
-# function to query for videos scheduled for frame deletion,
-# frames should be deleted if state_frames_required = False
-def get_videos_scheduled_for_frame_deletion():
-    return RawVideoFile.objects.filter(
-        state_frames_required=False
-    )
-
-def delete_frames_for_video(video:RawVideoFile):
-    # delete frames for video
-
-    # update state_frames_deleted
-    video.state_frames_extracted = False
-    video.save()
-
-    return video
-
-def delete_frames():
-    videos = get_videos_scheduled_for_frame_deletion()
-    for video in videos:
-        delete_frames_for_video(video)

endoreg_db/models/data_file/import_classes/raw_pdf.py

@@ -1,188 +0,0 @@
-# models/data_file/import_classes/raw_pdf.py
-# django db model "RawPdf"
-# Class to store raw pdf file using django file field
-# Class contains classmethod to create object from pdf file
-# objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
-# ------------------------------------------------------------------------------
-
-from django.db import models
-from django.core.files.storage import FileSystemStorage
-from django.core.files import File
-from django.conf import settings
-from django.utils import timezone
-from django.core.exceptions import ValidationError
-from django.core.validators import FileExtensionValidator
-from endoreg_db.utils.file_operations import get_uuid_filename
-
-from agl_report_reader.report_reader import ReportReader
-
-from endoreg_db.utils.hashs import get_pdf_hash
-from ..metadata import SensitiveMeta
-
-# setup logging to pdf_import.log
-import logging
-logger = logging.getLogger('pdf_import')
-
-import shutil
-
-# get pdf location from settings, default to ~/erc_data/raw_pdf and create if not exists
-PSEUDO_DIR_RAW_PDF = getattr(settings, 'PSEUDO_DIR_RAW_PDF', settings.BASE_DIR / 'erc_data/raw_pdf')
-
-class RawPdfFile(models.Model):
-    file = models.FileField(
-        upload_to='raw_pdf/',
-        validators=[FileExtensionValidator(allowed_extensions=['pdf'])],
-        storage=FileSystemStorage(location=PSEUDO_DIR_RAW_PDF.resolve().as_posix()),
-    )
-
-    pdf_hash = models.CharField(max_length=255, unique=True)
-    pdf_type = models.ForeignKey('PdfType', on_delete=models.CASCADE)
-    center = models.ForeignKey('Center', on_delete=models.CASCADE)
-
-    state_report_processing_required = models.BooleanField(default = True)
-    state_report_processed = models.BooleanField(default=False)
-
-    # report_file = models.OneToOneField("ReportFile", on_delete=models.CASCADE, null=True, blank=True)
-    sensitive_meta = models.OneToOneField(
-        'SensitiveMeta',
-        on_delete=models.CASCADE,
-        related_name='raw_pdf_file',
-        null=True,
-        blank=True,
-    )
-
-    text = models.TextField(blank=True, null=True)
-    anonymized_text = models.TextField(blank=True, null=True)
-
-    raw_meta = models.JSONField(blank=True, null=True)
-
-    created_at = models.DateTimeField(auto_now_add=True)
-
-    def __str__(self):
-        str_repr = f"RawPdfFile: {self.file.name}"
-        return str_repr
-
-    @classmethod
-    def create_from_file(
-        cls,
-        file_path,
-        center_name,
-        pdf_type_name,
-        destination_dir,
-        save=True,
-    ):
-        from endoreg_db.models import PdfType, Center
-        logger.info(f"Creating RawPdfFile object from file: {file_path}")
-        original_file_name = file_path.name
-
-        new_file_name, uuid = get_uuid_filename(file_path)
-
-        if not destination_dir.exists():
-            destination_dir.mkdir(parents=True)
-
-        pdf_hash = get_pdf_hash(file_path)
-
-        # check if pdf file already exists
-        if cls.objects.filter(pdf_hash=pdf_hash).exists():
-            logger.warning(f"RawPdfFile with hash {pdf_hash} already exists")
-            return None
-        
-        assert pdf_type_name is not None, "pdf_type_name is required"
-        assert center_name is not None, "center_name is required"
-
-        pdf_type = PdfType.objects.get(name=pdf_type_name)
-        center = Center.objects.get(name=center_name)
-
-        new_file_path = destination_dir / new_file_name
-
-        logger.info(f"Copying file to {new_file_path}")
-        success = shutil.copy(file_path, new_file_path)
-         
-        # validate copy operation by comparing hashs
-        assert get_pdf_hash(new_file_path) == pdf_hash, "Copy operation failed"
-
-        raw_pdf = cls(
-            file=new_file_path.resolve().as_posix(),
-            pdf_hash=pdf_hash,
-            pdf_type=pdf_type,
-            center=center,
-        )
-        logger.info(f"RawPdfFile object created: {raw_pdf}")
-
-        # remove source file
-        file_path.unlink()
-        logger.info(f"Source file removed: {file_path}")
-
-        if save:
-            raw_pdf.save()
-        
-
-        return raw_pdf
-
-    def process_file(self, verbose = False):
-        
-        pdf_path = self.file.path
-        rr_config = self.get_report_reader_config()
-
-        rr = ReportReader(**rr_config) #FIXME In future we need to pass a configuration file 
-        # This configuration file should be associated with pdf type 
-
-        text, anonymized_text, report_meta = rr.process_report(pdf_path, verbose=verbose)
-        if not self.sensitive_meta:
-            sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
-            sensitive_meta.save()
-            self.sensitive_meta = sensitive_meta
-
-        else: 
-            # update existing sensitive meta
-            sensitive_meta = self.sensitive_meta
-            sensitive_meta.update_from_dict(report_meta)
-
-        return text, anonymized_text, report_meta
-    
-    def update(self, save=True, verbose = True):
-        try:
-            self.text, self.anonymized_text, self.raw_meta = self.process_file(verbose = verbose)
-            self.state_report_processed = True
-            self.state_report_processing_required = False
-        
-            if save: 
-    
-                self.save()
-
-            return True
-
-        except:
-            logger.error(f"Error processing file: {self.file.path}")
-            return False
-
-    def save(self, *args, **kwargs):
-        if not self.file.name.endswith('.pdf'):
-            raise ValidationError('Only PDF files are allowed')
-        
-        if not self.pdf_hash:
-            self.pdf_hash = get_pdf_hash(self.file.path)
-
-        super().save(*args, **kwargs)
-
-
-    def get_report_reader_config(self):
-        if self.pdf_type.endoscope_info_line:
-            endoscope_info_line = self.pdf_type.endoscope_info_line.value
-        else:
-            endoscope_info_line = None
-        settings_dict = {
-            "locale": "de_DE",
-            "employee_first_names": [_.name for _ in self.center.first_names.all()],
-            "employee_last_names": [_.name for _ in self.center.last_names.all()],
-            "text_date_format":'%d.%m.%Y',
-            "flags": {
-                "patient_info_line": self.pdf_type.patient_info_line.value,
-                "endoscope_info_line": endoscope_info_line,
-                "examiner_info_line": self.pdf_type.examiner_info_line.value,
-                "cut_off_below": [_.value for _ in self.pdf_type.cut_off_below_lines.all()],
-                "cut_off_above": [_.value for _ in self.pdf_type.cut_off_above_lines.all()],
-            }
-        }
-
-        return settings_dict