endoreg-db 0.2.2__py3-none-any.whl → 0.3.0__py3-none-any.whl

endoreg_db/models/data_file/metadata/__init__.py

@@ -0,0 +1,133 @@
+from django.db import models
+import ffmpeg
+from pathlib import Path
+from .sensitive_meta import SensitiveMeta
+
+# import endoreg_center_id from django settings
+from django.conf import settings
+
+# check if endoreg_center_id is set
+if not hasattr(settings, 'ENDOREG_CENTER_ID'):
+    ENDOREG_CENTER_ID = 9999
+else:
+    ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
+
+# VideoMeta
+class VideoMeta(models.Model):
+    processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
+    endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
+    center = models.ForeignKey('Center', on_delete=models.CASCADE)
+    import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
+    ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
+
+    def __str__(self):
+
+        processor_name = self.processor.name if self.processor is not None else "None"
+        endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
+        center_name = self.center.name if self.center is not None else "None"
+
+        result_html = ""
+
+        result_html += f"Processor: {processor_name}<br>"
+        result_html += f"Endoscope: {endoscope_name}<br>"
+        result_html += f"Center: {center_name}<br>"
+
+        return result_html
+    
+    # import meta should be created when video meta is created
+    def save(self, *args, **kwargs):
+        if self.import_meta is None:
+            self.import_meta = VideoImportMeta.objects.create()
+        super(VideoMeta, self).save(*args, **kwargs)
+
+    def initialize_ffmpeg_meta(self, file_path):
+        """Initializes FFMpeg metadata for the video file if not already done."""
+        self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
+        self.save()
+
+    def update_meta(self, file_path):
+        """Updates the video metadata from the file."""
+        self.initialize_ffmpeg_meta(file_path)
+        self.save()
+
+    def get_endo_roi(self):
+        endo_roi = self.processor.get_roi_endoscope_image()
+        return endo_roi
+
+    def get_fps(self):
+        if not self.ffmpeg_meta:
+            return None
+    
+        return self.ffmpeg_meta.frame_rate
+            
+
+class FFMpegMeta(models.Model):
+    # Existing fields
+    duration = models.FloatField(blank=True, null=True)
+    width = models.IntegerField(blank=True, null=True)
+    height = models.IntegerField(blank=True, null=True)
+    frame_rate = models.FloatField(blank=True, null=True)
+    
+    # New fields for comprehensive information
+    video_codec = models.CharField(max_length=50, blank=True, null=True)
+    audio_codec = models.CharField(max_length=50, blank=True, null=True)
+    audio_channels = models.IntegerField(blank=True, null=True)
+    audio_sample_rate = models.IntegerField(blank=True, null=True)
+
+    # Existing __str__ method can be updated to include new fields
+
+    @classmethod
+    def create_from_file(cls, file_path: Path):
+        """Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
+        try:
+            probe = ffmpeg.probe(str(file_path))
+        except ffmpeg.Error as e:
+            print(e.stderr)
+            return None
+
+        video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
+        audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
+
+        # Check for the existence of a video stream
+        if video_stream is None:
+            print(f"No video stream found in {file_path}")
+            return None
+
+        # Extract and store video metadata
+        metadata = {
+            'duration': float(video_stream.get('duration', 0)),
+            'width': int(video_stream.get('width', 0)),
+            'height': int(video_stream.get('height', 0)),
+            'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
+            'video_codec': video_stream.get('codec_name', ''),
+        }
+
+        # If there are audio streams, extract and store audio metadata from the first stream
+        if audio_streams:
+            first_audio_stream = audio_streams[0]
+            metadata.update({
+                'audio_codec': first_audio_stream.get('codec_name', ''),
+                'audio_channels': int(first_audio_stream.get('channels', 0)),
+                'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
+            })
+
+        # Create and return the FFMpegMeta instance
+        return cls.objects.create(**metadata)
+
+class VideoImportMeta(models.Model):
+
+    video_anonymized = models.BooleanField(default=False)
+    video_patient_data_detected = models.BooleanField(default=False)
+    outside_detected = models.BooleanField(default=False)
+    patient_data_removed = models.BooleanField(default=False)
+    outside_removed = models.BooleanField(default=False)
+
+    def __str__(self):
+        result_html = ""
+
+        result_html += f"Video anonymized: {self.video_anonymized}<br>"
+        result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
+        result_html += f"Outside detected: {self.outside_detected}<br>"
+        result_html += f"Patient data removed: {self.patient_data_removed}<br>"
+        result_html += f"Outside removed: {self.outside_removed}<br>"
+        return result_html

endoreg_db/models/data_file/metadata/sensitive_meta.py

@@ -0,0 +1,13 @@
+from django.db import models
+
+class SensitiveMeta(models.Model):
+    examination_date = models.DateField(blank=True, null=True)
+    patient_first_name = models.CharField(max_length=255, blank=True, null=True)
+    patient_last_name = models.CharField(max_length=255, blank=True, null=True)
+    patient_dob = models.DateField(blank=True, null=True)
+    endoscope_type = models.CharField(max_length=255, blank=True, null=True)
+    endoscope_sn = models.CharField(max_length=255, blank=True, null=True)
+
+    @classmethod
+    def create_from_dict(cls, data: dict):
+        return cls.objects.create(**data)

endoreg_db/models/data_file/video/__init__.py

@@ -2,6 +2,6 @@ from .video import (
     Video,
     LegacyVideo,
 )
-from .import_meta import (
+from ..metadata import (
     VideoImportMeta,
 )

endoreg_db/models/data_file/video/import_meta.py

@@ -1,25 +1,25 @@
-from django.db import models
+# from django.db import models
 
-class VideoImportMeta(models.Model):
-    processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE)
-    endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
-    center = models.ForeignKey('Center', on_delete=models.CASCADE)
-    video_anonymized = models.BooleanField(default=False)
-    video_patient_data_detected = models.BooleanField(default=False)
-    outside_detected = models.BooleanField(default=False)
-    patient_data_removed = models.BooleanField(default=False)
-    outside_removed = models.BooleanField(default=False)
+# class VideoImportMeta(models.Model):
+#     processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE)
+#     endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
+#     center = models.ForeignKey('Center', on_delete=models.CASCADE)
+#     video_anonymized = models.BooleanField(default=False)
+#     video_patient_data_detected = models.BooleanField(default=False)
+#     outside_detected = models.BooleanField(default=False)
+#     patient_data_removed = models.BooleanField(default=False)
+#     outside_removed = models.BooleanField(default=False)
 
-    def __str__(self):
-        result_html = ""
+#     def __str__(self):
+#         result_html = ""
 
-        result_html += f"Processor: {self.processor.name}<br>"
-        result_html += f"Endoscope: {self.endoscope.name}<br>"
-        result_html += f"Center: {self.center.name}<br>"
-        result_html += f"Video anonymized: {self.video_anonymized}<br>"
-        result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
-        result_html += f"Outside detected: {self.outside_detected}<br>"
-        result_html += f"Patient data removed: {self.patient_data_removed}<br>"
-        result_html += f"Outside removed: {self.outside_removed}<br>"
-        return result_html
+#         result_html += f"Processor: {self.processor.name}<br>"
+#         result_html += f"Endoscope: {self.endoscope.name}<br>"
+#         result_html += f"Center: {self.center.name}<br>"
+#         result_html += f"Video anonymized: {self.video_anonymized}<br>"
+#         result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
+#         result_html += f"Outside detected: {self.outside_detected}<br>"
+#         result_html += f"Patient data removed: {self.patient_data_removed}<br>"
+#         result_html += f"Outside removed: {self.outside_removed}<br>"
+#         return result_html
     

endoreg_db/models/permissions/__init__.py

@@ -0,0 +1,44 @@
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.auth.models import Permission
+from endoreg_db.models import *  # Import your models here
+from django.db import transaction
+
+# Step 1: Define model to category mappings
+# Assuming every model class name is unique across your entire Django project
+model_categories = {
+    'SensitiveModel1': 'sensitive',
+    'SensitiveModel2': 'sensitive',
+    'DevelopmentModel1': 'development',
+    'DevelopmentModel2': 'development',
+    # Add all models you have, mapping them to either 'sensitive', 'development', or 'all'
+}
+
+# Step 2: Define permissions for each category
+category_permissions = {
+    'sensitive': ['view', 'edit', 'delete'],
+    'development': ['view', 'edit'],
+    'all': ['view'],
+}
+
+@transaction.atomic
+def create_permissions_for_all_models():
+    for model_class_name, category in model_categories.items():
+        model_class = globals().get(model_class_name)
+        if model_class is None:
+            print(f"Model {model_class_name} not found.")
+            continue
+        
+        content_type = ContentType.objects.get_for_model(model_class)
+        permissions = category_permissions.get(category, [])
+
+        for permission_codename in permissions:
+            permission_name = f"Can {permission_codename} {model_class_name}"
+            permission, created = Permission.objects.get_or_create(
+                codename=f"{permission_codename}_{model_class_name.lower()}",
+                defaults={'name': permission_name, 'content_type': content_type},
+            )
+            if created:
+                print(f"Created permission: {permission_name}")
+
+# Run the function to create and assign permissions based on categories
+create_permissions_for_all_models()

endoreg_db/utils/__init__.py

@@ -0,0 +1 @@
+from .dataloader import load_model_data_from_yaml

endoreg_db/utils/cropping.py

@@ -0,0 +1,29 @@
+from PIL import Image
+
+def crop_and_insert(image:Image, x, y, h, w, bg_color=(255, 255, 255)):
+    """
+    Crops a region from an inverted grayscale image and inserts it into a white image of the same size as the original.
+    
+    Parameters:
+    - fp: File path or a file object of the original image.
+    - x, y: The top-left coordinates of the rectangle to be cropped.
+    - h, w: The height and width of the rectangle to be cropped.
+    
+    Returns:
+    A PIL Image object containing the original image with the specified region replaced.
+    """
+    # Load the original image
+    original_image = image
+    
+    # Crop the specified region from the inverted image
+    crop_rectangle = (x, y, x + w, y + h)
+    cropped_content = original_image.crop(crop_rectangle)
+    
+    # Create a new white image of the same size as the original image
+    white_background = Image.new('RGB', original_image.size, bg_color)
+    
+    # Paste the cropped content onto the white image at the specified location
+    white_background.paste(cropped_content, (x, y))
+    
+    # The final image can be displayed or saved as needed
+    return white_background

endoreg_db/utils/dataloader.py

@@ -0,0 +1,69 @@
+import os
+import yaml
+
+def load_model_data_from_yaml(
+        command,
+        model_name,
+        metadata,
+        verbose,
+    ):
+
+    if verbose:
+        command.stdout.write(f"Start Loading {model_name}")
+    model = metadata["model"]
+    dir = metadata["dir"]
+    foreign_keys = metadata["foreign_keys"]
+    foreign_key_models = metadata["foreign_key_models"]
+
+    for file in [f for f in os.listdir(dir) if f.endswith('.yaml')]:
+        with open(os.path.join(dir, file), 'r') as f:
+            yaml_data = yaml.safe_load(f)
+        
+        load_data_with_foreign_keys(
+            command,
+            model,
+            yaml_data,
+            foreign_keys,
+            foreign_key_models,
+            verbose
+        )
+
+def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose):
+            # Since pathology types is a ManyToMany field, we need to hack arount
+            for entry in yaml_data:
+                fields = entry.get('fields', {})
+                name = fields.pop('name', None)
+                many_to_many_tuples = []
+                foreign_key_tuples = zip(foreign_keys, foreign_key_models)
+                for foreign_key, foreign_key_model in foreign_key_tuples:
+                    target_natural_key = fields.pop(foreign_key, None)
+   
+
+                    if isinstance(target_natural_key, list):
+                        # the field is a Many to X field.
+                        fk_objects = [foreign_key_model.objects.get_by_natural_key(_) for _ in target_natural_key]
+                        fk_tuple = (foreign_key, fk_objects)
+                        many_to_many_tuples.append(fk_tuple)
+                        continue
+                    # Use the natural key to look up the related object
+                    try:
+                        obj = foreign_key_model.objects.get_by_natural_key(target_natural_key)
+                    except model.DoesNotExist:
+                        command.stderr.write(command.style.ERROR(f'{model.__name__} with natural key {target_natural_key} does not exist. Skipping {name}.'))
+                        raise Exception(f'{model.__name__} with natural key {target_natural_key} does not exist. Skipping {name}.')
+
+                    # Assign the related object to the field
+                    fields[foreign_key] = obj
+
+                if name:
+                    obj, created = model.objects.get_or_create(name=name, defaults=fields)
+                else:
+                    obj, created = model.objects.get_or_create(**fields)
+                if many_to_many_tuples:
+                    for fk, fk_objects in many_to_many_tuples:
+                        getattr(obj, fk).set(fk_objects)
+
+                if created and verbose:
+                    command.stdout.write(command.style.SUCCESS(f'Created {model.__name__} {name}'))
+                elif verbose:
+                    command.stdout.write(command.style.WARNING(f'Skipped {model.__name__} {name}, already exists'))

endoreg_db/utils/file_operations.py

@@ -0,0 +1,30 @@
+import shutil
+from pathlib import Path
+from endoreg_db.utils.uuid import get_uuid
+
+def get_uuid_filename(file:Path) -> tuple[str, str]:
+    """
+    Returns a new filename with a uuid
+    """
+    # Get the file extension
+    file_extension = file.suffix
+    # Generate a new file name
+    uuid = get_uuid()
+    new_file_name = f"{uuid}{file_extension}"
+    return new_file_name, uuid
+    
+def rename_file_uuid(old_file:Path):
+    """
+    Rename a file by assigning a uuid while preserving file extension. Returns new filepath and uuid
+    """
+    # Get the file extension
+    file_extension = old_file.suffix
+    # Generate a new file name
+    uuid = get_uuid()
+    new_file_name = f"{uuid}{file_extension}"
+
+    # Rename the file
+    new_file = old_file.with_name(new_file_name)
+    shutil.move(old_file.resolve().as_posix(), new_file.resolve().as_posix())
+
+    return new_file, uuid

endoreg_db/utils/hashs.py

@@ -0,0 +1,16 @@
+import hashlib
+
+def get_video_hash(video_path):
+    """
+    Get the hash of a video file.
+    """
+    # Open the video file in read-binary mode:
+    with open(video_path, 'rb') as f:
+        # Create the hash object, passing in the video contents for hashing:
+        hash_object = hashlib.sha256(f.read())
+        # Get the hexadecimal representation of the hash
+        video_hash = hash_object.hexdigest()
+        assert len(video_hash) <= 255, "Hash length exceeds 255 characters"
+
+    return video_hash
+

endoreg_db/utils/legacy_ocr.py

@@ -0,0 +1,201 @@
+import pytesseract
+# import cv2
+from endoreg_db.models import EndoscopyProcessor
+import os
+from collections import Counter
+from tempfile import TemporaryDirectory
+import re
+from datetime import datetime
+from typing import Dict, List
+from icecream import ic
+import numpy as np
+
+N_FRAMES_MEAN_OCR = 2
+
+# Helper function to process date strings
+def process_date_text(date_text):
+    """
+    Processes a string of text that represents a date and returns a datetime.date object.
+
+    Args:
+        date_text (str): A string of text that represents a date.
+
+    Returns:
+        datetime.date: A datetime.date object representing the parsed date, or None if the text cannot be parsed.
+    """
+    try:
+        # Remove any non-digit characters
+        date_text_clean = re.sub(r'\D', '', date_text)
+        # Reformat to 'ddmmyyyy' if necessary
+        if len(date_text_clean) == 8:
+            return datetime.strptime(date_text_clean, "%d%m%Y").date()
+        elif len(date_text_clean) == 14:
+            return datetime.strptime(date_text_clean, "%d%m%Y%H%M%S").date()
+    except ValueError:
+        # Return None if the text cannot be parsed into a date
+        # set date to 1/1/1900
+        return datetime.strptime("01011900", "%d%m%Y").date()
+
+# Helper function to process patient names
+def process_name_text(name_text):
+    """
+    Remove all numbers, punctuation, and whitespace from a string of text and return the result.
+    """
+    name = re.sub(r'[0-9!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\s]+', '', name_text).strip()
+    # capitalize first letter of each word
+    name = ' '.join([word.capitalize() for word in name.split()])
+    return name
+
+
+# Helper function to process endoscope type text
+def process_general_text(endoscope_text):
+    """
+    This function takes in a string of text from an endoscope and returns a cleaned version of the text.
+    """
+    return ' '.join(endoscope_text.split())
+
+def roi_values_valid(roi):
+    """
+    Check if all values in an ROI dictionary are valid (>=0).
+    """
+    return all([value >= 0 for value in roi.values()])
+
+# Function to extract text from ROIs
+def extract_text_from_rois(image_path, processor:EndoscopyProcessor):
+    """
+    Extracts text from regions of interest (ROIs) in an image using OCR.
+
+    Args:
+        image_path (str): The path to the image file.
+        processor (EndoscopyProcessor): An instance of the EndoscopyProcessor class.
+
+    Returns:
+        dict: A dictionary containing the extracted text for each ROI.
+    """
+    # Read the image using OpenCV
+    image = cv2.imread(image_path)
+
+    # Initialize the dictionary to hold the extracted text
+    extracted_texts = {}
+
+    # Define your ROIs and their corresponding post-processing functions in tuples
+    rois_with_postprocessing = [
+        ('examination_date', processor.get_roi_examination_date, process_date_text),
+        ("patient_first_name", processor.get_roi_patient_first_name, process_name_text),
+        ('patient_last_name', processor.get_roi_patient_last_name, process_name_text),
+        ('patient_dob', processor.get_roi_patient_dob, process_date_text),
+        ('endoscope_type', processor.get_roi_endoscope_type, process_general_text),
+        ('endoscope_sn', processor.get_roi_endoscopy_sn, process_general_text),
+    ]
+
+    # Extract and post-process text for each ROI
+    for roi_name, roi_function, post_process in rois_with_postprocessing:
+        # Get the ROI dictionary
+        roi = roi_function()
+        
+        # Check if the ROI has values
+        
+        if roi_values_valid(roi):
+            # Crop the image to the ROI
+            x, y, w, h = roi['x'], roi['y'], roi['width'], roi['height']
+            roi_cropped = image[y:y+h, x:x+w]
+            # Convert to grayscale
+            gray = cv2.cvtColor(roi_cropped, cv2.COLOR_BGR2GRAY)
+
+            # Invert colors for white text on black background
+            gray = cv2.bitwise_not(gray)
+
+            # Binarize the image - using Otsu's method
+            _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+
+            # Dilate the image to improve the contour of the pixelated text
+            kernel = np.ones((2,2), np.uint8)
+            dilation = cv2.dilate(binary, kernel, iterations=1)
+
+            # OCR configuration: Recognize white text on black background without corrections
+            config = '--psm 10 -c tessedit_char_whitelist=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-üöäÜÖÄß'
+
+            # Use pytesseract to do OCR on the preprocessed ROI
+            text = pytesseract.image_to_string(dilation, config=config).strip()
+
+            # Post-process extracted text
+            processed_text = post_process(text)
+            # processed_text = text
+            
+            # Store the processed text in the dictionary
+            extracted_texts[roi_name] = processed_text
+
+        else:
+            ic(roi_name)
+            ic(roi)
+            ic("No values for this ROI")
+
+    return extracted_texts
+
+
+def get_most_frequent_values(rois_texts: Dict[str, List[str]]) -> Dict[str, str]:
+    """
+    Given a dictionary of ROIs and their corresponding texts, returns a dictionary of the most frequent text for each ROI.
+
+    Args:
+        rois_texts: A dictionary where the keys are the names of the ROIs and the values are lists of texts.
+
+    Returns:
+        A dictionary where the keys are the names of the ROIs and the values are the most frequent text for each ROI.
+    """
+    most_frequent = {}
+    for key in rois_texts.keys():
+        counter = Counter([text for text in rois_texts[key] if text])
+        ic(key)
+        ic(counter)
+        most_frequent[key], _ = counter.most_common(1)[0] if counter else (None, None)
+    return most_frequent
+
+def process_video(video_path, processor):
+    """
+    Processes a video file by extracting text from regions of interest (ROIs) in each frame.
+
+    Args:
+        video_path (str): The path to the video file to process.
+        processor (OCRProcessor): An instance of the OCRProcessor class that defines the ROIs to extract text from.
+
+    Returns:
+        dict: A dictionary containing the most frequent text values extracted from each ROI.
+    """
+    # Create a temporary directory to store frames
+    with TemporaryDirectory() as temp_dir:
+        ic(temp_dir)
+        # Capture the video
+        video = cv2.VideoCapture(video_path)
+        success, frame_number = True, 0
+        rois_texts = {roi_name: [] for roi_name in processor.get_rois().keys()}
+        frames_for_mean_extraction = 0
+
+        while success:
+            success, frame = video.read()
+
+            # Check if this is the 200th frame
+            if frame_number % 1000 == 0 and success:
+                frame_path = os.path.join(temp_dir, f"frame_{frame_number}.jpg")
+                cv2.imwrite(frame_path, frame)  # Save the frame as a JPEG file
+                # cv2.imwrite(f"_tmp/frame_{frame_number}.jpg", frame)
+                
+                # Extract text from ROIs
+                extracted_texts = extract_text_from_rois(frame_path, processor)
+                ic(extracted_texts)
+                
+                # Store the extracted text from each ROI
+                for key, text in extracted_texts.items():
+                    rois_texts[key].append(text)
+                frames_for_mean_extraction += 1
+
+            frame_number += 1
+
+            if frames_for_mean_extraction >= N_FRAMES_MEAN_OCR: break
+
+        # Release the video capture object
+        video.release()
+
+        # Get the most frequent values for each ROI
+        return get_most_frequent_values(rois_texts)
+