endoreg-db 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

endoreg_db/models/persons/patient.py

@@ -0,0 +1,58 @@
+from .person import Person
+from django import forms
+from django.forms import DateInput
+from rest_framework import serializers
+from ..patient_examination import PatientExamination
+from ..data_file import ReportFile
+from django.db import models
+
+class Patient(Person):
+    """
+    A class representing a patient.
+
+    Attributes inhereted from Person:
+        first_name (str): The first name of the patient.
+        last_name (str): The last name of the patient.
+        dob (datetime.date): The date of birth of the patient.
+        gender (str): The gender of the patient.
+        email (str): The email address of the patient.
+        phone (str): The phone number of the patient.
+
+    """
+    center = models.ForeignKey("Center", on_delete=models.CASCADE, blank=True, null=True)
+
+    def __str__(self):
+        return self.first_name + " " + self.last_name + " (" + str(self.dob) + ")"
+    
+    def get_unmatched_report_files(self): #field: self.report_files; filter: report_file.patient_examination = None
+        '''Returns all report files for this patient that are not matched to a patient examination.'''
+
+        return self.reportfile_set.filter(patient_examination=None)
+
+    def get_unmatched_video_files(self): #field: self.videos; filter: video.patient_examination = None
+        '''Returns all video files for this patient that are not matched to a patient examination.'''
+        return self.videos.filter(patient_examination=None)
+
+    def get_patient_examinations(self): #field: self.patient_examinations
+        '''Returns all patient examinations for this patient ordered by date (most recent is first).'''
+        return self.patient_examinations.order_by('-date')
+    
+    def create_examination_by_report_file(self, report_file:ReportFile):
+        '''Creates a patient examination for this patient based on the given report file.'''
+        patient_examination = PatientExamination(patient=self, report_file=report_file)
+        patient_examination.save()
+        return patient_examination
+        
+
+class PatientForm(forms.ModelForm):
+    class Meta:
+        model = Patient
+        fields = '__all__'
+        widgets = {
+            'dob': DateInput(attrs={'type': 'date'}),
+        }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        for field in self.fields.values():
+            field.widget.attrs['class'] = 'form-control'

endoreg_db/models/persons/person.py

@@ -0,0 +1,34 @@
+from abc import abstractmethod
+from django.db import models
+
+class Person(models.Model):
+    """
+    Abstract base class for a person.
+
+    Attributes:
+        first_name (str): The first name of the person.
+        last_name (str): The last name of the person.
+        dob (date): The date of birth of the person.
+        gender (str): The gender of the person.
+        email (str): The email address of the person.
+        phone (str): The phone number of the person.
+    """
+
+    first_name = models.CharField(max_length=255)
+    last_name = models.CharField(max_length=255)
+    dob = models.DateField("Date of Birth", blank=True, null=True)
+    gender = models.CharField(
+        max_length=10, choices=[('Male', 'Male'), ('Female', 'Female'), ('Other', 'Other')],
+        blank=True, null=True
+    )
+    email = models.EmailField(max_length=255, blank=True, null=True)
+    phone = models.CharField(max_length=255, blank=True, null=True)
+    is_real_person = models.BooleanField(default=True)
+
+    @abstractmethod
+    def __str__(self):
+        pass
+
+    class Meta:
+        abstract = True
+

endoreg_db/models/persons/portal_user_information.py

@@ -0,0 +1,29 @@
+from django.db import models
+
+# models.py in your main app
+
+from django.db import models
+from django.contrib.auth.models import User
+
+class ProfessionManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class Profession(models.Model):
+    objects = ProfessionManager()
+    name = models.CharField(max_length=100)
+    name_de = models.CharField(max_length=100, blank=True, null=True)
+    name_en = models.CharField(max_length=100, blank=True, null=True)
+    description = models.TextField(blank=True, null=True)
+
+    def __str__(self):
+        return self.name_de
+
+class PortalUserInfo(models.Model):
+    user = models.OneToOneField(User, on_delete=models.CASCADE)
+    profession = models.ForeignKey('Profession', on_delete=models.CASCADE, blank=True, null=True)
+    works_in_endoscopy = models.BooleanField(blank=True, null=True)
+    # Add other fields as needed
+
+    def __str__(self):
+        return self.user.username

endoreg_db/models/prediction/__init__.py

@@ -0,0 +1,2 @@
+from .image_classification import ImageClassificationPrediction
+from .video_prediction_meta import LegacyVideoPredictionMeta, VideoPredictionMeta

endoreg_db/models/prediction/image_classification.py

@@ -0,0 +1,37 @@
+from django.db import models
+
+class ImageClassificationPrediction(models.Model):
+    """
+    A class representing an image classification prediction.
+
+    Attributes:
+        label (Label): The label of the prediction.
+        frame (Frame): The frame of the prediction.
+        confidence (float): The confidence of the prediction.
+        date_created (datetime): The date the prediction was created.
+
+    """
+    label = models.ForeignKey("Label", on_delete=models.CASCADE, related_name="image_classification_predictions")
+    frame = models.ForeignKey("Frame", on_delete=models.CASCADE, blank=True, null=True, related_name="image_classification_predictions")
+    legacy_frame = models.ForeignKey("LegacyFrame", on_delete=models.CASCADE, blank=True, null=True, related_name="image_classification_predictions")
+    legacy_image = models.ForeignKey("LegacyImage", on_delete=models.CASCADE, blank=True, null=True, related_name="image_classification_predictions")
+    value = models.BooleanField()
+    confidence = models.FloatField()
+    model_meta = models.ForeignKey("ModelMeta", on_delete=models.CASCADE, related_name="image_classification_predictions")
+    date_created = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        unique_together = ('label', 'frame', 'model_meta')
+
+    def get_image_object(self):
+        """
+        Get the image of the prediction.
+        """
+        if self.frame:
+            return self.frame
+        elif self.legacy_frame:
+            return self.legacy_frame
+        elif self.legacy_image:
+            return self.legacy_image
+        else:
+            return None

endoreg_db/models/prediction/video_prediction_meta.py

@@ -0,0 +1,244 @@
+from django.db import models
+
+from endoreg_db.models.label.label import LabelSet
+from ..data_file.video import LegacyVideo, Video
+from ..data_file.frame import LegacyFrame, Frame
+from .image_classification import ImageClassificationPrediction
+from ..data_file.video_segment import LegacyLabelVideoSegment, LabelVideoSegment, find_segments_in_prediction_array
+from ..information_source import get_prediction_information_source
+import numpy as np
+import pickle
+
+DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
+DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
+
+class AbstractVideoPredictionMeta(models.Model):
+    model_meta = models.ForeignKey("ModelMeta", on_delete=models.CASCADE)
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    video = None  # Placeholder for the video field, to be defined in derived classes
+    prediction_array = models.BinaryField(blank=True, null=True)
+
+    class Meta:
+        abstract = True
+        unique_together = ('model_meta', 'video')
+
+    def __str__(self):
+        return f"Video {self.video.id} - {self.model_meta.name}"
+    
+    def get_labelset(self):
+        """
+        Get the labelset of the predictions.
+        """
+        return self.model_meta.labelset
+    
+    def get_video_model(self):
+        assert 1 == 2, "This method should be overridden in derived classes"
+
+    def get_frame_model(self):
+        assert 1 == 2, "This method should be overridden in derived classes"
+
+    def get_label_list(self):
+        """
+        Get the label list of the predictions.
+        """
+        labelset:LabelSet = self.get_labelset()
+        label_list = labelset.get_labels_in_order()
+        return label_list
+    
+    def get_video_segment_model(self):
+        assert 1 == 2, "This method should be overridden in derived classes"
+    
+    def save_prediction_array(self, prediction_array:np.array):
+        """
+        Save the prediction array to the database.
+        """
+        self.prediction_array = pickle.dumps(prediction_array)
+        self.save()
+
+    def get_prediction_array(self):
+        """
+        Get the prediction array from the database.
+        """
+        if self.prediction_array is None:
+            return None
+        else:
+            return pickle.loads(self.prediction_array)
+        
+    def calculate_prediction_array(self):
+        assert 1 == 2, "This method should be overridden in derived classes"
+
+    def apply_running_mean(self, confidence_array, window_size_in_seconds: int = None):
+        """
+        Apply a running mean filter to the confidence array for smoothing, assuming a padding
+        of 0.5 for the edges.
+
+        Args:
+        self: Object that has video and fps attributes, and to which this function belongs.
+        confidence_array: A 2D numpy array with dimensions (num_frames),
+                        containing confidence scores for each label at each frame.
+        window_size_in_seconds: The window size for the running mean in seconds.
+
+        Returns:
+        running_mean_array: A 2D numpy array with the same dimensions as confidence_array,
+                            containing the smoothed confidence scores.
+        """
+        video = self.video
+        fps = video.fps
+
+        if not window_size_in_seconds:
+            window_size_in_seconds = DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN
+
+        # Calculate window size in frames, ensuring at least one frame
+        window_size_in_frames = int(window_size_in_seconds * fps)
+        window_size_in_frames = max(window_size_in_frames, 1)
+
+        # Define the window for the running mean
+        window = np.ones(window_size_in_frames) / window_size_in_frames
+
+        # Create running mean array with the same shape as the confidence array
+        running_mean_array = np.zeros(confidence_array.shape)
+
+        # Calculate the padding size
+        pad_size = window_size_in_frames // 2
+
+        # Pad the array with 0.5 on both sides
+        padded_confidences = np.pad(confidence_array, (pad_size, pad_size), 'constant', constant_values=(0.5, 0.5))
+        
+        # Apply the running mean filter on the padded array
+        running_mean = np.convolve(padded_confidences, window, mode='same')
+        
+        # Remove the padding from the result to match the original shape
+        running_mean = running_mean[pad_size:-pad_size]
+
+        return running_mean
+    
+
+    def create_video_segments_for_label(self, segments, label):
+        """
+        Creates video segments for the given label and segments.
+        Segments is a list of tuples (start_frame_number, end_frame_number).
+        Labels is a Label object.
+        """
+        video = self.video
+        video_segment_model = self.get_video_segment_model()
+        information_source = get_prediction_information_source()
+
+        for segment in segments:
+            start_frame_number, end_frame_number = segment
+
+            video_segment = video_segment_model(
+                video=video,
+                prediction_meta=self,
+                start_frame_number=start_frame_number,
+                end_frame_number=end_frame_number,
+                source=information_source,
+                label=label,
+            )
+            video_segment.save()
+
+    def create_video_segments(self, segment_length_threshold_in_s:float=None):
+        if not segment_length_threshold_in_s:
+            segment_length_threshold_in_s = DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S
+
+        video = self.video
+        fps = video.fps
+        min_frame_length = int(segment_length_threshold_in_s * fps)
+
+        label_list = self.get_label_list()
+        
+        # if prediction array doesnt exist, create it
+        if self.prediction_array is None:
+            self.calculate_prediction_array()
+
+        prediction_array = self.get_prediction_array()
+
+        for i, label in enumerate(label_list):
+            # get predictions for this label
+            predictions = prediction_array[:, i]
+            # find segments of predictions that are longer than the threshold
+            # segments is a list of tuples (start_frame_number, end_frame_number)
+            segments = find_segments_in_prediction_array(predictions, min_frame_length)
+
+            # create video segments
+            self.create_video_segments_for_label(segments, label)
+
+import numpy as np
+class VideoPredictionMeta(AbstractVideoPredictionMeta):
+    video = models.OneToOneField("Video", on_delete=models.CASCADE, related_name="video_prediction_meta")
+
+    def get_video_model(self):
+        return Video
+    
+    def get_frame_model(self):
+        return Frame
+    
+    def get_video_segment_model(self):
+        return LabelVideoSegment
+    
+    def calculate_prediction_array(self, window_size_in_seconds:int=None):
+        """
+        Fetches all predictions for this video, labelset, and model meta.
+        """
+        video:Video = self.video
+
+        model_meta = self.model_meta
+        label_list = self.get_label_list()
+
+        prediction_array = np.zeros((video.get_frame_number, len(label_list)))
+        for i, label in enumerate(label_list):
+            # fetch all predictions for this label, video, and model meta ordered by ImageClassificationPrediction.frame.frame_number
+            predictions = ImageClassificationPrediction.objects.filter(label=label, frame__video=video, model_meta=model_meta).order_by('frame__frame_number')
+            confidences = np.array([prediction.confidence for prediction in predictions])
+            smooth_confidences = self.apply_running_mean(confidences, window_size_in_seconds)
+            # calculate binary predictions
+            binary_predictions = smooth_confidences > 0.5
+            # add to prediction array
+            prediction_array[:, i] = binary_predictions
+
+        # save prediction array
+        self.save_prediction_array(prediction_array)
+
+
+class LegacyVideoPredictionMeta(AbstractVideoPredictionMeta):
+    video = models.OneToOneField("LegacyVideo", on_delete=models.CASCADE, related_name="video_prediction_meta")
+
+    def get_video_model(self):
+        return LegacyVideo
+
+    def get_frame_model(self):
+        return LegacyFrame
+    
+    def get_video_segment_model(self):
+        return LegacyLabelVideoSegment
+    
+    def calculate_prediction_array(self, window_size_in_seconds:int=None):
+        """
+        Fetches all predictions for this video, labelset, and model meta.
+        """
+        video:LegacyVideo = self.video
+
+        model_meta = self.model_meta
+        label_list = self.get_label_list()
+
+        prediction_array = np.zeros((video.get_frame_number, len(label_list)))
+        for i, label in enumerate(label_list):
+            # fetch all predictions for this label, video, and model meta ordered by ImageClassificationPrediction.frame.frame_number
+            predictions = ImageClassificationPrediction.objects.filter(label=label, legacy_frame__video=video, model_meta=model_meta).order_by('legacy_frame__frame_number')
+            confidences = np.array([prediction.confidence for prediction in predictions])
+            smooth_confidences = self.apply_running_mean(confidences, window_size_in_seconds)
+            # calculate binary predictions
+            binary_predictions = smooth_confidences > 0.5
+            # add to prediction array
+            prediction_array[:, i] = binary_predictions
+
+        # save prediction array
+        self.save_prediction_array(prediction_array)
+
+    
+            
+
+
+
+
+

endoreg_db/models/unit.py

@@ -0,0 +1,20 @@
+from django.db import models
+
+class UnitManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class Unit(models.Model):
+    objects = UnitManager()
+
+    name = models.CharField(max_length=100) # e.g. "Centimeter"
+    name_de = models.CharField(max_length=100, blank=True, null=True) # e.g. "Zentimeter"
+    name_en = models.CharField(max_length=100, blank=True, null=True) # e.g. "Centimeter"
+    description = models.CharField(max_length=100, blank=True, null=True) # e.g. "centimeters", "millimeters", "inches"
+    abbreviation = models.CharField(max_length=10, blank=True, null=True) # e.g. "cm", "mm", "in"
+
+    def __str__(self):
+        return self.abbreviation
+    
+    def natural_key(self):
+        return (self.name,)

endoreg_db/queries/__init__.py

@@ -0,0 +1,5 @@
+from .annotations import (
+    generate_legacy_dataset_output
+)
+
+from .get import *

endoreg_db/queries/annotations/__init__.py

@@ -0,0 +1,3 @@
+from .legacy import (
+    get_legacy_annotations_for_labelset, generate_legacy_dataset_output
+)

endoreg_db/queries/annotations/legacy.py

@@ -0,0 +1,159 @@
+from ...models import LabelSet, ImageClassificationAnnotation
+from django.db.models import Q, F
+from django.db import models
+from icecream import ic
+from tqdm import tqdm
+from collections import defaultdict
+
+# def get_legacy_annotations_for_labelset(labelset_name, version=None):
+#     """
+#     Retrieve annotations for a given label set for training.
+
+#     Args:
+#     - labelset_name (str): The name of the label set.
+#     - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+#     Returns:
+#     - list[dict]: A list of dictionaries. Each dictionary represents an image and its annotations.
+#                   Format: [{"frame": <frame_object>, "annotations": [{"label": <label_name>, "value": <value>}, ...]}, ...]
+
+#     Example:
+#         annotations_for_training = get_annotations_for_labelset("YourLabelSetName", version=2)
+
+#     """
+
+#     # Fetch the label set based on the name and optionally the version
+#     if version:
+#         labelset = LabelSet.objects.get(name=labelset_name, version=version)
+#     else:
+#         labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
+#         if not labelset:
+#             raise ValueError(f"No label set found with the name: {labelset_name}")
+
+#     # Retrieve all labels in the label set
+#     labels_in_set = labelset.labels.all()
+
+#     # Get the most recent annotations for each frame/label combination
+#     annotations = ImageClassificationAnnotation.objects.filter(label__in=labels_in_set)
+#     annotations = annotations.annotate(
+#         latest_annotation=models.Window(
+#             expression=models.functions.RowNumber(),
+#             partition_by=[F('legacy_image'), F('label')],
+#             order_by=F('date_modified').desc()
+#         )
+#     ).filter(latest_annotation=1)
+
+#     # Organize the annotations by image/frame
+#     organized_annotations = []
+
+#     for annotation in tqdm(annotations):
+#         # ic(annotation)
+#         # Check if the frame is already in the organized list
+#         existing_entry = next((entry for entry in organized_annotations if entry['legacy_image'] == annotation.legacy_frame), None)
+
+#         if existing_entry:
+#             # Add this annotation to the existing frame's annotations
+#             existing_entry['annotations'].append({
+#                 "label": annotation.label.name,
+#                 "value": annotation.value
+#             })
+#         else:
+#             # Create a new entry for this frame
+#             organized_annotations.append({
+#                 "legacy_image": annotation.legacy_image,
+#                 "annotations": [{
+#                     "label": annotation.label.name,
+#                     "value": annotation.value
+#                 }]
+#             })
+
+#     return organized_annotations
+
+
+
+def get_legacy_annotations_for_labelset(labelset_name, version=None):
+    """
+    ... [rest of your docstring]
+    """
+
+    # Fetch the label set based on the name and optionally the version
+    if version:
+        labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    else:
+        labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
+        if not labelset:
+            raise ValueError(f"No label set found with the name: {labelset_name}")
+
+    # Retrieve all labels in the label set
+    labels_in_set = labelset.labels.all()
+
+    # Get the most recent annotations for each frame/label combination
+    annotations = (ImageClassificationAnnotation.objects
+                   .filter(label__in=labels_in_set)
+                   .select_related('legacy_image', 'label')  # Reduce number of queries
+                   .annotate(
+                        latest_annotation=models.Window(
+                            expression=models.functions.RowNumber(),
+                            partition_by=[F('legacy_image'), F('label')],
+                            order_by=F('date_modified').desc()
+                        )
+                    ).filter(latest_annotation=1))
+
+    # Organize the annotations by image/frame using a defaultdict
+    organized_annotations_dict = defaultdict(lambda: {
+        "legacy_image": None,
+        "annotations": []
+    })
+
+    for annotation in tqdm(annotations):
+        organized_entry = organized_annotations_dict[annotation.legacy_image.id]
+        organized_entry["legacy_image"] = annotation.legacy_image
+        organized_entry["annotations"].append({
+            "label": annotation.label.name,
+            "value": annotation.value
+        })
+
+    # Convert organized_annotations_dict to a list
+    organized_annotations = list(organized_annotations_dict.values())
+
+    return organized_annotations
+
+def generate_legacy_dataset_output(labelset_name, version=None):
+    """
+    Generate an output suitable for creating PyTorch datasets.
+
+    Args:
+    - labelset_name (str): The name of the label set.
+    - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+    Returns:
+    - list[dict]: A list of dictionaries, where each dictionary contains the file path and the labels.
+                  Format: [{"path": <file_path>, "labels": [<label_1_value>, <label_2_value>, ...]}, ...]
+    - labelset[LabelSet]: The label set that was used to generate the output.
+    """
+
+    # First, retrieve the organized annotations using the previously defined function
+    organized_annotations = get_legacy_annotations_for_labelset(labelset_name, version)
+
+    # Fetch all labels from the labelset for consistent ordering
+    labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    all_labels = labelset.get_labels_in_order()
+
+    dataset_output = []
+
+    for entry in organized_annotations:
+        # Prepare a dictionary for each frame
+        frame_data = {
+            "path": entry['legacy_image'].image.path,  # Assuming 'image' field stores the file path
+            "labels": [-1] * len(all_labels)  # Initialize with -1 for all labels
+        }
+
+        # Update the labels based on the annotations
+        for annotation in entry['annotations']:
+            index = next((i for i, label in enumerate(all_labels) if label.name == annotation['label']), None)
+            if index is not None:
+                frame_data['labels'][index] = int(annotation['value'])
+
+        dataset_output.append(frame_data)
+
+    return dataset_output, labelset

endoreg_db/queries/get/__init__.py

@@ -0,0 +1,6 @@
+from .annotation import *
+from .center import *
+from .model import *
+from .patient import *
+from .patient_examination import *
+from .video import *

endoreg_db/queries/get/center.py

@@ -0,0 +1,42 @@
+from endoreg_db.models import Center
+from typing import Optional
+
+def get_centers() -> Center:
+    """
+    Returns all Center objects from the database.
+    """
+    return Center.objects.all()
+
+def get_center_by_name(name) -> Optional[Center]:
+    """Retrieve a Center object by its name.
+
+    Args:
+        name (str): The name of the center to retrieve.
+
+    Returns:
+        Optional[Center]: The Center object with the given name, or None if it does not exist.
+    """
+    return Center.objects.get(name=name)
+
+def get_center_by_id(id) -> Optional[Center]:
+    """Retrieve a Center object by its id.
+
+    Args:
+        id (int): The id of the center to retrieve.
+
+    Returns:
+        Optional[Center]: The Center object with the given id, or None if it does not exist.
+    """
+    return Center.objects.get(id=id)
+
+def get_center_by_natural_key(name: str) -> Optional[Center]:
+    """
+    Retrieve a Center object by its natural key.
+
+    Args:
+        name: The name of the center to retrieve.
+
+    Returns:
+        The Center object with the given name, or None if it does not exist.
+    """
+    return Center.objects.get_by_natural_key(name=name)

endoreg_db/queries/get/model.py

@@ -0,0 +1,13 @@
+from endoreg_db.models import (
+    ModelMeta,
+)
+
+def get_latest_model_by_name(model_name):
+    """
+    Expects model name. Fetches models by name from database, sorts by version and returns latest version.
+    """
+    models = ModelMeta.objects.filter(name=model_name).order_by('-version')
+    if len(models) == 0:
+        return None
+    else:
+        return models[0]

endoreg_db/queries/get/patient.py

@@ -0,0 +1,14 @@
+from endoreg_db.models import Patient
+
+def get_patients() -> Patient:
+    """
+    Returns all Patient objects from the database.
+    """
+    return Patient.objects.all()
+
+def get_patients_without_dob() -> Patient:
+    """
+    Returns all Patient objects from the database without a date of birth.
+    """
+    return Patient.objects.filter(dob__isnull=True)
+