endoreg-db 0.6.0__py3-none-any.whl → 0.6.2__py3-none-any.whl

endoreg_db/models/data_file/video/video.py

@@ -0,0 +1,196 @@
+from django.db import models
+from endoreg_db.models.data_file.frame import Frame
+from django.core.validators import FileExtensionValidator
+from django.core.files.storage import FileSystemStorage
+from typing import TYPE_CHECKING
+import cv2
+from ..base_classes import AbstractVideoFile
+from ..base_classes.utils import (
+    VIDEO_DIR_NAME,
+    STORAGE_LOCATION,
+    FRAME_PROCESSING_BATCH_SIZE,
+)
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+    from endoreg_db.models import (
+        LabelVideoSegment,
+        VideoImportMeta,
+        SensitiveMeta,
+        Patient,
+        PatientExamination,
+        RawVideoFile,
+    )
+
+
+class Video(AbstractVideoFile):
+    file = models.FileField(
+        upload_to=VIDEO_DIR_NAME,
+        validators=[FileExtensionValidator(allowed_extensions=["mp4"])],  # FIXME
+        storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
+    )
+
+    patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+        related_name="videos",
+    )
+
+    examination = models.ForeignKey(
+        "PatientExamination",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="videos",
+    )
+
+    # Deprecate and move to video meta?
+    date = models.DateField(blank=True, null=True)
+    suffix = models.CharField(max_length=255, blank=True, null=True)
+    fps = models.FloatField(blank=True, null=True)
+    duration = models.FloatField(blank=True, null=True)
+    width = models.IntegerField(blank=True, null=True)
+    height = models.IntegerField(blank=True, null=True)
+    endoscope_image_x = models.IntegerField(blank=True, null=True)
+    endoscope_image_y = models.IntegerField(blank=True, null=True)
+    endoscope_image_width = models.IntegerField(blank=True, null=True)
+    endoscope_image_height = models.IntegerField(blank=True, null=True)
+
+    state_frames_extracted = models.BooleanField(default=False)
+
+    meta = models.JSONField(blank=True, null=True)
+
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.CASCADE,
+        related_name="videos",
+        null=True,
+        blank=True,
+    )
+
+    import_meta = models.OneToOneField(
+        "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
+    )
+
+    if TYPE_CHECKING:
+        import_meta: "VideoImportMeta"
+        patient: "Patient"
+        examination: "PatientExamination"
+        frames: "QuerySet[Frame]"
+        label_video_segments: (
+            "QuerySet[LabelVideoSegment]"  # Foreign key in LabelVideoSegment.video
+        )
+        sensitive_meta: "SensitiveMeta"
+        raw_videos: "QuerySet[RawVideoFile]"
+
+    # override the save method to set fps if not set
+    def save(self, *args, **kwargs):
+        if not self.fps:
+            self.fps = self.get_fps()
+        super().save(*args, **kwargs)
+
+    def label_segments_to_frame_annotations(self):
+        """
+        Generate annotations for all label video segments.
+        """
+        for lvs in self.label_video_segments.all():
+            lvs.generate_annotations()
+
+    def sync_from_raw_video(self):
+        """
+        Sync metadata from the associated raw video file.
+        """
+        from endoreg_db.models import RawVideoFile, LabelVideoSegment
+
+        raw_video: RawVideoFile = self.raw_videos.first()
+
+        assert isinstance(raw_video, RawVideoFile), "Raw video is not a file"
+
+        self.predictions = raw_video.predictions
+        self.readable_predictions = raw_video.readable_predictions
+        self.sequences = raw_video.sequences
+
+        label_video_segments = raw_video.label_video_segments.all()
+
+        label_video_segments = [
+            LabelVideoSegment.from_raw(
+                video=self, raw_label_video_segment=raw_label_video_segment
+            )
+            # LabelVideoSegment.from_raw(self, raw_label_video_segment)
+            for raw_label_video_segment in label_video_segments
+        ]
+
+        for lvs in label_video_segments:
+            lvs.save()
+
+        self.state_histology_required = raw_video.state_histology_required
+        self.state_histology_available = raw_video.state_histology_available
+        self.state_follow_up_intervention_required = (
+            raw_video.state_follow_up_intervention_required
+        )
+        self.state_follow_up_intervention_available = (
+            raw_video.state_follow_up_intervention_available
+        )
+        self.state_dataset_complete = raw_video.state_dataset_complete
+
+        self.save()
+
+    ## Deprecated
+    # def get_roi_endoscope_image(self):
+    #     return {
+    #         "x": self.endoscope_image_content_x,
+    #         "y": self.endoscope_image_content_y,
+    #         "width": self.endoscope_image_content_width,
+    #         "height": self.endoscope_image_content_height,
+    #     }
+
+    # def initialize_metadata_in_db(self, video_meta=None):
+    #     if not video_meta:
+    #         video_meta = self.meta
+    #     self.set_examination_date_from_video_meta(video_meta)
+    #     self.patient, created = self.get_or_create_patient(video_meta)
+    #     self.save()
+
+    # def get_or_create_patient(self, video_meta=None):
+    #     from ...persons import Patient
+
+    #     if not video_meta:
+    #         video_meta = self.meta
+
+    #     patient_first_name = video_meta["patient_first_name"]
+    #     patient_last_name = video_meta["patient_last_name"]
+    #     patient_dob = video_meta["patient_dob"]
+
+    #     # assert that we got all the necessary information
+    #     assert patient_first_name and patient_last_name and patient_dob, (
+    #         "Missing patient information"
+    #     )
+
+    #     patient, created = Patient.objects.get_or_create(
+    #         first_name=patient_first_name, last_name=patient_last_name, dob=patient_dob
+    #     )
+
+    #     return patient, created
+
+    # DEPRECATED
+    # def set_examination_date_from_video_meta(self, video_meta=None):
+    #     if not video_meta:
+    #         video_meta = self.meta
+    #     date_str = video_meta["examination_date"]  # e.g. 2020-01-01
+    #     if date_str:
+    #         self.date = date.fromisoformat(date_str)
+    #         self.save()
+
+    def initialize_video_specs(self):
+        """
+        Initialize and save video metadata like framerate, dimensions, and duration.
+        """
+        video = cv2.VideoCapture(self.file.path)
+        self.fps = video.get(cv2.CAP_PROP_FPS)
+        self.width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
+        self.height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        self.duration = video.get(cv2.CAP_PROP_FRAME_COUNT) / self.fps
+        self.save()
+        video.release()

endoreg_db/models/data_file/video_segment.py

@@ -0,0 +1,214 @@
+from django.db import models
+import numpy as np
+from ..annotation import ImageClassificationAnnotation
+from typing import TYPE_CHECKING, Union
+from tqdm import tqdm
+
+if TYPE_CHECKING:
+    from endoreg_db.models import (
+        RawVideoFile,
+        Video,
+        Label,
+        InformationSource,
+        VideoPredictionMeta,
+        RawVideoPredictionMeta,
+    )
+
+
+def find_segments_in_prediction_array(prediction_array: np.array, min_frame_len: int):
+    """
+    Expects a prediction array of shape (num_frames) and a minimum frame length.
+    Returns a list of tuples (start_frame_number, end_frame_number) that represent the segments.
+    """
+    # Add False to the beginning and end to detect changes at the array boundaries
+    padded_prediction = np.pad(
+        prediction_array, (1, 1), "constant", constant_values=False
+    )
+
+    # Find the start points and end points of the segments
+    diffs = np.diff(padded_prediction.astype(int))
+    segment_starts = np.where(diffs == 1)[0]
+    segment_ends = np.where(diffs == -1)[0]
+
+    # Filter segments based on min_frame_len
+    segments = [
+        (start, end)
+        for start, end in zip(segment_starts, segment_ends)
+        if end - start >= min_frame_len
+    ]
+
+    return segments
+
+
+class AbstractLabelVideoSegment(models.Model):
+    start_frame_number = models.IntegerField()
+    end_frame_number = models.IntegerField()
+    source = models.ForeignKey(
+        "InformationSource", on_delete=models.SET_NULL, null=True
+    )
+    label = models.ForeignKey("Label", on_delete=models.SET_NULL, null=True, blank=True)
+
+    if TYPE_CHECKING:
+        label: "Label"
+        source: "InformationSource"
+        prediction_meta: Union["RawVideoPredictionMeta", "VideoPredictionMeta"]
+        video: Union[Video, RawVideoFile]
+
+    class Meta:
+        abstract = True
+
+    def __str__(self):
+        video = self.video
+        label = self.label
+
+        str_repr = (
+            video.file.path
+            + " Label - "
+            + label.name
+            + " - "
+            + str(self.start_frame_number)
+            + " - "
+            + str(self.end_frame_number)
+        )
+        assert isinstance(str_repr, str), "String representation is not a string"
+        return str_repr
+
+    def get_model_meta(self):
+        return self.prediction_meta.model_meta
+
+    def get_frames(self):
+        video = self.video
+        return video.get_frame_range(self.start_frame_number, self.end_frame_number)
+
+    def get_annotations(self):
+        frames = self.get_frames()
+        annotations = ImageClassificationAnnotation.objects.filter(
+            frame__in=frames, label=self.label
+        )
+
+        return annotations
+
+    def get_segment_len_in_s(self):
+        return (self.end_frame_number - self.start_frame_number) / self.video.get_fps()
+
+    def get_frames_without_annotation(self, n_frames: int):
+        """
+        Get a frame without an annotation.
+        """
+        frames = self.get_frames()
+        annotations = ImageClassificationAnnotation.objects.filter(
+            frame__in=frames, label=self.label
+        )
+
+        annotated_frames = [annotation.frame for annotation in annotations]
+        frames_without_annotation = [
+            frame for frame in frames if frame not in annotated_frames
+        ]
+
+        # draw n random frames
+        if len(frames_without_annotation) > n_frames:
+            frames_without_annotation = np.random.choice(
+                frames_without_annotation, n_frames, replace=False
+            )
+
+        return frames_without_annotation
+
+
+class LabelVideoSegment(AbstractLabelVideoSegment):
+    video = models.ForeignKey(
+        "Video", on_delete=models.CASCADE, related_name="label_video_segments"
+    )
+    prediction_meta = models.ForeignKey(
+        "VideoPredictionMeta",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="label_video_segments",
+    )
+
+    if TYPE_CHECKING:
+        video: "Video"
+        label: "Label"
+        source: "InformationSource"
+        prediction_meta: "VideoPredictionMeta"
+
+    @classmethod
+    def from_raw(cls, video: "Video", raw_label_video_segment: "LabelRawVideoSegment"):
+        from endoreg_db.models import VideoPredictionMeta
+
+        raw_video_prediction_meta = raw_label_video_segment.prediction_meta
+
+        prediction_meta = VideoPredictionMeta.from_raw(
+            video=video, raw_video_prediction_meta=raw_video_prediction_meta
+        )
+        existing_segment = cls.objects.filter(
+            video=video,
+            start_frame_number=raw_label_video_segment.start_frame_number,
+            end_frame_number=raw_label_video_segment.end_frame_number,
+            source=raw_label_video_segment.source,
+            label=raw_label_video_segment.label,
+            prediction_meta=prediction_meta,
+        ).first()
+        if existing_segment:
+            return existing_segment
+        segment = cls(
+            start_frame_number=raw_label_video_segment.start_frame_number,
+            end_frame_number=raw_label_video_segment.end_frame_number,
+            source=raw_label_video_segment.source,
+            label=raw_label_video_segment.label,
+            video=video,
+            prediction_meta=prediction_meta,
+        )
+
+        segment.save()
+        return segment
+
+    def get_video_model(self):
+        from endoreg_db.models import Video
+
+        return Video
+
+    def generate_annotations(self):
+        """
+        Generate annotations for the segment.
+        """
+        from endoreg_db.models import InformationSource
+
+        frames = self.get_frames()
+        model_meta = self.get_model_meta()
+        information_source, _created = InformationSource.objects.get_or_create(
+            name="prediction"
+        )
+
+        for frame in tqdm(frames):
+            ImageClassificationAnnotation.objects.get_or_create(
+                frame=frame,
+                label=self.label,
+                model_meta=model_meta,
+                value=1,
+                information_source=information_source,
+            )
+
+
+class LabelRawVideoSegment(AbstractLabelVideoSegment):
+    video = models.ForeignKey(
+        "RawVideoFile", on_delete=models.CASCADE, related_name="label_video_segments"
+    )
+    prediction_meta = models.ForeignKey(
+        "RawVideoPredictionMeta",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="label_video_segments",
+    )
+
+    if TYPE_CHECKING:
+        video: "RawVideoFile"
+        label: "Label"
+        source: "InformationSource"
+        prediction_meta: "RawVideoPredictionMeta"
+
+    def get_video_model(self):
+        from endoreg_db.models import RawVideoFile
+
+        return RawVideoFile

endoreg_db/models/disease.py

@@ -0,0 +1,79 @@
+from django.db import models
+from typing import List
+
+class DiseaseManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class Disease(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+    name_de = models.CharField(max_length=255, blank=True, null=True)
+    name_en = models.CharField(max_length=255, blank=True, null=True)
+    subcategories = models.JSONField(default=dict)
+    numerical_descriptors = models.JSONField(default=dict)
+
+    objects = DiseaseManager()
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name
+    
+    
+
+        
+
+
+    
+    def get_classifications(self)->List['DiseaseClassification']:
+        classifications: List[DiseaseClassification] = [_ for _ in self.disease_classifications.all()]
+        return classifications
+    
+class DiseaseClassificationManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class DiseaseClassification(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+    name_de = models.CharField(max_length=255, blank=True, null=True)
+    name_en = models.CharField(max_length=255, blank=True, null=True)
+
+    disease = models.ForeignKey(
+        Disease, on_delete=models.CASCADE,
+        related_name='disease_classifications'
+    )
+
+    objects = DiseaseClassificationManager()
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name
+    
+    def get_choices(self)->List['DiseaseClassificationChoice']:
+        choices:List[DiseaseClassificationChoice] = [_ for _ in self.disease_classification_choices.all()]
+        return choices
+    
+class DiseaseClassificationChoiceManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class DiseaseClassificationChoice(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+    name_de = models.CharField(max_length=255, blank=True, null=True)
+    name_en = models.CharField(max_length=255, blank=True, null=True)
+
+    disease_classification = models.ForeignKey(
+        DiseaseClassification, on_delete=models.CASCADE,
+        related_name='disease_classification_choices'
+    )
+
+    objects = DiseaseClassificationChoiceManager()
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name

endoreg_db/models/emission/__init__.py

@@ -0,0 +1,5 @@
+from .emission_factor import EmissionFactor
+
+__all__ = [
+    'EmissionFactor',
+]

endoreg_db/models/emission/emission_factor.py

@@ -0,0 +1,85 @@
+from django.db import models
+from typing import List
+class EmissionFactorManager(models.Manager):
+    """
+    Manager for EmissionFactor with custom query methods.
+    """
+    def get_by_natural_key(self, name: str) -> "EmissionFactor":
+        return self.get(name=name)
+
+# get debug from settings
+# from django.conf import settings
+# DEBUG = settings.DEBUG
+
+class EmissionFactor(models.Model):
+    """
+    Represents an emission factor with associated unit and value.
+
+    Attributes:
+        name (str): The name of the emission factor.
+        name_de (str): The German name of the emission factor.
+        name_en (str): The English name of the emission factor.
+        unit (ForeignKey): The unit associated with the emission factor.
+        value (float): The value of the emission factor.
+    """
+    objects = EmissionFactorManager()
+
+    name = models.CharField(max_length=255)
+    name_de = models.CharField(max_length=255, null=True)
+    name_en = models.CharField(max_length=255, null=True)
+    unit = models.ForeignKey("Unit", on_delete=models.SET_NULL, null=True)
+    value = models.FloatField()
+    
+    def natural_key(self) -> tuple:
+        """
+        Returns the natural key for the emission factor.
+
+        Returns:
+            tuple: The natural key consisting of the name.
+        """
+        return (self.name,)
+
+    def __str__(self, verbose: bool = False) -> str:
+        """
+        String representation of the emission factor.
+
+        Args:
+            verbose (bool): If True, includes additional details.
+
+        Returns:
+            str: The string representation of the emission factor.
+        """
+        result = f"{self.name}:	{self.value} per {self.unit}"
+        if verbose:
+            result += "\n\tSources:"
+            for source in self.sources():
+                result += f"\n\t\t{source}"
+
+        return result
+    
+    def get_reference_products(self) -> List["ReferenceProduct"]:
+        """
+        Retrieves all reference products associated with the emission factor.
+
+        Returns:
+            list: A list of ReferenceProduct instances associated with this emission factor.
+        """
+        from endoreg_db.models import ReferenceProduct
+
+        reference_products = []
+        reference_products += ReferenceProduct.objects.filter(emission_factor_total=self)
+        reference_products += ReferenceProduct.objects.filter(emission_factor_package=self)
+        reference_products += ReferenceProduct.objects.filter(emission_factor_product=self)
+
+        return reference_products
+
+    def sources(self) -> List:
+        """
+        Retrieves all sources related to the emission factor.
+
+        Returns:
+            list: A list of sources related to the emission factor.
+        """
+        sources = []
+        sources.extend(self.get_reference_products())
+        return sources

endoreg_db/models/event.py

@@ -0,0 +1,73 @@
+from django.db import models
+from typing import List
+
+class Event(models.Model):
+    """
+    A class representing an event.
+
+    Attributes:
+        name (str): The name of the event.
+        name_de (str): The German name of the event.
+        name_en (str): The English name of the event.
+        description (str): A description of the event.
+    """
+    name = models.CharField(max_length=100, unique=True)
+    name_de = models.CharField(max_length=100, blank=True, null=True)
+    name_en = models.CharField(max_length=100, blank=True, null=True)
+    description = models.TextField(blank=True, null=True)
+    event_classification = models.ForeignKey(
+        'EventClassification', on_delete=models.CASCADE, related_name='events',
+        null=True, blank=True
+    )
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name
+    
+
+class EventClassificationManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+    
+class EventClassification(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+    name_de = models.CharField(max_length=255, blank=True, null=True)
+    name_en = models.CharField(max_length=255, blank=True, null=True)
+    
+    objects = EventClassificationManager()
+    
+    def natural_key(self):
+        return (self.name,)
+    
+    def __str__(self):
+        return self.name
+    
+    def get_choices(self)->List['EventClassificationChoice']:
+        choices:List[EventClassificationChoice] = [_ for _ in self.event_classification_choices.all()]
+        return choices
+    
+class EventClassificationChoiceManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+    
+class EventClassificationChoice(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+    name_de = models.CharField(max_length=255, blank=True, null=True)
+    name_en = models.CharField(max_length=255, blank=True, null=True)
+    subcategories = models.JSONField(default=dict)
+    numerical_descriptors = models.JSONField(default=dict)
+    
+    event_classification = models.ForeignKey(
+        EventClassification, on_delete=models.CASCADE,
+        related_name='event_classification_choices'
+    )
+    
+    objects = EventClassificationChoiceManager()
+    
+    def natural_key(self):
+        return (self.name,)
+    
+    def __str__(self):
+        return self.name

endoreg_db/models/examination/__init__.py

@@ -0,0 +1,9 @@
+from .examination import Examination
+from .examination_type import ExaminationType
+from .examination_time import ExaminationTime
+from .examination_time_type import ExaminationTimeType
+from .examination_indication import(
+    ExaminationIndication,
+    ExaminationIndicationClassification,
+    ExaminationIndicationClassificationChoice
+)