endoreg-db 0.5.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

endoreg_db/models/data_file/video/video.py

@@ -1,25 +1,196 @@
-from ..base_classes import AbstractVideo
 from django.db import models
-
 from endoreg_db.models.data_file.frame import Frame
-from endoreg_db.models.data_file.frame import LegacyFrame
+from django.core.validators import FileExtensionValidator
+from django.core.files.storage import FileSystemStorage
+from typing import TYPE_CHECKING
+import cv2
+from ..base_classes import AbstractVideoFile
+from ..base_classes.utils import (
+    VIDEO_DIR_NAME,
+    STORAGE_LOCATION,
+    FRAME_PROCESSING_BATCH_SIZE,
+)
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+    from endoreg_db.models import (
+        LabelVideoSegment,
+        VideoImportMeta,
+        SensitiveMeta,
+        Patient,
+        PatientExamination,
+        RawVideoFile,
+    )
+
+
+class Video(AbstractVideoFile):
+    file = models.FileField(
+        upload_to=VIDEO_DIR_NAME,
+        validators=[FileExtensionValidator(allowed_extensions=["mp4"])],  # FIXME
+        storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
+    )
+
+    patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+        related_name="videos",
+    )
+
+    examination = models.ForeignKey(
+        "PatientExamination",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="videos",
+    )
+
+    # Deprecate and move to video meta?
+    date = models.DateField(blank=True, null=True)
+    suffix = models.CharField(max_length=255, blank=True, null=True)
+    fps = models.FloatField(blank=True, null=True)
+    duration = models.FloatField(blank=True, null=True)
+    width = models.IntegerField(blank=True, null=True)
+    height = models.IntegerField(blank=True, null=True)
+    endoscope_image_x = models.IntegerField(blank=True, null=True)
+    endoscope_image_y = models.IntegerField(blank=True, null=True)
+    endoscope_image_width = models.IntegerField(blank=True, null=True)
+    endoscope_image_height = models.IntegerField(blank=True, null=True)
+
+    state_frames_extracted = models.BooleanField(default=False)
+
+    meta = models.JSONField(blank=True, null=True)
+
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.CASCADE,
+        related_name="videos",
+        null=True,
+        blank=True,
+    )
+
+    import_meta = models.OneToOneField(
+        "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
+    )
+
+    if TYPE_CHECKING:
+        import_meta: "VideoImportMeta"
+        patient: "Patient"
+        examination: "PatientExamination"
+        frames: "QuerySet[Frame]"
+        label_video_segments: (
+            "QuerySet[LabelVideoSegment]"  # Foreign key in LabelVideoSegment.video
+        )
+        sensitive_meta: "SensitiveMeta"
+        raw_videos: "QuerySet[RawVideoFile]"
+
+    # override the save method to set fps if not set
+    def save(self, *args, **kwargs):
+        if not self.fps:
+            self.fps = self.get_fps()
+        super().save(*args, **kwargs)
+
+    def label_segments_to_frame_annotations(self):
+        """
+        Generate annotations for all label video segments.
+        """
+        for lvs in self.label_video_segments.all():
+            lvs.generate_annotations()
+
+    def sync_from_raw_video(self):
+        """
+        Sync metadata from the associated raw video file.
+        """
+        from endoreg_db.models import RawVideoFile, LabelVideoSegment
+
+        raw_video: RawVideoFile = self.raw_videos.first()
+
+        assert isinstance(raw_video, RawVideoFile), "Raw video is not a file"
+
+        self.predictions = raw_video.predictions
+        self.readable_predictions = raw_video.readable_predictions
+        self.sequences = raw_video.sequences
+
+        label_video_segments = raw_video.label_video_segments.all()
+
+        label_video_segments = [
+            LabelVideoSegment.from_raw(
+                video=self, raw_label_video_segment=raw_label_video_segment
+            )
+            # LabelVideoSegment.from_raw(self, raw_label_video_segment)
+            for raw_label_video_segment in label_video_segments
+        ]
+
+        for lvs in label_video_segments:
+            lvs.save()
+
+        self.state_histology_required = raw_video.state_histology_required
+        self.state_histology_available = raw_video.state_histology_available
+        self.state_follow_up_intervention_required = (
+            raw_video.state_follow_up_intervention_required
+        )
+        self.state_follow_up_intervention_available = (
+            raw_video.state_follow_up_intervention_available
+        )
+        self.state_dataset_complete = raw_video.state_dataset_complete
+
+        self.save()
+
+    ## Deprecated
+    # def get_roi_endoscope_image(self):
+    #     return {
+    #         "x": self.endoscope_image_content_x,
+    #         "y": self.endoscope_image_content_y,
+    #         "width": self.endoscope_image_content_width,
+    #         "height": self.endoscope_image_content_height,
+    #     }
+
+    # def initialize_metadata_in_db(self, video_meta=None):
+    #     if not video_meta:
+    #         video_meta = self.meta
+    #     self.set_examination_date_from_video_meta(video_meta)
+    #     self.patient, created = self.get_or_create_patient(video_meta)
+    #     self.save()
+
+    # def get_or_create_patient(self, video_meta=None):
+    #     from ...persons import Patient
 
-BATCH_SIZE = 1000
+    #     if not video_meta:
+    #         video_meta = self.meta
 
-class Video(AbstractVideo):
-    import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
-    def get_video_model(self):
-        return Video
+    #     patient_first_name = video_meta["patient_first_name"]
+    #     patient_last_name = video_meta["patient_last_name"]
+    #     patient_dob = video_meta["patient_dob"]
 
-    def get_frame_model(self):
-        return Frame
+    #     # assert that we got all the necessary information
+    #     assert patient_first_name and patient_last_name and patient_dob, (
+    #         "Missing patient information"
+    #     )
 
+    #     patient, created = Patient.objects.get_or_create(
+    #         first_name=patient_first_name, last_name=patient_last_name, dob=patient_dob
+    #     )
 
-class LegacyVideo(AbstractVideo):
-    file = models.FileField(upload_to="legacy_videos", blank=True, null=True)
+    #     return patient, created
 
-    def get_video_model(self):
-        return LegacyVideo
+    # DEPRECATED
+    # def set_examination_date_from_video_meta(self, video_meta=None):
+    #     if not video_meta:
+    #         video_meta = self.meta
+    #     date_str = video_meta["examination_date"]  # e.g. 2020-01-01
+    #     if date_str:
+    #         self.date = date.fromisoformat(date_str)
+    #         self.save()
 
-    def get_frame_model(self):
-        return LegacyFrame
+    def initialize_video_specs(self):
+        """
+        Initialize and save video metadata like framerate, dimensions, and duration.
+        """
+        video = cv2.VideoCapture(self.file.path)
+        self.fps = video.get(cv2.CAP_PROP_FPS)
+        self.width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
+        self.height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        self.duration = video.get(cv2.CAP_PROP_FRAME_COUNT) / self.fps
+        self.save()
+        video.release()

endoreg_db/models/data_file/video_segment.py

@@ -1,6 +1,19 @@
 from django.db import models
 import numpy as np
 from ..annotation import ImageClassificationAnnotation
+from typing import TYPE_CHECKING, Union
+from tqdm import tqdm
+
+if TYPE_CHECKING:
+    from endoreg_db.models import (
+        RawVideoFile,
+        Video,
+        Label,
+        InformationSource,
+        VideoPredictionMeta,
+        RawVideoPredictionMeta,
+    )
+
 
 def find_segments_in_prediction_array(prediction_array: np.array, min_frame_len: int):
     """
@@ -8,100 +21,194 @@ def find_segments_in_prediction_array(prediction_array: np.array, min_frame_len:
     Returns a list of tuples (start_frame_number, end_frame_number) that represent the segments.
     """
     # Add False to the beginning and end to detect changes at the array boundaries
-    padded_prediction = np.pad(prediction_array, (1, 1), 'constant', constant_values=False)
-    
+    padded_prediction = np.pad(
+        prediction_array, (1, 1), "constant", constant_values=False
+    )
+
     # Find the start points and end points of the segments
     diffs = np.diff(padded_prediction.astype(int))
     segment_starts = np.where(diffs == 1)[0]
     segment_ends = np.where(diffs == -1)[0]
-    
+
     # Filter segments based on min_frame_len
-    segments = [(start, end) for start, end in zip(segment_starts, segment_ends) if end - start >= min_frame_len]
-    
+    segments = [
+        (start, end)
+        for start, end in zip(segment_starts, segment_ends)
+        if end - start >= min_frame_len
+    ]
+
     return segments
 
+
 class AbstractLabelVideoSegment(models.Model):
-    video = None # Placeholder for the video field, to be defined in derived classes
-    prediction_meta = None # Placeholder for the prediction_meta field, to be defined in derived classes
     start_frame_number = models.IntegerField()
     end_frame_number = models.IntegerField()
-    source = models.ForeignKey("InformationSource", on_delete=models.CASCADE)
-    label = models.ForeignKey("Label", on_delete=models.CASCADE)
+    source = models.ForeignKey(
+        "InformationSource", on_delete=models.SET_NULL, null=True
+    )
+    label = models.ForeignKey("Label", on_delete=models.SET_NULL, null=True, blank=True)
+
+    if TYPE_CHECKING:
+        label: "Label"
+        source: "InformationSource"
+        prediction_meta: Union["RawVideoPredictionMeta", "VideoPredictionMeta"]
+        video: Union[Video, RawVideoFile]
 
     class Meta:
         abstract = True
 
     def __str__(self):
-        return self.video.file.path + " Label - " + self.label.name + " - " + str(self.start_frame_number) + " - " + str(self.end_frame_number)
-    
+        video = self.video
+        label = self.label
+
+        str_repr = (
+            video.file.path
+            + " Label - "
+            + label.name
+            + " - "
+            + str(self.start_frame_number)
+            + " - "
+            + str(self.end_frame_number)
+        )
+        assert isinstance(str_repr, str), "String representation is not a string"
+        return str_repr
+
+    def get_model_meta(self):
+        return self.prediction_meta.model_meta
+
     def get_frames(self):
-        return self.video.get_frame_range(self.start_frame_number, self.end_frame_number)
-    
-    def get_annotations(self) -> ImageClassificationAnnotation.objects:
+        video = self.video
+        return video.get_frame_range(self.start_frame_number, self.end_frame_number)
+
+    def get_annotations(self):
         frames = self.get_frames()
-        annotations = ImageClassificationAnnotation.objects.filter(frame__in=frames, label=self.label)
+        annotations = ImageClassificationAnnotation.objects.filter(
+            frame__in=frames, label=self.label
+        )
 
         return annotations
-    
-    def get_frames_without_annotation(self):
-        """
-        Get a frame without an annotation.
-        """
-        assert 1 == 2, "This method should be overridden in derived classes"
 
     def get_segment_len_in_s(self):
-        return (self.end_frame_number - self.start_frame_number) / self.video.fps
+        return (self.end_frame_number - self.start_frame_number) / self.video.get_fps()
 
-class LegacyLabelVideoSegment(AbstractLabelVideoSegment):
-    video = models.ForeignKey("LegacyVideo", on_delete=models.CASCADE)
-    prediction_meta = models.ForeignKey("LegacyVideoPredictionMeta", on_delete=models.CASCADE, related_name="video_segments")
-
-    def get_video_model(self):
-        from endoreg_db.models.data_file.video import LegacyVideo
-        return LegacyVideo
-
-    def get_annotations(self) -> ImageClassificationAnnotation.objects:
-        frames = self.get_frames()
-        annotations = ImageClassificationAnnotation.objects.filter(legacy_frame__in=frames, label=self.label)
-
-        return annotations
-
-    def get_frames_without_annotation(self, n_frames):
+    def get_frames_without_annotation(self, n_frames: int):
         """
         Get a frame without an annotation.
         """
         frames = self.get_frames()
-        annotations = ImageClassificationAnnotation.objects.filter(legacy_frame__in=frames, label=self.label)
+        annotations = ImageClassificationAnnotation.objects.filter(
+            frame__in=frames, label=self.label
+        )
 
-        annotated_frames = [annotation.legacy_frame for annotation in annotations]
-        frames_without_annotation = [frame for frame in frames if frame not in annotated_frames]
+        annotated_frames = [annotation.frame for annotation in annotations]
+        frames_without_annotation = [
+            frame for frame in frames if frame not in annotated_frames
+        ]
 
         # draw n random frames
         if len(frames_without_annotation) > n_frames:
-            frames_without_annotation = np.random.choice(frames_without_annotation, n_frames, replace=False)
+            frames_without_annotation = np.random.choice(
+                frames_without_annotation, n_frames, replace=False
+            )
 
         return frames_without_annotation
-    
+
+
 class LabelVideoSegment(AbstractLabelVideoSegment):
-    video = models.ForeignKey("Video", on_delete=models.CASCADE)
-    prediction_meta = models.ForeignKey("VideoPredictionMeta", on_delete=models.CASCADE, related_name="video_segments")
+    video = models.ForeignKey(
+        "Video", on_delete=models.CASCADE, related_name="label_video_segments"
+    )
+    prediction_meta = models.ForeignKey(
+        "VideoPredictionMeta",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="label_video_segments",
+    )
+
+    if TYPE_CHECKING:
+        video: "Video"
+        label: "Label"
+        source: "InformationSource"
+        prediction_meta: "VideoPredictionMeta"
+
+    @classmethod
+    def from_raw(cls, video: "Video", raw_label_video_segment: "LabelRawVideoSegment"):
+        from endoreg_db.models import VideoPredictionMeta
+
+        raw_video_prediction_meta = raw_label_video_segment.prediction_meta
+
+        prediction_meta = VideoPredictionMeta.from_raw(
+            video=video, raw_video_prediction_meta=raw_video_prediction_meta
+        )
+        existing_segment = cls.objects.filter(
+            video=video,
+            start_frame_number=raw_label_video_segment.start_frame_number,
+            end_frame_number=raw_label_video_segment.end_frame_number,
+            source=raw_label_video_segment.source,
+            label=raw_label_video_segment.label,
+            prediction_meta=prediction_meta,
+        ).first()
+        if existing_segment:
+            return existing_segment
+        segment = cls(
+            start_frame_number=raw_label_video_segment.start_frame_number,
+            end_frame_number=raw_label_video_segment.end_frame_number,
+            source=raw_label_video_segment.source,
+            label=raw_label_video_segment.label,
+            video=video,
+            prediction_meta=prediction_meta,
+        )
+
+        segment.save()
+        return segment
 
     def get_video_model(self):
-        from endoreg_db.models.data_file.video import Video
+        from endoreg_db.models import Video
+
         return Video
 
-    def get_frames_without_annotation(self, n_frames):
+    def generate_annotations(self):
         """
-        Get a frame without an annotation.
+        Generate annotations for the segment.
         """
-        frames = self.get_frames()
-        annotations = ImageClassificationAnnotation.objects.filter(frame__in=frames, label=self.label)
+        from endoreg_db.models import InformationSource
 
-        annotated_frames = [annotation.frame for annotation in annotations]
-        frames_without_annotation = [frame for frame in frames if frame not in annotated_frames]
+        frames = self.get_frames()
+        model_meta = self.get_model_meta()
+        information_source, _created = InformationSource.objects.get_or_create(
+            name="prediction"
+        )
+
+        for frame in tqdm(frames):
+            ImageClassificationAnnotation.objects.get_or_create(
+                frame=frame,
+                label=self.label,
+                model_meta=model_meta,
+                value=1,
+                information_source=information_source,
+            )
+
+
+class LabelRawVideoSegment(AbstractLabelVideoSegment):
+    video = models.ForeignKey(
+        "RawVideoFile", on_delete=models.CASCADE, related_name="label_video_segments"
+    )
+    prediction_meta = models.ForeignKey(
+        "RawVideoPredictionMeta",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="label_video_segments",
+    )
+
+    if TYPE_CHECKING:
+        video: "RawVideoFile"
+        label: "Label"
+        source: "InformationSource"
+        prediction_meta: "RawVideoPredictionMeta"
 
-        # draw n random frames
-        if len(frames_without_annotation) > n_frames:
-            frames_without_annotation = np.random.choice(frames_without_annotation, n_frames, replace=False)
+    def get_video_model(self):
+        from endoreg_db.models import RawVideoFile
 
-        return frames_without_annotation
+        return RawVideoFile

endoreg_db/models/disease.py

@@ -1,4 +1,5 @@
 from django.db import models
+from typing import List
 
 class DiseaseManager(models.Manager):
     def get_by_natural_key(self, name):
@@ -8,6 +9,8 @@ class Disease(models.Model):
     name = models.CharField(max_length=255, unique=True)
     name_de = models.CharField(max_length=255, blank=True, null=True)
     name_en = models.CharField(max_length=255, blank=True, null=True)
+    subcategories = models.JSONField(default=dict)
+    numerical_descriptors = models.JSONField(default=dict)
 
     objects = DiseaseManager()
 
@@ -17,6 +20,16 @@ class Disease(models.Model):
     def __str__(self):
         return self.name
     
+    
+
+        
+
+
+    
+    def get_classifications(self)->List['DiseaseClassification']:
+        classifications: List[DiseaseClassification] = [_ for _ in self.disease_classifications.all()]
+        return classifications
+    
 class DiseaseClassificationManager(models.Manager):
     def get_by_natural_key(self, name):
         return self.get(name=name)
@@ -26,7 +39,10 @@ class DiseaseClassification(models.Model):
     name_de = models.CharField(max_length=255, blank=True, null=True)
     name_en = models.CharField(max_length=255, blank=True, null=True)
 
-    disease = models.ForeignKey(Disease, on_delete=models.CASCADE)
+    disease = models.ForeignKey(
+        Disease, on_delete=models.CASCADE,
+        related_name='disease_classifications'
+    )
 
     objects = DiseaseClassificationManager()
 
@@ -36,6 +52,10 @@ class DiseaseClassification(models.Model):
     def __str__(self):
         return self.name
     
+    def get_choices(self)->List['DiseaseClassificationChoice']:
+        choices:List[DiseaseClassificationChoice] = [_ for _ in self.disease_classification_choices.all()]
+        return choices
+    
 class DiseaseClassificationChoiceManager(models.Manager):
     def get_by_natural_key(self, name):
         return self.get(name=name)
@@ -45,7 +65,10 @@ class DiseaseClassificationChoice(models.Model):
     name_de = models.CharField(max_length=255, blank=True, null=True)
     name_en = models.CharField(max_length=255, blank=True, null=True)
 
-    disease_classification = models.ForeignKey(DiseaseClassification, on_delete=models.CASCADE)
+    disease_classification = models.ForeignKey(
+        DiseaseClassification, on_delete=models.CASCADE,
+        related_name='disease_classification_choices'
+    )
 
     objects = DiseaseClassificationChoiceManager()
 

endoreg_db/models/emission/__init__.py

@@ -1 +1,5 @@
-from .emission_factor import EmissionFactor
+from .emission_factor import EmissionFactor
+
+__all__ = [
+    'EmissionFactor',
+]

endoreg_db/models/emission/emission_factor.py

@@ -1,10 +1,27 @@
 from django.db import models
-
+from typing import List
 class EmissionFactorManager(models.Manager):
-    def get_by_natural_key(self, name):
+    """
+    Manager for EmissionFactor with custom query methods.
+    """
+    def get_by_natural_key(self, name: str) -> "EmissionFactor":
         return self.get(name=name)
-    
+
+# get debug from settings
+# from django.conf import settings
+# DEBUG = settings.DEBUG
+
 class EmissionFactor(models.Model):
+    """
+    Represents an emission factor with associated unit and value.
+
+    Attributes:
+        name (str): The name of the emission factor.
+        name_de (str): The German name of the emission factor.
+        name_en (str): The English name of the emission factor.
+        unit (ForeignKey): The unit associated with the emission factor.
+        value (float): The value of the emission factor.
+    """
     objects = EmissionFactorManager()
 
     name = models.CharField(max_length=255)
@@ -13,8 +30,56 @@ class EmissionFactor(models.Model):
     unit = models.ForeignKey("Unit", on_delete=models.SET_NULL, null=True)
     value = models.FloatField()
     
-    def natural_key(self):
+    def natural_key(self) -> tuple:
+        """
+        Returns the natural key for the emission factor.
+
+        Returns:
+            tuple: The natural key consisting of the name.
+        """
         return (self.name,)
+
+    def __str__(self, verbose: bool = False) -> str:
+        """
+        String representation of the emission factor.
+
+        Args:
+            verbose (bool): If True, includes additional details.
+
+        Returns:
+            str: The string representation of the emission factor.
+        """
+        result = f"{self.name}:	{self.value} per {self.unit}"
+        if verbose:
+            result += "\n\tSources:"
+            for source in self.sources():
+                result += f"\n\t\t{source}"
+
+        return result
     
-    def __str__(self):
-        return self.name
+    def get_reference_products(self) -> List["ReferenceProduct"]:
+        """
+        Retrieves all reference products associated with the emission factor.
+
+        Returns:
+            list: A list of ReferenceProduct instances associated with this emission factor.
+        """
+        from endoreg_db.models import ReferenceProduct
+
+        reference_products = []
+        reference_products += ReferenceProduct.objects.filter(emission_factor_total=self)
+        reference_products += ReferenceProduct.objects.filter(emission_factor_package=self)
+        reference_products += ReferenceProduct.objects.filter(emission_factor_product=self)
+
+        return reference_products
+
+    def sources(self) -> List:
+        """
+        Retrieves all sources related to the emission factor.
+
+        Returns:
+            list: A list of sources related to the emission factor.
+        """
+        sources = []
+        sources.extend(self.get_reference_products())
+        return sources