endoreg-db 0.6.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/models/metadata/video_prediction_meta.py

@@ -0,0 +1,270 @@
+from typing import TYPE_CHECKING, Optional, List, Tuple
+from django.db import models
+import logging
+
+from endoreg_db.models.label import LabelSet
+
+from ..label.label_video_segment import (
+    LabelVideoSegment,
+
+)
+from ..utils import find_segments_in_prediction_array
+
+import numpy as np
+import pickle
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
+DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
+
+if TYPE_CHECKING:
+    from endoreg_db.models import ModelMeta, Label
+    from ..media.video.video_file import VideoFile
+
+
+class VideoPredictionMeta(models.Model):
+    """
+    Stores metadata about predictions made by a model for a specific video.
+
+    Must be associated with exactly one `VideoFile`.
+    """
+    model_meta = models.ForeignKey("ModelMeta", on_delete=models.CASCADE)
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    prediction_array = models.BinaryField(blank=True, null=True)
+
+    video_file = models.ForeignKey(
+        "VideoFile",
+        on_delete=models.CASCADE,
+        related_name="video_prediction_meta",
+        null=False,
+        blank=False,
+    )
+
+    if TYPE_CHECKING:
+        model_meta: "ModelMeta"
+        video_file: "VideoFile"
+        label_video_segments: "models.Manager[LabelVideoSegment]"
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=['model_meta', 'video_file'], name='unique_prediction_per_video_model')
+        ]
+        indexes = [
+            models.Index(fields=["model_meta", "video_file"]),
+        ]
+
+    def get_video(self) -> "VideoFile":
+        """Returns the associated VideoFile instance."""
+        if self.video_file:
+            return self.video_file
+        else:
+            raise ValueError("VideoPredictionMeta is not associated with a VideoFile.")
+
+    def __str__(self):
+        try:
+            video_obj = self.get_video()
+            return f"Prediction Meta for Video {video_obj.uuid} - {self.model_meta.name}"
+        except ValueError:
+            return f"Prediction Meta {self.pk} (Error: No VideoFile) - {self.model_meta.name}"
+        except Exception as e:
+            logger.warning("Error generating string representation for VideoPredictionMeta %s: %s", self.pk, e)
+            return f"Prediction Meta {self.pk} (Error: {e})"
+
+    def get_labelset(self) -> Optional["LabelSet"]:
+        """Get the labelset associated with the model."""
+        return self.model_meta.labelset
+
+    def get_label_list(self) -> list["Label"]:
+        """Get the ordered list of labels from the model's labelset."""
+        labelset = self.get_labelset()
+        if labelset:
+            return labelset.get_labels_in_order()
+        return []
+
+    def save_prediction_array(self, prediction_array: np.array):
+        """
+        Save the prediction array to the database.
+        """
+        self.prediction_array = pickle.dumps(prediction_array)
+        self.save(update_fields=['prediction_array', 'date_modified'])
+
+    def get_prediction_array(self):
+        """
+        Get the prediction array from the database.
+        """
+        if self.prediction_array is None:
+            return None
+        else:
+            try:
+                return pickle.loads(self.prediction_array)
+            except (pickle.UnpicklingError, TypeError, EOFError) as e:
+                logger.error(f"Error unpickling prediction array for {self}: {e}")
+                return None
+
+    def calculate_prediction_array(self, window_size_in_seconds: int = None):
+        """
+        Fetches all predictions for the associated video, labelset, and model meta,
+        applies smoothing, and saves the resulting binary prediction array.
+        """
+        from ..label import ImageClassificationAnnotation
+
+        video_obj = self.get_video()
+        model_meta = self.model_meta
+        label_list = self.get_label_list()
+        num_frames = video_obj.frame_count
+
+        if num_frames is None or num_frames <= 0:
+            logger.warning(f"Cannot calculate prediction array for {video_obj} with invalid frame count ({num_frames}).")
+            return
+
+        if not label_list:
+            logger.warning(f"No labels found for model {model_meta}. Cannot calculate prediction array.")
+            return
+
+        prediction_array = np.zeros((num_frames, len(label_list)))
+
+        base_pred_qs = ImageClassificationAnnotation.objects.filter(
+            model_meta=model_meta,
+            frame__video_file=video_obj
+        )
+
+        for i, label in enumerate(label_list):
+            predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list(
+                "frame__frame_number", "float_value"
+            )
+
+            confidences = np.full(num_frames, 0.5)
+            found_predictions = False
+            for frame_num, confidence in predictions:
+                if 0 <= frame_num < num_frames:
+                    confidences[frame_num] = confidence
+                    found_predictions = True
+                else:
+                    logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames-1}). Skipping.")
+
+            if not found_predictions:
+                logger.warning(f"No predictions found for label '{label.name}' in {video_obj}. Using default confidence.")
+
+            smooth_confidences = self.apply_running_mean(
+                confidences, window_size_in_seconds
+            )
+            binary_predictions = smooth_confidences > 0.5
+            prediction_array[:, i] = binary_predictions
+
+        self.save_prediction_array(prediction_array)
+        logger.info(f"Calculated and saved prediction array for {self}")
+
+    def apply_running_mean(self, confidence_array, window_size_in_seconds: int = None):
+        """
+        Apply a running mean filter to the confidence array for smoothing, assuming a padding
+        of 0.5 for the edges.
+        """
+        video_obj = self.get_video()
+        fps = video_obj.get_fps()
+
+        if fps is None or fps <= 0:
+            logger.warning(f"Invalid FPS ({fps}) for {video_obj}. Cannot apply running mean. Returning original array.")
+            return confidence_array
+
+        if not window_size_in_seconds:
+            window_size_in_seconds = DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN
+
+        window_size_in_frames = int(window_size_in_seconds * fps)
+        window_size_in_frames = max(window_size_in_frames, 1)
+
+        window = np.ones(window_size_in_frames) / window_size_in_frames
+        pad_size = window_size_in_frames // 2
+
+        padded_confidences = np.pad(
+            confidence_array,
+            (pad_size, pad_size),
+            "constant",
+            constant_values=(0.5, 0.5),
+        )
+
+        running_mean = np.convolve(padded_confidences, window, mode="same")
+        start_index = pad_size
+        end_index = start_index + len(confidence_array)
+        running_mean = running_mean[start_index:end_index]
+
+        if running_mean.shape != confidence_array.shape:
+            logger.warning(f"Running mean output shape {running_mean.shape} differs from input {confidence_array.shape}. Check padding/slicing.")
+            return confidence_array
+
+        return running_mean
+
+    def create_video_segments_for_label(self, segments: List[Tuple[int, int]], label: "Label"):
+        """
+        Creates LabelVideoSegment instances for the given label and segments.
+        """
+        from endoreg_db.models import InformationSource
+
+        video_obj = self.get_video()
+        information_source, _ = InformationSource.objects.get_or_create(name="prediction")
+
+        segments_to_create = []
+        for start_frame, end_frame in segments:
+            segment_data = {
+                "start_frame_number": start_frame,
+                "end_frame_number": end_frame,
+                "source": information_source,
+                "label": label,
+                "prediction_meta": self,
+                "video_file": video_obj,
+            }
+            if not LabelVideoSegment.objects.filter(
+                video_file=video_obj,
+                prediction_meta=self,
+                label=label,
+                start_frame_number=start_frame,
+                end_frame_number=end_frame
+            ).exists():
+                segments_to_create.append(LabelVideoSegment(**segment_data))
+
+        if segments_to_create:
+            LabelVideoSegment.objects.bulk_create(segments_to_create)
+            logger.info(f"Created {len(segments_to_create)} video segments for label '{label.name}' in {video_obj}.")
+        else:
+            logger.info(f"No new video segments needed for label '{label.name}' in {video_obj}.")
+
+    def create_video_segments(self, segment_length_threshold_in_s: float = None):
+        """
+        Generates LabelVideoSegments based on the stored prediction array.
+        """
+        if not segment_length_threshold_in_s:
+            segment_length_threshold_in_s = DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S
+
+        video_obj = self.get_video()
+        fps = video_obj.get_fps()
+
+        if fps is None or fps <= 0:
+            logger.warning(f"Cannot create video segments for {video_obj} with invalid FPS ({fps}).")
+            return
+
+        min_frame_length = int(segment_length_threshold_in_s * fps)
+        min_frame_length = max(min_frame_length, 1)
+
+        label_list = self.get_label_list()
+
+        prediction_array = self.get_prediction_array()
+        if prediction_array is None:
+            logger.info(f"Prediction array not found for {self}. Calculating...")
+            self.calculate_prediction_array()
+            prediction_array = self.get_prediction_array()
+            if prediction_array is None:
+                logger.error(f"Failed to get or calculate prediction array for {self}. Cannot create segments.")
+                return
+
+        if prediction_array.shape[1] != len(label_list):
+            logger.warning(f"Prediction array shape {prediction_array.shape} incompatible with label list length {len(label_list)} for {self}.")
+            return
+
+        logger.info(f"Creating video segments for {self} (min length: {min_frame_length} frames)...")
+        for i, label in enumerate(label_list):
+            binary_predictions = prediction_array[:, i].astype(bool)
+            segments = find_segments_in_prediction_array(binary_predictions, min_frame_length)
+            if segments:
+                self.create_video_segments_for_label(segments, label)
+        logger.info(f"Finished creating video segments for {self}.")

endoreg_db/models/other/__init__.py

@@ -9,6 +9,17 @@ from .distribution import (
     MultipleCategoricalValueDistribution,
     DateValueDistribution,
 )
+from .emission import EmissionFactor
+
+from .gender import Gender
+from .information_source import (
+    InformationSource,
+    InformationSourceType,
+)
+
+from .unit import Unit
+
+from .tag import Tag
 
 __all__ = [
     'Material',
@@ -20,4 +31,10 @@ __all__ = [
     'SingleCategoricalValueDistribution',
     'MultipleCategoricalValueDistribution',
     'DateValueDistribution',
+    "Gender",
+    "InformationSource",
+    "InformationSourceType",
+    "Unit",
+    "EmissionFactor",
+    "Tag",
 ]

endoreg_db/models/other/distribution/date_value_distribution.py

@@ -22,8 +22,6 @@ class DateValueDistribution(BaseValueDistribution):
     """
     objects = DateValueDistributionManager()
     name = models.CharField(max_length=100)
-    name_de = models.CharField(max_length=100, blank=True, null=True)
-    name_en = models.CharField(max_length=100, blank=True, null=True)
     description = models.TextField(blank=True, null=True)
     DISTRIBUTION_CHOICES = [
         ('uniform', 'Uniform'),

endoreg_db/models/other/distribution/numeric_value_distribution.py

@@ -41,9 +41,38 @@ class NumericValueDistribution(BaseValueDistribution):
         assert isinstance(lab_value, LabValue)
         default_normal_range_dict = lab_value.get_normal_range(patient.age(), patient.gender)
         assert isinstance(default_normal_range_dict, dict)
+
         if self.distribution_type == 'uniform':
             assert self.min_descriptor and self.max_descriptor
-            assert "min" in default_normal_range_dict and "max" in default_normal_range_dict
+            
+            min_key_needed = self.min_descriptor.split('_')[0]
+            max_key_needed = self.max_descriptor.split('_')[0]
+
+            min_val_from_range = default_normal_range_dict.get(min_key_needed)
+            max_val_from_range = default_normal_range_dict.get(max_key_needed)
+
+            if min_val_from_range is None:
+                raise ValueError(
+                    f"Cannot generate value for LabValue '{lab_value.name}' using distribution "
+                    f"'{getattr(self, 'name', self.pk)}'. "
+                    f"Required normal range component '{min_key_needed}' derived from min_descriptor "
+                    f"'{self.min_descriptor}' is None. "
+                    f"Patient context: age={patient.age()}, gender='{patient.gender.name if patient.gender else None}'. "
+                    f"LabValue '{lab_value.name}' is gender-dependent: {lab_value.normal_range_gender_dependent}. "
+                    f"Check LabValue's default_normal_range definition for this patient context."
+                )
+            
+            if max_val_from_range is None:
+                raise ValueError(
+                    f"Cannot generate value for LabValue '{lab_value.name}' using distribution "
+                    f"'{getattr(self, 'name', self.pk)}'. "
+                    f"Required normal range component '{max_key_needed}' derived from max_descriptor "
+                    f"'{self.max_descriptor}' is None. "
+                    f"Patient context: age={patient.age()}, gender='{patient.gender.name if patient.gender else None}'. "
+                    f"LabValue '{lab_value.name}' is gender-dependent: {lab_value.normal_range_gender_dependent}. "
+                    f"Check LabValue's default_normal_range definition for this patient context."
+                )
+            
             value = self._generate_value_uniform(default_normal_range_dict)
             
             return value
@@ -94,4 +123,3 @@ class NumericValueDistribution(BaseValueDistribution):
         # generate value
         return np.random.uniform(result_dict["min"], result_dict["max"])
 
-    

endoreg_db/models/{emission → other/emission}/emission_factor.py

@@ -1,5 +1,10 @@
 from django.db import models
-from typing import List
+from typing import TYPE_CHECKING, List
+
+if TYPE_CHECKING:
+    from ...administration import ReferenceProduct
+    from ..unit import Unit
+
 class EmissionFactorManager(models.Manager):
     """
     Manager for EmissionFactor with custom query methods.
@@ -17,18 +22,22 @@ class EmissionFactor(models.Model):
 
     Attributes:
         name (str): The name of the emission factor.
-        name_de (str): The German name of the emission factor.
-        name_en (str): The English name of the emission factor.
         unit (ForeignKey): The unit associated with the emission factor.
         value (float): The value of the emission factor.
     """
     objects = EmissionFactorManager()
 
     name = models.CharField(max_length=255)
-    name_de = models.CharField(max_length=255, null=True)
-    name_en = models.CharField(max_length=255, null=True)
     unit = models.ForeignKey("Unit", on_delete=models.SET_NULL, null=True)
     value = models.FloatField()
+
+    if TYPE_CHECKING:
+        unit: "Unit"
+        reference_products: models.QuerySet["ReferenceProduct"]
+        reference_product_package: models.QuerySet["ReferenceProduct"]
+        reference_product_product: models.QuerySet["ReferenceProduct"]
+
+        
     
     def natural_key(self) -> tuple:
         """
@@ -64,7 +73,7 @@ class EmissionFactor(models.Model):
         Returns:
             list: A list of ReferenceProduct instances associated with this emission factor.
         """
-        from endoreg_db.models import ReferenceProduct
+        from ...administration.product import ReferenceProduct
 
         reference_products = []
         reference_products += ReferenceProduct.objects.filter(emission_factor_total=self)

endoreg_db/models/{persons → other}/gender.py

@@ -1,4 +1,8 @@
 from django.db import models
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..administration import Patient
 
 class GenderManager(models.Manager):
     def get_by_natural_key(self, name):
@@ -9,14 +13,15 @@ class Gender(models.Model):
     objects = GenderManager()
 
     name = models.CharField(max_length=255)
-    name_de = models.CharField(max_length=255, null=True)
-    name_en = models.CharField(max_length=255, null=True)
     abbreviation = models.CharField(max_length=255, null=True)
     description = models.TextField(blank=True, null=True)
 
+    if TYPE_CHECKING:
+        patients: models.QuerySet["Patient"]
+
     def natural_key(self):
         return (self.name,)
     
     def __str__(self):
-        return self.name
+        return str(self.name)
 

endoreg_db/models/other/information_source.py

@@ -0,0 +1,159 @@
+from django.db import models
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from endoreg_db.models import InformationSourceType
+
+def get_prediction_information_source():
+    """
+    Returns the InformationSource instance with the name "prediction".
+    
+    Raises:
+        AssertionError: If no InformationSource with the name "prediction" exists.
+    """
+    _source = InformationSource.objects.get(name="prediction")
+
+    # make sure to return only one object
+    assert _source, "No prediction information source found"
+    return _source
+
+
+class InformationSourceManager(models.Manager):
+    def get_by_natural_key(self, name):
+        """
+        Retrieves a model instance using its natural key.
+        
+        Args:
+            name: The natural key value corresponding to the model's 'name' field.
+        
+        Returns:
+            The model instance that matches the provided natural key.
+        """
+        return self.get(name=name)
+
+
+class InformationSource(models.Model):
+    objects = InformationSourceManager()
+
+    name = models.CharField(max_length=100)
+
+    url = models.URLField(blank=True, null=True)
+    description = models.TextField(blank=True, null=True)
+    date = models.DateField(blank=True, null=True)
+    date_created = models.DateField(auto_now_add=True)
+    date_modified = models.DateField(auto_now=True)
+    abbreviation = models.CharField(max_length=100, blank=True, null=True, unique=True)
+    
+    if TYPE_CHECKING:
+        #information_source_types: models.QuerySet["InformationSourceType"]
+        # Avoid self-referential import; use forward references instead
+        pass
+    class Meta:
+        verbose_name = "Information Source"
+        verbose_name_plural = "Information Sources"
+
+        # add name and abbreviation as index
+        indexes = [
+            models.Index(fields=["name"]),
+            models.Index(fields=["abbreviation"]),
+        ] 
+
+
+    def natural_key(self):
+        """
+        Returns the natural key tuple for the information source.
+        
+        The tuple contains the object's name, which uniquely identifies it for 
+        serialization and natural key lookup.
+        """
+        return (self.name,)
+
+    def __str__(self):
+        """
+        Return the name of the InformationSource as its string representation.
+        """
+        return str(self.name)
+
+class InformationSourceTypeManager(models.Manager):
+    def get_by_natural_key(self, name):
+        """
+        Retrieve an instance of the model by its natural key, which is the 'name' field.
+        
+        Parameters:
+            name (str): The value of the 'name' field to look up.
+        
+        Returns:
+            The model instance with the specified name.
+        """
+        return self.get(name=name)
+    
+class InformationSourceType(models.Model):
+    objects = InformationSourceTypeManager()
+
+    name = models.CharField(max_length=100, unique=True)
+    description = models.TextField(blank=True, null=True)
+
+    information_sources = models.ManyToManyField(
+        InformationSource,
+        related_name="information_source_types",
+        blank=True,
+    )
+
+    class Meta:
+        verbose_name = "Information Source Type"
+        verbose_name_plural = "Information Source Types"
+
+    # information_sources: models.QuerySet["InformationSource"]
+
+    @classmethod
+    def get_prediction_type(cls) -> "InformationSourceType":
+        """
+        Return the InformationSourceType instance with the name "prediction".
+        
+        Returns:
+            InformationSourceType: The instance representing the "prediction" information source type.
+        
+        Raises:
+            InformationSourceType.DoesNotExist: If no such instance exists.
+
+        """
+        try:
+            return cls.objects.get(name="prediction")
+        except cls.DoesNotExist as e:
+            raise cls.DoesNotExist(
+                "The 'prediction' InformationSourceType was not found. "
+                "Please check your data fixtures or initial data migrations."
+            ) from e
+
+    @classmethod
+    def get_manual_annotation_type(cls) -> "InformationSourceType":
+        """
+
+        Return the InformationSourceType instance representing manual annotation.
+        
+        Returns:
+            InformationSourceType: The instance with name "annotation".
+        
+        Raises:
+            AssertionError: If no InformationSourceType with name "annotation" exists.
+
+        """
+        try:
+            return cls.objects.get(name="manual_annotation")
+        except cls.DoesNotExist as e:
+            raise cls.DoesNotExist(
+                "The 'manual_annotation' InformationSourceType was not found. "
+                "Please check your data fixtures or initial data migrations."
+            ) from e
+
+
+    def natural_key(self):
+        """
+        Return a tuple containing the name of the information source type for natural key serialization.
+        """
+        return (self.name,)
+
+    def __str__(self):
+        """
+        Return the name of the InformationSourceType as its string representation.
+        """
+        return str(self.name)

endoreg_db/models/other/material.py

@@ -1,4 +1,10 @@
 from django.db import models
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from .emission import EmissionFactor
+    from ..administration.product.product_material import ProductMaterial
 
 class MaterialManager(models.Manager):
     def get_by_natural_key(self, name):
@@ -8,10 +14,12 @@ class Material(models.Model):
     objects = MaterialManager()
 
     name = models.CharField(max_length=255)
-    name_de = models.CharField(max_length=255, null=True)
-    name_en = models.CharField(max_length=255, null=True)
     emission_factor = models.ForeignKey("EmissionFactor", on_delete=models.SET_NULL, null=True)
 
+    if TYPE_CHECKING:
+        emission_factor: "EmissionFactor"
+        material_product_materials: models.QuerySet["ProductMaterial"]
+
     def natural_key(self):
         return (self.name,)
     

endoreg_db/models/other/resource.py

@@ -8,10 +8,14 @@ class Resource(models.Model):
     objects = ResourceManager()
 
     name = models.CharField(max_length=255)
-    name_de = models.CharField(max_length=255, null=True)
-    name_en = models.CharField(max_length=255, null=True)
 
     def natural_key(self):
+        """
+        Return a tuple representing the natural key for this resource instance.
+        
+        Returns:
+            tuple: A one-element tuple containing the resource's name.
+        """
         return (self.name,)
     
     def __str__(self):

endoreg_db/models/other/tag.py

@@ -0,0 +1,27 @@
+from django.db import models
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from endoreg_db.models import RequirementSet
+
+class TagManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+    
+class Tag(models.Model):
+    name = models.CharField(max_length=100, unique=True)
+
+    objects = TagManager()
+    
+    if TYPE_CHECKING:
+        requirement_sets: "models.ManyToManyField[RequirementSet]"
+
+    class Meta:
+        verbose_name = "Tag"
+        verbose_name_plural = "Tags"
+
+    def __str__(self):
+        return self.name
+
+    def natural_key(self):
+        return (self.name,)

endoreg_db/models/other/transport_route.py

@@ -1,4 +1,12 @@
 from django.db import models
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from endoreg_db.models import (
+        Product,
+        EmissionFactor,
+        Unit,
+    )
 
 class TransportRouteManager(models.Manager):
     def get_by_natural_key(self, name):
@@ -9,11 +17,14 @@ class TransportRoute(models.Model):
 
     distance = models.FloatField()
     name = models.CharField(max_length=255)
-    name_de = models.CharField(max_length=255, null=True)
-    name_en = models.CharField(max_length=255, null=True)
     emission_factor = models.ForeignKey("EmissionFactor", on_delete=models.SET_NULL, null=True)
     unit = models.ForeignKey("Unit", on_delete=models.SET_NULL, null=True)
 
+    if TYPE_CHECKING:
+        emission_factor: "EmissionFactor"
+        unit: "Unit"
+        products: models.QuerySet["Product"]
+
     def natural_key(self):
         return (self.name,)