endoreg-db 0.3.6__py3-none-any.whl → 0.8.6.1__py3-none-any.whl

endoreg_db/models/metadata/video_prediction_meta.py

@@ -0,0 +1,270 @@
+from typing import TYPE_CHECKING, Optional, List, Tuple
+from django.db import models
+import logging
+
+from endoreg_db.models.label import LabelSet
+
+from ..label.label_video_segment import (
+    LabelVideoSegment,
+
+)
+from ..utils import find_segments_in_prediction_array
+
+import numpy as np
+import pickle
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
+DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
+
+if TYPE_CHECKING:
+    from endoreg_db.models import ModelMeta, Label
+    from ..media.video.video_file import VideoFile
+
+
+class VideoPredictionMeta(models.Model):
+    """
+    Stores metadata about predictions made by a model for a specific video.
+
+    Must be associated with exactly one `VideoFile`.
+    """
+    model_meta = models.ForeignKey("ModelMeta", on_delete=models.CASCADE)
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    prediction_array = models.BinaryField(blank=True, null=True)
+
+    video_file = models.ForeignKey(
+        "VideoFile",
+        on_delete=models.CASCADE,
+        related_name="video_prediction_meta",
+        null=False,
+        blank=False,
+    )
+
+    if TYPE_CHECKING:
+        model_meta: "ModelMeta"
+        video_file: "VideoFile"
+        label_video_segments: "models.Manager[LabelVideoSegment]"
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=['model_meta', 'video_file'], name='unique_prediction_per_video_model')
+        ]
+        indexes = [
+            models.Index(fields=["model_meta", "video_file"]),
+        ]
+
+    def get_video(self) -> "VideoFile":
+        """Returns the associated VideoFile instance."""
+        if self.video_file:
+            return self.video_file
+        else:
+            raise ValueError("VideoPredictionMeta is not associated with a VideoFile.")
+
+    def __str__(self):
+        try:
+            video_obj = self.get_video()
+            return f"Prediction Meta for Video {video_obj.uuid} - {self.model_meta.name}"
+        except ValueError:
+            return f"Prediction Meta {self.pk} (Error: No VideoFile) - {self.model_meta.name}"
+        except Exception as e:
+            logger.warning("Error generating string representation for VideoPredictionMeta %s: %s", self.pk, e)
+            return f"Prediction Meta {self.pk} (Error: {e})"
+
+    def get_labelset(self) -> Optional["LabelSet"]:
+        """Get the labelset associated with the model."""
+        return self.model_meta.labelset
+
+    def get_label_list(self) -> list["Label"]:
+        """Get the ordered list of labels from the model's labelset."""
+        labelset = self.get_labelset()
+        if labelset:
+            return labelset.get_labels_in_order()
+        return []
+
+    def save_prediction_array(self, prediction_array: np.array):
+        """
+        Save the prediction array to the database.
+        """
+        self.prediction_array = pickle.dumps(prediction_array)
+        self.save(update_fields=['prediction_array', 'date_modified'])
+
+    def get_prediction_array(self):
+        """
+        Get the prediction array from the database.
+        """
+        if self.prediction_array is None:
+            return None
+        else:
+            try:
+                return pickle.loads(self.prediction_array)
+            except (pickle.UnpicklingError, TypeError, EOFError) as e:
+                logger.error(f"Error unpickling prediction array for {self}: {e}")
+                return None
+
+    def calculate_prediction_array(self, window_size_in_seconds: int = None):
+        """
+        Fetches all predictions for the associated video, labelset, and model meta,
+        applies smoothing, and saves the resulting binary prediction array.
+        """
+        from ..label import ImageClassificationAnnotation
+
+        video_obj = self.get_video()
+        model_meta = self.model_meta
+        label_list = self.get_label_list()
+        num_frames = video_obj.frame_count
+
+        if num_frames is None or num_frames <= 0:
+            logger.warning(f"Cannot calculate prediction array for {video_obj} with invalid frame count ({num_frames}).")
+            return
+
+        if not label_list:
+            logger.warning(f"No labels found for model {model_meta}. Cannot calculate prediction array.")
+            return
+
+        prediction_array = np.zeros((num_frames, len(label_list)))
+
+        base_pred_qs = ImageClassificationAnnotation.objects.filter(
+            model_meta=model_meta,
+            frame__video_file=video_obj
+        )
+
+        for i, label in enumerate(label_list):
+            predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list(
+                "frame__frame_number", "float_value"
+            )
+
+            confidences = np.full(num_frames, 0.5)
+            found_predictions = False
+            for frame_num, confidence in predictions:
+                if 0 <= frame_num < num_frames:
+                    confidences[frame_num] = confidence
+                    found_predictions = True
+                else:
+                    logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames-1}). Skipping.")
+
+            if not found_predictions:
+                logger.warning(f"No predictions found for label '{label.name}' in {video_obj}. Using default confidence.")
+
+            smooth_confidences = self.apply_running_mean(
+                confidences, window_size_in_seconds
+            )
+            binary_predictions = smooth_confidences > 0.5
+            prediction_array[:, i] = binary_predictions
+
+        self.save_prediction_array(prediction_array)
+        logger.info(f"Calculated and saved prediction array for {self}")
+
+    def apply_running_mean(self, confidence_array, window_size_in_seconds: int = None):
+        """
+        Apply a running mean filter to the confidence array for smoothing, assuming a padding
+        of 0.5 for the edges.
+        """
+        video_obj = self.get_video()
+        fps = video_obj.get_fps()
+
+        if fps is None or fps <= 0:
+            logger.warning(f"Invalid FPS ({fps}) for {video_obj}. Cannot apply running mean. Returning original array.")
+            return confidence_array
+
+        if not window_size_in_seconds:
+            window_size_in_seconds = DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN
+
+        window_size_in_frames = int(window_size_in_seconds * fps)
+        window_size_in_frames = max(window_size_in_frames, 1)
+
+        window = np.ones(window_size_in_frames) / window_size_in_frames
+        pad_size = window_size_in_frames // 2
+
+        padded_confidences = np.pad(
+            confidence_array,
+            (pad_size, pad_size),
+            "constant",
+            constant_values=(0.5, 0.5),
+        )
+
+        running_mean = np.convolve(padded_confidences, window, mode="same")
+        start_index = pad_size
+        end_index = start_index + len(confidence_array)
+        running_mean = running_mean[start_index:end_index]
+
+        if running_mean.shape != confidence_array.shape:
+            logger.warning(f"Running mean output shape {running_mean.shape} differs from input {confidence_array.shape}. Check padding/slicing.")
+            return confidence_array
+
+        return running_mean
+
+    def create_video_segments_for_label(self, segments: List[Tuple[int, int]], label: "Label"):
+        """
+        Creates LabelVideoSegment instances for the given label and segments.
+        """
+        from endoreg_db.models import InformationSource
+
+        video_obj = self.get_video()
+        information_source, _ = InformationSource.objects.get_or_create(name="prediction")
+
+        segments_to_create = []
+        for start_frame, end_frame in segments:
+            segment_data = {
+                "start_frame_number": start_frame,
+                "end_frame_number": end_frame,
+                "source": information_source,
+                "label": label,
+                "prediction_meta": self,
+                "video_file": video_obj,
+            }
+            if not LabelVideoSegment.objects.filter(
+                video_file=video_obj,
+                prediction_meta=self,
+                label=label,
+                start_frame_number=start_frame,
+                end_frame_number=end_frame
+            ).exists():
+                segments_to_create.append(LabelVideoSegment(**segment_data))
+
+        if segments_to_create:
+            LabelVideoSegment.objects.bulk_create(segments_to_create)
+            logger.info(f"Created {len(segments_to_create)} video segments for label '{label.name}' in {video_obj}.")
+        else:
+            logger.info(f"No new video segments needed for label '{label.name}' in {video_obj}.")
+
+    def create_video_segments(self, segment_length_threshold_in_s: float = None):
+        """
+        Generates LabelVideoSegments based on the stored prediction array.
+        """
+        if not segment_length_threshold_in_s:
+            segment_length_threshold_in_s = DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S
+
+        video_obj = self.get_video()
+        fps = video_obj.get_fps()
+
+        if fps is None or fps <= 0:
+            logger.warning(f"Cannot create video segments for {video_obj} with invalid FPS ({fps}).")
+            return
+
+        min_frame_length = int(segment_length_threshold_in_s * fps)
+        min_frame_length = max(min_frame_length, 1)
+
+        label_list = self.get_label_list()
+
+        prediction_array = self.get_prediction_array()
+        if prediction_array is None:
+            logger.info(f"Prediction array not found for {self}. Calculating...")
+            self.calculate_prediction_array()
+            prediction_array = self.get_prediction_array()
+            if prediction_array is None:
+                logger.error(f"Failed to get or calculate prediction array for {self}. Cannot create segments.")
+                return
+
+        if prediction_array.shape[1] != len(label_list):
+            logger.warning(f"Prediction array shape {prediction_array.shape} incompatible with label list length {len(label_list)} for {self}.")
+            return
+
+        logger.info(f"Creating video segments for {self} (min length: {min_frame_length} frames)...")
+        for i, label in enumerate(label_list):
+            binary_predictions = prediction_array[:, i].astype(bool)
+            segments = find_segments_in_prediction_array(binary_predictions, min_frame_length)
+            if segments:
+                self.create_video_segments_for_label(segments, label)
+        logger.info(f"Finished creating video segments for {self}.")

endoreg_db/models/other/__init__.py

@@ -1,5 +1,40 @@
-from .material import Material
-from .resource import Resource
-from .transport_route import TransportRoute
-from .waste import Waste
-from .distribution import *
+from .material import Material
+from .resource import Resource
+from .transport_route import TransportRoute
+from .waste import Waste
+from .distribution import (
+    BaseValueDistribution,
+    NumericValueDistribution,
+    SingleCategoricalValueDistribution,
+    MultipleCategoricalValueDistribution,
+    DateValueDistribution,
+)
+from .emission import EmissionFactor
+
+from .gender import Gender
+from .information_source import (
+    InformationSource,
+    InformationSourceType,
+)
+
+from .unit import Unit
+
+from .tag import Tag
+
+__all__ = [
+    'Material',
+    'Resource',
+    'TransportRoute',
+    'Waste',
+    'BaseValueDistribution',
+    'NumericValueDistribution',
+    'SingleCategoricalValueDistribution',
+    'MultipleCategoricalValueDistribution',
+    'DateValueDistribution',
+    "Gender",
+    "InformationSource",
+    "InformationSourceType",
+    "Unit",
+    "EmissionFactor",
+    "Tag",
+]

endoreg_db/models/other/distribution/__init__.py

@@ -0,0 +1,44 @@
+'''Module for distribution models.'''
+
+from .base_value_distribution import BaseValueDistribution
+from .numeric_value_distribution import NumericValueDistribution
+from .single_categorical_value_distribution import SingleCategoricalValueDistribution
+from .multiple_categorical_value_distribution import MultipleCategoricalValueDistribution
+from .date_value_distribution import DateValueDistribution
+
+
+__all__ = [
+    'BaseValueDistribution',
+    'NumericValueDistribution',
+    'SingleCategoricalValueDistribution',
+    'MultipleCategoricalValueDistribution',
+    'DateValueDistribution',
+]
+
+# Example Usage
+# Numeric distribution for age
+# age_distribution = NumericValueDistribution.objects.create(
+#     name='Age Distribution',
+#     distribution_type='normal',
+#     min_value=0,
+#     max_value=100,
+#     mean=50,
+#     std_dev=15
+# )
+
+# # Single categorical distribution for gender
+# gender_distribution = SingleCategoricalValueDistribution.objects.create(
+#     name='Gender Distribution',
+#     categories={'male': 0.5, 'female': 0.5}
+# )
+
+# # Multiple categorical distribution for symptoms
+# symptoms_distribution = MultipleCategoricalValueDistribution.objects.create(
+#     name='Symptoms Distribution',
+#     categories={'fever': 0.3, 'cough': 0.4, 'fatigue': 0.2, 'nausea': 0.1},
+#     min_count=1,
+#     max_count=3,
+#     count_distribution_type='normal',
+#     count_mean=2,
+#     count_std_dev=0.5
+# )

endoreg_db/models/other/distribution/base_value_distribution.py

@@ -0,0 +1,20 @@
+from django.db import models
+
+class BaseValueDistribution(models.Model):
+    """
+    Abstract base class for value distributions.
+    """
+    name = models.CharField(max_length=100)
+
+    class Meta:
+        abstract = True
+
+    def generate_value(self):
+        """
+        Generate a value based on the distribution rules.
+        Must be implemented by subclasses.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+    
+    def natural_key(self):
+        return (self.name,)

endoreg_db/models/other/distribution/date_value_distribution.py

@@ -0,0 +1,89 @@
+'''Model for date value distribution'''
+
+from datetime import date, timedelta
+from django.db import models
+import numpy as np
+
+from .base_value_distribution import BaseValueDistribution
+
+class DateValueDistributionManager(models.Manager):
+    '''Object manager for DateValueDistribution'''
+    def get_by_natural_key(self, name):
+        '''Retrieve a DateValueDistribution by its natural key.'''
+        return self.get(name=name)
+
+
+class DateValueDistribution(BaseValueDistribution):
+    """
+    Assign date values based on specified distribution.
+    Expects distribution_type (uniform, normal) and mode (date, timedelta) and based on this either
+    date_min, date_max, date_mean, date_std_dev or
+    timedelta_days_min, timedelta_days_max, timedelta_days_mean, timedelta_days_std_dev
+    """
+    objects = DateValueDistributionManager()
+    name = models.CharField(max_length=100)
+    description = models.TextField(blank=True, null=True)
+    DISTRIBUTION_CHOICES = [
+        ('uniform', 'Uniform'),
+        ('normal', 'Normal'),
+    ]
+    MODE_CHOICES = [
+        ('date', 'Date'),
+        ('timedelta', 'Timedelta'),
+    ]
+
+    distribution_type = models.CharField(max_length=20, choices=DISTRIBUTION_CHOICES)
+    mode = models.CharField(max_length=20, choices=MODE_CHOICES)
+
+    # Date-related fields
+    date_min = models.DateField(blank=True, null=True)
+    date_max = models.DateField(blank=True, null=True)
+    date_mean = models.DateField(blank=True, null=True)
+    date_std_dev = models.IntegerField(blank=True, null=True)  # Standard deviation in days
+
+    # Timedelta-related fields
+    timedelta_days_min = models.IntegerField(blank=True, null=True)
+    timedelta_days_max = models.IntegerField(blank=True, null=True)
+    timedelta_days_mean = models.IntegerField(blank=True, null=True)
+    timedelta_days_std_dev = models.IntegerField(blank=True, null=True)
+
+    def generate_value(self):
+        if self.mode == 'date':
+            return self._generate_date_value()
+        elif self.mode == 'timedelta':
+            return self._generate_timedelta_value()
+        else:
+            raise ValueError("Unsupported mode")
+
+    def _generate_date_value(self):
+        #UNTESTED
+        if self.distribution_type == 'uniform':
+            start_date = self.date_min.toordinal()
+            end_date = self.date_max.toordinal()
+            random_ordinal = np.random.randint(start_date, end_date)
+            return date.fromordinal(random_ordinal)
+        elif self.distribution_type == 'normal':
+            mean_ordinal = self.date_mean.toordinal()
+            std_dev_days = self.date_std_dev
+            random_ordinal = int(np.random.normal(mean_ordinal, std_dev_days))
+            random_ordinal = np.clip(random_ordinal, self.date_min.toordinal(), self.date_max.toordinal())
+            return date.fromordinal(random_ordinal)
+        else:
+            raise ValueError("Unsupported distribution type")
+
+    def _generate_timedelta_value(self):
+        if self.distribution_type == 'uniform':
+            random_days = np.random.randint(self.timedelta_days_min, self.timedelta_days_max + 1)
+            
+            
+        elif self.distribution_type == 'normal':
+            random_days = int(np.random.normal(self.timedelta_days_mean, self.timedelta_days_std_dev))
+            random_days = np.clip(random_days, self.timedelta_days_min, self.timedelta_days_max)
+            
+        else:
+            raise ValueError("Unsupported distribution type")
+        
+        current_date = date.today()
+        generated_date = current_date - timedelta(days=random_days)
+        print(generated_date)
+        return(generated_date)

endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py

@@ -0,0 +1,32 @@
+from django.db import models
+import numpy as np
+from .base_value_distribution import BaseValueDistribution
+
+class MultipleCategoricalValueDistributionManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class MultipleCategoricalValueDistribution(BaseValueDistribution):
+    """
+    Multiple categorical value distribution model.
+    Assigns a specific number or varying number of values based on probabilities.
+    """
+    objects = MultipleCategoricalValueDistributionManager()
+    categories = models.JSONField()  # { "category": "probability", ... }
+    min_count = models.IntegerField()
+    max_count = models.IntegerField()
+    count_distribution_type = models.CharField(max_length=20, choices=[('uniform', 'Uniform'), ('normal', 'Normal')])
+    count_mean = models.FloatField(null=True, blank=True)
+    count_std_dev = models.FloatField(null=True, blank=True)
+
+    def generate_value(self):
+        if self.count_distribution_type == 'uniform':
+            count = np.random.randint(self.min_count, self.max_count + 1)
+        elif self.count_distribution_type == 'normal':
+            count = int(np.random.normal(self.count_mean, self.count_std_dev))
+            count = np.clip(count, self.min_count, self.max_count)
+        else:
+            raise ValueError("Unsupported count distribution type")
+
+        categories, probabilities = zip(*self.categories.items())
+        return list(np.random.choice(categories, size=count, p=probabilities))

endoreg_db/models/other/distribution/numeric_value_distribution.py

@@ -0,0 +1,125 @@
+'''Model for numeric value distribution'''
+
+from django.db import models
+import numpy as np
+from .base_value_distribution import BaseValueDistribution
+from scipy.stats import skewnorm
+
+
+class NumericValueDistributionManager(models.Manager):
+    '''Object manager for NumericValueDistribution'''
+    def get_by_natural_key(self, name):
+        '''Retrieve a NumericValueDistribution by its natural key.'''
+        return self.get(name=name)
+
+class NumericValueDistribution(BaseValueDistribution):
+    """
+    Numeric value distribution model.
+    Supports uniform, normal, and skewed normal distributions with hard limits.
+    """
+    objects = NumericValueDistributionManager()
+    DISTRIBUTION_CHOICES = [
+        ('uniform', 'Uniform'),
+        ('normal', 'Normal'),
+        ('skewed_normal', 'Skewed Normal'),
+    ]
+
+    distribution_type = models.CharField(max_length=20, choices=DISTRIBUTION_CHOICES)
+    min_descriptor = models.CharField(
+        max_length=20
+    )
+
+    max_descriptor = models.CharField(
+        max_length=20
+    )
+
+    def generate_value(self, lab_value, patient):
+        '''Generate a value based on the distribution rules.'''
+        #FIXME
+        from endoreg_db.models import LabValue, Patient
+        assert isinstance(patient, Patient)
+        assert isinstance(lab_value, LabValue)
+        default_normal_range_dict = lab_value.get_normal_range(patient.age(), patient.gender)
+        assert isinstance(default_normal_range_dict, dict)
+
+        if self.distribution_type == 'uniform':
+            assert self.min_descriptor and self.max_descriptor
+            
+            min_key_needed = self.min_descriptor.split('_')[0]
+            max_key_needed = self.max_descriptor.split('_')[0]
+
+            min_val_from_range = default_normal_range_dict.get(min_key_needed)
+            max_val_from_range = default_normal_range_dict.get(max_key_needed)
+
+            if min_val_from_range is None:
+                raise ValueError(
+                    f"Cannot generate value for LabValue '{lab_value.name}' using distribution "
+                    f"'{getattr(self, 'name', self.pk)}'. "
+                    f"Required normal range component '{min_key_needed}' derived from min_descriptor "
+                    f"'{self.min_descriptor}' is None. "
+                    f"Patient context: age={patient.age()}, gender='{patient.gender.name if patient.gender else None}'. "
+                    f"LabValue '{lab_value.name}' is gender-dependent: {lab_value.normal_range_gender_dependent}. "
+                    f"Check LabValue's default_normal_range definition for this patient context."
+                )
+            
+            if max_val_from_range is None:
+                raise ValueError(
+                    f"Cannot generate value for LabValue '{lab_value.name}' using distribution "
+                    f"'{getattr(self, 'name', self.pk)}'. "
+                    f"Required normal range component '{max_key_needed}' derived from max_descriptor "
+                    f"'{self.max_descriptor}' is None. "
+                    f"Patient context: age={patient.age()}, gender='{patient.gender.name if patient.gender else None}'. "
+                    f"LabValue '{lab_value.name}' is gender-dependent: {lab_value.normal_range_gender_dependent}. "
+                    f"Check LabValue's default_normal_range definition for this patient context."
+                )
+            
+            value = self._generate_value_uniform(default_normal_range_dict)
+            
+            return value
+        
+        elif self.distribution_type == 'normal':
+            value = np.random.normal(self.mean, self.std_dev)
+            return np.clip(value, self.min_value, self.max_value)
+        elif self.distribution_type == 'skewed_normal':
+            value = skewnorm.rvs(a=self.skewness, loc=self.mean, scale=self.std_dev)
+            return np.clip(value, self.min_value, self.max_value)
+        else:
+            raise ValueError("Unsupported distribution type")
+
+    def parse_value_descriptor(self, value_descriptor:str):
+        '''Parse the value descriptor string into a dict with a lambda function.'''
+        # strings of shape f"{value_key}_{operator}_{value}"
+        # extract value_key, operator, value
+        value_key, operator, value = value_descriptor.split("_")
+        value = float(value)
+
+        operator_functions = {
+            "+": lambda x: x + value,
+            "-": lambda x: x - value,
+            "x": lambda x: x * value,
+            "/": lambda x: x / value,
+        }
+
+        return {value_key: operator_functions[operator]}
+
+        # create dict with {value_key: lambda x: x operator value}
+
+    def _generate_value_uniform(self, default_normal_range_dict:dict):
+        value_function_dict = self.parse_value_descriptor(
+            self.min_descriptor
+        )
+
+        _ = self.parse_value_descriptor(
+            self.max_descriptor
+        )
+
+        value_function_dict.update(_)
+
+        result_dict = {
+            key: value_function(default_normal_range_dict[key]) 
+            for key, value_function in value_function_dict.items()
+        }
+
+        # generate value
+        return np.random.uniform(result_dict["min"], result_dict["max"])
+

endoreg_db/models/other/distribution/single_categorical_value_distribution.py

@@ -0,0 +1,22 @@
+from django.db import models
+import numpy as np
+from .base_value_distribution import BaseValueDistribution
+
+
+class SingleCategoricalValueDistributionManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class SingleCategoricalValueDistribution(BaseValueDistribution):
+    """
+    Single categorical value distribution model.
+    Assigns a single value based on specified probabilities.
+    """
+    objects = SingleCategoricalValueDistributionManager()
+    categories = models.JSONField()  # { "category": "probability", ... }
+
+    def generate_value(self):
+        categories, probabilities = zip(*self.categories.items())
+        return np.random.choice(categories, p=probabilities)
+    
+

endoreg_db/models/other/emission/__init__.py

@@ -0,0 +1,5 @@
+from .emission_factor import EmissionFactor
+
+__all__ = [
+    'EmissionFactor',
+]