endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/models/metadata/video_prediction_meta.py

@@ -0,0 +1,319 @@
+import logging
+import pickle
+from typing import TYPE_CHECKING, List, Optional, Tuple
+
+import numpy as np
+from django.db import models
+
+from endoreg_db.models.label import LabelSet
+
+from ..label.label_video_segment import LabelVideoSegment
+from ..utils import find_segments_in_prediction_array
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
+DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
+
+if TYPE_CHECKING:
+    from endoreg_db.models import Label, ModelMeta
+
+    from ..media.video.video_file import VideoFile
+
+
+class VideoPredictionMeta(models.Model):
+    """
+    Stores metadata about predictions made by a model for a specific video.
+
+    Must be associated with exactly one `VideoFile`.
+    """
+
+    model_meta = models.ForeignKey("ModelMeta", on_delete=models.CASCADE)
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    prediction_array = models.BinaryField(blank=True, null=True)
+
+    video_file = models.ForeignKey(
+        "VideoFile",
+        on_delete=models.CASCADE,
+        related_name="video_prediction_meta",
+        null=False,
+        blank=False,
+    )
+
+    if TYPE_CHECKING:
+        model_meta: models.ForeignKey["ModelMeta"]
+        video_file: models.ForeignKey["VideoFile|None"]
+        label_video_segments: "models.Manager[LabelVideoSegment]"
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["model_meta", "video_file"],
+                name="unique_prediction_per_video_model",
+            )
+        ]
+        indexes = [
+            models.Index(fields=["model_meta", "video_file"]),
+        ]
+
+    @property
+    def video_file_safe(self) -> "VideoFile":
+        """Returns the associated VideoFile instance."""
+        video_file = self.video_file
+        if not video_file:
+            raise ValueError("VideoPredictionMeta is not associated with a VideoFile.")
+        return video_file
+
+    def get_video(self) -> "VideoFile":
+        """Returns the associated VideoFile instance."""
+        if self.video_file:
+            return self.video_file
+        else:
+            raise ValueError("VideoPredictionMeta is not associated with a VideoFile.")
+
+    def __str__(self):
+        try:
+            video_obj = self.get_video()
+            return f"Prediction Meta for Video {video_obj.video_hash} - {self.model_meta.name}"
+        except ValueError:
+            return f"Prediction Meta {self.pk} (Error: No VideoFile) - {self.model_meta.name}"
+        except Exception as e:
+            logger.warning(
+                "Error generating string representation for VideoPredictionMeta %s: %s",
+                self.pk,
+                e,
+            )
+            return f"Prediction Meta {self.pk} (Error: {e})"
+
+    def get_labelset(self) -> Optional["LabelSet"]:
+        """Get the labelset associated with the model."""
+        return self.model_meta.labelset
+
+    def get_label_list(self) -> list["Label"]:
+        """Get the ordered list of labels from the model's labelset."""
+        labelset = self.get_labelset()
+        if labelset:
+            return labelset.get_labels_in_order()
+        return []
+
+    def save_prediction_array(self, prediction_array: np.typing.NDArray):
+        """
+        Save the prediction array to the database.
+        """
+        self.prediction_array = pickle.dumps(prediction_array)
+        self.save(update_fields=["prediction_array", "date_modified"])
+
+    def get_prediction_array(self):
+        """
+        Get the prediction array from the database.
+        """
+        if self.prediction_array is None:
+            return None
+        else:
+            try:
+                return pickle.loads(self.prediction_array)
+            except (pickle.UnpicklingError, TypeError, EOFError) as e:
+                logger.error(f"Error unpickling prediction array for {self}: {e}")
+                return None
+
+    def calculate_prediction_array(self, window_size_in_seconds: Optional[int] = None):
+        """
+        Fetches all predictions for the associated video, labelset, and model meta,
+        applies smoothing, and saves the resulting binary prediction array.
+        """
+        from ..label import ImageClassificationAnnotation
+
+        video_obj = self.get_video()
+        model_meta = self.model_meta
+        label_list = self.get_label_list()
+        num_frames = video_obj.frame_count
+
+        if num_frames is None or num_frames <= 0:
+            logger.warning(
+                f"Cannot calculate prediction array for {video_obj} with invalid frame count ({num_frames})."
+            )
+            return
+
+        if not label_list:
+            logger.warning(
+                f"No labels found for model {model_meta}. Cannot calculate prediction array."
+            )
+            return
+
+        prediction_array = np.zeros((num_frames, len(label_list)))
+
+        base_pred_qs = ImageClassificationAnnotation.objects.filter(
+            model_meta=model_meta, frame__video_file=video_obj
+        )
+
+        for i, label in enumerate(label_list):
+            predictions = (
+                base_pred_qs.filter(label=label)
+                .order_by("frame__frame_number")
+                .values_list("frame__frame_number", "float_value")
+            )
+
+            confidences = np.full(num_frames, 0.5)
+            found_predictions = False
+            for frame_num, confidence in predictions:
+                if 0 <= frame_num < num_frames:
+                    confidences[frame_num] = confidence
+                    found_predictions = True
+                else:
+                    logger.warning(
+                        f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames - 1}). Skipping."
+                    )
+
+            if not found_predictions:
+                logger.warning(
+                    f"No predictions found for label '{label.name}' in {video_obj}. Using default confidence."
+                )
+
+            smooth_confidences = self.apply_running_mean(
+                confidences, window_size_in_seconds
+            )
+            binary_predictions = smooth_confidences > 0.5
+            prediction_array[:, i] = binary_predictions
+
+        self.save_prediction_array(prediction_array)
+        logger.info(f"Calculated and saved prediction array for {self}")
+
+    def apply_running_mean(
+        self, confidence_array, window_size_in_seconds: Optional[float] = None
+    ):
+        """
+        Apply a running mean filter to the confidence array for smoothing, assuming a padding
+        of 0.5 for the edges.
+        """
+        video_obj = self.get_video()
+        fps = video_obj.get_fps()
+
+        if fps is None or fps <= 0:
+            logger.warning(
+                f"Invalid FPS ({fps}) for {video_obj}. Cannot apply running mean. Returning original array."
+            )
+            return confidence_array
+
+        if not window_size_in_seconds:
+            window_size_in_seconds = DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN
+
+        window_size_in_frames = int(window_size_in_seconds * fps)
+        window_size_in_frames = max(window_size_in_frames, 1)
+
+        window = np.ones(window_size_in_frames) / window_size_in_frames
+        pad_size = window_size_in_frames // 2
+
+        padded_confidences = np.pad(
+            confidence_array,
+            (pad_size, pad_size),
+            "constant",
+            constant_values=(0.5, 0.5),
+        )
+
+        running_mean = np.convolve(padded_confidences, window, mode="same")
+        start_index = pad_size
+        end_index = start_index + len(confidence_array)
+        running_mean = running_mean[start_index:end_index]
+
+        if running_mean.shape != confidence_array.shape:
+            logger.warning(
+                f"Running mean output shape {running_mean.shape} differs from input {confidence_array.shape}. Check padding/slicing."
+            )
+            return confidence_array
+
+        return running_mean
+
+    def create_video_segments_for_label(
+        self, segments: List[Tuple[int, int]], label: "Label"
+    ):
+        """
+        Creates LabelVideoSegment instances for the given label and segments.
+        """
+        from endoreg_db.models import InformationSource
+
+        video_obj = self.get_video()
+        information_source, _ = InformationSource.objects.get_or_create(
+            name="prediction"
+        )
+
+        segments_to_create = []
+        for start_frame, end_frame in segments:
+            segment_data = {
+                "start_frame_number": start_frame,
+                "end_frame_number": end_frame,
+                "source": information_source,
+                "label": label,
+                "prediction_meta": self,
+                "video_file": video_obj,
+            }
+            if not LabelVideoSegment.objects.filter(
+                video_file=video_obj,
+                prediction_meta=self,
+                label=label,
+                start_frame_number=start_frame,
+                end_frame_number=end_frame,
+            ).exists():
+                segments_to_create.append(LabelVideoSegment(**segment_data))
+
+        if segments_to_create:
+            LabelVideoSegment.objects.bulk_create(segments_to_create)
+            logger.info(
+                f"Created {len(segments_to_create)} video segments for label '{label.name}' in {video_obj}."
+            )
+        else:
+            logger.info(
+                f"No new video segments needed for label '{label.name}' in {video_obj}."
+            )
+
+    def create_video_segments(
+        self, segment_length_threshold_in_s: Optional[float] = None
+    ):
+        """
+        Generates LabelVideoSegments based on the stored prediction array.
+        """
+        if not segment_length_threshold_in_s:
+            segment_length_threshold_in_s = DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S
+
+        video_obj = self.get_video()
+        fps = video_obj.get_fps()
+
+        if fps is None or fps <= 0:
+            logger.warning(
+                f"Cannot create video segments for {video_obj} with invalid FPS ({fps})."
+            )
+            return
+
+        min_frame_length = int(segment_length_threshold_in_s * fps)
+        min_frame_length = max(min_frame_length, 1)
+
+        label_list = self.get_label_list()
+
+        prediction_array = self.get_prediction_array()
+        if prediction_array is None:
+            logger.info(f"Prediction array not found for {self}. Calculating...")
+            self.calculate_prediction_array()
+            prediction_array = self.get_prediction_array()
+            if prediction_array is None:
+                logger.error(
+                    f"Failed to get or calculate prediction array for {self}. Cannot create segments."
+                )
+                return
+
+        if prediction_array.shape[1] != len(label_list):
+            logger.warning(
+                f"Prediction array shape {prediction_array.shape} incompatible with label list length {len(label_list)} for {self}."
+            )
+            return
+
+        logger.info(
+            f"Creating video segments for {self} (min length: {min_frame_length} frames)..."
+        )
+        for i, label in enumerate(label_list):
+            binary_predictions = prediction_array[:, i].astype(bool)
+            segments = find_segments_in_prediction_array(
+                binary_predictions, min_frame_length
+            )
+            if segments:
+                self.create_video_segments_for_label(segments, label)
+        logger.info(f"Finished creating video segments for {self}.")

endoreg_db/models/operation_log.py

@@ -0,0 +1,63 @@
+from django.conf import settings
+from django.db import models
+
+
+class OperationLog(models.Model):
+    """
+    Lightweight log of user-triggered operations (audit-like).
+    """
+
+    # actor_id – internal Django user ID (primary key)
+    # Who did it
+    actor_user = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="operation_logs",
+    )
+    actor_username = models.CharField(max_length=150, blank=True)
+    actor_email = models.EmailField(blank=True)
+    actor_keycloak_id = models.CharField(
+        max_length=255,
+        blank=True,
+        help_text="Keycloak subject/ID if you later want to store it.",
+    )
+
+    # What happened
+    action = models.CharField(
+        max_length=100,
+        help_text="e.g. 'anonymization.start', 'anonymization.validate'",
+    )
+    http_method = models.CharField(max_length=10, blank=True)
+    path = models.CharField(max_length=512, blank=True)
+
+    # On what resource
+    resource_type = models.CharField(
+        max_length=50,
+        blank=True,
+        help_text="e.g. 'video', 'pdf'",
+    )
+    resource_id = models.IntegerField(
+        null=True,
+        blank=True,
+        help_text="ID of VideoFile / RawPdfFile etc.",
+    )
+
+    # State before/after
+    status_before = models.CharField(max_length=50, blank=True)
+    status_after = models.CharField(max_length=50, blank=True)
+
+    # Extra info
+    meta = models.JSONField(null=True, blank=True)
+
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    #
+    class Meta:
+        verbose_name = "Operation Log"
+        verbose_name_plural = "Operation Logs"
+        ordering = ["-created_at"]
+
+    def __str__(self) -> str:
+        return f"[{self.created_at.isoformat()}] {self.action} by {self.actor_username or 'unknown'}"

endoreg_db/models/other/__init__.py

@@ -0,0 +1,40 @@
+from .material import Material
+from .resource import Resource
+from .transport_route import TransportRoute
+from .waste import Waste
+from .distribution import (
+    BaseValueDistribution,
+    NumericValueDistribution,
+    SingleCategoricalValueDistribution,
+    MultipleCategoricalValueDistribution,
+    DateValueDistribution,
+)
+from .emission import EmissionFactor
+
+from .gender import Gender
+from .information_source import (
+    InformationSource,
+    InformationSourceType,
+)
+
+from .unit import Unit
+
+from .tag import Tag
+
+__all__ = [
+    "Material",
+    "Resource",
+    "TransportRoute",
+    "Waste",
+    "BaseValueDistribution",
+    "NumericValueDistribution",
+    "SingleCategoricalValueDistribution",
+    "MultipleCategoricalValueDistribution",
+    "DateValueDistribution",
+    "Gender",
+    "InformationSource",
+    "InformationSourceType",
+    "Unit",
+    "EmissionFactor",
+    "Tag",
+]

endoreg_db/models/other/distribution/__init__.py

@@ -0,0 +1,46 @@
+"""Module for distribution models."""
+
+from .base_value_distribution import BaseValueDistribution
+from .numeric_value_distribution import NumericValueDistribution
+from .single_categorical_value_distribution import SingleCategoricalValueDistribution
+from .multiple_categorical_value_distribution import (
+    MultipleCategoricalValueDistribution,
+)
+from .date_value_distribution import DateValueDistribution
+
+
+__all__ = [
+    "BaseValueDistribution",
+    "NumericValueDistribution",
+    "SingleCategoricalValueDistribution",
+    "MultipleCategoricalValueDistribution",
+    "DateValueDistribution",
+]
+
+# Example Usage
+# Numeric distribution for age
+# age_distribution = NumericValueDistribution.objects.create(
+#     name='Age Distribution',
+#     distribution_type='normal',
+#     min_value=0,
+#     max_value=100,
+#     mean=50,
+#     std_dev=15
+# )
+
+# # Single categorical distribution for gender
+# gender_distribution = SingleCategoricalValueDistribution.objects.create(
+#     name='Gender Distribution',
+#     categories={'male': 0.5, 'female': 0.5}
+# )
+
+# # Multiple categorical distribution for symptoms
+# symptoms_distribution = MultipleCategoricalValueDistribution.objects.create(
+#     name='Symptoms Distribution',
+#     categories={'fever': 0.3, 'cough': 0.4, 'fatigue': 0.2, 'nausea': 0.1},
+#     min_count=1,
+#     max_count=3,
+#     count_distribution_type='normal',
+#     count_mean=2,
+#     count_std_dev=0.5
+# )

endoreg_db/models/other/distribution/base_value_distribution.py

@@ -0,0 +1,22 @@
+from django.db import models
+
+
+class BaseValueDistribution(models.Model):
+    """
+    Abstract base class for value distributions.
+    """
+
+    name = models.CharField(max_length=100)
+
+    class Meta:
+        abstract = True
+
+    def generate_value(self):
+        """
+        Generate a value based on the distribution rules.
+        Must be implemented by subclasses.
+        """
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def natural_key(self):
+        return (self.name,)

endoreg_db/models/other/distribution/date_value_distribution.py

@@ -0,0 +1,163 @@
+"""Model for date value distribution"""
+
+from datetime import date, timedelta
+
+import numpy as np
+from django.db import models
+
+from .base_value_distribution import BaseValueDistribution
+
+
+class DateValueDistributionManager(models.Manager):
+    """Object manager for DateValueDistribution"""
+
+    def get_by_natural_key(self, name):
+        """Retrieve a DateValueDistribution by its natural key."""
+        return self.get(name=name)
+
+
+class DateValueDistribution(BaseValueDistribution):
+    """
+    Assign date values based on specified distribution.
+    Expects distribution_type (uniform, normal) and mode (date, timedelta) and based on this either
+    date_min, date_max, date_mean, date_std_dev or
+    timedelta_days_min, timedelta_days_max, timedelta_days_mean, timedelta_days_std_dev
+    """
+
+    objects = DateValueDistributionManager()
+    name = models.CharField(max_length=100)
+    description = models.TextField(blank=True, null=True)
+    DISTRIBUTION_CHOICES = [
+        ("uniform", "Uniform"),
+        ("normal", "Normal"),
+    ]
+    MODE_CHOICES = [
+        ("date", "Date"),
+        ("timedelta", "Timedelta"),
+    ]
+
+    distribution_type = models.CharField(max_length=20, choices=DISTRIBUTION_CHOICES)
+    mode = models.CharField(max_length=20, choices=MODE_CHOICES)
+
+    # Date-related fields
+    date_min = models.DateField(blank=True, null=True)
+    date_max = models.DateField(blank=True, null=True)
+    date_mean = models.DateField(blank=True, null=True)
+    date_std_dev = models.IntegerField(
+        blank=True, null=True
+    )  # Standard deviation in days
+
+    # Timedelta-related fields
+    timedelta_days_min = models.IntegerField(blank=True, null=True)
+    timedelta_days_max = models.IntegerField(blank=True, null=True)
+    timedelta_days_mean = models.IntegerField(blank=True, null=True)
+    timedelta_days_std_dev = models.IntegerField(blank=True, null=True)
+
+    # create *_safe properties for dates and timedeltas
+    @property
+    def date_min_safe(self):
+        _date = self.date_min
+        if _date is None:
+            raise ValueError("date_min is not set")
+        return _date
+
+    @property
+    def date_max_safe(self):
+        _date = self.date_max
+        if _date is None:
+            raise ValueError("date_max is not set")
+        return _date
+
+    @property
+    def date_mean_safe(self):
+        _date = self.date_mean
+        if _date is None:
+            raise ValueError("date_mean is not set")
+        return _date
+
+    @property
+    def date_std_dev_safe(self):
+        _std_dev = self.date_std_dev
+        if _std_dev is None:
+            raise ValueError("date_std_dev is not set")
+        return _std_dev
+
+    @property
+    def timedelta_days_min_safe(self):
+        _min = self.timedelta_days_min
+        if _min is None:
+            raise ValueError("timedelta_days_min is not set")
+        return _min
+
+    @property
+    def timedelta_days_max_safe(self):
+        _max = self.timedelta_days_max
+        if _max is None:
+            raise ValueError("timedelta_days_max is not set")
+        return _max
+
+    @property
+    def timedelta_days_mean_safe(self):
+        _mean = self.timedelta_days_mean
+        if _mean is None:
+            raise ValueError("timedelta_days_mean is not set")
+        return _mean
+
+    @property
+    def timedelta_days_std_dev_safe(self):
+        _std_dev = self.timedelta_days_std_dev
+        if _std_dev is None:
+            raise ValueError("timedelta_days_std_dev is not set")
+        return _std_dev
+
+    def generate_value(self):
+        if self.mode == "date":
+            return self._generate_date_value()
+        elif self.mode == "timedelta":
+            return self._generate_timedelta_value()
+        else:
+            raise ValueError("Unsupported mode")
+
+    def _generate_date_value(self):
+        # UNTESTED
+        if self.distribution_type == "uniform":
+            start_date = self.date_min_safe.toordinal()
+            end_date = self.date_max_safe.toordinal()
+            random_ordinal = np.random.randint(start_date, end_date)
+            return date.fromordinal(random_ordinal)
+        elif self.distribution_type == "normal":
+            mean_ordinal = self.date_mean_safe.toordinal()
+            std_dev_days = self.date_std_dev_safe
+            random_ordinal = int(np.random.normal(mean_ordinal, std_dev_days))
+            random_ordinal = np.clip(
+                random_ordinal,
+                self.date_min_safe.toordinal(),
+                self.date_max_safe.toordinal(),
+            )
+            return date.fromordinal(random_ordinal)
+        else:
+            raise ValueError("Unsupported distribution type")
+
+    def _generate_timedelta_value(self):
+        if self.distribution_type == "uniform":
+            random_days = np.random.randint(
+                self.timedelta_days_min_safe, self.timedelta_days_max_safe + 1
+            )
+
+        elif self.distribution_type == "normal":
+            random_days = int(
+                np.random.normal(
+                    self.timedelta_days_mean_safe, self.timedelta_days_std_dev_safe
+                )
+            )
+            random_days = np.clip(
+                random_days, self.timedelta_days_min_safe, self.timedelta_days_max_safe
+            )
+
+        else:
+            raise ValueError("Unsupported distribution type")
+
+        current_date = date.today()
+        generated_date = current_date - timedelta(days=random_days)
+
+        return generated_date