endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/models/state/audit_ledger.py

@@ -0,0 +1,155 @@
+# stats/models.py
+import hashlib
+import json
+import uuid
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+
+"""
+
+AuditLedger Model
+
+AuditLedger is a model that tracks changes to other models in the database.
+It stores a hash of the previous state, the current state, and the action taken.
+This allows for a complete audit trail of changes made to the database.
+The model includes fields for the timestamp, user who made the change, object type,
+object primary key, action taken, and the data associated with the change.
+The save method computes the hash of the current state and the previous state
+before saving the record to the database.
+The hash is computed using SHA-256 and includes the timestamp, user ID,
+object type, object primary key, action taken, and the data associated with the change.
+The hash is stored in the database to ensure data integrity and to allow for
+verification of the data.
+The model also includes a method to retrieve the last hash from the database
+to ensure that the current hash is always based on the most recent state of the database.
+The model is designed to be immutable, meaning that once a record is created,
+it cannot be modified. This ensures that the audit trail is complete and accurate.
+
+Raises:
+    RuntimeError: _description_
+
+Returns:
+    _type_: _description_
+"""
+
+# TODO implement later on
+
+
+class AuditLedger(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    ts = models.DateTimeField(default=timezone.now, editable=False)
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT)
+    object_type = models.CharField(max_length=80)  # e.g. 'VideoFile'
+    object_pk = models.CharField(max_length=40)  # UUID or int
+    action = models.CharField(max_length=40)  # 'created' | 'validated' | …
+    data = models.JSONField()  # snapshot / diff / metadata
+    prev_hash = models.CharField(max_length=64, editable=False)
+    hash = models.CharField(max_length=64, editable=False)
+
+    class Meta:
+        ordering = ["ts"]
+        indexes = [models.Index(fields=["object_type", "object_pk"])]
+
+    # ------------------------------------------------------
+    def save(self, *args, **kw):
+        """
+        Saves a new immutable audit record, computing and linking cryptographic hashes.
+
+        Raises:
+            RuntimeError: If an attempt is made to modify an existing audit record.
+        """
+        if self._state.adding:  # only on INSERT
+            self.prev_hash = self._last_hash()
+            self.hash = self._compute_hash()
+        else:
+            raise RuntimeError("AuditLedger rows are immutable")
+        super().save(*args, **kw)
+
+    # ------------------------------------------------------
+    def _last_hash(self) -> str:
+        """
+        Retrieves the hash of the most recent audit record.
+
+        Returns:
+            The SHA-256 hash of the latest `AuditLedger` entry by timestamp, or a string of 64 zeros if no records exist.
+        """
+        last = AuditLedger.objects.order_by("-ts").first()
+        return last.hash if last else "0" * 64
+
+    def _compute_hash(self) -> str:
+        """
+        Computes the SHA-256 hash of the current audit record's data.
+
+        The hash is generated from a JSON-serialized payload containing the timestamp, user ID, object type and primary key, action, associated data, and the previous record's hash. This ensures the integrity and immutability of the audit trail.
+
+        Returns:
+            The hexadecimal SHA-256 hash string representing the current audit record.
+        """
+        payload = {
+            "ts": self.ts.isoformat(),
+            "uid": str(self.id),
+            "obj": f"{self.object_type}:{self.object_pk}",
+            "act": self.action,
+            "data": self.data,
+            "prev": self.prev_hash,
+        }
+        return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+
+    def log_validation(self, user, instance, action: str, extra=None):
+        """
+        Creates an audit record for a validation action performed by a user on a specific model instance.
+
+        Args:
+            user: The user performing the action.
+            instance: The model instance being validated.
+            action: The action performed (e.g., 'validated').
+            extra: Optional additional data to include in the audit record.
+        """
+        AuditLedger.objects.create(
+            user=user,
+            object_type=instance.__class__.__name__,
+            object_pk=str(instance.pk),
+            action=action,
+            data=extra or {},
+        )
+
+    @classmethod
+    def _distinct(self, object_type: str, action: str):
+        """
+        Returns the number of distinct objects of a given type that have a specific audit action recorded.
+
+        Args:
+            object_type: The type of object to filter by (e.g., 'VideoFile').
+            action: The audit action to filter by (e.g., 'validated').
+
+        Returns:
+            The count of unique object primary keys matching the specified type and action.
+        """
+        return (
+            AuditLedger.objects.filter(object_type=object_type, action=action)
+            .values("object_pk")
+            .distinct()
+            .count()
+        )
+
+    def collect_counters(self):
+        """
+        Aggregates and returns summary statistics for audit actions and object types.
+
+        Returns:
+            dict: A dictionary containing counts of distinct cases, videos, annotations,
+            anonymizations, images, and breakdowns of video statuses based on audit records.
+        """
+        return {
+            "totalCases": AuditLedger._distinct("VideoFile", "created"),
+            "totalVideos": AuditLedger._distinct("VideoFile", "created"),
+            "totalAnnotations": AuditLedger.objects.filter(
+                action="annotation_added"
+            ).count(),
+            "totalAnonymizations": AuditLedger._distinct("VideoFile", "anonymized"),
+            "totalImages": AuditLedger._distinct("Image", "created"),
+            "videosCompleted": AuditLedger._distinct("VideoFile", "validated"),
+            "videosAnonym": AuditLedger._distinct("VideoFile", "anonymized"),
+        }

endoreg_db/models/state/label_video_segment.py

@@ -0,0 +1,31 @@
+from typing import TYPE_CHECKING
+
+from django.db import models
+
+from .abstract import AbstractState
+
+
+class LabelVideoSegmentState(AbstractState):
+    """State for label video segment data."""
+
+    prediction = models.BooleanField(default=False)
+    annotation = models.BooleanField(default=False)
+    frames_extracted = models.BooleanField(default=False)
+    is_validated = models.BooleanField(default=False)
+
+    origin = models.OneToOneField(
+        "LabelVideoSegment",
+        on_delete=models.CASCADE,
+        related_name="state",
+        null=True,
+        blank=True,
+    )
+
+    class Meta:
+        verbose_name = "Label Video Segment State"
+        verbose_name_plural = "Label Video Segment States"
+
+    if TYPE_CHECKING:
+        from endoreg_db.models import LabelVideoSegment
+
+        origin: models.OneToOneField["LabelVideoSegment|None"]

endoreg_db/models/state/processing_history/__init__.py

@@ -0,0 +1,3 @@
+from .processing_history import ProcessingHistory
+
+__all__ = ["ProcessingHistory"]

endoreg_db/models/state/processing_history/processing_history.py

@@ -0,0 +1,136 @@
+from typing import Optional
+from logging import getLogger
+from pathlib import Path
+from django.db import models
+from endoreg_db.utils.file_operations import get_content_hash_filename
+
+logger = getLogger(__name__)
+
+
+class ProcessingHistory(models.Model):
+    """
+    Processing history keyed by a stable file *content hash*.
+
+    - For videos: use VideoFile.video_hash (raw mp4 bytes)
+    - For reports: use RawPdfFile.pdf_hash (raw pdf bytes)
+
+    We *optionally* link back to a concrete model instance using
+    (content_type, object_id), but the logical identity is file_hash.
+    """
+
+    file_hash = models.CharField(
+        max_length=64,
+        primary_key=True,
+        help_text="Content hash of the original file (e.g. video_hash/pdf_hash).",
+        blank=True,
+    )
+
+    created_at = models.DateTimeField(auto_now_add=True)
+    success = models.BooleanField(default=False, blank=True)
+
+    object_id = models.PositiveBigIntegerField(null=True, blank=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+
+    # ------------------------------------------------------------------
+    # Core helpers
+    # ------------------------------------------------------------------
+    @staticmethod
+    def get_content_hash(path: Path):
+        return get_content_hash_filename(path)
+
+    @classmethod
+    def get_or_create_for_hash(
+        cls,
+        *,
+        file_hash: str,
+        obj: Optional[models.Model] = None,
+        success: Optional[bool] = None,
+    ) -> "ProcessingHistory":
+        """
+        Get or create history row for a given *file hash*.
+
+        - PK is file_hash
+        - Optionally set/overwrite:
+            - content_type/object_id
+            - success flag
+        """
+        defaults: dict[str, object] = {}
+        if success is not None:
+            defaults["success"] = success
+
+        ph, created = cls.objects.get_or_create(
+            file_hash=file_hash,
+            defaults=defaults,
+        )
+
+        changed: list[str] = []
+
+        if obj is not None:
+            if ph.object_id != obj.pk:
+                ph.object_id = obj.pk
+                changed.append("object_id")
+
+        if success is not None and ph.success != success:
+            ph.success = success
+            changed.append("success")
+
+        if changed:
+            ph.save(update_fields=changed)
+
+        if created:
+            logger.info(
+                "Created ProcessingHistory for hash=%s (success=%s).",
+                file_hash,
+                ph.success,
+            )
+
+        return ph
+
+    @classmethod
+    def has_history_for_hash(
+        cls,
+        *,
+        file_hash: str,
+        success: Optional[bool] = None,
+    ) -> bool:
+        qs = cls.objects.filter(file_hash=file_hash)
+        if success is not None:
+            qs = qs.filter(success=success)
+        return qs.exists()
+
+    # ------------------------------------------------------------------
+    # Convenience methods
+    # ------------------------------------------------------------------
+    @classmethod
+    def update_status(
+        cls,
+        *,
+        file_hash: str,
+        success: bool,
+        obj: Optional[models.Model] = None,
+    ) -> "ProcessingHistory":
+        return cls.get_or_create_for_hash(
+            file_hash=file_hash,
+            obj=obj,
+            success=success,
+        )
+
+    @classmethod
+    def mark_success(
+        cls,
+        *,
+        file_hash: str,
+        obj: Optional[models.Model] = None,
+    ) -> "ProcessingHistory":
+        return cls.update_status(file_hash=file_hash, success=True, obj=obj)
+
+    @classmethod
+    def mark_failure(
+        cls,
+        *,
+        file_hash: str,
+        obj: Optional[models.Model] = None,
+    ) -> "ProcessingHistory":
+        return cls.update_status(file_hash=file_hash, success=False, obj=obj)

endoreg_db/models/state/raw_pdf.py

@@ -0,0 +1,219 @@
+"""
+Defines state tracking models related to report processing, including extraction of text and metadata, AI predictions, and anonymization status for RawPdfFile instances.
+"""
+
+import logging
+from typing import TYPE_CHECKING
+
+from django.db import models, transaction
+
+from endoreg_db.models.state.anonymization import AnonymizationState
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from ..media import RawPdfFile
+
+
+class RawPdfState(models.Model):
+    """
+    Tracks the processing state of a RawPdfFile instance.
+    Uses BooleanFields for clear, distinct states.
+    """
+
+    text_meta_extracted = models.BooleanField(
+        default=False, help_text="True if text metadata (OCR) has been extracted."
+    )
+
+    # AI / Annotation related states
+    initial_prediction_completed = models.BooleanField(
+        default=False, help_text="True if initial AI prediction has run."
+    )
+
+    # Processing state
+    sensitive_meta_processed = models.BooleanField(
+        default=False,
+        help_text="True if the video has been fully processed, meaning a anonymized person was created.",
+    )
+
+    # Anonymization state
+    anonymized = models.BooleanField(
+        default=False, help_text="True if the anonymized video file has been created."
+    )
+    anonymization_validated = models.BooleanField(
+        default=False,
+        help_text="True if the anonymization process has been validated and confirmed.",
+    )
+
+    # Processing state
+    processing_started = models.BooleanField(
+        default=False,
+        help_text="True if the processing has started, but not yet completed.",
+    )
+    processing_error = models.BooleanField(
+        default=False, help_text="True if an error occurred during processing."
+    )
+
+    # Timestamps
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+
+    was_created = models.BooleanField(
+        default=True, help_text="True if this state was created for the first time."
+    )
+
+    # report metadata extraction state
+    pdf_meta_extracted = models.BooleanField(
+        default=False, help_text="True if report metadata has been extracted."
+    )
+
+    if TYPE_CHECKING:
+        raw_pdf_file: "RawPdfFile"
+
+    def __str__(self):
+        """
+        Return a string summarizing the RawPdfState instance, including the related report file UUID and key processing state flags with timestamps.
+        """
+        try:
+            uuid = self.raw_pdf_file.pk
+        except Exception:
+            uuid = None
+
+        states = [
+            f"TextMetaExtracted={self.text_meta_extracted}",
+            f"PredictionDone={self.initial_prediction_completed}",
+            f"Anonymized={self.anonymized}",
+            f"AnonymizationValidated={self.anonymization_validated}",
+            f"SensitiveMetaProcessed={self.sensitive_meta_processed}",
+            f"DateCreated={self.date_created.isoformat()}",
+            f"DateModified={self.date_modified.isoformat()}",
+        ]
+        return f"RawPdfState(Pdf:{uuid}): {', '.join(states)}"
+
+    @property
+    def anonymization_status(self) -> AnonymizationState:
+        """
+        Determines the current anonymization workflow status for the report processing state.
+
+        Returns:
+            AnonymizationStatus: The current status, reflecting progress or failure in the anonymization process.
+        """
+        if self.anonymization_validated:
+            return AnonymizationState.VALIDATED  #  Validation in Frontend completed -> Views related to this /home/admin/endoreg-db/endoreg_db/views/anonymization/validate.py
+        if self.sensitive_meta_processed:
+            return (
+                AnonymizationState.DONE_PROCESSING_ANONYMIZATION
+            )  # /home/admin/endoreg-db/endoreg_db/services/pdf_import.py
+        if (
+            self.processing_started
+            and not self.processing_error
+            and not self.anonymized
+        ):
+            return AnonymizationState.PROCESSING_ANONYMIZING
+        if getattr(self, "processing_error", False):
+            return (
+                AnonymizationState.FAILED
+            )  # /home/admin/endoreg-db/endoreg_db/services/pdf_import.py
+        if self.processing_started:
+            return (
+                AnonymizationState.STARTED
+            )  # /home/admin/endoreg-db/endoreg_db/services/pdf_import.py
+        if self.anonymized:
+            return AnonymizationState.ANONYMIZED
+        return AnonymizationState.NOT_STARTED
+
+    def mark_processing_not_started(self) -> None:
+        """
+        Mark the processing as not started and optionally save the updated state.
+
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        with transaction.atomic():
+            self.processing_started = False
+            self.anonymized = False
+            self.was_created = False
+            self.sensitive_meta_processed = False
+            self.anonymization_validated = False
+            self.save()
+
+    def mark_processing_started(self, *, save: bool = True) -> None:
+        """
+        Mark the processing as started and optionally save the updated state.
+
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        self.processing_started = True
+        self.save(update_fields=["processing_started", "date_modified"])
+
+    # ---- Single‑responsibility mutators ---------------------------------
+    def mark_sensitive_meta_processed(self, *, save: bool = True) -> None:
+        """
+        Mark the sensitive metadata processing step as completed for this report state.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.sensitive_meta_processed = True
+        if save:
+            self.save(update_fields=["sensitive_meta_processed", "date_modified"])
+
+    def mark_anonymization_validated(self, *, save: bool = True) -> None:
+        """
+        Mark the anonymization as validated for this report processing state.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.anonymization_validated = True
+        if save:
+            self.save(update_fields=["anonymization_validated", "date_modified"])
+
+    def mark_anonymized(self, *, save: bool = True) -> None:
+        """
+        Mark the report as anonymized and optionally save the updated state.
+
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        with transaction.atomic():
+            self.anonymized = True
+            self.save(update_fields=["anonymized", "date_modified"])
+
+    def mark_initial_prediction_completed(self, *, save: bool = True) -> None:
+        """
+        Mark the initial AI prediction step as completed for this report processing state.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.initial_prediction_completed = True
+        if save:
+            self.save(update_fields=["initial_prediction_completed", "date_modified"])
+
+    def mark_pdf_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the report metadata extraction step as completed for this state.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.pdf_meta_extracted = True
+        if save:
+            self.save(update_fields=["pdf_meta_extracted", "date_modified"])
+
+    def mark_text_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the text metadata extraction step as completed for this report processing state.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.text_meta_extracted = True
+        if save:
+            self.save(update_fields=["text_meta_extracted", "date_modified"])
+
+    class Meta:
+        verbose_name = "Raw report Processing State"
+        verbose_name_plural = "Raw report Processing States"

endoreg_db/models/state/sensitive_meta.py

@@ -0,0 +1,50 @@
+from typing import TYPE_CHECKING
+
+from django.db import models
+
+from .abstract import AbstractState
+
+
+class SensitiveMetaState(AbstractState):
+    """State for sensitive meta data."""
+
+    dob_verified = models.BooleanField(default=False)
+    names_verified = models.BooleanField(default=False)
+
+    origin = models.OneToOneField(
+        "SensitiveMeta",
+        on_delete=models.CASCADE,
+        related_name="state",
+        null=True,
+        blank=True,
+    )
+
+    if TYPE_CHECKING:
+        from endoreg_db.models import SensitiveMeta
+
+        origin: models.OneToOneField["SensitiveMeta|None"]
+
+    @property
+    def is_verified(self) -> bool:
+        """
+        Return True if both date of birth and names have been verified; otherwise, return False.
+        """
+        return self.dob_verified and self.names_verified
+
+    def mark_dob_verified(self):
+        """
+        Set the date of birth verification status to True and persist the change to the database.
+        """
+        self.dob_verified = True
+        self.save(update_fields=["dob_verified"])
+
+    def mark_names_verified(self):
+        """
+        Mark the names as verified and persist the change to the database.
+        """
+        self.names_verified = True
+        self.save(update_fields=["names_verified"])
+
+    class Meta:
+        verbose_name = "Sensitive Meta State"
+        verbose_name_plural = "Sensitive Meta States"