endoreg-db 0.4.5__py3-none-any.whl → 0.8.6.3__py3-none-any.whl

endoreg_db/models/state/__init__.py

@@ -0,0 +1,12 @@
+from .sensitive_meta import SensitiveMetaState
+from .video import VideoState, AnonymizationStatus
+from .label_video_segment import LabelVideoSegmentState
+from .raw_pdf import RawPdfState
+
+__all__ = [
+    "SensitiveMetaState",
+    "VideoState",
+    "LabelVideoSegmentState",
+    "AnonymizationStatus",
+    "RawPdfState",
+]

endoreg_db/models/state/abstract.py

@@ -0,0 +1,11 @@
+from django.db import models
+
+class AbstractState(models.Model):
+    """Abstract base class for all states."""
+
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+
+    class Meta:
+        abstract = True

endoreg_db/models/state/audit_ledger.py

@@ -0,0 +1,150 @@
+# stats/models.py
+import hashlib, json, uuid
+from django.db import models
+from django.utils import timezone
+from django.conf import settings
+
+"""
+
+AuditLedger Model
+
+AuditLedger is a model that tracks changes to other models in the database.
+It stores a hash of the previous state, the current state, and the action taken.
+This allows for a complete audit trail of changes made to the database.
+The model includes fields for the timestamp, user who made the change, object type,
+object primary key, action taken, and the data associated with the change.
+The save method computes the hash of the current state and the previous state
+before saving the record to the database.
+The hash is computed using SHA-256 and includes the timestamp, user ID,
+object type, object primary key, action taken, and the data associated with the change.
+The hash is stored in the database to ensure data integrity and to allow for
+verification of the data.
+The model also includes a method to retrieve the last hash from the database
+to ensure that the current hash is always based on the most recent state of the database.
+The model is designed to be immutable, meaning that once a record is created,
+it cannot be modified. This ensures that the audit trail is complete and accurate.
+
+Raises:
+    RuntimeError: _description_
+
+Returns:
+    _type_: _description_
+"""
+
+
+class AuditLedger(models.Model):
+    id           = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    ts           = models.DateTimeField(default=timezone.now, editable=False)
+    user         = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT)
+    object_type  = models.CharField(max_length=80)            # e.g. 'VideoFile'
+    object_pk    = models.CharField(max_length=40)            # UUID or int
+    action       = models.CharField(max_length=40)            # 'created' | 'validated' | …
+    data         = models.JSONField()                         # snapshot / diff / metadata
+    prev_hash    = models.CharField(max_length=64, editable=False)
+    hash         = models.CharField(max_length=64, editable=False)
+
+    class Meta:
+        ordering = ['ts']
+        indexes  = [models.Index(fields=['object_type', 'object_pk'])]
+
+    # ------------------------------------------------------
+    def save(self, *args, **kw):
+        """
+        Saves a new immutable audit record, computing and linking cryptographic hashes.
+        
+        Raises:
+            RuntimeError: If an attempt is made to modify an existing audit record.
+        """
+        if self._state.adding:                         # only on INSERT
+            self.prev_hash = self._last_hash()
+            self.hash      = self._compute_hash()
+        else:
+            raise RuntimeError("AuditLedger rows are immutable")
+        super().save(*args, **kw)
+
+    # ------------------------------------------------------
+    def _last_hash(self) -> str:
+        """
+        Retrieves the hash of the most recent audit record.
+        
+        Returns:
+            The SHA-256 hash of the latest `AuditLedger` entry by timestamp, or a string of 64 zeros if no records exist.
+        """
+        last = AuditLedger.objects.order_by('-ts').first()
+        return last.hash if last else '0' * 64
+
+    def _compute_hash(self) -> str:
+        """
+        Computes the SHA-256 hash of the current audit record's data.
+        
+        The hash is generated from a JSON-serialized payload containing the timestamp, user ID, object type and primary key, action, associated data, and the previous record's hash. This ensures the integrity and immutability of the audit trail.
+        
+        Returns:
+            The hexadecimal SHA-256 hash string representing the current audit record.
+        """
+        payload = {
+            'ts':   self.ts.isoformat(),
+            'uid':  str(self.user_id),
+            'obj':  f'{self.object_type}:{self.object_pk}',
+            'act':  self.action,
+            'data': self.data,
+            'prev': self.prev_hash,
+        }
+        return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+    
+    def log_validation(user, instance, action: str, extra=None):
+        """
+        Creates an audit record for a validation action performed by a user on a specific model instance.
+        
+        Args:
+            user: The user performing the action.
+            instance: The model instance being validated.
+            action: The action performed (e.g., 'validated').
+            extra: Optional additional data to include in the audit record.
+        """
+        AuditLedger.objects.create(
+            user=user,
+            object_type=instance.__class__.__name__,
+            object_pk=str(instance.pk),
+            action=action,
+            data=extra or {},
+        )
+
+    def _distinct(object_type: str, action: str):
+        """
+        Returns the number of distinct objects of a given type that have a specific audit action recorded.
+        
+        Args:
+            object_type: The type of object to filter by (e.g., 'VideoFile').
+            action: The audit action to filter by (e.g., 'validated').
+        
+        Returns:
+            The count of unique object primary keys matching the specified type and action.
+        """
+        return (
+            AuditLedger.objects
+            .filter(object_type=object_type, action=action)
+            .values('object_pk')
+            .distinct()
+            .count()
+        )
+
+    def collect_counters():
+        """
+        Aggregates and returns summary statistics for audit actions and object types.
+        
+        Returns:
+            dict: A dictionary containing counts of distinct cases, videos, annotations,
+            anonymizations, images, and breakdowns of video statuses based on audit records.
+        """
+        return {
+            "totalCases":           AuditLedger._distinct("VideoFile", "created"),
+            "totalVideos":          AuditLedger._distinct("VideoFile", "created"),
+            "totalAnnotations":     AuditLedger.objects.filter(action="annotation_added").count(),
+            "totalAnonymizations":  AuditLedger._distinct("VideoFile", "anonymized"),
+            "totalImages":          AuditLedger._distinct("Image",     "created"),
+            # video breakdown
+            "videosCompleted":      AuditLedger._distinct("VideoFile", "validated"),
+            "videosAnonym":         AuditLedger._distinct("VideoFile", "anonymized"),
+            # add more as needed …
+        }

endoreg_db/models/state/label_video_segment.py

@@ -0,0 +1,22 @@
+from django.db import models
+from .abstract import AbstractState
+
+class LabelVideoSegmentState(AbstractState):
+    """State for label video segment data."""
+
+    prediction = models.BooleanField(default=False)
+    annotation = models.BooleanField(default=False)
+    frames_extracted = models.BooleanField(default=False)
+    is_validated = models.BooleanField(default=False)
+
+    origin = models.OneToOneField(
+        "LabelVideoSegment",
+        on_delete=models.CASCADE,
+        related_name="state",
+        null=True,
+        blank=True,
+    )
+
+    class Meta:
+        verbose_name = "Label Video Segment State"
+        verbose_name_plural = "Label Video Segment States"

endoreg_db/models/state/raw_pdf.py

@@ -0,0 +1,187 @@
+"""
+Defines state tracking models related to PDF processing, including extraction of text and metadata, AI predictions, and anonymization status for RawPdfFile instances.
+"""
+from django.db import models
+from typing import TYPE_CHECKING
+import logging
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from ..media import RawPdfFile
+    
+
+class AnonymizationStatus(str, Enum):
+    NOT_STARTED             = "not_started"
+    PROCESSING_ANONYMIZING  = "processing_anonymization"
+    DONE                    = "done"
+    VALIDATED               = "validated"
+    FAILED                  = "failed"
+    STARTED                = "started"
+
+
+class RawPdfState(models.Model):
+    """
+    Tracks the processing state of a RawPdfFile instance.
+    Uses BooleanFields for clear, distinct states.
+    """
+    text_meta_extracted = models.BooleanField(default=False, help_text="True if text metadata (OCR) has been extracted.")
+
+    # AI / Annotation related states
+    initial_prediction_completed = models.BooleanField(default=False, help_text="True if initial AI prediction has run.")
+
+    # Processing state
+    sensitive_meta_processed = models.BooleanField(default=False, help_text="True if the video has been fully processed, meaning a anonymized person was created.")
+
+    # Anonymization state
+    anonymized = models.BooleanField(default=False, help_text="True if the anonymized video file has been created.")
+    anonymization_validated = models.BooleanField(default=False, help_text="True if the anonymization process has been validated and confirmed.")
+    
+    # Processing state
+    processing_started = models.BooleanField(default=False, help_text="True if the processing has started, but not yet completed.")
+    processing_error = models.BooleanField(default=False, help_text="True if an error occurred during processing.")
+    
+    # Timestamps
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    
+    was_created = models.BooleanField(default=True, help_text="True if this state was created for the first time.")
+
+    # PDF metadata extraction state
+    pdf_meta_extracted = models.BooleanField(default=False, help_text="True if PDF metadata has been extracted.")
+
+    if TYPE_CHECKING:
+        raw_pdf_file: "RawPdfFile"
+
+    def __str__(self):
+        """
+        Return a string summarizing the RawPdfState instance, including the related PDF file UUID and key processing state flags with timestamps.
+        """
+        try:
+            uuid = self.raw_pdf_file.id
+        except Exception:
+            uuid = None
+
+        states = [
+            f"TextMetaExtracted={self.text_meta_extracted}",
+            f"PredictionDone={self.initial_prediction_completed}",
+            f"Anonymized={self.anonymized}",
+            f"AnonymizationValidated={self.anonymization_validated}",
+            f"SensitiveMetaProcessed={self.sensitive_meta_processed}",
+            f"DateCreated={self.date_created.isoformat()}",
+            f"DateModified={self.date_modified.isoformat()}"
+        ]
+        return f"RawPdfState(Pdf:{uuid}): {', '.join(states)}"
+
+    @property
+    def anonymization_status(self) -> AnonymizationStatus:
+        """
+        Determines the current anonymization workflow status for the PDF processing state.
+        
+        Returns:
+            AnonymizationStatus: The current status, reflecting progress or failure in the anonymization process.
+        """
+        if getattr(self, "processing_error", False):
+            return AnonymizationStatus.FAILED
+        if self.anonymization_validated:
+            return AnonymizationStatus.VALIDATED
+        if self.sensitive_meta_processed:
+            return AnonymizationStatus.DONE
+        if self.anonymized:
+            return AnonymizationStatus.DONE
+        if self.initial_prediction_completed:
+            return AnonymizationStatus.PROCESSING_ANONYMIZING
+        if self.pdf_meta_extracted:
+            return AnonymizationStatus.NOT_STARTED
+        if self.processing_started:
+            return AnonymizationStatus.STARTED
+        return AnonymizationStatus.NOT_STARTED
+    
+
+    
+    def mark_processing_started(self, *, save: bool = True) -> None:
+        """
+        Mark the processing as started and optionally save the updated state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        self.processing_started = True
+        if save:
+            self.save(update_fields=["processing_started", "date_modified"])
+
+    # ---- Single‑responsibility mutators ---------------------------------
+    def mark_sensitive_meta_processed(self, *, save: bool = True) -> None:
+        """
+        Mark the sensitive metadata processing step as completed for this PDF state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.sensitive_meta_processed = True
+        if save:
+            self.save(update_fields=["sensitive_meta_processed", "date_modified"])
+
+    def mark_anonymization_validated(self, *, save: bool = True) -> None:
+        """
+        Mark the anonymization as validated for this PDF processing state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.anonymization_validated = True
+        if save:
+            self.save(update_fields=["anonymization_validated", "date_modified"])
+
+    def mark_anonymized(self, *, save: bool = True) -> None:
+        """
+        Mark the PDF as anonymized and optionally save the updated state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        self.anonymized = True
+        if save:
+            self.save(update_fields=["anonymized", "date_modified"])
+
+    def mark_initial_prediction_completed(self, *, save: bool = True) -> None:
+        """
+        Mark the initial AI prediction step as completed for this PDF processing state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.initial_prediction_completed = True
+        if save:
+            self.save(update_fields=["initial_prediction_completed", "date_modified"])
+
+    def mark_pdf_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the PDF metadata extraction step as completed for this state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.pdf_meta_extracted = True
+        if save:
+            self.save(update_fields=["pdf_meta_extracted", "date_modified"])
+
+    def mark_text_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the text metadata extraction step as completed for this PDF processing state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.text_meta_extracted = True
+        if save:
+            self.save(update_fields=["text_meta_extracted", "date_modified"])
+    
+    
+
+    class Meta:
+        verbose_name = "Raw PDF Processing State"
+        verbose_name_plural = "Raw PDF Processing States"
+
+

endoreg_db/models/state/sensitive_meta.py

@@ -0,0 +1,46 @@
+from django.db import models
+from .abstract import AbstractState
+from typing import TYPE_CHECKING
+
+class SensitiveMetaState(AbstractState):
+    """State for sensitive meta data."""
+
+    dob_verified = models.BooleanField(default=False)
+    names_verified = models.BooleanField(default=False)
+
+    origin = models.OneToOneField(
+        "SensitiveMeta",
+        on_delete=models.CASCADE,
+        related_name="state",
+        null=True,
+        blank=True,
+    )
+
+    if TYPE_CHECKING:
+        from endoreg_db.models import SensitiveMeta
+
+        origin: "SensitiveMeta"
+
+    @property
+    def is_verified(self) -> bool:
+        """
+        Return True if both date of birth and names have been verified; otherwise, return False.
+        """
+        return self.dob_verified and self.names_verified
+
+    def mark_dob_verified(self):
+        """
+        Set the date of birth verification status to True and persist the change to the database.
+        """
+        self.dob_verified = True
+        self.save(update_fields=["dob_verified"])
+
+    def mark_names_verified(self):
+        """
+        Mark the names as verified and persist the change to the database.
+        """
+        self.names_verified = True
+        self.save(update_fields=["names_verified"])
+    class Meta:
+        verbose_name = "Sensitive Meta State"
+        verbose_name_plural = "Sensitive Meta States"

endoreg_db/models/state/video.py

@@ -0,0 +1,232 @@
+"""
+Defines state tracking models related to video processing.
+"""
+from django.db import models
+from .abstract import AbstractState
+from typing import TYPE_CHECKING, Optional
+import logging
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from ..media import VideoFile
+    
+
+class AnonymizationStatus(str, Enum):
+    NOT_STARTED             = "not_started"
+    EXTRACTING_FRAMES       = "extracting_frames"
+    PROCESSING_ANONYMIZING  = "processing_anonymization"
+    DONE                    = "done"
+    VALIDATED               = "validated"
+    FAILED                  = "failed"
+    STARTED                = "started"
+
+class VideoState(models.Model):
+    """
+    Tracks the processing state of a VideoFile instance.
+    Uses BooleanFields for clear, distinct states.
+    """
+    # Frame related states
+    if TYPE_CHECKING:
+        video_file: Optional["VideoFile"]
+        
+    frames_extracted = models.BooleanField(default=False, help_text="True if raw frames have been extracted to files.")
+    frames_initialized = models.BooleanField(default=False, help_text="True if Frame DB objects have been created.")
+    frame_count = models.PositiveIntegerField(null=True, blank=True, help_text="Number of frames extracted/initialized.")
+
+    # Metadata related states
+    video_meta_extracted = models.BooleanField(default=False, help_text="True if VideoMeta (technical specs) has been extracted.")
+    text_meta_extracted = models.BooleanField(default=False, help_text="True if text metadata (OCR) has been extracted.")
+
+    # AI / Annotation related states
+    initial_prediction_completed = models.BooleanField(default=False, help_text="True if initial AI prediction has run.")
+    lvs_created = models.BooleanField(default=False, help_text="True if LabelVideoSegments have been created from predictions.")
+    frame_annotations_generated = models.BooleanField(default=False, help_text="True if frame-level annotations have been generated from segments.")
+    
+    # Processing state
+    sensitive_meta_processed = models.BooleanField(default=False, help_text="True if the video has been fully processed, meaning a anonymized person was created.")
+
+    # Anonymization state
+    anonymized = models.BooleanField(default=False, help_text="True if the anonymized video file has been created.")
+    anonymization_validated = models.BooleanField(default=False, help_text="True if the anonymization process has been validated and confirmed.")
+    anonymization_status: AnonymizationStatus
+    
+    processing_started = models.BooleanField(default=False, help_text="True if the processing has started, but not yet completed.")
+    
+    # Timestamps
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    
+    # Segment Annotation State
+    segment_annotations_created = models.BooleanField(default=False, help_text="True if segment annotations have been created from LabelVideoSegments.")
+    segment_annotations_validated = models.BooleanField(default=False, help_text="True if segment annotations have been validated.")
+    
+    was_created = models.BooleanField(default=True, help_text="True if this state was created for the first time.")
+    
+    objects = models.Manager()
+
+    def __str__(self):
+        # Find the related VideoFile's UUID if possible
+        video_uuid = "Unknown"
+        try:
+            # Access the related VideoFile via the reverse relation 'video_file'
+            if hasattr(self, 'video_file') and self.video_file:
+                video_uuid = self.video_file.uuid
+        except Exception:
+            pass # Ignore errors if relation doesn't exist or causes issues
+
+        states = [
+            f"FramesExtracted={self.frames_extracted}",
+            f"FramesInit={self.frames_initialized}",
+            f"VideoMetaExtracted={self.video_meta_extracted}",
+            f"TextMetaExtracted={self.text_meta_extracted}",
+            f"PredictionDone={self.initial_prediction_completed}",
+            f"LvsCreated={self.lvs_created}",
+            f"Anonymized={self.anonymized}",
+            f"AnonymizationValidated={self.anonymization_validated}",
+            f"SensitiveMetaProcessed={self.sensitive_meta_processed}",
+            f"FrameCount={self.frame_count}" if self.frame_count is not None else "FrameCount=None",
+            f"SegmentAnnotationsCreated={self.segment_annotations_created}",
+            f"SegmentAnnotationsValidated={self.segment_annotations_validated}",
+            f"DateCreated={self.date_created.isoformat()}",
+            f"DateModified={self.date_modified.isoformat()}"
+        ]
+        return f"VideoState(Video:{video_uuid}): {', '.join(states)}"
+
+    @property
+    def anonymization_status(self) -> AnonymizationStatus:
+        """
+        Fast, side‑effect‑free status resolution used by API & UI.
+        """
+        if self.anonymization_validated:
+            return AnonymizationStatus.VALIDATED
+        if self.sensitive_meta_processed:
+            return AnonymizationStatus.DONE
+        if self.frames_extracted and not self.anonymized:
+            return AnonymizationStatus.PROCESSING_ANONYMIZING
+        if self.was_created and not self.frames_extracted:
+            return AnonymizationStatus.EXTRACTING_FRAMES
+        if getattr(self, "processing_error", False):
+            return AnonymizationStatus.FAILED
+        if self.processing_started:
+            return AnonymizationStatus.STARTED
+        return AnonymizationStatus.NOT_STARTED
+
+    # ---- Single‑responsibility mutators ---------------------------------
+    def mark_sensitive_meta_processed(self, *, save: bool = True) -> None:
+        self.sensitive_meta_processed = True
+        if save:
+            self.save(update_fields=["sensitive_meta_processed", "date_modified"])
+
+    def mark_anonymization_validated(self, *, save: bool = True) -> None:
+        """
+        Mark the anonymization process as validated for this video state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.anonymization_validated = True
+        if save:
+            self.save(update_fields=["anonymization_validated", "date_modified"])
+
+    def mark_frames_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having its frames extracted.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.frames_extracted = True
+        if save:
+            self.save(update_fields=["frames_extracted", "date_modified"])
+
+    def mark_frames_not_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having no extracted frames.
+        
+        If `save` is True, updates the database record for this state.
+        """
+        self.frames_extracted = False
+        if save:
+            self.save(update_fields=["frames_extracted", "date_modified"])
+
+    def mark_anonymized(self, *, save: bool = True) -> None:
+        """
+        Mark the video as anonymized by setting the anonymized flag to True.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.anonymized = True
+        if save:
+            self.save(update_fields=["anonymized", "date_modified"])
+
+    def mark_initial_prediction_completed(self, *, save: bool = True) -> None:
+        """
+        Mark the initial AI prediction as completed for this video state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.initial_prediction_completed = True
+        if save:
+            self.save(update_fields=["initial_prediction_completed", "date_modified"])
+
+    def mark_video_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video metadata as extracted for this video state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.video_meta_extracted = True
+        if save:
+            self.save(update_fields=["video_meta_extracted", "date_modified"])
+
+    def mark_text_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having its text metadata extracted.
+        
+        Parameters:
+        	save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.text_meta_extracted = True
+        if save:
+            self.save(update_fields=["text_meta_extracted", "date_modified"])
+    
+    def get_or_create_state(self):
+        """
+        Get the current state of the video, or create a new one if it doesn't exist.
+        
+        Returns:
+            VideoState: The current or newly created state.
+        """
+        if not hasattr(self, 'video_file'):
+            raise ValueError("This method requires a related VideoFile instance.")
+        
+        # If the state already exists, return it
+        if self.video_file.state:
+            return self.video_file.state
+        
+        # Otherwise, create a new state
+        new_state = VideoState(video_file=self.video_file)
+        new_state.save()
+        return new_state
+    
+    def mark_processing_started(self, *, save: bool = True) -> None:
+        """
+        Mark the processing as started for this video state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.processing_started = True
+        if save:
+            self.save(update_fields=["processing_started", "date_modified"])
+    
+    
+
+    class Meta:
+        verbose_name = "Video Processing State"
+        verbose_name_plural = "Video Processing States"