endoreg-db 0.8.5.1__py3-none-any.whl

endoreg_db/models/rule/rule_applicator.py

@@ -0,0 +1,224 @@
+from .rule import Rule
+import random
+from django.core.exceptions import ValidationError
+
+class RuleApplicator:
+    """
+    A class to apply different types of rules to a Django model instance based on the Rule configuration.
+    """
+
+    def get_rule_by_name(self, rule_name):
+        """
+        A helper method to fetch a rule by name. This can be further customized.
+        """
+        # get rule by name which is natural key
+        rule = Rule.objects.get_by_natural_key(rule_name)
+        return rule
+
+
+    def apply(self, obj, rule):
+        """
+        Applies a specified rule to an object based on the rule type and attributes.
+
+        Parameters:
+        obj (Django model instance): The object to which the rule is applied.
+        rule (Rule): The rule instance containing the rule_type and attribute details.
+        """
+        rule_type_method = self.get_rule_type_method(rule.rule_type.name)
+        if rule_type_method:
+            rule_type_method(obj, rule)
+
+        else:
+            raise ValueError(f"Unsupported rule type: {rule.rule_type.name}")
+        
+    def apply_rule_by_name(self, obj, rule_name):
+        """
+        A helper method to apply a rule and get a value. This can be further customized.
+        """
+        rule = self.get_rule_by_name(rule_name)
+        self.apply(obj, rule)
+
+    def apply_rules_by_name(self, obj, rule_names):
+        """
+        A helper method to apply multiple rules and get values. This can be further customized.
+        """
+        rules = [self.get_rule_by_name(rule_name) for rule_name in rule_names]
+        for rule in rules:
+            self.apply(obj, rule)
+            
+
+    def get_rule_type_method(self, rule_type_name):
+        """
+        Maps rule type name to the corresponding method.
+        """
+        return getattr(self, f"handle_{rule_type_name}", None)
+
+    def parse_attribute_path(self, obj, attribute_key):
+        """
+        Parses the attribute path and applies the value to the correct attribute of a nested object.
+        """
+        parts = attribute_key.split('.')
+        model_name = parts[0]
+        if model_name.lower() != obj.__class__.__name__.lower():
+            raise ValidationError(f"Model type mismatch: expected {model_name}, got {obj.__class__.__name__}")
+
+        # Navigate through the nested attributes
+        target = obj
+        for part in parts[1:-1]:
+            target = getattr(target, part)
+
+        return target, parts[-1]
+
+    def set_attribute_value(self, obj, rule):
+        """
+        Generic method to set attribute value considering nested paths.
+        """
+        target, attribute = self.parse_attribute_path(obj, rule.attribute_key)
+        setattr(target, attribute, rule.attribute_dict['value'])
+        target.save()
+
+    #####
+
+    def handle_case_attribute_set_value(self, obj, rule):
+        """
+        Sets a specific attribute to a given value.
+        """
+        self.set_attribute_value(obj, rule.attribute_key, rule.attribute_dict["value"])
+
+    def handle_case_attribute_set_value_range_uniform(self, obj, rule):
+        """
+        Sets an attribute to a value within a specified range, selected uniformly.
+        """
+        if not hasattr(rule, 'attribute_dtype') or rule.attribute_dtype.name not in ['numeric', 'ordered_categorical']:
+            raise ValidationError("Attribute dtype must be numeric or ordered_categorical")
+        min_val = rule.attribute_dict["value_min"]
+        max_val = rule.attribute_dict["value_max"]
+        value = random.uniform(min_val, max_val)
+        self.set_attribute_value(obj, rule.attribute_key, value)
+
+    def handle_case_attribute_set_value_range_norm_dist(self, obj, rule):
+        """
+        Sets an attribute to a value within a specified range, based on a normal distribution.
+        """
+        if rule.attribute_dtype.name in ['float', "integer"]:
+            raise ValidationError("Attribute dtype must be float or integer")
+        mean = rule.attribute_dict["value_mean"]
+        std_dev = rule.attribute_dict["value_sd"]
+        value = random.normalvariate(mean, std_dev)
+        self.set_attribute_value(obj, rule.attribute_key, value)
+        
+
+    def handle_case_attribute_set_from_list_uniform(self, obj, rule):
+        """
+        Selects an attribute value uniformly from a list of choices.
+        """
+        choices = rule.attribute_dict["value_choices"]
+        value = random.choice(choices)
+        self.set_attribute_value(obj, rule.attribute_key, value)
+
+    def handle_case_attribute_set_from_prop_tuple_list(self, obj, rule):
+        """
+        Selects an attribute value based on a list of value-probability tuples.
+        """
+        prop_list = rule.attribute_dict["value_prop_tuple_list"]
+        total = sum(prop for _, prop in prop_list)
+        pick = random.uniform(0, total)
+        current = 0
+        for value, prop in prop_list:
+            current += prop
+            if current > pick:
+                self.set_attribute_value(obj, rule.attribute_key, value)
+                break
+
+
+    #####
+
+    def handle_case_add_patient(self, obj, rule):
+        """Function to add a patient to a case. If the patient already exists, raise an error. Requires the following
+        rules in the rule_dict:
+        - gender_rule
+        - dob_rule
+        - event_rules
+        - disease_rules
+        """
+        from endoreg_db.models import Patient, PatientEvent, PatientDisease, Case
+
+        # Check if the 
+        obj:Case = obj
+        if obj.patient:
+            raise ValueError("Patient already exists in the case")
+        
+        patient = Patient()
+        self.apply_rule_by_name(patient, rule.attribute_dict["patient_gender_rule"])
+        self.apply_rule_by_name(patient, rule.attribute_dict["patient_dob_rule"])
+        self.apply_rules_by_name(patient, rule.attribute_dict["patient_event_rules"])
+        self.apply_rules_by_name(patient, rule.attribute_dict["patient_disease_rules"])
+
+    def handle_case_add_polyp(self, obj, rule):
+        """
+        Adds a polyp to the case, processing various polyp characteristics. Requires Rules for each attribute:
+        - polyp_location_organ_rule (single rule)
+        - polyp_location_organ_part_rule (single rule)
+        - polyp_morphology_classification_shape_choices_rule (list of rules)
+        - polyp_morphology_classification_chromo_choices_rule (list of rules)
+        - polyp_intervention_type_rule (single rule)
+        - polyp_intervention_instrument_rule (single rule)
+        - polyp_size_mm_rule (single rule)
+
+        """
+        # Example: create and add a polyp instance based on the rules
+        from endoreg_db.models import Polyp  # Ensure you have a Polyp model and adjust the import
+        from endoreg_db.models import Location
+        from endoreg_db.models import EndoscopicIntervention
+        from endoreg_db.models import PolypMorphology
+
+        assert obj.examination.type != None, "Examination type is required"
+
+        location = Location()
+        location.organ = self.apply_rule_by_name(location, rule.attribute_dict["polyp_location_organ_rule"])
+        location.organ_part = self.apply_rule_by_name(location, rule.attribute_dict["polyp_location_organ_part_rule"])
+        location.save()
+
+        morphology = PolypMorphology()
+        # M2M to ClassificationChoice (linked to Classification which has name as natural key)
+        morphology.shape_classification_choices = self.apply_rules_by_name(morphology, rule.attribute_dict["polyp_morphology_classification_shape_choices_rules"])
+        # M2M to ClassificationChoice (linked to Classification which has name as natural key)
+        morphology.chromo_classification_choices = self.apply_rules_by_name(morphology, rule.attribute_dict["polyp_morphology_classification_chromo_choices_rules"])
+        morphology.save()
+
+        intervention = EndoscopicIntervention()
+        intervention.type = self.apply_rule_by_name(intervention, rule.attribute_dict["polyp_intervention_type_rule"])
+        intervention.instrument = self.apply_rule_by_name(intervention, rule.attribute_dict["polyp_intervention_instrument_rule"])
+        intervention.save()
+
+        polyp = Polyp()
+        polyp.size_mm = self.apply_rule_by_name(polyp, rule.attribute_dict["polyp_size_mm_rule"])
+        polyp.location = location
+        polyp.morphology = morphology
+        polyp.intervention = intervention
+        
+        polyp.save()
+
+        obj.polyps.add(polyp)
+        obj.save()
+
+
+    # def handle_case_add_esophageal_varices(self, obj, rule):
+    #     """
+    #     Handles adding esophageal varices to a case based on the rule.
+    #     """
+    #     # Similar to the polyp case, create and add esophageal varices based on the attributes
+    #     from endoreg_db.models import EsophagealVarices  # Ensure this model exists
+    #     varices = EsophagealVarices()
+    #     varices.origin = self.apply_rule_by_name(obj, rule.attribute_dict["esophageal_varices_origin_rule"])
+    #     varices.classification = self.apply_rule_by_name(obj, rule.attribute_dict["esophageal_varices_classification_rule"])
+    #     varices.location_extent = self.apply_rule_by_name(obj, rule.attribute_dict["esophageal_varices_location_extent_rule"])
+    #     varices.save()
+    #     # Assuming a relationship setup
+    #     obj.esophageal_varices.add(varices)
+    #     obj.save()
+
+
+    
+    
+        

endoreg_db/models/rule/rule_attribute_dtype.py

@@ -0,0 +1,17 @@
+from django.db import models
+
+
+class RuleAttributeDtypeManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class RuleAttributeDType(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+
+    objects = RuleAttributeDtypeManager()
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name

endoreg_db/models/rule/rule_type.py

@@ -0,0 +1,20 @@
+from django.db import models
+
+class RuleTypeManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class RuleType(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+
+    objects = RuleTypeManager()
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        verbose_name = 'Rule Type'
+        verbose_name_plural = 'Rule Types'

endoreg_db/models/rule/ruleset.py

@@ -0,0 +1,17 @@
+from django.db import models
+
+class RulesetManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class Ruleset(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+    rules = models.ManyToManyField('Rule')
+
+    objects = RulesetManager()
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name

endoreg_db/models/state/__init__.py

@@ -0,0 +1,12 @@
+from .sensitive_meta import SensitiveMetaState
+from .video import VideoState, AnonymizationStatus
+from .label_video_segment import LabelVideoSegmentState
+from .raw_pdf import RawPdfState
+
+__all__ = [
+    "SensitiveMetaState",
+    "VideoState",
+    "LabelVideoSegmentState",
+    "AnonymizationStatus",
+    "RawPdfState",
+]

endoreg_db/models/state/abstract.py

@@ -0,0 +1,11 @@
+from django.db import models
+
+class AbstractState(models.Model):
+    """Abstract base class for all states."""
+
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+
+    class Meta:
+        abstract = True

endoreg_db/models/state/audit_ledger.py

@@ -0,0 +1,150 @@
+# stats/models.py
+import hashlib, json, uuid
+from django.db import models
+from django.utils import timezone
+from django.conf import settings
+
+"""
+
+AuditLedger Model
+
+AuditLedger is a model that tracks changes to other models in the database.
+It stores a hash of the previous state, the current state, and the action taken.
+This allows for a complete audit trail of changes made to the database.
+The model includes fields for the timestamp, user who made the change, object type,
+object primary key, action taken, and the data associated with the change.
+The save method computes the hash of the current state and the previous state
+before saving the record to the database.
+The hash is computed using SHA-256 and includes the timestamp, user ID,
+object type, object primary key, action taken, and the data associated with the change.
+The hash is stored in the database to ensure data integrity and to allow for
+verification of the data.
+The model also includes a method to retrieve the last hash from the database
+to ensure that the current hash is always based on the most recent state of the database.
+The model is designed to be immutable, meaning that once a record is created,
+it cannot be modified. This ensures that the audit trail is complete and accurate.
+
+Raises:
+    RuntimeError: _description_
+
+Returns:
+    _type_: _description_
+"""
+
+
+class AuditLedger(models.Model):
+    id           = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    ts           = models.DateTimeField(default=timezone.now, editable=False)
+    user         = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT)
+    object_type  = models.CharField(max_length=80)            # e.g. 'VideoFile'
+    object_pk    = models.CharField(max_length=40)            # UUID or int
+    action       = models.CharField(max_length=40)            # 'created' | 'validated' | …
+    data         = models.JSONField()                         # snapshot / diff / metadata
+    prev_hash    = models.CharField(max_length=64, editable=False)
+    hash         = models.CharField(max_length=64, editable=False)
+
+    class Meta:
+        ordering = ['ts']
+        indexes  = [models.Index(fields=['object_type', 'object_pk'])]
+
+    # ------------------------------------------------------
+    def save(self, *args, **kw):
+        """
+        Saves a new immutable audit record, computing and linking cryptographic hashes.
+        
+        Raises:
+            RuntimeError: If an attempt is made to modify an existing audit record.
+        """
+        if self._state.adding:                         # only on INSERT
+            self.prev_hash = self._last_hash()
+            self.hash      = self._compute_hash()
+        else:
+            raise RuntimeError("AuditLedger rows are immutable")
+        super().save(*args, **kw)
+
+    # ------------------------------------------------------
+    def _last_hash(self) -> str:
+        """
+        Retrieves the hash of the most recent audit record.
+        
+        Returns:
+            The SHA-256 hash of the latest `AuditLedger` entry by timestamp, or a string of 64 zeros if no records exist.
+        """
+        last = AuditLedger.objects.order_by('-ts').first()
+        return last.hash if last else '0' * 64
+
+    def _compute_hash(self) -> str:
+        """
+        Computes the SHA-256 hash of the current audit record's data.
+        
+        The hash is generated from a JSON-serialized payload containing the timestamp, user ID, object type and primary key, action, associated data, and the previous record's hash. This ensures the integrity and immutability of the audit trail.
+        
+        Returns:
+            The hexadecimal SHA-256 hash string representing the current audit record.
+        """
+        payload = {
+            'ts':   self.ts.isoformat(),
+            'uid':  str(self.user_id),
+            'obj':  f'{self.object_type}:{self.object_pk}',
+            'act':  self.action,
+            'data': self.data,
+            'prev': self.prev_hash,
+        }
+        return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+    
+    def log_validation(user, instance, action: str, extra=None):
+        """
+        Creates an audit record for a validation action performed by a user on a specific model instance.
+        
+        Args:
+            user: The user performing the action.
+            instance: The model instance being validated.
+            action: The action performed (e.g., 'validated').
+            extra: Optional additional data to include in the audit record.
+        """
+        AuditLedger.objects.create(
+            user=user,
+            object_type=instance.__class__.__name__,
+            object_pk=str(instance.pk),
+            action=action,
+            data=extra or {},
+        )
+
+    def _distinct(object_type: str, action: str):
+        """
+        Returns the number of distinct objects of a given type that have a specific audit action recorded.
+        
+        Args:
+            object_type: The type of object to filter by (e.g., 'VideoFile').
+            action: The audit action to filter by (e.g., 'validated').
+        
+        Returns:
+            The count of unique object primary keys matching the specified type and action.
+        """
+        return (
+            AuditLedger.objects
+            .filter(object_type=object_type, action=action)
+            .values('object_pk')
+            .distinct()
+            .count()
+        )
+
+    def collect_counters():
+        """
+        Aggregates and returns summary statistics for audit actions and object types.
+        
+        Returns:
+            dict: A dictionary containing counts of distinct cases, videos, annotations,
+            anonymizations, images, and breakdowns of video statuses based on audit records.
+        """
+        return {
+            "totalCases":           AuditLedger._distinct("VideoFile", "created"),
+            "totalVideos":          AuditLedger._distinct("VideoFile", "created"),
+            "totalAnnotations":     AuditLedger.objects.filter(action="annotation_added").count(),
+            "totalAnonymizations":  AuditLedger._distinct("VideoFile", "anonymized"),
+            "totalImages":          AuditLedger._distinct("Image",     "created"),
+            # video breakdown
+            "videosCompleted":      AuditLedger._distinct("VideoFile", "validated"),
+            "videosAnonym":         AuditLedger._distinct("VideoFile", "anonymized"),
+            # add more as needed …
+        }

endoreg_db/models/state/label_video_segment.py

@@ -0,0 +1,22 @@
+from django.db import models
+from .abstract import AbstractState
+
+class LabelVideoSegmentState(AbstractState):
+    """State for label video segment data."""
+
+    prediction = models.BooleanField(default=False)
+    annotation = models.BooleanField(default=False)
+    frames_extracted = models.BooleanField(default=False)
+    is_validated = models.BooleanField(default=False)
+
+    origin = models.OneToOneField(
+        "LabelVideoSegment",
+        on_delete=models.CASCADE,
+        related_name="state",
+        null=True,
+        blank=True,
+    )
+
+    class Meta:
+        verbose_name = "Label Video Segment State"
+        verbose_name_plural = "Label Video Segment States"

endoreg_db/models/state/raw_pdf.py

@@ -0,0 +1,187 @@
+"""
+Defines state tracking models related to PDF processing, including extraction of text and metadata, AI predictions, and anonymization status for RawPdfFile instances.
+"""
+from django.db import models
+from typing import TYPE_CHECKING
+import logging
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from ..media import RawPdfFile
+    
+
+class AnonymizationStatus(str, Enum):
+    NOT_STARTED             = "not_started"
+    PROCESSING_ANONYMIZING  = "processing_anonymization"
+    DONE                    = "done"
+    VALIDATED               = "validated"
+    FAILED                  = "failed"
+    STARTED                = "started"
+
+
+class RawPdfState(models.Model):
+    """
+    Tracks the processing state of a RawPdfFile instance.
+    Uses BooleanFields for clear, distinct states.
+    """
+    text_meta_extracted = models.BooleanField(default=False, help_text="True if text metadata (OCR) has been extracted.")
+
+    # AI / Annotation related states
+    initial_prediction_completed = models.BooleanField(default=False, help_text="True if initial AI prediction has run.")
+
+    # Processing state
+    sensitive_meta_processed = models.BooleanField(default=False, help_text="True if the video has been fully processed, meaning a anonymized person was created.")
+
+    # Anonymization state
+    anonymized = models.BooleanField(default=False, help_text="True if the anonymized video file has been created.")
+    anonymization_validated = models.BooleanField(default=False, help_text="True if the anonymization process has been validated and confirmed.")
+    
+    # Processing state
+    processing_started = models.BooleanField(default=False, help_text="True if the processing has started, but not yet completed.")
+    processing_error = models.BooleanField(default=False, help_text="True if an error occurred during processing.")
+    
+    # Timestamps
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    
+    was_created = models.BooleanField(default=True, help_text="True if this state was created for the first time.")
+
+    # PDF metadata extraction state
+    pdf_meta_extracted = models.BooleanField(default=False, help_text="True if PDF metadata has been extracted.")
+
+    if TYPE_CHECKING:
+        raw_pdf_file: "RawPdfFile"
+
+    def __str__(self):
+        """
+        Return a string summarizing the RawPdfState instance, including the related PDF file UUID and key processing state flags with timestamps.
+        """
+        try:
+            uuid = self.raw_pdf_file.id
+        except Exception:
+            uuid = None
+
+        states = [
+            f"TextMetaExtracted={self.text_meta_extracted}",
+            f"PredictionDone={self.initial_prediction_completed}",
+            f"Anonymized={self.anonymized}",
+            f"AnonymizationValidated={self.anonymization_validated}",
+            f"SensitiveMetaProcessed={self.sensitive_meta_processed}",
+            f"DateCreated={self.date_created.isoformat()}",
+            f"DateModified={self.date_modified.isoformat()}"
+        ]
+        return f"RawPdfState(Pdf:{uuid}): {', '.join(states)}"
+
+    @property
+    def anonymization_status(self) -> AnonymizationStatus:
+        """
+        Determines the current anonymization workflow status for the PDF processing state.
+        
+        Returns:
+            AnonymizationStatus: The current status, reflecting progress or failure in the anonymization process.
+        """
+        if getattr(self, "processing_error", False):
+            return AnonymizationStatus.FAILED
+        if self.anonymization_validated:
+            return AnonymizationStatus.VALIDATED
+        if self.sensitive_meta_processed:
+            return AnonymizationStatus.DONE
+        if self.anonymized:
+            return AnonymizationStatus.DONE
+        if self.initial_prediction_completed:
+            return AnonymizationStatus.PROCESSING_ANONYMIZING
+        if self.pdf_meta_extracted:
+            return AnonymizationStatus.NOT_STARTED
+        if self.processing_started:
+            return AnonymizationStatus.STARTED
+        return AnonymizationStatus.NOT_STARTED
+    
+
+    
+    def mark_processing_started(self, *, save: bool = True) -> None:
+        """
+        Mark the processing as started and optionally save the updated state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        self.processing_started = True
+        if save:
+            self.save(update_fields=["processing_started", "date_modified"])
+
+    # ---- Single‑responsibility mutators ---------------------------------
+    def mark_sensitive_meta_processed(self, *, save: bool = True) -> None:
+        """
+        Mark the sensitive metadata processing step as completed for this PDF state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.sensitive_meta_processed = True
+        if save:
+            self.save(update_fields=["sensitive_meta_processed", "date_modified"])
+
+    def mark_anonymization_validated(self, *, save: bool = True) -> None:
+        """
+        Mark the anonymization as validated for this PDF processing state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.anonymization_validated = True
+        if save:
+            self.save(update_fields=["anonymization_validated", "date_modified"])
+
+    def mark_anonymized(self, *, save: bool = True) -> None:
+        """
+        Mark the PDF as anonymized and optionally save the updated state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        self.anonymized = True
+        if save:
+            self.save(update_fields=["anonymized", "date_modified"])
+
+    def mark_initial_prediction_completed(self, *, save: bool = True) -> None:
+        """
+        Mark the initial AI prediction step as completed for this PDF processing state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.initial_prediction_completed = True
+        if save:
+            self.save(update_fields=["initial_prediction_completed", "date_modified"])
+
+    def mark_pdf_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the PDF metadata extraction step as completed for this state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.pdf_meta_extracted = True
+        if save:
+            self.save(update_fields=["pdf_meta_extracted", "date_modified"])
+
+    def mark_text_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the text metadata extraction step as completed for this PDF processing state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.text_meta_extracted = True
+        if save:
+            self.save(update_fields=["text_meta_extracted", "date_modified"])
+    
+    
+
+    class Meta:
+        verbose_name = "Raw PDF Processing State"
+        verbose_name_plural = "Raw PDF Processing States"
+
+