endoreg-db 0.8.6.4__py3-none-any.whl

endoreg_db/models/state/sensitive_meta.py

@@ -0,0 +1,46 @@
+from django.db import models
+from .abstract import AbstractState
+from typing import TYPE_CHECKING
+
+class SensitiveMetaState(AbstractState):
+    """State for sensitive meta data."""
+
+    dob_verified = models.BooleanField(default=False)
+    names_verified = models.BooleanField(default=False)
+
+    origin = models.OneToOneField(
+        "SensitiveMeta",
+        on_delete=models.CASCADE,
+        related_name="state",
+        null=True,
+        blank=True,
+    )
+
+    if TYPE_CHECKING:
+        from endoreg_db.models import SensitiveMeta
+
+        origin: "SensitiveMeta"
+
+    @property
+    def is_verified(self) -> bool:
+        """
+        Return True if both date of birth and names have been verified; otherwise, return False.
+        """
+        return self.dob_verified and self.names_verified
+
+    def mark_dob_verified(self):
+        """
+        Set the date of birth verification status to True and persist the change to the database.
+        """
+        self.dob_verified = True
+        self.save(update_fields=["dob_verified"])
+
+    def mark_names_verified(self):
+        """
+        Mark the names as verified and persist the change to the database.
+        """
+        self.names_verified = True
+        self.save(update_fields=["names_verified"])
+    class Meta:
+        verbose_name = "Sensitive Meta State"
+        verbose_name_plural = "Sensitive Meta States"

endoreg_db/models/state/video.py

@@ -0,0 +1,232 @@
+"""
+Defines state tracking models related to video processing.
+"""
+from django.db import models
+from .abstract import AbstractState
+from typing import TYPE_CHECKING, Optional
+import logging
+from enum import Enum
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from ..media import VideoFile
+    
+
+class AnonymizationStatus(str, Enum):
+    NOT_STARTED             = "not_started"
+    EXTRACTING_FRAMES       = "extracting_frames"
+    PROCESSING_ANONYMIZING  = "processing_anonymization"
+    DONE                    = "done"
+    VALIDATED               = "validated"
+    FAILED                  = "failed"
+    STARTED                = "started"
+
+class VideoState(models.Model):
+    """
+    Tracks the processing state of a VideoFile instance.
+    Uses BooleanFields for clear, distinct states.
+    """
+    # Frame related states
+    if TYPE_CHECKING:
+        video_file: Optional["VideoFile"]
+        
+    frames_extracted = models.BooleanField(default=False, help_text="True if raw frames have been extracted to files.")
+    frames_initialized = models.BooleanField(default=False, help_text="True if Frame DB objects have been created.")
+    frame_count = models.PositiveIntegerField(null=True, blank=True, help_text="Number of frames extracted/initialized.")
+
+    # Metadata related states
+    video_meta_extracted = models.BooleanField(default=False, help_text="True if VideoMeta (technical specs) has been extracted.")
+    text_meta_extracted = models.BooleanField(default=False, help_text="True if text metadata (OCR) has been extracted.")
+
+    # AI / Annotation related states
+    initial_prediction_completed = models.BooleanField(default=False, help_text="True if initial AI prediction has run.")
+    lvs_created = models.BooleanField(default=False, help_text="True if LabelVideoSegments have been created from predictions.")
+    frame_annotations_generated = models.BooleanField(default=False, help_text="True if frame-level annotations have been generated from segments.")
+    
+    # Processing state
+    sensitive_meta_processed = models.BooleanField(default=False, help_text="True if the video has been fully processed, meaning a anonymized person was created.")
+
+    # Anonymization state
+    anonymized = models.BooleanField(default=False, help_text="True if the anonymized video file has been created.")
+    anonymization_validated = models.BooleanField(default=False, help_text="True if the anonymization process has been validated and confirmed.")
+    anonymization_status: AnonymizationStatus
+    
+    processing_started = models.BooleanField(default=False, help_text="True if the processing has started, but not yet completed.")
+    
+    # Timestamps
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    
+    # Segment Annotation State
+    segment_annotations_created = models.BooleanField(default=False, help_text="True if segment annotations have been created from LabelVideoSegments.")
+    segment_annotations_validated = models.BooleanField(default=False, help_text="True if segment annotations have been validated.")
+    
+    was_created = models.BooleanField(default=True, help_text="True if this state was created for the first time.")
+    
+    objects = models.Manager()
+
+    def __str__(self):
+        # Find the related VideoFile's UUID if possible
+        video_uuid = "Unknown"
+        try:
+            # Access the related VideoFile via the reverse relation 'video_file'
+            if hasattr(self, 'video_file') and self.video_file:
+                video_uuid = self.video_file.uuid
+        except Exception:
+            pass # Ignore errors if relation doesn't exist or causes issues
+
+        states = [
+            f"FramesExtracted={self.frames_extracted}",
+            f"FramesInit={self.frames_initialized}",
+            f"VideoMetaExtracted={self.video_meta_extracted}",
+            f"TextMetaExtracted={self.text_meta_extracted}",
+            f"PredictionDone={self.initial_prediction_completed}",
+            f"LvsCreated={self.lvs_created}",
+            f"Anonymized={self.anonymized}",
+            f"AnonymizationValidated={self.anonymization_validated}",
+            f"SensitiveMetaProcessed={self.sensitive_meta_processed}",
+            f"FrameCount={self.frame_count}" if self.frame_count is not None else "FrameCount=None",
+            f"SegmentAnnotationsCreated={self.segment_annotations_created}",
+            f"SegmentAnnotationsValidated={self.segment_annotations_validated}",
+            f"DateCreated={self.date_created.isoformat()}",
+            f"DateModified={self.date_modified.isoformat()}"
+        ]
+        return f"VideoState(Video:{video_uuid}): {', '.join(states)}"
+
+    @property
+    def anonymization_status(self) -> AnonymizationStatus:
+        """
+        Fast, side‑effect‑free status resolution used by API & UI.
+        """
+        if self.anonymization_validated:
+            return AnonymizationStatus.VALIDATED
+        if self.sensitive_meta_processed:
+            return AnonymizationStatus.DONE
+        if self.frames_extracted and not self.anonymized:
+            return AnonymizationStatus.PROCESSING_ANONYMIZING
+        if self.was_created and not self.frames_extracted:
+            return AnonymizationStatus.EXTRACTING_FRAMES
+        if getattr(self, "processing_error", False):
+            return AnonymizationStatus.FAILED
+        if self.processing_started:
+            return AnonymizationStatus.STARTED
+        return AnonymizationStatus.NOT_STARTED
+
+    # ---- Single‑responsibility mutators ---------------------------------
+    def mark_sensitive_meta_processed(self, *, save: bool = True) -> None:
+        self.sensitive_meta_processed = True
+        if save:
+            self.save(update_fields=["sensitive_meta_processed", "date_modified"])
+
+    def mark_anonymization_validated(self, *, save: bool = True) -> None:
+        """
+        Mark the anonymization process as validated for this video state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.anonymization_validated = True
+        if save:
+            self.save(update_fields=["anonymization_validated", "date_modified"])
+
+    def mark_frames_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having its frames extracted.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.frames_extracted = True
+        if save:
+            self.save(update_fields=["frames_extracted", "date_modified"])
+
+    def mark_frames_not_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having no extracted frames.
+        
+        If `save` is True, updates the database record for this state.
+        """
+        self.frames_extracted = False
+        if save:
+            self.save(update_fields=["frames_extracted", "date_modified"])
+
+    def mark_anonymized(self, *, save: bool = True) -> None:
+        """
+        Mark the video as anonymized by setting the anonymized flag to True.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.anonymized = True
+        if save:
+            self.save(update_fields=["anonymized", "date_modified"])
+
+    def mark_initial_prediction_completed(self, *, save: bool = True) -> None:
+        """
+        Mark the initial AI prediction as completed for this video state.
+        
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.initial_prediction_completed = True
+        if save:
+            self.save(update_fields=["initial_prediction_completed", "date_modified"])
+
+    def mark_video_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video metadata as extracted for this video state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.video_meta_extracted = True
+        if save:
+            self.save(update_fields=["video_meta_extracted", "date_modified"])
+
+    def mark_text_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having its text metadata extracted.
+        
+        Parameters:
+        	save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.text_meta_extracted = True
+        if save:
+            self.save(update_fields=["text_meta_extracted", "date_modified"])
+    
+    def get_or_create_state(self):
+        """
+        Get the current state of the video, or create a new one if it doesn't exist.
+        
+        Returns:
+            VideoState: The current or newly created state.
+        """
+        if not hasattr(self, 'video_file'):
+            raise ValueError("This method requires a related VideoFile instance.")
+        
+        # If the state already exists, return it
+        if self.video_file.state:
+            return self.video_file.state
+        
+        # Otherwise, create a new state
+        new_state = VideoState(video_file=self.video_file)
+        new_state.save()
+        return new_state
+    
+    def mark_processing_started(self, *, save: bool = True) -> None:
+        """
+        Mark the processing as started for this video state.
+        
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.processing_started = True
+        if save:
+            self.save(update_fields=["processing_started", "date_modified"])
+    
+    
+
+    class Meta:
+        verbose_name = "Video Processing State"
+        verbose_name_plural = "Video Processing States"

endoreg_db/models/upload_job.py

@@ -0,0 +1,99 @@
+import uuid
+from django.db import models
+
+
+class UploadJob(models.Model):
+    """
+    Tracks file upload jobs and their processing status.
+    Supports both PDF and video file uploads with asynchronous processing.
+    """
+    
+    class Status(models.TextChoices):
+        PENDING = 'pending', 'Pending'
+        PROCESSING = 'processing', 'Processing'
+        ANONYMIZED = 'anonymized', 'Anonymized'
+        ERROR = 'error', 'Error'
+
+    id = models.UUIDField(
+        primary_key=True, 
+        default=uuid.uuid4, 
+        editable=False,
+        help_text="Unique identifier for the upload job"
+    )
+    
+    file = models.FileField(
+        upload_to='uploads/%Y/%m/%d/',
+        help_text="Uploaded file (PDF or video)"
+    )
+    
+    status = models.CharField(
+        max_length=20,
+        choices=Status.choices,
+        default=Status.PENDING,
+        help_text="Current processing status of the upload"
+    )
+    
+    content_type = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text="MIME type of the uploaded file"
+    )
+    
+    sensitive_meta = models.ForeignKey(
+        'SensitiveMeta',
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        help_text="Link to the created SensitiveMeta record after processing"
+    )
+    
+    error_detail = models.TextField(
+        blank=True,
+        help_text="Error message if processing failed"
+    )
+    
+    created_at = models.DateTimeField(
+        auto_now_add=True,
+        help_text="When the upload job was created"
+    )
+    
+    updated_at = models.DateTimeField(
+        auto_now=True,
+        help_text="When the upload job was last updated"
+    )
+
+    class Meta:
+        ordering = ['-created_at']
+        verbose_name = "Upload Job"
+        verbose_name_plural = "Upload Jobs"
+
+    def __str__(self):
+        return f"UploadJob {self.id} - {self.status} ({self.content_type})"
+
+    @property
+    def is_complete(self):
+        """Returns True if the job has finished processing (success or error)."""
+        return self.status in [self.Status.ANONYMIZED, self.Status.ERROR]
+
+    @property
+    def is_successful(self):
+        """Returns True if the job completed successfully."""
+        return self.status == self.Status.ANONYMIZED
+
+    def mark_processing(self):
+        """Mark the job as processing."""
+        self.status = self.Status.PROCESSING
+        self.save(update_fields=['status', 'updated_at'])
+
+    def mark_completed(self, sensitive_meta=None):
+        """Mark the job as successfully completed."""
+        self.status = self.Status.ANONYMIZED
+        if sensitive_meta:
+            self.sensitive_meta = sensitive_meta
+        self.save(update_fields=['status', 'sensitive_meta', 'updated_at'])
+
+    def mark_error(self, error_detail: str):
+        """Mark the job as failed with error details."""
+        self.status = self.Status.ERROR
+        self.error_detail = error_detail
+        self.save(update_fields=['status', 'error_detail', 'updated_at'])

endoreg_db/models/utils.py

@@ -0,0 +1,135 @@
+from ..utils import (
+    data_paths,
+    DJANGO_NAME_SALT,
+)
+from django.core.files import File
+from django.core.files.storage import FileSystemStorage
+import io
+import os
+from tqdm import tqdm
+import numpy as np
+import cv2
+from typing import TYPE_CHECKING, List, Tuple
+from pathlib import Path
+if TYPE_CHECKING:
+    from ..models.media import VideoFile
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+STORAGE_DIR = data_paths["storage"]
+FILE_STORAGE = FileSystemStorage(location = STORAGE_DIR)
+VIDEO_DIR = data_paths["video"]
+TMP_VIDEO_DIR = VIDEO_DIR / "tmp"
+ANONYM_VIDEO_DIR = data_paths["video_export"]
+FRAME_DIR = data_paths["frame"]
+WEIGHTS_DIR = data_paths["weights"]
+PDF_DIR = data_paths["raw_pdf"]
+DOCUMENT_DIR = data_paths["pdf"]
+
+TEST_RUN = os.environ.get("TEST_RUN", "False")
+TEST_RUN = TEST_RUN.lower() == "true"
+
+TEST_RUN_FRAME_NUMBER = int(os.environ.get("TEST_RUN_FRAME_NUMBER", "500"))
+
+
+def prepare_bulk_frames(frame_paths: List[Path]):
+    """
+    Reads the frame paths into memory as Django File objects.
+    This avoids 'seek of closed file' errors by using BytesIO for each frame.
+    """
+    for path in frame_paths:
+        frame_number = int(path.stem.split("_")[1])
+        with open(path, "rb") as f:
+            content = f.read()
+        file_obj = File(io.BytesIO(content), name=path.name)
+        yield frame_number, file_obj
+
+
+def find_segments_in_prediction_array(prediction_array: np.array, min_frame_len: int):
+    """
+    Expects a prediction array of shape (num_frames) and a minimum frame length.
+    Returns a list of tuples (start_frame_number, end_frame_number) that represent the segments.
+    """
+    # Add False to the beginning and end to detect changes at the array boundaries
+    padded_prediction = np.pad(
+        prediction_array, (1, 1), "constant", constant_values=False
+    )
+
+    # Find the start points and end points of the segments
+    diffs = np.diff(padded_prediction.astype(int))
+    segment_starts = np.where(diffs == 1)[0]
+    segment_ends = np.where(diffs == -1)[0]
+
+    # Filter segments based on min_frame_len
+    segments = [
+        (start, end)
+        for start, end in zip(segment_starts, segment_ends)
+        if end - start >= min_frame_len
+    ]
+
+    return segments
+
+def anonymize_frame(
+    raw_frame_path: Path, target_frame_path: Path, endo_roi, all_black: bool = False, censor_color: Tuple[int, int, int] = (0, 0, 0) # Added censor_color param
+):
+    """
+    Anonymize the frame by blacking out pixels outside the endoscope ROI or making the whole frame black.
+    """
+    frame = cv2.imread(raw_frame_path.as_posix())
+    if frame is None:
+        # Raise error instead of returning None/frame
+        raise FileNotFoundError(f"Could not read frame at {raw_frame_path}")
+
+    # make black frame with same size as original frame
+    new_frame = np.zeros_like(frame)
+
+    if not all_black:
+        # Validate ROI dictionary keys
+        required_keys = {"x", "y", "width", "height"}
+        if not required_keys.issubset(endo_roi):
+            raise ValueError(f"Invalid endo_roi dictionary provided: {endo_roi}. Missing keys.")
+
+        x = endo_roi["x"]
+        y = endo_roi["y"]
+        width = endo_roi["width"]
+        height = endo_roi["height"]
+
+        # Add boundary checks for ROI coordinates
+        h_orig, w_orig, _ = frame.shape
+        x1, y1 = max(0, x), max(0, y)
+        x2, y2 = min(w_orig, x + width), min(h_orig, y + height)
+
+        if x1 >= x2 or y1 >= y2:
+            logger.warning(f"ROI [{x},{y},{width},{height}] is outside or invalid for frame dimensions {w_orig}x{h_orig}. Resulting frame might be all black.")
+        else:
+            # copy valid endoscope roi part to black frame
+            new_frame[y1:y2, x1:x2] = frame[y1:y2, x1:x2]
+    else:
+        # If all_black, fill with censor_color (defaults to black)
+        new_frame[:] = censor_color
+
+    # Check if writing the anonymized frame was successful
+    success = cv2.imwrite(target_frame_path.as_posix(), new_frame)
+    if not success:
+        raise IOError(f"Failed to write anonymized frame to {target_frame_path}")
+
+
+__all__ = [
+    "DJANGO_NAME_SALT",
+    "data_paths",
+    "FILE_STORAGE",
+    "VIDEO_DIR",
+    "TMP_VIDEO_DIR",
+    "ANONYM_VIDEO_DIR",
+    "FRAME_DIR",
+    "WEIGHTS_DIR",
+    "PDF_DIR",
+    "DOCUMENT_DIR",
+    "prepare_bulk_frames",
+    "anonymize_frame",
+    "find_segments_in_prediction_array",
+    "TEST_RUN",
+    "TEST_RUN_FRAME_NUMBER",
+]

endoreg_db/queries/__init__.py

@@ -0,0 +1,5 @@
+from .annotations import (
+    generate_legacy_dataset_output
+)
+
+from .get import *

endoreg_db/queries/annotations/__init__.py

@@ -0,0 +1,3 @@
+from .legacy import (
+    get_legacy_annotations_for_labelset, generate_legacy_dataset_output
+)

endoreg_db/queries/annotations/legacy.py

@@ -0,0 +1,158 @@
+from ...models import LabelSet, ImageClassificationAnnotation
+from django.db.models import Q, F
+from django.db import models
+from tqdm import tqdm
+from collections import defaultdict
+
+# def get_legacy_annotations_for_labelset(labelset_name, version=None):
+#     """
+#     Retrieve annotations for a given label set for training.
+
+#     Args:
+#     - labelset_name (str): The name of the label set.
+#     - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+#     Returns:
+#     - list[dict]: A list of dictionaries. Each dictionary represents an image and its annotations.
+#                   Format: [{"frame": <frame_object>, "annotations": [{"label": <label_name>, "value": <value>}, ...]}, ...]
+
+#     Example:
+#         annotations_for_training = get_annotations_for_labelset("YourLabelSetName", version=2)
+
+#     """
+
+#     # Fetch the label set based on the name and optionally the version
+#     if version:
+#         labelset = LabelSet.objects.get(name=labelset_name, version=version)
+#     else:
+#         labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
+#         if not labelset:
+#             raise ValueError(f"No label set found with the name: {labelset_name}")
+
+#     # Retrieve all labels in the label set
+#     labels_in_set = labelset.labels.all()
+
+#     # Get the most recent annotations for each frame/label combination
+#     annotations = ImageClassificationAnnotation.objects.filter(label__in=labels_in_set)
+#     annotations = annotations.annotate(
+#         latest_annotation=models.Window(
+#             expression=models.functions.RowNumber(),
+#             partition_by=[F('legacy_image'), F('label')],
+#             order_by=F('date_modified').desc()
+#         )
+#     ).filter(latest_annotation=1)
+
+#     # Organize the annotations by image/frame
+#     organized_annotations = []
+
+#     for annotation in tqdm(annotations):
+#         # ic(annotation)
+#         # Check if the frame is already in the organized list
+#         existing_entry = next((entry for entry in organized_annotations if entry['legacy_image'] == annotation.legacy_frame), None)
+
+#         if existing_entry:
+#             # Add this annotation to the existing frame's annotations
+#             existing_entry['annotations'].append({
+#                 "label": annotation.label.name,
+#                 "value": annotation.value
+#             })
+#         else:
+#             # Create a new entry for this frame
+#             organized_annotations.append({
+#                 "legacy_image": annotation.legacy_image,
+#                 "annotations": [{
+#                     "label": annotation.label.name,
+#                     "value": annotation.value
+#                 }]
+#             })
+
+#     return organized_annotations
+
+
+
+def get_legacy_annotations_for_labelset(labelset_name, version=None):
+    """
+    ... [rest of your docstring]
+    """
+
+    # Fetch the label set based on the name and optionally the version
+    if version:
+        labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    else:
+        labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
+        if not labelset:
+            raise ValueError(f"No label set found with the name: {labelset_name}")
+
+    # Retrieve all labels in the label set
+    labels_in_set = labelset.labels.all()
+
+    # Get the most recent annotations for each frame/label combination
+    annotations = (ImageClassificationAnnotation.objects
+                   .filter(label__in=labels_in_set)
+                   .select_related('legacy_image', 'label')  # Reduce number of queries
+                   .annotate(
+                        latest_annotation=models.Window(
+                            expression=models.functions.RowNumber(),
+                            partition_by=[F('legacy_image'), F('label')],
+                            order_by=F('date_modified').desc()
+                        )
+                    ).filter(latest_annotation=1))
+
+    # Organize the annotations by image/frame using a defaultdict
+    organized_annotations_dict = defaultdict(lambda: {
+        "legacy_image": None,
+        "annotations": []
+    })
+
+    for annotation in tqdm(annotations):
+        organized_entry = organized_annotations_dict[annotation.legacy_image.id]
+        organized_entry["legacy_image"] = annotation.legacy_image
+        organized_entry["annotations"].append({
+            "label": annotation.label.name,
+            "value": annotation.value
+        })
+
+    # Convert organized_annotations_dict to a list
+    organized_annotations = list(organized_annotations_dict.values())
+
+    return organized_annotations
+
+def generate_legacy_dataset_output(labelset_name, version=None):
+    """
+    Generate an output suitable for creating PyTorch datasets.
+
+    Args:
+    - labelset_name (str): The name of the label set.
+    - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+    Returns:
+    - list[dict]: A list of dictionaries, where each dictionary contains the file path and the labels.
+                  Format: [{"path": <file_path>, "labels": [<label_1_value>, <label_2_value>, ...]}, ...]
+    - labelset[LabelSet]: The label set that was used to generate the output.
+    """
+
+    # First, retrieve the organized annotations using the previously defined function
+    organized_annotations = get_legacy_annotations_for_labelset(labelset_name, version)
+
+    # Fetch all labels from the labelset for consistent ordering
+    labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    all_labels = labelset.get_labels_in_order()
+
+    dataset_output = []
+
+    for entry in organized_annotations:
+        # Prepare a dictionary for each frame
+        frame_data = {
+            "path": entry['legacy_image'].image.path,  # Assuming 'image' field stores the file path
+            "labels": [-1] * len(all_labels)  # Initialize with -1 for all labels
+        }
+
+        # Update the labels based on the annotations
+        for annotation in entry['annotations']:
+            index = next((i for i, label in enumerate(all_labels) if label.name == annotation['label']), None)
+            if index is not None:
+                frame_data['labels'][index] = int(annotation['value'])
+
+        dataset_output.append(frame_data)
+
+    return dataset_output, labelset