endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/models/state/video.py

@@ -0,0 +1,251 @@
+"""
+Defines state tracking models related to video processing.
+"""
+
+import logging
+from typing import TYPE_CHECKING
+
+from django.db import models, transaction
+
+from endoreg_db.models.state.anonymization import AnonymizationState
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from ..media import VideoFile
+
+
+class VideoState(models.Model):
+    """
+    Tracks the processing state of a VideoFile instance.
+    Uses BooleanFields for clear, distinct states.
+    """
+
+    # Frame related states
+    if TYPE_CHECKING:
+        video_file: models.OneToOneField["VideoFile"]
+
+    frames_extracted = models.BooleanField(
+        default=False, help_text="True if raw frames have been extracted to files."
+    )
+    frames_initialized = models.BooleanField(
+        default=False, help_text="True if Frame DB objects have been created."
+    )
+    frame_count = models.PositiveIntegerField(
+        null=True, blank=True, help_text="Number of frames extracted/initialized."
+    )
+
+    # Metadata related states
+    video_meta_extracted = models.BooleanField(
+        default=False,
+        help_text="True if VideoMeta (technical specs) has been extracted.",
+    )
+    text_meta_extracted = models.BooleanField(
+        default=False, help_text="True if text metadata (OCR) has been extracted."
+    )
+
+    # AI / Annotation related states
+    initial_prediction_completed = models.BooleanField(
+        default=False, help_text="True if initial AI prediction has run."
+    )
+    lvs_created = models.BooleanField(
+        default=False,
+        help_text="True if LabelVideoSegments have been created from predictions.",
+    )
+    frame_annotations_generated = models.BooleanField(
+        default=False,
+        help_text="True if frame-level annotations have been generated from segments.",
+    )
+
+    # Processing state
+    sensitive_meta_processed = models.BooleanField(
+        default=False,
+        help_text="True if the video has been fully processed, meaning a anonymized person was created.",
+    )
+
+    # Anonymization state
+    anonymized = models.BooleanField(
+        default=False, help_text="True if the anonymized video file has been created."
+    )
+    anonymization_validated = models.BooleanField(
+        default=False,
+        help_text="True if the anonymization process has been validated and confirmed.",
+    )
+
+    processing_started = models.BooleanField(
+        default=False,
+        help_text="True if the processing has started, but not yet completed.",
+    )
+
+    # Timestamps
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+
+    # Segment Annotation State
+    segment_annotations_created = models.BooleanField(
+        default=False,
+        help_text="True if segment annotations have been created from LabelVideoSegments.",
+    )
+    segment_annotations_validated = models.BooleanField(
+        default=False, help_text="True if segment annotations have been validated."
+    )
+
+    was_created = models.BooleanField(
+        default=True, help_text="True if this state was created for the first time."
+    )
+
+    objects = models.Manager()
+
+    @property
+    def anonymization_status(self) -> AnonymizationState:
+        """
+        Fast, side‑effect‑free status resolution used by API & UI.
+        """
+        if self.anonymization_validated:
+            return AnonymizationState.VALIDATED  # Validation in Frontend completed -> Views related to this /home/admin/endoreg-db/endoreg_db/views/anonymization/validate.py
+        if self.sensitive_meta_processed:
+            return AnonymizationState.DONE_PROCESSING_ANONYMIZATION  # /home/admin/endoreg-db/endoreg_db/services/video_import.py /home/admin/endoreg-db/endoreg_db/views/video/reimport.py
+        if self.frames_extracted and not self.anonymized:
+            return AnonymizationState.PROCESSING_ANONYMIZING
+        if self.was_created and not self.frames_extracted:
+            return AnonymizationState.EXTRACTING_FRAMES
+        if getattr(self, "processing_error", False):
+            return AnonymizationState.FAILED
+        if self.processing_started:
+            return AnonymizationState.STARTED
+        if self.anonymized:
+            return AnonymizationState.ANONYMIZED
+
+        return AnonymizationState.NOT_STARTED
+
+    def mark_processing_not_started(self) -> None:
+        """
+        Mark the processing as not started and optionally save the updated state.
+
+        Parameters:
+            save (bool): If True, persist the change to the database immediately. Defaults to True.
+        """
+        with transaction.atomic():
+            self.processing_started = False
+            self.anonymized = False
+            self.was_created = False
+            self.sensitive_meta_processed = False
+            self.anonymization_validated = False
+            self.frames_extracted = False
+            self.save()
+
+    # ---- Single‑responsibility mutators ---------------------------------
+    def mark_sensitive_meta_processed(self, *, save: bool = True) -> None:
+        self.sensitive_meta_processed = True
+        if save:
+            self.save(update_fields=["sensitive_meta_processed", "date_modified"])
+
+    def mark_anonymization_validated(self, *, save: bool = True) -> None:
+        """
+        Mark the anonymization process as validated for this video state.
+
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.anonymization_validated = True
+        if save:
+            self.save(update_fields=["anonymization_validated", "date_modified"])
+
+    def mark_frames_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having its frames extracted.
+
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.frames_extracted = True
+        if save:
+            self.save(update_fields=["frames_extracted", "date_modified"])
+
+    def mark_frames_not_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having no extracted frames.
+
+        If `save` is True, updates the database record for this state.
+        """
+        self.frames_extracted = False
+        if save:
+            self.save(update_fields=["frames_extracted", "date_modified"])
+
+    def mark_anonymized(self, *, save: bool = True) -> None:
+        """
+        Mark the video as anonymized by setting the anonymized flag to True.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        with transaction.atomic():
+            self.anonymized = True
+            self.save(update_fields=["anonymized", "date_modified"])
+
+    def mark_initial_prediction_completed(self, *, save: bool = True) -> None:
+        """
+        Mark the initial AI prediction as completed for this video state.
+
+        Parameters:
+            save (bool): If True, persist the change to the database immediately.
+        """
+        self.initial_prediction_completed = True
+        if save:
+            self.save(update_fields=["initial_prediction_completed", "date_modified"])
+
+    def mark_video_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video metadata as extracted for this video state.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.video_meta_extracted = True
+        if save:
+            self.save(update_fields=["video_meta_extracted", "date_modified"])
+
+    def mark_text_meta_extracted(self, *, save: bool = True) -> None:
+        """
+        Mark the video as having its text metadata extracted.
+
+        Parameters:
+                save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.text_meta_extracted = True
+        if save:
+            self.save(update_fields=["text_meta_extracted", "date_modified"])
+
+    def get_or_create_state(self):
+        """
+        Get the current state of the video, or create a new one if it doesn't exist.
+
+        Returns:
+            VideoState: The current or newly created state.
+        """
+        if not hasattr(self, "video_file"):
+            raise ValueError("This method requires a related VideoFile instance.")
+
+        # If the state already exists, return it
+        if self.video_file.state:
+            return self.video_file.state
+
+        # Otherwise, create a new state
+        new_state = VideoState(video_file=self.video_file)
+        new_state.save()
+        return new_state
+
+    def mark_processing_started(self, *, save: bool = True) -> None:
+        """
+        Mark the processing as started for this video state.
+
+        Parameters:
+            save (bool): If True, immediately saves the updated state to the database.
+        """
+        self.processing_started = True
+        if save:
+            self.save(update_fields=["processing_started", "date_modified"])
+
+    class Meta:
+        verbose_name = "Video Processing State"
+        verbose_name_plural = "Video Processing States"

endoreg_db/models/upload_job.py

@@ -0,0 +1,100 @@
+import uuid
+from typing import TYPE_CHECKING, cast
+
+from django.db import models
+
+
+class UploadJob(models.Model):
+    """
+    Tracks file upload jobs and their processing status.
+    Supports both report and video file uploads with asynchronous processing.
+    """
+
+    class Status(models.TextChoices):
+        PENDING = "pending", "Pending"
+        PROCESSING = "processing", "Processing"
+        ANONYMIZED = "anonymized", "Anonymized"
+        ERROR = "error", "Error"
+
+    id = models.UUIDField(
+        primary_key=True,
+        default=uuid.uuid4,
+        editable=False,
+        help_text="Unique identifier for the upload job",
+    )
+
+    file = models.FileField(
+        upload_to="uploads/%Y/%m/%d/", help_text="Uploaded file (report or video)"
+    )
+
+    status = models.CharField(
+        max_length=20,
+        choices=Status.choices,
+        default=Status.PENDING,
+        help_text="Current processing status of the upload",
+    )
+
+    content_type = models.CharField(
+        max_length=100, blank=True, help_text="MIME type of the uploaded file"
+    )
+
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        help_text="Link to the created SensitiveMeta record after processing",
+    )
+
+    error_detail = models.TextField(
+        blank=True, help_text="Error message if processing failed"
+    )
+
+    created_at = models.DateTimeField(
+        auto_now_add=True, help_text="When the upload job was created"
+    )
+
+    updated_at = models.DateTimeField(
+        auto_now=True, help_text="When the upload job was last updated"
+    )
+
+    if TYPE_CHECKING:
+        from django.db.models.fields.files import FieldFile
+
+        file = cast(FieldFile, file)
+
+    class Meta:
+        ordering = ["-created_at"]
+        verbose_name = "Upload Job"
+        verbose_name_plural = "Upload Jobs"
+
+    def __str__(self):
+        return f"UploadJob {self.id} - {self.status} ({self.content_type})"
+
+    @property
+    def is_complete(self):
+        """Returns True if the job has finished processing (success or error)."""
+        return self.status in [self.Status.ANONYMIZED, self.Status.ERROR]
+
+    @property
+    def is_successful(self):
+        """Returns True if the job completed successfully."""
+        return self.status == self.Status.ANONYMIZED
+
+    def mark_processing(self):
+        """Mark the job as processing."""
+        self.status = self.Status.PROCESSING
+        self.save(update_fields=["status", "updated_at"])
+
+    def mark_completed(self, sensitive_meta=None):
+        """Mark the job as successfully completed."""
+        self.status = self.Status.ANONYMIZED
+        if sensitive_meta:
+            self.sensitive_meta = sensitive_meta
+        self.save(update_fields=["status", "sensitive_meta", "updated_at"])
+
+    def mark_error(self, error_detail: str):
+        """Mark the job as failed with error details."""
+        self.status = self.Status.ERROR
+        self.error_detail = error_detail
+        self.save(update_fields=["status", "error_detail", "updated_at"])

endoreg_db/models/utils.py

@@ -0,0 +1,138 @@
+import io
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Tuple
+
+import cv2
+import numpy as np
+from django.core.files import File
+from django.core.files.storage import FileSystemStorage
+
+from ..utils import DJANGO_NAME_SALT, data_paths
+
+if TYPE_CHECKING:
+    pass
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+STORAGE_DIR = data_paths["storage"]
+FILE_STORAGE = FileSystemStorage(location=str(STORAGE_DIR))
+TRANSCODING_DIR = data_paths["transcoding"]
+FRAME_DIR = data_paths["frame"]
+WEIGHTS_DIR = data_paths["weights"]
+REPORT_DIR = data_paths["import_report"]
+DOCUMENT_DIR = data_paths[
+    "documents"
+]  
+ANONYM_VIDEO_DIR = data_paths["anonym_video"]
+TEST_RUN = os.environ.get("TEST_RUN", "False")
+TEST_RUN = TEST_RUN.lower() == "true"
+
+
+
+def prepare_bulk_frames(frame_paths: List[Path]):
+    """
+    Reads the frame paths into memory as Django File objects.
+    This avoids 'seek of closed file' errors by using BytesIO for each frame.
+    """
+    for path in frame_paths:
+        frame_number = int(path.stem.split("_")[1])
+        with open(path, "rb") as f:
+            content = f.read()
+        file_obj = File(io.BytesIO(content), name=path.name)
+        yield frame_number, file_obj
+
+
+def find_segments_in_prediction_array(prediction_array: np.array, min_frame_len: int):
+    """
+    Expects a prediction array of shape (num_frames) and a minimum frame length.
+    Returns a list of tuples (start_frame_number, end_frame_number) that represent the segments.
+    """
+    # Add False to the beginning and end to detect changes at the array boundaries
+    padded_prediction = np.pad(
+        prediction_array, (1, 1), "constant", constant_values=False
+    )
+
+    # Find the start points and end points of the segments
+    diffs = np.diff(padded_prediction.astype(int))
+    segment_starts = np.where(diffs == 1)[0]
+    segment_ends = np.where(diffs == -1)[0]
+
+    # Filter segments based on min_frame_len
+    segments = [
+        (start, end)
+        for start, end in zip(segment_starts, segment_ends)
+        if end - start >= min_frame_len
+    ]
+
+    return segments
+
+
+def anonymize_frame(
+    raw_frame_path: Path,
+    target_frame_path: Path,
+    endo_roi,
+    all_black: bool = False,
+    censor_color: Tuple[int, int, int] = (0, 0, 0),  # Added censor_color param
+):
+    """
+    Anonymize the frame by blacking out pixels outside the endoscope ROI or making the whole frame black.
+    """
+    frame = cv2.imread(raw_frame_path.as_posix())
+    if frame is None:
+        # Raise error instead of returning None/frame
+        raise FileNotFoundError(f"Could not read frame at {raw_frame_path}")
+
+    # make black frame with same size as original frame
+    new_frame = np.zeros_like(frame)
+
+    if not all_black:
+        # Validate ROI dictionary keys
+        required_keys = {"x", "y", "width", "height"}
+        if not required_keys.issubset(endo_roi):
+            raise ValueError(
+                f"Invalid endo_roi dictionary provided: {endo_roi}. Missing keys."
+            )
+
+        x = endo_roi["x"]
+        y = endo_roi["y"]
+        width = endo_roi["width"]
+        height = endo_roi["height"]
+
+        # Add boundary checks for ROI coordinates
+        h_orig, w_orig, _ = frame.shape
+        x1, y1 = max(0, x), max(0, y)
+        x2, y2 = min(w_orig, x + width), min(h_orig, y + height)
+
+        if x1 >= x2 or y1 >= y2:
+            logger.warning(
+                f"ROI [{x},{y},{width},{height}] is outside or invalid for frame dimensions {w_orig}x{h_orig}. Resulting frame might be all black."
+            )
+        else:
+            # copy valid endoscope roi part to black frame
+            new_frame[y1:y2, x1:x2] = frame[y1:y2, x1:x2]
+    else:
+        # If all_black, fill with censor_color (defaults to black)
+        new_frame[:] = censor_color
+
+    # Check if writing the anonymized frame was successful
+    success = cv2.imwrite(target_frame_path.as_posix(), new_frame)
+    if not success:
+        raise IOError(f"Failed to write anonymized frame to {target_frame_path}")
+
+
+__all__ = [
+    "DJANGO_NAME_SALT",
+    "data_paths",
+    "FILE_STORAGE",
+    "ANONYM_VIDEO_DIR",
+    "FRAME_DIR",
+    "WEIGHTS_DIR",
+    "REPORT_DIR",
+    "prepare_bulk_frames",
+    "anonymize_frame",
+    "find_segments_in_prediction_array",
+    "TEST_RUN",
+]

endoreg_db/queries/__init__.py

@@ -0,0 +1,3 @@
+from .annotations import generate_legacy_dataset_output
+
+__all__ = ["generate_legacy_dataset_output"]

endoreg_db/queries/annotations/__init__.py

@@ -0,0 +1 @@
+from .legacy import get_legacy_annotations_for_labelset, generate_legacy_dataset_output

endoreg_db/queries/annotations/legacy.py

@@ -0,0 +1,169 @@
+from ...models import LabelSet, ImageClassificationAnnotation
+from django.db.models import F
+from django.db import models
+from tqdm import tqdm
+from collections import defaultdict
+
+# def get_legacy_annotations_for_labelset(labelset_name, version=None):
+#     """
+#     Retrieve annotations for a given label set for training.
+
+#     Args:
+#     - labelset_name (str): The name of the label set.
+#     - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+#     Returns:
+#     - list[dict]: A list of dictionaries. Each dictionary represents an image and its annotations.
+#                   Format: [{"frame": <frame_object>, "annotations": [{"label": <label_name>, "value": <value>}, ...]}, ...]
+
+#     Example:
+#         annotations_for_training = get_annotations_for_labelset("YourLabelSetName", version=2)
+
+#     """
+
+#     # Fetch the label set based on the name and optionally the version
+#     if version:
+#         labelset = LabelSet.objects.get(name=labelset_name, version=version)
+#     else:
+#         labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
+#         if not labelset:
+#             raise ValueError(f"No label set found with the name: {labelset_name}")
+
+#     # Retrieve all labels in the label set
+#     labels_in_set = labelset.labels.all()
+
+#     # Get the most recent annotations for each frame/label combination
+#     annotations = ImageClassificationAnnotation.objects.filter(label__in=labels_in_set)
+#     annotations = annotations.annotate(
+#         latest_annotation=models.Window(
+#             expression=models.functions.RowNumber(),
+#             partition_by=[F('legacy_image'), F('label')],
+#             order_by=F('date_modified').desc()
+#         )
+#     ).filter(latest_annotation=1)
+
+#     # Organize the annotations by image/frame
+#     organized_annotations = []
+
+#     for annotation in tqdm(annotations):
+#         # ic(annotation)
+#         # Check if the frame is already in the organized list
+#         existing_entry = next((entry for entry in organized_annotations if entry['legacy_image'] == annotation.legacy_frame), None)
+
+#         if existing_entry:
+#             # Add this annotation to the existing frame's annotations
+#             existing_entry['annotations'].append({
+#                 "label": annotation.label.name,
+#                 "value": annotation.value
+#             })
+#         else:
+#             # Create a new entry for this frame
+#             organized_annotations.append({
+#                 "legacy_image": annotation.legacy_image,
+#                 "annotations": [{
+#                     "label": annotation.label.name,
+#                     "value": annotation.value
+#                 }]
+#             })
+
+#     return organized_annotations
+
+
+def get_legacy_annotations_for_labelset(labelset_name, version=None):
+    """
+    ... [rest of your docstring]
+    """
+
+    # Fetch the label set based on the name and optionally the version
+    if version:
+        labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    else:
+        labelset = (
+            LabelSet.objects.filter(name=labelset_name).order_by("-version").first()
+        )
+        if not labelset:
+            raise ValueError(f"No label set found with the name: {labelset_name}")
+
+    # Retrieve all labels in the label set
+    labels_in_set = labelset.labels.all()
+
+    # Get the most recent annotations for each frame/label combination
+    annotations = (
+        ImageClassificationAnnotation.objects.filter(label__in=labels_in_set)
+        .select_related("legacy_image", "label")  # Reduce number of queries
+        .annotate(
+            latest_annotation=models.Window(
+                expression=models.functions.RowNumber(),
+                partition_by=[F("legacy_image"), F("label")],
+                order_by=F("date_modified").desc(),
+            )
+        )
+        .filter(latest_annotation=1)
+    )
+
+    # Organize the annotations by image/frame using a defaultdict
+    organized_annotations_dict = defaultdict(
+        lambda: {"legacy_image": None, "annotations": []}
+    )
+
+    for annotation in tqdm(annotations):
+        organized_entry = organized_annotations_dict[annotation.legacy_image.id]
+        organized_entry["legacy_image"] = annotation.legacy_image
+        organized_entry["annotations"].append(
+            {"label": annotation.label.name, "value": annotation.value}
+        )
+
+    # Convert organized_annotations_dict to a list
+    organized_annotations = list(organized_annotations_dict.values())
+
+    return organized_annotations
+
+
+def generate_legacy_dataset_output(labelset_name, version=None):
+    """
+    Generate an output suitable for creating PyTorch datasets.
+
+    Args:
+    - labelset_name (str): The name of the label set.
+    - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+    Returns:
+    - list[dict]: A list of dictionaries, where each dictionary contains the file path and the labels.
+                  Format: [{"path": <file_path>, "labels": [<label_1_value>, <label_2_value>, ...]}, ...]
+    - labelset[LabelSet]: The label set that was used to generate the output.
+    """
+
+    # First, retrieve the organized annotations using the previously defined function
+    organized_annotations = get_legacy_annotations_for_labelset(labelset_name, version)
+
+    # Fetch all labels from the labelset for consistent ordering
+    labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    all_labels = labelset.get_labels_in_order()
+
+    dataset_output = []
+
+    for entry in organized_annotations:
+        # Prepare a dictionary for each frame
+        frame_data = {
+            "path": entry[
+                "legacy_image"
+            ].image.path,  # Assuming 'image' field stores the file path
+            "labels": [-1] * len(all_labels),  # Initialize with -1 for all labels
+        }
+
+        # Update the labels based on the annotations
+        for annotation in entry["annotations"]:
+            index = next(
+                (
+                    i
+                    for i, label in enumerate(all_labels)
+                    if label.name == annotation["label"]
+                ),
+                None,
+            )
+            if index is not None:
+                frame_data["labels"][index] = int(annotation["value"])
+
+        dataset_output.append(frame_data)
+
+    return dataset_output, labelset

endoreg_db/root_urls.py

@@ -0,0 +1,27 @@
+# endoreg_db/root_urls.py
+from django.urls import include, path
+from django.http import HttpResponse
+from django.conf import settings
+from django.conf.urls.static import static
+
+# Import raw API urlpatterns (no prefix) from your API urls package
+from endoreg_db.urls import urlpatterns as api_urlpatterns
+
+
+def public_home(_request):
+    return HttpResponse("Public home – no login required.")
+
+
+urlpatterns = [
+    path("", public_home, name="public_home"),
+    # path("admin/", admin.site.urls),
+    # Mount ALL API endpoints under /api/
+    path("api/", include((api_urlpatterns, "endoreg_db"), namespace="api")),
+    # OIDC (mozilla-django-oidc provides /oidc/authenticate/ and /oidc/callback/)
+    path("oidc/", include("mozilla_django_oidc.urls")),
+]
+
+# Serve static/media only in DEBUG (at root, not under /api/)
+if settings.DEBUG:
+    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)