endoreg-db 0.3.6__py3-none-any.whl → 0.8.6.1__py3-none-any.whl

endoreg_db/models/upload_job.py

@@ -0,0 +1,99 @@
+import uuid
+from django.db import models
+
+
+class UploadJob(models.Model):
+    """
+    Tracks file upload jobs and their processing status.
+    Supports both PDF and video file uploads with asynchronous processing.
+    """
+    
+    class Status(models.TextChoices):
+        PENDING = 'pending', 'Pending'
+        PROCESSING = 'processing', 'Processing'
+        ANONYMIZED = 'anonymized', 'Anonymized'
+        ERROR = 'error', 'Error'
+
+    id = models.UUIDField(
+        primary_key=True, 
+        default=uuid.uuid4, 
+        editable=False,
+        help_text="Unique identifier for the upload job"
+    )
+    
+    file = models.FileField(
+        upload_to='uploads/%Y/%m/%d/',
+        help_text="Uploaded file (PDF or video)"
+    )
+    
+    status = models.CharField(
+        max_length=20,
+        choices=Status.choices,
+        default=Status.PENDING,
+        help_text="Current processing status of the upload"
+    )
+    
+    content_type = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text="MIME type of the uploaded file"
+    )
+    
+    sensitive_meta = models.ForeignKey(
+        'SensitiveMeta',
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        help_text="Link to the created SensitiveMeta record after processing"
+    )
+    
+    error_detail = models.TextField(
+        blank=True,
+        help_text="Error message if processing failed"
+    )
+    
+    created_at = models.DateTimeField(
+        auto_now_add=True,
+        help_text="When the upload job was created"
+    )
+    
+    updated_at = models.DateTimeField(
+        auto_now=True,
+        help_text="When the upload job was last updated"
+    )
+
+    class Meta:
+        ordering = ['-created_at']
+        verbose_name = "Upload Job"
+        verbose_name_plural = "Upload Jobs"
+
+    def __str__(self):
+        return f"UploadJob {self.id} - {self.status} ({self.content_type})"
+
+    @property
+    def is_complete(self):
+        """Returns True if the job has finished processing (success or error)."""
+        return self.status in [self.Status.ANONYMIZED, self.Status.ERROR]
+
+    @property
+    def is_successful(self):
+        """Returns True if the job completed successfully."""
+        return self.status == self.Status.ANONYMIZED
+
+    def mark_processing(self):
+        """Mark the job as processing."""
+        self.status = self.Status.PROCESSING
+        self.save(update_fields=['status', 'updated_at'])
+
+    def mark_completed(self, sensitive_meta=None):
+        """Mark the job as successfully completed."""
+        self.status = self.Status.ANONYMIZED
+        if sensitive_meta:
+            self.sensitive_meta = sensitive_meta
+        self.save(update_fields=['status', 'sensitive_meta', 'updated_at'])
+
+    def mark_error(self, error_detail: str):
+        """Mark the job as failed with error details."""
+        self.status = self.Status.ERROR
+        self.error_detail = error_detail
+        self.save(update_fields=['status', 'error_detail', 'updated_at'])

endoreg_db/models/utils.py

@@ -0,0 +1,135 @@
+from ..utils import (
+    data_paths,
+    DJANGO_NAME_SALT,
+)
+from django.core.files import File
+from django.core.files.storage import FileSystemStorage
+import io
+import os
+from tqdm import tqdm
+import numpy as np
+import cv2
+from typing import TYPE_CHECKING, List, Tuple
+from pathlib import Path
+if TYPE_CHECKING:
+    from ..models.media import VideoFile
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+STORAGE_DIR = data_paths["storage"]
+FILE_STORAGE = FileSystemStorage(location = STORAGE_DIR)
+VIDEO_DIR = data_paths["video"]
+TMP_VIDEO_DIR = VIDEO_DIR / "tmp"
+ANONYM_VIDEO_DIR = data_paths["video_export"]
+FRAME_DIR = data_paths["frame"]
+WEIGHTS_DIR = data_paths["weights"]
+PDF_DIR = data_paths["raw_pdf"]
+DOCUMENT_DIR = data_paths["pdf"]
+
+TEST_RUN = os.environ.get("TEST_RUN", "False")
+TEST_RUN = TEST_RUN.lower() == "true"
+
+TEST_RUN_FRAME_NUMBER = int(os.environ.get("TEST_RUN_FRAME_NUMBER", "500"))
+
+
+def prepare_bulk_frames(frame_paths: List[Path]):
+    """
+    Reads the frame paths into memory as Django File objects.
+    This avoids 'seek of closed file' errors by using BytesIO for each frame.
+    """
+    for path in frame_paths:
+        frame_number = int(path.stem.split("_")[1])
+        with open(path, "rb") as f:
+            content = f.read()
+        file_obj = File(io.BytesIO(content), name=path.name)
+        yield frame_number, file_obj
+
+
+def find_segments_in_prediction_array(prediction_array: np.array, min_frame_len: int):
+    """
+    Expects a prediction array of shape (num_frames) and a minimum frame length.
+    Returns a list of tuples (start_frame_number, end_frame_number) that represent the segments.
+    """
+    # Add False to the beginning and end to detect changes at the array boundaries
+    padded_prediction = np.pad(
+        prediction_array, (1, 1), "constant", constant_values=False
+    )
+
+    # Find the start points and end points of the segments
+    diffs = np.diff(padded_prediction.astype(int))
+    segment_starts = np.where(diffs == 1)[0]
+    segment_ends = np.where(diffs == -1)[0]
+
+    # Filter segments based on min_frame_len
+    segments = [
+        (start, end)
+        for start, end in zip(segment_starts, segment_ends)
+        if end - start >= min_frame_len
+    ]
+
+    return segments
+
+def anonymize_frame(
+    raw_frame_path: Path, target_frame_path: Path, endo_roi, all_black: bool = False, censor_color: Tuple[int, int, int] = (0, 0, 0) # Added censor_color param
+):
+    """
+    Anonymize the frame by blacking out pixels outside the endoscope ROI or making the whole frame black.
+    """
+    frame = cv2.imread(raw_frame_path.as_posix())
+    if frame is None:
+        # Raise error instead of returning None/frame
+        raise FileNotFoundError(f"Could not read frame at {raw_frame_path}")
+
+    # make black frame with same size as original frame
+    new_frame = np.zeros_like(frame)
+
+    if not all_black:
+        # Validate ROI dictionary keys
+        required_keys = {"x", "y", "width", "height"}
+        if not required_keys.issubset(endo_roi):
+            raise ValueError(f"Invalid endo_roi dictionary provided: {endo_roi}. Missing keys.")
+
+        x = endo_roi["x"]
+        y = endo_roi["y"]
+        width = endo_roi["width"]
+        height = endo_roi["height"]
+
+        # Add boundary checks for ROI coordinates
+        h_orig, w_orig, _ = frame.shape
+        x1, y1 = max(0, x), max(0, y)
+        x2, y2 = min(w_orig, x + width), min(h_orig, y + height)
+
+        if x1 >= x2 or y1 >= y2:
+            logger.warning(f"ROI [{x},{y},{width},{height}] is outside or invalid for frame dimensions {w_orig}x{h_orig}. Resulting frame might be all black.")
+        else:
+            # copy valid endoscope roi part to black frame
+            new_frame[y1:y2, x1:x2] = frame[y1:y2, x1:x2]
+    else:
+        # If all_black, fill with censor_color (defaults to black)
+        new_frame[:] = censor_color
+
+    # Check if writing the anonymized frame was successful
+    success = cv2.imwrite(target_frame_path.as_posix(), new_frame)
+    if not success:
+        raise IOError(f"Failed to write anonymized frame to {target_frame_path}")
+
+
+__all__ = [
+    "DJANGO_NAME_SALT",
+    "data_paths",
+    "FILE_STORAGE",
+    "VIDEO_DIR",
+    "TMP_VIDEO_DIR",
+    "ANONYM_VIDEO_DIR",
+    "FRAME_DIR",
+    "WEIGHTS_DIR",
+    "PDF_DIR",
+    "DOCUMENT_DIR",
+    "prepare_bulk_frames",
+    "anonymize_frame",
+    "find_segments_in_prediction_array",
+    "TEST_RUN",
+    "TEST_RUN_FRAME_NUMBER",
+]

endoreg_db/queries/__init__.py

@@ -1,5 +1,5 @@
-from .annotations import (
-    generate_legacy_dataset_output
-)
-
+from .annotations import (
+    generate_legacy_dataset_output
+)
+
 from .get import *

endoreg_db/queries/annotations/__init__.py

@@ -1,3 +1,3 @@
-from .legacy import (
-    get_legacy_annotations_for_labelset, generate_legacy_dataset_output
+from .legacy import (
+    get_legacy_annotations_for_labelset, generate_legacy_dataset_output
 )

endoreg_db/queries/annotations/legacy.py

@@ -1,159 +1,158 @@
-from ...models import LabelSet, ImageClassificationAnnotation
-from django.db.models import Q, F
-from django.db import models
-from icecream import ic
-from tqdm import tqdm
-from collections import defaultdict
-
-# def get_legacy_annotations_for_labelset(labelset_name, version=None):
-#     """
-#     Retrieve annotations for a given label set for training.
-
-#     Args:
-#     - labelset_name (str): The name of the label set.
-#     - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
-
-#     Returns:
-#     - list[dict]: A list of dictionaries. Each dictionary represents an image and its annotations.
-#                   Format: [{"frame": <frame_object>, "annotations": [{"label": <label_name>, "value": <value>}, ...]}, ...]
-
-#     Example:
-#         annotations_for_training = get_annotations_for_labelset("YourLabelSetName", version=2)
-
-#     """
-
-#     # Fetch the label set based on the name and optionally the version
-#     if version:
-#         labelset = LabelSet.objects.get(name=labelset_name, version=version)
-#     else:
-#         labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
-#         if not labelset:
-#             raise ValueError(f"No label set found with the name: {labelset_name}")
-
-#     # Retrieve all labels in the label set
-#     labels_in_set = labelset.labels.all()
-
-#     # Get the most recent annotations for each frame/label combination
-#     annotations = ImageClassificationAnnotation.objects.filter(label__in=labels_in_set)
-#     annotations = annotations.annotate(
-#         latest_annotation=models.Window(
-#             expression=models.functions.RowNumber(),
-#             partition_by=[F('legacy_image'), F('label')],
-#             order_by=F('date_modified').desc()
-#         )
-#     ).filter(latest_annotation=1)
-
-#     # Organize the annotations by image/frame
-#     organized_annotations = []
-
-#     for annotation in tqdm(annotations):
-#         # ic(annotation)
-#         # Check if the frame is already in the organized list
-#         existing_entry = next((entry for entry in organized_annotations if entry['legacy_image'] == annotation.legacy_frame), None)
-
-#         if existing_entry:
-#             # Add this annotation to the existing frame's annotations
-#             existing_entry['annotations'].append({
-#                 "label": annotation.label.name,
-#                 "value": annotation.value
-#             })
-#         else:
-#             # Create a new entry for this frame
-#             organized_annotations.append({
-#                 "legacy_image": annotation.legacy_image,
-#                 "annotations": [{
-#                     "label": annotation.label.name,
-#                     "value": annotation.value
-#                 }]
-#             })
-
-#     return organized_annotations
-
-
-
-def get_legacy_annotations_for_labelset(labelset_name, version=None):
-    """
-    ... [rest of your docstring]
-    """
-
-    # Fetch the label set based on the name and optionally the version
-    if version:
-        labelset = LabelSet.objects.get(name=labelset_name, version=version)
-    else:
-        labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
-        if not labelset:
-            raise ValueError(f"No label set found with the name: {labelset_name}")
-
-    # Retrieve all labels in the label set
-    labels_in_set = labelset.labels.all()
-
-    # Get the most recent annotations for each frame/label combination
-    annotations = (ImageClassificationAnnotation.objects
-                   .filter(label__in=labels_in_set)
-                   .select_related('legacy_image', 'label')  # Reduce number of queries
-                   .annotate(
-                        latest_annotation=models.Window(
-                            expression=models.functions.RowNumber(),
-                            partition_by=[F('legacy_image'), F('label')],
-                            order_by=F('date_modified').desc()
-                        )
-                    ).filter(latest_annotation=1))
-
-    # Organize the annotations by image/frame using a defaultdict
-    organized_annotations_dict = defaultdict(lambda: {
-        "legacy_image": None,
-        "annotations": []
-    })
-
-    for annotation in tqdm(annotations):
-        organized_entry = organized_annotations_dict[annotation.legacy_image.id]
-        organized_entry["legacy_image"] = annotation.legacy_image
-        organized_entry["annotations"].append({
-            "label": annotation.label.name,
-            "value": annotation.value
-        })
-
-    # Convert organized_annotations_dict to a list
-    organized_annotations = list(organized_annotations_dict.values())
-
-    return organized_annotations
-
-def generate_legacy_dataset_output(labelset_name, version=None):
-    """
-    Generate an output suitable for creating PyTorch datasets.
-
-    Args:
-    - labelset_name (str): The name of the label set.
-    - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
-
-    Returns:
-    - list[dict]: A list of dictionaries, where each dictionary contains the file path and the labels.
-                  Format: [{"path": <file_path>, "labels": [<label_1_value>, <label_2_value>, ...]}, ...]
-    - labelset[LabelSet]: The label set that was used to generate the output.
-    """
-
-    # First, retrieve the organized annotations using the previously defined function
-    organized_annotations = get_legacy_annotations_for_labelset(labelset_name, version)
-
-    # Fetch all labels from the labelset for consistent ordering
-    labelset = LabelSet.objects.get(name=labelset_name, version=version)
-    all_labels = labelset.get_labels_in_order()
-
-    dataset_output = []
-
-    for entry in organized_annotations:
-        # Prepare a dictionary for each frame
-        frame_data = {
-            "path": entry['legacy_image'].image.path,  # Assuming 'image' field stores the file path
-            "labels": [-1] * len(all_labels)  # Initialize with -1 for all labels
-        }
-
-        # Update the labels based on the annotations
-        for annotation in entry['annotations']:
-            index = next((i for i, label in enumerate(all_labels) if label.name == annotation['label']), None)
-            if index is not None:
-                frame_data['labels'][index] = int(annotation['value'])
-
-        dataset_output.append(frame_data)
-
-    return dataset_output, labelset
+from ...models import LabelSet, ImageClassificationAnnotation
+from django.db.models import Q, F
+from django.db import models
+from tqdm import tqdm
+from collections import defaultdict
+
+# def get_legacy_annotations_for_labelset(labelset_name, version=None):
+#     """
+#     Retrieve annotations for a given label set for training.
+
+#     Args:
+#     - labelset_name (str): The name of the label set.
+#     - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+#     Returns:
+#     - list[dict]: A list of dictionaries. Each dictionary represents an image and its annotations.
+#                   Format: [{"frame": <frame_object>, "annotations": [{"label": <label_name>, "value": <value>}, ...]}, ...]
+
+#     Example:
+#         annotations_for_training = get_annotations_for_labelset("YourLabelSetName", version=2)
+
+#     """
+
+#     # Fetch the label set based on the name and optionally the version
+#     if version:
+#         labelset = LabelSet.objects.get(name=labelset_name, version=version)
+#     else:
+#         labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
+#         if not labelset:
+#             raise ValueError(f"No label set found with the name: {labelset_name}")
+
+#     # Retrieve all labels in the label set
+#     labels_in_set = labelset.labels.all()
+
+#     # Get the most recent annotations for each frame/label combination
+#     annotations = ImageClassificationAnnotation.objects.filter(label__in=labels_in_set)
+#     annotations = annotations.annotate(
+#         latest_annotation=models.Window(
+#             expression=models.functions.RowNumber(),
+#             partition_by=[F('legacy_image'), F('label')],
+#             order_by=F('date_modified').desc()
+#         )
+#     ).filter(latest_annotation=1)
+
+#     # Organize the annotations by image/frame
+#     organized_annotations = []
+
+#     for annotation in tqdm(annotations):
+#         # ic(annotation)
+#         # Check if the frame is already in the organized list
+#         existing_entry = next((entry for entry in organized_annotations if entry['legacy_image'] == annotation.legacy_frame), None)
+
+#         if existing_entry:
+#             # Add this annotation to the existing frame's annotations
+#             existing_entry['annotations'].append({
+#                 "label": annotation.label.name,
+#                 "value": annotation.value
+#             })
+#         else:
+#             # Create a new entry for this frame
+#             organized_annotations.append({
+#                 "legacy_image": annotation.legacy_image,
+#                 "annotations": [{
+#                     "label": annotation.label.name,
+#                     "value": annotation.value
+#                 }]
+#             })
+
+#     return organized_annotations
+
+
+
+def get_legacy_annotations_for_labelset(labelset_name, version=None):
+    """
+    ... [rest of your docstring]
+    """
+
+    # Fetch the label set based on the name and optionally the version
+    if version:
+        labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    else:
+        labelset = LabelSet.objects.filter(name=labelset_name).order_by('-version').first()
+        if not labelset:
+            raise ValueError(f"No label set found with the name: {labelset_name}")
+
+    # Retrieve all labels in the label set
+    labels_in_set = labelset.labels.all()
+
+    # Get the most recent annotations for each frame/label combination
+    annotations = (ImageClassificationAnnotation.objects
+                   .filter(label__in=labels_in_set)
+                   .select_related('legacy_image', 'label')  # Reduce number of queries
+                   .annotate(
+                        latest_annotation=models.Window(
+                            expression=models.functions.RowNumber(),
+                            partition_by=[F('legacy_image'), F('label')],
+                            order_by=F('date_modified').desc()
+                        )
+                    ).filter(latest_annotation=1))
+
+    # Organize the annotations by image/frame using a defaultdict
+    organized_annotations_dict = defaultdict(lambda: {
+        "legacy_image": None,
+        "annotations": []
+    })
+
+    for annotation in tqdm(annotations):
+        organized_entry = organized_annotations_dict[annotation.legacy_image.id]
+        organized_entry["legacy_image"] = annotation.legacy_image
+        organized_entry["annotations"].append({
+            "label": annotation.label.name,
+            "value": annotation.value
+        })
+
+    # Convert organized_annotations_dict to a list
+    organized_annotations = list(organized_annotations_dict.values())
+
+    return organized_annotations
+
+def generate_legacy_dataset_output(labelset_name, version=None):
+    """
+    Generate an output suitable for creating PyTorch datasets.
+
+    Args:
+    - labelset_name (str): The name of the label set.
+    - version (int, optional): The version of the label set. If not specified, the latest version is fetched.
+
+    Returns:
+    - list[dict]: A list of dictionaries, where each dictionary contains the file path and the labels.
+                  Format: [{"path": <file_path>, "labels": [<label_1_value>, <label_2_value>, ...]}, ...]
+    - labelset[LabelSet]: The label set that was used to generate the output.
+    """
+
+    # First, retrieve the organized annotations using the previously defined function
+    organized_annotations = get_legacy_annotations_for_labelset(labelset_name, version)
+
+    # Fetch all labels from the labelset for consistent ordering
+    labelset = LabelSet.objects.get(name=labelset_name, version=version)
+    all_labels = labelset.get_labels_in_order()
+
+    dataset_output = []
+
+    for entry in organized_annotations:
+        # Prepare a dictionary for each frame
+        frame_data = {
+            "path": entry['legacy_image'].image.path,  # Assuming 'image' field stores the file path
+            "labels": [-1] * len(all_labels)  # Initialize with -1 for all labels
+        }
+
+        # Update the labels based on the annotations
+        for annotation in entry['annotations']:
+            index = next((i for i, label in enumerate(all_labels) if label.name == annotation['label']), None)
+            if index is not None:
+                frame_data['labels'][index] = int(annotation['value'])
+
+        dataset_output.append(frame_data)
+
+    return dataset_output, labelset

endoreg_db/renames.yml

@@ -0,0 +1,8 @@
+date_created: created_at
+date_modified: updated_at
+last_update: updated_at
+date_updated: updated_at
+createdOn: created_at
+updatedOn: updated_at
+modified_at: updated_at
+lastModified: updated_at

endoreg_db/root_urls.py

@@ -0,0 +1,9 @@
+from django.urls import include, path
+
+# Import raw API urlpatterns (no prefix) from the URLs package
+from endoreg_db.urls import urlpatterns as api_urlpatterns
+
+# Mount the API under /api/
+urlpatterns = [
+    path("api/", include((api_urlpatterns, "endoreg_db"), namespace="api")),
+]

endoreg_db/schemas/examination_evaluation.py

@@ -0,0 +1,27 @@
+# app/schemas/evaluation.py
+from __future__ import annotations
+from pydantic import BaseModel, ConfigDict, Field
+from typing import List, Optional
+
+class RequirementEval(BaseModel):
+    id: int
+    name: str
+    description: Optional[str] = None
+    satisfied: bool
+    message: Optional[str] = None  # optional explanation
+
+class RequirementSetEval(BaseModel):
+    id: int
+    name: str
+    type: Optional[str] = None  # e.g., "all", "any", "exactly_1"
+    is_satisfied: bool
+    requirements: List[RequirementEval] = Field(default_factory=list)
+    linked_sets: List["RequirementSetEval"] = Field(default_factory=list)
+
+class ExaminationEvalReport(BaseModel):
+    examination_id: int
+    summary: dict
+    sets: List[RequirementSetEval]
+    errors: List[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(from_attributes=True)