endoreg-db 0.6.3__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/models/upload_job.py

@@ -0,0 +1,99 @@
+import uuid
+from django.db import models
+
+
+class UploadJob(models.Model):
+    """
+    Tracks file upload jobs and their processing status.
+    Supports both PDF and video file uploads with asynchronous processing.
+    """
+    
+    class Status(models.TextChoices):
+        PENDING = 'pending', 'Pending'
+        PROCESSING = 'processing', 'Processing'
+        ANONYMIZED = 'anonymized', 'Anonymized'
+        ERROR = 'error', 'Error'
+
+    id = models.UUIDField(
+        primary_key=True, 
+        default=uuid.uuid4, 
+        editable=False,
+        help_text="Unique identifier for the upload job"
+    )
+    
+    file = models.FileField(
+        upload_to='uploads/%Y/%m/%d/',
+        help_text="Uploaded file (PDF or video)"
+    )
+    
+    status = models.CharField(
+        max_length=20,
+        choices=Status.choices,
+        default=Status.PENDING,
+        help_text="Current processing status of the upload"
+    )
+    
+    content_type = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text="MIME type of the uploaded file"
+    )
+    
+    sensitive_meta = models.ForeignKey(
+        'SensitiveMeta',
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        help_text="Link to the created SensitiveMeta record after processing"
+    )
+    
+    error_detail = models.TextField(
+        blank=True,
+        help_text="Error message if processing failed"
+    )
+    
+    created_at = models.DateTimeField(
+        auto_now_add=True,
+        help_text="When the upload job was created"
+    )
+    
+    updated_at = models.DateTimeField(
+        auto_now=True,
+        help_text="When the upload job was last updated"
+    )
+
+    class Meta:
+        ordering = ['-created_at']
+        verbose_name = "Upload Job"
+        verbose_name_plural = "Upload Jobs"
+
+    def __str__(self):
+        return f"UploadJob {self.id} - {self.status} ({self.content_type})"
+
+    @property
+    def is_complete(self):
+        """Returns True if the job has finished processing (success or error)."""
+        return self.status in [self.Status.ANONYMIZED, self.Status.ERROR]
+
+    @property
+    def is_successful(self):
+        """Returns True if the job completed successfully."""
+        return self.status == self.Status.ANONYMIZED
+
+    def mark_processing(self):
+        """Mark the job as processing."""
+        self.status = self.Status.PROCESSING
+        self.save(update_fields=['status', 'updated_at'])
+
+    def mark_completed(self, sensitive_meta=None):
+        """Mark the job as successfully completed."""
+        self.status = self.Status.ANONYMIZED
+        if sensitive_meta:
+            self.sensitive_meta = sensitive_meta
+        self.save(update_fields=['status', 'sensitive_meta', 'updated_at'])
+
+    def mark_error(self, error_detail: str):
+        """Mark the job as failed with error details."""
+        self.status = self.Status.ERROR
+        self.error_detail = error_detail
+        self.save(update_fields=['status', 'error_detail', 'updated_at'])

endoreg_db/models/utils.py

@@ -0,0 +1,135 @@
+from ..utils import (
+    data_paths,
+    DJANGO_NAME_SALT,
+)
+from django.core.files import File
+from django.core.files.storage import FileSystemStorage
+import io
+import os
+from tqdm import tqdm
+import numpy as np
+import cv2
+from typing import TYPE_CHECKING, List, Tuple
+from pathlib import Path
+if TYPE_CHECKING:
+    from ..models.media import VideoFile
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+STORAGE_DIR = data_paths["storage"]
+FILE_STORAGE = FileSystemStorage(location = STORAGE_DIR)
+VIDEO_DIR = data_paths["video"]
+TMP_VIDEO_DIR = VIDEO_DIR / "tmp"
+ANONYM_VIDEO_DIR = data_paths["video_export"]
+FRAME_DIR = data_paths["frame"]
+WEIGHTS_DIR = data_paths["weights"]
+PDF_DIR = data_paths["raw_pdf"]
+DOCUMENT_DIR = data_paths["pdf"]
+
+TEST_RUN = os.environ.get("TEST_RUN", "False")
+TEST_RUN = TEST_RUN.lower() == "true"
+
+TEST_RUN_FRAME_NUMBER = int(os.environ.get("TEST_RUN_FRAME_NUMBER", "500"))
+
+
+def prepare_bulk_frames(frame_paths: List[Path]):
+    """
+    Reads the frame paths into memory as Django File objects.
+    This avoids 'seek of closed file' errors by using BytesIO for each frame.
+    """
+    for path in frame_paths:
+        frame_number = int(path.stem.split("_")[1])
+        with open(path, "rb") as f:
+            content = f.read()
+        file_obj = File(io.BytesIO(content), name=path.name)
+        yield frame_number, file_obj
+
+
+def find_segments_in_prediction_array(prediction_array: np.array, min_frame_len: int):
+    """
+    Expects a prediction array of shape (num_frames) and a minimum frame length.
+    Returns a list of tuples (start_frame_number, end_frame_number) that represent the segments.
+    """
+    # Add False to the beginning and end to detect changes at the array boundaries
+    padded_prediction = np.pad(
+        prediction_array, (1, 1), "constant", constant_values=False
+    )
+
+    # Find the start points and end points of the segments
+    diffs = np.diff(padded_prediction.astype(int))
+    segment_starts = np.where(diffs == 1)[0]
+    segment_ends = np.where(diffs == -1)[0]
+
+    # Filter segments based on min_frame_len
+    segments = [
+        (start, end)
+        for start, end in zip(segment_starts, segment_ends)
+        if end - start >= min_frame_len
+    ]
+
+    return segments
+
+def anonymize_frame(
+    raw_frame_path: Path, target_frame_path: Path, endo_roi, all_black: bool = False, censor_color: Tuple[int, int, int] = (0, 0, 0) # Added censor_color param
+):
+    """
+    Anonymize the frame by blacking out pixels outside the endoscope ROI or making the whole frame black.
+    """
+    frame = cv2.imread(raw_frame_path.as_posix())
+    if frame is None:
+        # Raise error instead of returning None/frame
+        raise FileNotFoundError(f"Could not read frame at {raw_frame_path}")
+
+    # make black frame with same size as original frame
+    new_frame = np.zeros_like(frame)
+
+    if not all_black:
+        # Validate ROI dictionary keys
+        required_keys = {"x", "y", "width", "height"}
+        if not required_keys.issubset(endo_roi):
+            raise ValueError(f"Invalid endo_roi dictionary provided: {endo_roi}. Missing keys.")
+
+        x = endo_roi["x"]
+        y = endo_roi["y"]
+        width = endo_roi["width"]
+        height = endo_roi["height"]
+
+        # Add boundary checks for ROI coordinates
+        h_orig, w_orig, _ = frame.shape
+        x1, y1 = max(0, x), max(0, y)
+        x2, y2 = min(w_orig, x + width), min(h_orig, y + height)
+
+        if x1 >= x2 or y1 >= y2:
+            logger.warning(f"ROI [{x},{y},{width},{height}] is outside or invalid for frame dimensions {w_orig}x{h_orig}. Resulting frame might be all black.")
+        else:
+            # copy valid endoscope roi part to black frame
+            new_frame[y1:y2, x1:x2] = frame[y1:y2, x1:x2]
+    else:
+        # If all_black, fill with censor_color (defaults to black)
+        new_frame[:] = censor_color
+
+    # Check if writing the anonymized frame was successful
+    success = cv2.imwrite(target_frame_path.as_posix(), new_frame)
+    if not success:
+        raise IOError(f"Failed to write anonymized frame to {target_frame_path}")
+
+
+__all__ = [
+    "DJANGO_NAME_SALT",
+    "data_paths",
+    "FILE_STORAGE",
+    "VIDEO_DIR",
+    "TMP_VIDEO_DIR",
+    "ANONYM_VIDEO_DIR",
+    "FRAME_DIR",
+    "WEIGHTS_DIR",
+    "PDF_DIR",
+    "DOCUMENT_DIR",
+    "prepare_bulk_frames",
+    "anonymize_frame",
+    "find_segments_in_prediction_array",
+    "TEST_RUN",
+    "TEST_RUN_FRAME_NUMBER",
+]

endoreg_db/models/video_metadata.py

@@ -0,0 +1,66 @@
+"""
+Video Metadata Model
+
+Stores analysis results for videos (sensitive frames, detection statistics).
+Created as part of Phase 1.1: Video Correction API Endpoints.
+"""
+from django.db import models
+from endoreg_db.models.media import VideoFile
+
+
+class VideoMetadata(models.Model):
+    """
+    Stores analysis results for videos after sensitive frame detection.
+    
+    This model holds the output of frame analysis operations (MiniCPM, OCR+LLM)
+    and provides metrics for the correction UI.
+    """
+    video = models.OneToOneField(
+        VideoFile, 
+        on_delete=models.CASCADE, 
+        related_name='metadata',
+        help_text="Video file this metadata belongs to"
+    )
+    
+    # Analysis Results
+    sensitive_frame_count = models.IntegerField(
+        null=True, 
+        blank=True,
+        help_text="Number of frames detected as containing sensitive information"
+    )
+    sensitive_ratio = models.FloatField(
+        null=True, 
+        blank=True,
+        help_text="Ratio of sensitive frames to total frames (0.0-1.0)"
+    )
+    sensitive_frame_ids = models.TextField(
+        null=True, 
+        blank=True,
+        help_text="JSON array of sensitive frame indices (0-based)"
+    )
+    
+    # Metadata
+    analyzed_at = models.DateTimeField(
+        auto_now=True,
+        help_text="Timestamp of last analysis"
+    )
+    
+    class Meta:
+        db_table = 'video_metadata'
+        verbose_name = 'Video Metadata'
+        verbose_name_plural = 'Video Metadata'
+    
+    def __str__(self):
+        return f"Metadata for {self.video.uuid} ({self.sensitive_frame_count or 0} sensitive frames)"
+    
+    @property
+    def has_analysis(self) -> bool:
+        """Check if this video has been analyzed."""
+        return self.sensitive_frame_count is not None
+    
+    @property
+    def sensitive_percentage(self) -> float:
+        """Get sensitivity as percentage (0-100)."""
+        if self.sensitive_ratio is not None:
+            return self.sensitive_ratio * 100
+        return 0.0

endoreg_db/models/video_processing.py

@@ -0,0 +1,153 @@
+"""
+Video Processing History Model
+
+Tracks all video correction operations (masking, frame removal, reprocessing).
+Created as part of Phase 1.1: Video Correction API Endpoints.
+"""
+from django.db import models
+from django.utils import timezone
+from endoreg_db.models.media import VideoFile
+
+
+class VideoProcessingHistory(models.Model):
+    """
+    History of all video processing operations.
+    
+    Stores configuration and results of masking, frame removal, and reprocessing
+    operations for audit trail and download access.
+    """
+    
+    # Operation Types
+    OPERATION_MASKING = 'mask_overlay'
+    OPERATION_FRAME_REMOVAL = 'frame_removal'
+    OPERATION_ANALYSIS = 'analysis'
+    OPERATION_REPROCESSING = 'reprocessing'
+    
+    OPERATION_CHOICES = [
+        (OPERATION_MASKING, 'Mask Overlay'),
+        (OPERATION_FRAME_REMOVAL, 'Frame Removal'),
+        (OPERATION_ANALYSIS, 'Sensitivity Analysis'),
+        (OPERATION_REPROCESSING, 'Full Reprocessing'),
+    ]
+    
+    # Status Types
+    STATUS_PENDING = 'pending'
+    STATUS_RUNNING = 'running'
+    STATUS_SUCCESS = 'success'
+    STATUS_FAILURE = 'failure'
+    STATUS_CANCELLED = 'cancelled'
+    
+    STATUS_CHOICES = [
+        (STATUS_PENDING, 'Pending'),
+        (STATUS_RUNNING, 'Running'),
+        (STATUS_SUCCESS, 'Success'),
+        (STATUS_FAILURE, 'Failure'),
+        (STATUS_CANCELLED, 'Cancelled'),
+    ]
+    
+    video = models.ForeignKey(
+        VideoFile, 
+        on_delete=models.CASCADE, 
+        related_name='processing_history',
+        help_text="Video file this operation was performed on"
+    )
+    
+    operation = models.CharField(
+        max_length=50,
+        choices=OPERATION_CHOICES,
+        help_text="Type of processing operation"
+    )
+    
+    status = models.CharField(
+        max_length=20,
+        choices=STATUS_CHOICES,
+        default=STATUS_PENDING,
+        help_text="Current status of the operation"
+    )
+    
+    # Configuration & Results
+    config = models.JSONField(
+        default=dict,
+        help_text="Operation configuration (mask settings, frame list, etc.)"
+    )
+    
+    output_file = models.CharField(
+        max_length=500,
+        blank=True,
+        help_text="Path to output file (relative to MEDIA_ROOT)"
+    )
+    
+    details = models.TextField(
+        blank=True,
+        help_text="Additional details or error messages"
+    )
+    
+    # Celery Integration
+    task_id = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text="Celery task ID for progress tracking"
+    )
+    
+    # Timestamps
+    created_at = models.DateTimeField(
+        auto_now_add=True,
+        help_text="When the operation was started"
+    )
+    
+    completed_at = models.DateTimeField(
+        null=True,
+        blank=True,
+        help_text="When the operation completed (success or failure)"
+    )
+    
+    class Meta:
+        db_table = 'video_processing_history'
+        verbose_name = 'Video Processing History'
+        verbose_name_plural = 'Video Processing Histories'
+        ordering = ['-created_at']
+        indexes = [
+            models.Index(fields=['video', '-created_at']),
+            models.Index(fields=['task_id']),
+            models.Index(fields=['status']),
+        ]
+    
+    def __str__(self):
+        return f"{self.get_operation_display()} on {self.video.uuid} - {self.get_status_display()}"
+    
+    def mark_running(self, save=True):
+        """Mark operation as running."""
+        self.status = self.STATUS_RUNNING
+        if save:
+            self.save(update_fields=['status'])
+    
+    def mark_success(self, output_file=None, details=None, save=True):
+        """Mark operation as successful."""
+        self.status = self.STATUS_SUCCESS
+        self.completed_at = timezone.now()
+        if output_file:
+            self.output_file = output_file
+        if details:
+            self.details = details
+        if save:
+            self.save(update_fields=['status', 'completed_at', 'output_file', 'details'])
+    
+    def mark_failure(self, error_message, save=True):
+        """Mark operation as failed."""
+        self.status = self.STATUS_FAILURE
+        self.completed_at = timezone.now()
+        self.details = error_message
+        if save:
+            self.save(update_fields=['status', 'completed_at', 'details'])
+    
+    @property
+    def duration(self):
+        """Calculate operation duration if completed."""
+        if self.completed_at and self.created_at:
+            return (self.completed_at - self.created_at).total_seconds()
+        return None
+    
+    @property
+    def is_complete(self) -> bool:
+        """Check if operation is in a terminal state."""
+        return self.status in [self.STATUS_SUCCESS, self.STATUS_FAILURE, self.STATUS_CANCELLED]

endoreg_db/renames.yml

@@ -0,0 +1,8 @@
+date_created: created_at
+date_modified: updated_at
+last_update: updated_at
+date_updated: updated_at
+createdOn: created_at
+updatedOn: updated_at
+modified_at: updated_at
+lastModified: updated_at

endoreg_db/root_urls.py

@@ -0,0 +1,9 @@
+from django.urls import include, path
+
+# Import raw API urlpatterns (no prefix) from the URLs package
+from endoreg_db.urls import urlpatterns as api_urlpatterns
+
+# Mount the API under /api/
+urlpatterns = [
+    path("api/", include((api_urlpatterns, "endoreg_db"), namespace="api")),
+]

endoreg_db/schemas/examination_evaluation.py

@@ -0,0 +1,27 @@
+# app/schemas/evaluation.py
+from __future__ import annotations
+from pydantic import BaseModel, ConfigDict, Field
+from typing import List, Optional
+
+class RequirementEval(BaseModel):
+    id: int
+    name: str
+    description: Optional[str] = None
+    satisfied: bool
+    message: Optional[str] = None  # optional explanation
+
+class RequirementSetEval(BaseModel):
+    id: int
+    name: str
+    type: Optional[str] = None  # e.g., "all", "any", "exactly_1"
+    is_satisfied: bool
+    requirements: List[RequirementEval] = Field(default_factory=list)
+    linked_sets: List["RequirementSetEval"] = Field(default_factory=list)
+
+class ExaminationEvalReport(BaseModel):
+    examination_id: int
+    summary: dict
+    sets: List[RequirementSetEval]
+    errors: List[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(from_attributes=True)