endoreg-db 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

endoreg_db/utils/pydantic_models/__init__.py

@@ -0,0 +1,6 @@
+'''Module for Pydantic models.'''
+from .db_config import DbConfig
+
+__all__ = [
+    'DbConfig'
+]

endoreg_db/utils/pydantic_models/db_config.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+from typing import Optional, Union
+import yaml
+from icecream import ic
+from pydantic import BaseModel
+
+
+class DbConfig(BaseModel):
+    """Database configuration model."""
+
+    engine: str = "django.db.backends.postgresql"
+    host: str
+    port: int
+    user: str
+    password: Optional[str] = None
+    password_file: str = "/etc/secrets/vault/SCRT_local_password_maintenance_password"
+    name: str
+
+    # class options should allow for unused excess data
+
+    @classmethod
+    def from_file(cls, path: Union[Path, str]) -> "DbConfig":
+        """Create a DbConfig instance from a YAML configuration file."""
+        if isinstance(path, str):
+            filepath = Path(path)
+        else:
+            filepath = path
+        assert filepath.exists(), f"Missing Config {filepath}"
+
+        with open(filepath, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f)
+
+        obj = cls(**cfg)
+
+        return obj
+
+    def sync_password(self):
+        """Load the password from the configured password file."""
+        with open(self.password_file, "r", encoding="utf-8") as f:
+            self.password = f.read().strip()
+
+    def validate(self):
+        """Fully validate all database configuration fields."""
+        self.sync_password()
+
+        assert self.host, "Missing Host"
+        assert self.port, "Missing Port"
+        assert self.user, "Missing User"
+        assert self.password, "Missing Password"
+        assert self.name, "Missing Database"
+
+    def to_file(self, target: str = "./conf/db.yml", ask_override: bool = True):
+        """Export the configuration to a YAML file."""
+        ic(target)
+
+        with open(target, "w") as f:
+            yaml.safe_dump(self.model_dump(), f)

endoreg_db/utils/uuid.py

@@ -0,0 +1,4 @@
+import uuid
+
+def get_uuid():
+    return uuid.uuid4()

endoreg_db/utils/validate_endo_roi.py

@@ -0,0 +1,19 @@
+def validate_endo_roi(endo_roi_dict:dict):
+    """
+    Validate endoscope ROI dictionary. The dictionary must have the following
+    keys: x, y, width, height. The values must be greater than or equal to 0
+    for x and y, and greater than 0 for width and height.
+    """
+    for key, value in endo_roi_dict.items():
+        if key == "x":
+            assert value >= 0, f"Endoscope ROI x value must be greater than or equal to 0. Got {value}"
+        elif key == "y":
+            assert value >= 0, f"Endoscope ROI y value must be greater than or equal to 0. Got {value}"
+        elif key == "width":
+            assert value > 0, f"Endoscope ROI width value must be greater than 0. Got {value}"
+        elif key == "height":
+            assert value > 0, f"Endoscope ROI height value must be greater than 0. Got {value}"
+        else:
+            raise ValueError(f"Endoscope ROI key {key} not recognized")
+
+    return True

endoreg_db/utils/validate_subcategory_dict.py

@@ -0,0 +1,91 @@
+from endoreg_db.models import Unit
+
+def validate_subcategory_dict(self, subcategory_dict:dict=None):
+    if subcategory_dict is None:
+        return False
+    
+    if not isinstance(subcategory_dict, dict):
+        return False
+
+    # check if key choices exists and is a list of strings
+    if "choices" not in subcategory_dict:
+        return False
+    
+    if not isinstance(subcategory_dict["choices"], list):
+        return False
+    
+    for choice in subcategory_dict["choices"]:
+        if not isinstance(choice, str):
+            return False
+        
+    # check if key default exists and is a string
+    if "default" not in subcategory_dict:
+        return False
+    
+    if not isinstance(subcategory_dict["default"], str):
+        return False
+    
+    # check if default is in choices
+    if subcategory_dict["default"] not in subcategory_dict["choices"]:
+        return False
+    
+    if not subcategory_dict["required"]:
+        return False
+    
+    return True
+
+
+def validate_numerical_descriptor(self, numerical_descriptor_dict:dict=None):
+    # Check if numerical_descriptor_dict is None
+    error_message = ""
+    if numerical_descriptor_dict is None:
+        error_message = "numerical_descriptor_dict is None"
+        return False, error_message
+    
+    # Check if numerical_descriptor_dict is a dictionary
+    if not isinstance(numerical_descriptor_dict, dict):
+        error_message = "numerical_descriptor_dict is not a dictionary"
+        return False, error_message
+    
+    # if key unit exists and is a string and Unit object with that name exists
+    if "unit" not in numerical_descriptor_dict:
+        error_message = "unit key does not exist in numerical_descriptor_dict"
+        return False, error_message
+    
+    elif not isinstance(numerical_descriptor_dict["unit"], str):
+        error_message = "unit key is not a string"
+        return False, error_message
+
+    elif not Unit.objects.filter(name=numerical_descriptor_dict["unit"]).exists():
+        error_message = "Unit object with that name does not exist"
+        return False, error_message
+    
+    
+    if not "required" in numerical_descriptor_dict:
+        error_message = "required key does not exist in numerical_descriptor_dict"
+        return False, error_message
+    
+    elif not isinstance(numerical_descriptor_dict["required"], bool):
+        error_message = "required key is not a boolean"
+        return False, error_message
+    
+    # check if min, max, mean, std exist and are either None or float
+    for key in ["min", "max", "mean", "std", "default"]:
+        if key not in numerical_descriptor_dict:
+            error_message = f"{key} key does not exist in numerical_descriptor_dict"
+            return False, error_message
+        
+        if numerical_descriptor_dict[key] is not None and not isinstance(numerical_descriptor_dict[key], float):
+            error_message = f"{key} key is not a float"
+            return False, error_message
+        
+    # check if distribution exists and is either "normal" or "uniform"
+    if "distribution" not in numerical_descriptor_dict:
+        error_message = "distribution key does not exist in numerical_descriptor_dict"
+        return False, error_message
+    
+    if numerical_descriptor_dict["distribution"] not in ["normal", "uniform"]:
+        error_message = "distribution key is not either 'normal' or 'uniform'"
+        return False, error_message
+    
+    return True, None

endoreg_db/utils/video/__init__.py

@@ -0,0 +1,13 @@
+from .transcode_videofile import (
+    transcode_videofile,
+    transcode_videofile_if_required,
+)
+
+from .extract_frames import extract_frames, initialize_frame_objects
+
+__all__ = [
+    "extract_frames",
+    "initialize_frame_objects",
+    "transcode_videofile",
+    "transcode_videofile_if_required",
+]

endoreg_db/utils/video/extract_frames.py

@@ -0,0 +1,121 @@
+import os
+import shutil
+from pathlib import Path
+from icecream import ic
+import subprocess
+from django.db import transaction
+from tqdm import tqdm
+from typing import TYPE_CHECKING, Union, List
+
+if TYPE_CHECKING:
+    from endoreg_db.models import RawVideoFile, Video
+
+from django.core.files import File
+import io
+
+
+def prepare_bulk_frames(frame_paths: List[Path]):
+    """
+    Reads the frame paths into memory as Django File objects.
+    This avoids 'seek of closed file' errors by using BytesIO for each frame.
+    """
+    for path in frame_paths:
+        frame_number = int(path.stem.split("_")[1])
+        with open(path, "rb") as f:
+            content = f.read()
+        file_obj = File(io.BytesIO(content), name=path.name)
+        yield frame_number, file_obj
+
+
+def extract_frames(
+    video: Union["RawVideoFile", "Video"],
+    quality: int = 2,
+    overwrite: bool = False,
+    ext="jpg",
+    verbose=False,
+) -> List[Path]:
+    """
+    Extract frames from the video file and save them to the frame_dir.
+    For this, ffmpeg must be available in in the current environment.
+    """
+    frame_dir = Path(video.frame_dir)
+    ic(f"Extracting frames to {frame_dir}")
+    if not frame_dir.exists():
+        frame_dir.mkdir(parents=True, exist_ok=True)
+
+    if not overwrite and len(list(frame_dir.glob(f"*.{ext}"))) > 0:
+        video.state_frames_extracted = True  # Mark frames as extracted
+        extracted_paths = sorted(frame_dir.glob(f"*.{ext}"))
+        return extracted_paths
+
+    video_path = Path(video.file.path).resolve().as_posix()
+
+    frame_path_string = frame_dir.resolve().as_posix()
+    command = [
+        "ffmpeg",
+        "-i",
+        video_path,  #
+        "-q:v",
+        str(quality),
+        os.path.join(frame_path_string, f"frame_%07d.{ext}"),
+    ]
+
+    # Ensure FFmpeg is available
+    if not shutil.which("ffmpeg"):
+        raise EnvironmentError(
+            "FFmpeg could not be found. Ensure it is installed and in your PATH."
+        )
+
+    # Extract frames from the video file
+    # Execute the command
+    process = subprocess.Popen(
+        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+    )
+    stdout_data, stderr_data = process.communicate()
+
+    if process.returncode != 0:
+        raise Exception(f"Error extracting frames: {stderr_data}")
+
+    if verbose and stdout_data:
+        print(stdout_data)
+
+    # After extracting frames with ffmpeg, parse frame filenames and batch-create
+    extracted_paths = sorted(frame_dir.glob(f"*.{ext}"))
+
+    return extracted_paths
+
+
+def initialize_frame_objects(
+    video: Union["RawVideoFile", "Video"], extracted_paths: List[Path]
+):
+    """
+    Initialize frame objects for the extracted frames.
+    """
+    if video.state_frames_initialized:
+        return
+    video.frame_count = len(extracted_paths)
+    frames_to_create = []
+    batch_size = int(os.environ.get("DJANGO_FFMPEG_EXTRACT_FRAME_BATCHSIZE", "500"))
+    for i, (frame_number, file_obj) in tqdm(
+        enumerate(prepare_bulk_frames(extracted_paths), start=1)
+    ):
+        frame_obj_instance = video.create_frame_object(
+            frame_number, image_file=file_obj, extracted=True
+        )
+        frames_to_create.append(frame_obj_instance)
+
+        if i % batch_size == 0:
+            with transaction.atomic():
+                video.bulk_create_frames(frames_to_create)
+            frames_to_create.clear()
+
+    if frames_to_create:
+        with transaction.atomic():
+            video.bulk_create_frames(frames_to_create)
+
+    video.set_frames_extracted(True)
+    video.save()
+
+    frame_dir = extracted_paths[0].parent
+    ic(f"Removing frame directory: {frame_dir}")
+    shutil.rmtree(frame_dir)

endoreg_db/utils/video/transcode_videofile.py

@@ -0,0 +1,111 @@
+import shutil
+import subprocess
+import os
+from pathlib import Path
+from icecream import ic
+
+
+def get_transcoded_file_path(source_file_path: Path, suffix: str = "mp4"):
+    """
+    Method to get the transcoded file path.
+
+    Args:
+        source_file_path (Path): Source file path.
+        suffix (str): Suffix of the transcoded file.
+
+    Returns:
+        transcoded_file_path (Path): Transcoded file path.
+    """
+    transcoded_file_name = f"{source_file_path.stem}_transcoded.{suffix}"
+    transcoded_file_path = source_file_path.parent / transcoded_file_name
+    return transcoded_file_path
+
+
+def check_require_transcode(
+    filepath: Path, transcoded_file_path: Path, target_suffix=".mp4"
+):
+    """
+    Checks whether a video file requires transcoding.\
+    We check if the current suffix of the file path matches the target suffix\
+    and if the transcoded file path exists.\
+    If the current suffix does not match the target suffix and the transcoded file path does not exist,\
+    transcoding is required.
+    """
+    current_suffix = filepath.suffix
+
+    require_transcode = False
+    if not current_suffix == target_suffix and not transcoded_file_path.exists():
+        if not transcoded_file_path.exists():
+            require_transcode = True
+    return require_transcode
+
+
+def transcode_videofile_if_required(filepath: Path):
+    """
+    Perform transcoding on a video file if required.
+    This method checks whether a transcoded version (with an ".mp4" suffix) of the given
+    video file exists or needs to be produced. It first computes the expected transcoded file path,
+    then uses a class-specific check (check_require_transcode) to decide if transcoding is necessary.
+    If so, it transcodes the video file by calling the class method transcode_videofile and returns
+    the path of the transcoded file after ensuring that the resulting file path matches the computed one.
+    If transcoding is not required, the original file path is returned.
+    Args:
+        filepath (Path): The path to the original video file that may require transcoding.
+    Returns:
+        Path: The path to the transcoded video file if transcoding was performed; otherwise, the original file path.
+    """
+
+    transcoded_file_path = get_transcoded_file_path(filepath, suffix=".mp4")
+    if check_require_transcode(filepath, transcoded_file_path):
+        transcoded_path = transcode_videofile(
+            filepath, transcoded_path=transcoded_file_path
+        )
+        assert transcoded_file_path == transcoded_path
+        return transcoded_path
+    else:
+        return filepath
+
+
+def transcode_videofile(filepath: Path, transcoded_path: Path):
+    """
+    Transcodes a video to a compatible MP4 format using ffmpeg.
+    If the transcoded file exists, it is returned.
+
+    Parameters
+    ----------
+    mov_file : str
+        The full path to the video file.
+
+    Returns
+    -------
+    transcoded_path : str
+        The full path to the transcoded video file.
+    """
+    ic("Transcoding video")
+    ic(f"Input path: {filepath}")
+
+    # if filepath suffix is .mp4 or .MP4 we dont need to transcode and can copy the file
+    if filepath.suffix.lower() in [".mp4"]:
+        shutil.copyfile(filepath, transcoded_path)
+        return transcoded_path
+
+    ic(f"Transcoded path: {transcoded_path}")
+    if os.path.exists(transcoded_path):
+        return transcoded_path
+
+    # Run ffmpeg to transcode the video using H264 and AAC
+    # TODO Document settings, check if we need to change them
+    command = [
+        "ffmpeg",
+        "-i",
+        filepath.resolve().as_posix(),
+        "-c:v",
+        "libx264",
+        "-preset",
+        "fast",
+        "-c:a",
+        "aac",
+        transcoded_path,
+    ]
+    subprocess.run(command, check=True)
+    return transcoded_path

endoreg_db/views/__init__.py

@@ -0,0 +1,2 @@
+# Import the views so they are available for Django
+from .patient_views import start_examination

endoreg_db/views/csrf.py

@@ -0,0 +1,7 @@
+from django.http import JsonResponse
+from django.middleware.csrf import get_token
+
+# New view to return the CSRF token in JSON format
+def csrf_token_view(request):
+    token = get_token(request)
+    return JsonResponse({'csrf_token': token})

endoreg_db/views/patient_views.py

@@ -0,0 +1,90 @@
+from django.shortcuts import render
+from django.contrib.admin.views.decorators import staff_member_required
+from django.http import JsonResponse
+from django.http import JsonResponse
+from django.views.decorators.http import require_GET
+from django.core.exceptions import ObjectDoesNotExist
+from rest_framework import viewsets
+from ..models import Patient
+from ..serializers import PatientSerializer
+from rest_framework.permissions import IsAuthenticatedOrReadOnly
+from endoreg_db.models import (
+    FindingLocationClassification,
+    FindingLocationClassificationChoice,
+    FindingMorphologyClassification,
+    FindingMorphologyClassificationType
+)
+
+@staff_member_required  # Ensures only staff members can access the page
+def start_examination(request):
+    return render(request, 'admin/start_examination.html')  # Loads the simple HTML page
+
+#from ..models.patient.patient_finding_location import PatientFindingLocation
+from ..models import FindingLocationClassification, FindingLocationClassificationChoice  # Correct models
+
+
+#need to implement one with json data after tesing whethe rthis works or not
+"""def get_location_choices(request, location_id):
+   
+    try:
+        # Ensure the location exists
+        location = FindingLocationClassification.objects.get(id=location_id)
+        # Get only choices related to the selected location classification
+        #location_choices = FindingLocationClassificationChoice.objects.filter(location_classification=location)
+        #its many to may relation so
+        location_choices = location.choices.all()
+        
+    except FindingLocationClassification.DoesNotExist:
+        location_choices = []
+
+    # Get previously selected values to retain them after reloading
+    selected_location = int(location_id) if location_id else None
+
+    return render(request, 'admin/patient_finding_intervention.html', {
+        "location_choices": location_choices,  # Pass updated choices to the template
+        "selected_location": location_id,  # Keep previous selection
+    })
+"""
+
+class PatientViewSet(viewsets.ModelViewSet):
+    """API endpoint for managing patients."""
+    queryset = Patient.objects.all()
+    serializer_class = PatientSerializer
+    #permission_classes = [IsAuthenticatedOrReadOnly] 
+
+    def perform_create(self, serializer):
+        serializer.save()
+    
+    def update(self, request, *args, **kwargs):
+        # custom edit logic here if needed
+        return super().update(request, *args, **kwargs)
+
+    def destroy(self, request, *args, **kwargs):
+        # custom delete logic here if needed
+        return super().destroy(request, *args, **kwargs)
+
+@require_GET
+def get_location_choices(request, location_id):
+    """Fetch location choices dynamically based on FindingLocationClassification."""
+    try:
+        location = FindingLocationClassification.objects.get(id=location_id)
+        location_choices = location.choices.all()
+        data = [{"id": choice.id, "name": choice.name} for choice in location_choices]
+        return JsonResponse({"location_choices": data})
+    except FindingLocationClassification.DoesNotExist:
+        return JsonResponse({"error": "Location classification not found", "location_choices": []}, status=404)
+
+@require_GET
+def get_morphology_choices(request, morphology_id):
+    """Fetch morphology choices dynamically based on FindingMorphologyClassification."""
+    try:
+        morphology_classification = FindingMorphologyClassification.objects.get(id=morphology_id)
+        morphology_choices = FindingMorphologyClassificationType.objects.filter(
+            id=morphology_classification.classification_type_id
+        )
+        data = [{"id": choice.id, "name": choice.name} for choice in morphology_choices]
+        return JsonResponse({"morphology_choices": data})
+    except ObjectDoesNotExist:
+        return JsonResponse({"error": "Morphology classification not found", "morphology_choices": []}, status=404)
+    except Exception as e:
+        return JsonResponse({"error": "Internal server error", "morphology_choices": []}, status=500)

endoreg_db/views/raw_video_meta_validation_views.py

@@ -0,0 +1,38 @@
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from rest_framework import status
+from ..models import RawVideoFile
+from ..serializers.raw_video_meta_validation import VideoFileForMetaSerializer
+
+class VideoFileForMetaView(APIView):
+    """
+    API endpoint to fetch video metadata step-by-step.
+    If last_id is not provided  Returns the first video.
+     If last_id is given Returns the next available video.
+    """
+
+    ##need to change this fucntion , like the previous one
+
+    def get(self, request):
+        """
+        Handles:
+        First video if last_id is nt in query params.
+        Next video where id > last_id` if provided.
+        """
+        last_id = request.GET.get("last_id")  # Get last_id from query params (e.g., ?last_id=2)
+
+        try:
+            # If last_id is provided, fetch the next video where id > last_id
+            # id__gt is orm syntax which is equal to SELECT * FROM rawvideofile WHERE id > 2 ORDER BY id ASC LIMIT 1;
+
+            query_filter = {} if last_id is None else {"id__gt": int(last_id)}
+            video_entry = RawVideoFile.objects.select_related("sensitive_meta").filter(**query_filter).order_by('id').first()
+
+            if not video_entry:
+                return Response({"message": "No more videos available."}, status=status.HTTP_404_NOT_FOUND)
+
+            serializer = VideoFileForMetaSerializer(video_entry)
+            return Response(serializer.data, status=status.HTTP_200_OK)
+
+        except Exception as e:
+            return Response({"error": f"Internal server error: {str(e)}"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

endoreg_db/views/report_views.py

@@ -0,0 +1,96 @@
+from django.shortcuts import render
+from django.contrib.admin.views.decorators import staff_member_required
+
+@staff_member_required  # Ensures only staff members can access the page
+def start_examination(request):
+    return render(request, 'admin/start_examination.html')  # Loads the simple HTML page
+
+from django.shortcuts import render
+#from ..models.patient.patient_finding_location import PatientFindingLocation
+from ..models import FindingLocationClassification, FindingLocationClassificationChoice  # Correct models
+
+
+#need to implement one with json data after tesing whethe rthis works or not
+"""def get_location_choices(request, location_id):
+   
+    try:
+        # Ensure the location exists
+        location = FindingLocationClassification.objects.get(id=location_id)
+        # Get only choices related to the selected location classification
+        #location_choices = FindingLocationClassificationChoice.objects.filter(location_classification=location)
+        #its many to may relation so
+        location_choices = location.choices.all()
+        
+    except FindingLocationClassification.DoesNotExist:
+        location_choices = []
+
+    # Get previously selected values to retain them after reloading
+    selected_location = int(location_id) if location_id else None
+
+    return render(request, 'admin/patient_finding_intervention.html', {
+        "location_choices": location_choices,  # Pass updated choices to the template
+        "selected_location": location_id,  # Keep previous selection
+    })
+"""
+from django.shortcuts import render
+from rest_framework import viewsets
+from ..models import Patient
+from ..serializers import PatientSerializer
+from rest_framework.permissions import IsAuthenticatedOrReadOnly
+
+class PatientViewSet(viewsets.ModelViewSet):
+    """API endpoint for managing patients."""
+    queryset = Patient.objects.all()
+    serializer_class = PatientSerializer
+    permission_classes = [IsAuthenticatedOrReadOnly] 
+
+    def perform_create(self, serializer):
+        serializer.save()
+
+from django.http import JsonResponse
+from ..models import FindingLocationClassification, FindingLocationClassificationChoice
+
+def get_location_choices(request, location_id):
+    """
+    Fetch location choices dynamically based on the selected FindingLocationClassification (Location).
+    """
+    try:
+        location = FindingLocationClassification.objects.get(id=location_id)
+        location_choices = location.choices.all()  # Get choices via Many-to-Many relationship
+        data = [{"id": choice.id, "name": choice.name} for choice in location_choices]
+    except FindingLocationClassification.DoesNotExist:
+        data = []
+
+    return JsonResponse({"location_choices": data})
+
+from django.http import JsonResponse
+from ..models import FindingMorphologyClassification, FindingMorphologyClassificationChoice, FindingMorphologyClassificationType
+from django.core.exceptions import ObjectDoesNotExist
+
+
+
+def get_morphology_choices(request, morphology_id):
+    """
+    Fetch morphology choices dynamically based on the selected FindingMorphologyClassification.
+    """
+    try:
+        # Find the selected Morphology Classification
+        morphology_classification = FindingMorphologyClassification.objects.get(id=morphology_id)
+
+        # Fetch choices from FindingMorphologyClassificationType using classification_type_id
+        morphology_choices = FindingMorphologyClassificationType.objects.filter(
+            id=morphology_classification.classification_type_id
+        )
+
+        #  Cpnvert QuerySet to JSON
+        data = [{"id": choice.id, "name": choice.name} for choice in morphology_choices]
+
+        return JsonResponse({"morphology_choices": data})  #  Always return JSON
+
+    except ObjectDoesNotExist:
+        return JsonResponse({"error": "Morphology classification not found", "morphology_choices": []}, status=404)
+
+    except Exception as e:
+        print(f"Error fetching morphology choices: {e}")  # Debugging Log
+        return JsonResponse({"error": "Internal server error", "morphology_choices": []}, status=500)
+