endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/utils/calc_duration_seconds.py

@@ -0,0 +1,24 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from endoreg_db.models import VideoFile
+
+
+def _calc_duration_vf(obj: "VideoFile") -> float:
+    """
+    Calculate duration of a VideoFile.
+    """
+    if not obj.ffmpeg_meta:
+        raise ValueError("ffmpeg_meta is missing, cannot calculate duration.")
+
+    fps = obj.get_fps()
+    frame_count = obj.frame_count  # TODO similar implementation as in get_fps
+
+    duration = frame_count / fps if fps > 0 else -1
+    if duration > 0:
+        return duration
+    else:
+        raise ValueError(
+            f"Invalid duration calculated for video {obj.video_hash}: {duration}. "
+            "Ensure the video file is valid and accessible."
+        )

endoreg_db/utils/case_generator/__init__.py

@@ -0,0 +1,3 @@
+"""Legacy case generator utilities based on deprecated case templates."""
+
+__all__: list[str] = []

endoreg_db/utils/case_generator/lab_sample_factory.py

@@ -0,0 +1,32 @@
+from datetime import datetime, timezone
+from endoreg_db.models import Patient, PatientLabSample, PatientLabSampleType
+
+
+class LabSampleFactory:
+    """
+    Provides methods to generate lab samples.
+    """
+
+    def __init__(self):
+        """
+        Initializes the LabSampleFactory.
+        """
+        pass
+
+    def create_generic_lab_sample(self, patient: Patient):
+        """
+        Generates a generic lab sample for a given patient.
+
+        Args:
+            patient (Patient): The patient for whom the lab sample is generated.
+
+        Returns:
+            PatientLabSample: The created lab sample instance.
+        """
+        sample_type = PatientLabSampleType.objects.get(name="generic")
+
+        lab_sample = PatientLabSample.objects.create(
+            patient=patient, sample_type=sample_type, date=datetime.now(tz=timezone.utc)
+        )
+
+        return lab_sample

endoreg_db/utils/check_video_files.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+Simple video file existence checker and path corrector for VideoFile records.
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+# Parse command-line arguments and environment variables for configuration
+parser = argparse.ArgumentParser(
+    description="Simple video file existence checker and path corrector for VideoFile records."
+)
+parser.add_argument(
+    "--django-base",
+    type=str,
+    default=os.environ.get(
+        "ENDOREG_DJANGO_PROJECT_PATH",
+        str(Path(__file__).resolve().parent.parent.parent),
+    ),
+    help="Path to the Django project base (default: env ENDOREG_DJANGO_PROJECT_PATH or project root)",
+)
+parser.add_argument(
+    "--django-settings",
+    type=str,
+    default=os.environ.get("DJANGO_SETTINGS_MODULE", "dev.dev_settings"),
+    help="Django settings module (default: env DJANGO_SETTINGS_MODULE or dev.dev_settings)",
+)
+parser.add_argument(
+    "--storage-dir",
+    type=str,
+    default=os.environ.get("ENDOREG_STORAGE_DIR", "./storage"),
+    help="Path to the storage directory (default: ./storage or $ENDOREG_STORAGE_DIR)",
+)
+args, unknown = parser.parse_known_args()
+
+sys.path.insert(0, args.django_base)
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.django_settings)
+
+try:
+    import django
+
+    django.setup()
+    from endoreg_db.models import VideoFile
+
+    DJANGO_AVAILABLE = True
+except Exception as e:
+    print(f"Django not available: {e}")
+    DJANGO_AVAILABLE = False
+
+
+def find_video_files():
+    """Find all video files in storage directory."""
+    storage_dir = Path(args.storage_dir)
+    video_files = []
+
+    for pattern in ["**/*.mp4", "**/*.avi", "**/*.mov", "**/*.mkv"]:
+        video_files.extend(storage_dir.glob(pattern))
+
+    return video_files
+
+
+def check_video_file_accessibility(file_path):
+    """Check if a video file is accessible and valid."""
+    try:
+        if not file_path.exists():
+            return False, "File does not exist"
+
+        if file_path.stat().st_size == 0:
+            return False, "File is empty (0 bytes)"
+
+        if not os.access(file_path, os.R_OK):
+            return False, "File is not readable"
+
+        # Try to read first few bytes to check if it's actually a file
+        with open(file_path, "rb") as f:
+            header = f.read(8)
+            if len(header) < 8:
+                return False, "File too small or corrupted"
+
+        return True, f"OK - {file_path.stat().st_size / (1024 * 1024):.1f} MB"
+
+    except Exception as e:
+        return False, f"Error checking file: {e}"
+
+
+def main():
+    print("🔍 VIDEO FILE EXISTENCE CHECKER")
+    print("=" * 40)
+
+    # Find all video files
+    print("1. Scanning for video files...")
+    video_files = find_video_files()
+    print(f"Found {len(video_files)} video files in storage directory")
+
+    if not video_files:
+        print("❌ No video files found in storage directory!")
+        return
+
+    # Check each file
+    print("\n2. Checking file accessibility...")
+    accessible_files = []
+
+    for video_file in video_files[:10]:  # Check first 10
+        accessible, message = check_video_file_accessibility(video_file)
+        status = "✅" if accessible else "❌"
+        print(f"{status} {video_file.name}: {message}")
+
+        if accessible:
+            accessible_files.append(video_file)
+
+    if not accessible_files:
+        print("\n❌ No accessible video files found!")
+        return
+
+    print(f"\n✅ Found {len(accessible_files)} accessible video files")
+
+    # If Django is available, check database records
+    if DJANGO_AVAILABLE:
+        print("\n3. Checking database records...")
+        try:
+            video_5 = VideoFile.objects.get(pk=5)
+            print("📋 Video ID 5 found in database:")
+            print(f"   UUID: {video_5.video_hash}")
+
+            # Check different file path attributes
+            for attr in ["raw_file", "processed_file"]:
+                if hasattr(video_5, attr):
+                    file_field = getattr(video_5, attr)
+                    if file_field:
+                        try:
+                            file_path = Path(file_field.path)
+                            accessible, message = check_video_file_accessibility(
+                                file_path
+                            )
+                            status = "✅" if accessible else "❌"
+                            print(f"   {attr}: {status} {file_path} ({message})")
+                        except Exception as e:
+                            print(f"   {attr}: ❌ Error accessing path: {e}")
+                    else:
+                        print(f"   {attr}: ❌ No file set")
+
+            # Check if UUID matches any found files
+            uuid_str = str(video_5.video_hash)
+            matching_files = [f for f in accessible_files if uuid_str in str(f)]
+
+            if matching_files:
+                print(f"\n💡 Found matching files for UUID {uuid_str}:")
+                for match in matching_files:
+                    accessible, message = check_video_file_accessibility(match)
+                    print(f"   ✅ {match} ({message})")
+
+                print("\n🔧 SOLUTION: Update VideoFile record to use:")
+                print(f"   {matching_files[0]}")
+                print("\n🐍 Django command to fix:")
+                print("   video = VideoFile.objects.get(pk=5)")
+                print(
+                    f"   video.raw_file.name = '{matching_files[0].relative_to(Path(args.storage_dir))}'"
+                )
+                print("   video.save()")
+            else:
+                print(f"\n❌ No files found matching UUID {uuid_str}")
+
+        except Exception as e:
+            print(f"❌ Error checking database: {e}")
+
+    print("\n4. 🎯 QUICK TEST RECOMMENDATION:")
+    print("   Use this accessible file for testing:")
+    print(f"   {accessible_files[0]}")
+    print(f"   Size: {accessible_files[0].stat().st_size / (1024 * 1024):.1f} MB")
+
+
+if __name__ == "__main__":
+    main()

endoreg_db/utils/cropping.py

@@ -0,0 +1,30 @@
+from PIL import Image
+
+
+def crop_and_insert(image: Image, x, y, h, w, bg_color=(255, 255, 255)):
+    """
+    Crops a region from an inverted grayscale image and inserts it into a white image of the same size as the original.
+
+    Parameters:
+    - fp: File path or a file object of the original image.
+    - x, y: The top-left coordinates of the rectangle to be cropped.
+    - h, w: The height and width of the rectangle to be cropped.
+
+    Returns:
+    A PIL Image object containing the original image with the specified region replaced.
+    """
+    # Load the original image
+    original_image = image
+
+    # Crop the specified region from the inverted image
+    crop_rectangle = (x, y, x + w, y + h)
+    cropped_content = original_image.crop(crop_rectangle)
+
+    # Create a new white image of the same size as the original image
+    white_background = Image.new("RGB", original_image.size, bg_color)
+
+    # Paste the cropped content onto the white image at the specified location
+    white_background.paste(cropped_content, (x, y))
+
+    # The final image can be displayed or saved as needed
+    return white_background

endoreg_db/utils/dataloader.py

@@ -0,0 +1,285 @@
+import os
+from datetime import UTC, datetime
+
+import yaml
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import OperationalError, transaction
+
+from endoreg_db.utils.paths import STORAGE_DIR
+
+_WARNING_LOG_PATH = None
+
+
+def _get_warning_log_path():
+    """Return the path used for warning logs, creating it on first access."""
+    global _WARNING_LOG_PATH
+    if _WARNING_LOG_PATH is None:
+        log_dir = STORAGE_DIR / "logs"
+        log_dir.mkdir(parents=True, exist_ok=True)
+        timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+        _WARNING_LOG_PATH = log_dir / f"dataloader_warnings_{timestamp}.log"
+    return _WARNING_LOG_PATH
+
+
+def _record_warning(command, message, verbose, context):
+    """Write a warning to stdout (when verbose) and append it to the log file."""
+    prefix = f"[{context}] " if context else ""
+    full_message = f"{prefix}{message}"
+
+    if verbose:
+        command.stdout.write(command.style.WARNING(full_message))
+
+    log_path = _get_warning_log_path()
+    with open(log_path, "a", encoding="utf-8") as log_file:
+        log_file.write(f"{datetime.now(UTC).isoformat()}Z {full_message}\n")
+
+
+def load_model_data_from_yaml(command, model_name, metadata, verbose):
+    """
+    Load model data from YAML files.
+
+    Args:
+        command: Command object for stdout writing.
+        model_name: Name of the model being loaded.
+        metadata: Metadata including directory and foreign key information.
+        verbose: Boolean indicating whether to print verbose output.
+    """
+    if verbose:
+        command.stdout.write(f"Start loading {model_name}")
+
+    warning_log_path = _get_warning_log_path()
+    if verbose:
+        command.stdout.write(f"Warning log file: {warning_log_path}")
+    model = metadata["model"]
+    dir_path = metadata["dir"]
+    foreign_keys = metadata["foreign_keys"]
+    foreign_key_models = metadata["foreign_key_models"]
+    validators = metadata.get("validators", [])
+
+    _files = [f for f in os.listdir(dir_path) if f.endswith(".yaml")]
+    # sort
+    _files.sort()
+    for file in _files:
+        with open(os.path.join(dir_path, file), "r", encoding="utf-8") as file:
+            yaml_data = yaml.safe_load(file)
+
+        load_data_with_foreign_keys(
+            command,
+            model,
+            yaml_data,
+            foreign_keys,
+            foreign_key_models,
+            validators,
+            verbose,
+            log_context=model_name or model.__name__,
+        )
+
+
+def load_data_with_foreign_keys(
+    command,
+    model,
+    yaml_data,
+    foreign_keys,
+    foreign_key_models,
+    validators,
+    verbose,
+    log_context=None,
+):
+    """
+    Load YAML data into Django model instances with FK and M2M support.
+
+    Processes each YAML entry to create or update a model instance. For each entry, the
+    function extracts field data and uses the presence of a 'name' field to decide whether
+    to update an existing instance or create a new one. Foreign key fields listed in
+    foreign_keys are handled by retrieving related objects via natural keys. When a field
+    contains a list, it is treated as a many-to-many relationship and the corresponding
+    objects are set after the instance is saved. Missing or unresolved foreign keys trigger
+    warnings if verbose output is enabled.
+
+    Parameters:
+        model: The Django model class representing the data.
+        yaml_data: A list of dictionaries representing YAML entries.
+        foreign_keys: A list of foreign key field names to process from each entry.
+        foreign_key_models: The corresponding Django model classes for each foreign key.
+        validators: A sequence of callables invoked before persisting each entry. Each
+            validator receives a shallow copy of the entry's field dictionary along with
+            the original entry and model for context.
+        verbose: If True, prints detailed output and warnings during processing.
+        log_context: Label that identifies the source dataset inside the warning log.
+    """
+    context_label = log_context or getattr(model, "__name__", "dataloader")
+
+    for entry in yaml_data:
+        raw_fields = entry.get("fields", {})
+
+        for validator in validators:
+            validator(dict(raw_fields), entry=entry, model=model)
+
+        fields = dict(raw_fields)
+        name = fields.pop("name", None)
+
+        if getattr(model, "_meta", None) and model._meta.model_name == "requirement":
+            requirement_types = fields.get("requirement_types", [])
+
+            if not requirement_types:
+                raise ValueError(
+                    f"Requirement '{name}' must define at least one requirement_types entry."
+                )
+
+        ####################
+        # TODO REMOVE AFTER TRANSLATION SUPPORT IS ADDED
+        SKIP_NAMES = [
+            "name_de",  # German name, not used
+            "name_en",  # English name, not used
+            "description_de",  # German description
+            "description_en",  # English description
+        ]
+
+        # Remove fields that are not needed
+        for skip_name in SKIP_NAMES:
+            if skip_name in fields:
+                fields.pop(skip_name)
+        # ########################
+
+        m2m_relationships = {}  # Store many-to-many relationships
+        # print(entry)
+
+        # Handle foreign keys and many-to-many relationships
+        for fk_field, fk_model in zip(foreign_keys, foreign_key_models):
+            # Skip fields that are not in the data
+            if fk_field not in fields:
+                continue
+
+            target_keys = fields.pop(fk_field, None)
+
+            # Ensure the foreign key exists
+            if target_keys is None:
+                _record_warning(
+                    command,
+                    f"Foreign key {fk_field} not found in fields",
+                    verbose,
+                    context_label,
+                )
+                continue  # Skip if no foreign key provided
+
+            # Process many-to-many fields or foreign keys
+            if isinstance(target_keys, list):  # Assume many-to-many relationship
+                related_objects = []
+                for key in target_keys:
+                    try:
+                        obj = fk_model.objects.get_by_natural_key(key)
+                    except ObjectDoesNotExist:
+                        _record_warning(
+                            command,
+                            f"{fk_model.__name__} with key {key} not found",
+                            verbose,
+                            context_label,
+                        )
+                        continue
+                    related_objects.append(obj)
+                m2m_relationships[fk_field] = related_objects
+            else:  # Single foreign key relationship
+                if model.__name__ == "ModelMeta" and fk_field == "labelset":
+                    labelset_version = fields.pop("labelset_version", None)
+
+                    if isinstance(target_keys, (tuple, list)):
+                        labelset_name = target_keys[0] if target_keys else None
+                        if len(target_keys) > 1 and labelset_version in (None, ""):
+                            labelset_version = target_keys[1]
+                    else:
+                        labelset_name = target_keys
+
+                    if not labelset_name:
+                        _record_warning(
+                            command,
+                            "LabelSet name missing for ModelMeta entry",
+                            verbose,
+                            context_label,
+                        )
+                        continue
+
+                    queryset = fk_model.objects.filter(name=labelset_name)
+                    if labelset_version not in (None, "", -1):
+                        try:
+                            version_value = int(labelset_version)
+                        except (TypeError, ValueError):
+                            version_value = labelset_version
+                        queryset = queryset.filter(version=version_value)
+
+                    obj = queryset.order_by("-version").first()
+                    if obj is None:
+                        _record_warning(
+                            command,
+                            f"LabelSet '{labelset_name}' (version={labelset_version}) not found",
+                            verbose,
+                            context_label,
+                        )
+                        continue
+                    fields[fk_field] = obj
+                else:
+                    try:
+                        obj = fk_model.objects.get_by_natural_key(target_keys)
+                    except ObjectDoesNotExist:
+                        _record_warning(
+                            command,
+                            f"{fk_model.__name__} with key {target_keys} not found",
+                            verbose,
+                            context_label,
+                        )
+                        continue
+                    fields[fk_field] = obj
+
+        # Create or update the main object (avoid update_or_create to prevent SQLite locks)
+        version_value = fields.get("version")
+
+        def _save_instance():
+            if name is None:
+                # Try to find an existing object by all provided fields
+                obj = model.objects.filter(**fields).first()
+                if obj is None:
+                    obj = model.objects.create(**fields)
+                    created = True
+                else:
+                    created = False
+            else:
+                lookup_kwargs = {"name": name}
+                if model.__name__ == "LabelSet" and version_value is not None:
+                    lookup_kwargs["version"] = version_value
+
+                obj = model.objects.filter(**lookup_kwargs).first()
+                if obj is None:
+                    obj = model.objects.create(name=name, **fields)
+                    created = True
+                else:
+                    # Update fields
+                    for k, v in fields.items():
+                        setattr(obj, k, v)
+                    obj.save()
+                    created = False
+            return obj, created
+
+        try:
+            # Attempt save inside a transaction for consistency
+            with transaction.atomic():
+                obj, created = _save_instance()
+        except OperationalError:
+            # Retry once on SQLite lock
+            obj, created = _save_instance()
+
+        if created and verbose:
+            command.stdout.write(
+                command.style.SUCCESS(f"Created {model.__name__} {name}")
+            )
+        elif verbose:
+            pass
+
+        # Set many-to-many relationships
+        for field_name, related_objs in m2m_relationships.items():
+            if related_objs:  # Only set if there are objects to set
+                getattr(obj, field_name).set(related_objs)
+                if verbose:
+                    command.stdout.write(
+                        command.style.SUCCESS(
+                            f"Set {len(related_objs)} {field_name} for {model.__name__} {name}"
+                        )
+                    )

endoreg_db/utils/dates.py

@@ -0,0 +1,59 @@
+from datetime import datetime, date, timedelta
+from random import randint
+from calendar import monthrange
+from django.utils import timezone
+
+# TODO replace used random_day_by_year function implementation when
+# creating pseudo patients with new function "random date by age_at_date and examination_date"
+
+
+def random_day_by_age_at_date(age_at_date: int, examination_date: date) -> date:
+    """
+    Return a random birth day for a patient with the specified age at the specified examination date.
+    """
+    examination_year = examination_date.year
+    latest_birthdate = examination_date.replace(year=examination_year - age_at_date)
+    valid_dates = [latest_birthdate - timedelta(days=i) for i in range(365)]
+
+    select = randint(0, len(valid_dates) - 1)
+    birth_date = valid_dates[select]
+
+    return birth_date
+
+
+def random_day_by_year(year: int) -> date:
+    """
+    Return a random birth day within the specified year.
+    """
+    month = randint(1, 12)
+    day = randint(1, monthrange(year, month)[1])
+
+    return date(year, month, day)
+
+
+def random_day_by_month_year(month: int, year) -> date:
+    """
+    Return a random birth day within the specified month and year.
+    """
+
+    day = randint(1, monthrange(year, month)[1])
+    return date(year, month, day)
+
+
+def ensure_aware_datetime(dt):
+    """
+    Ensures a datetime object is timezone-aware.
+    If the datetime is naive (has no timezone info), the current timezone is applied.
+
+    Args:
+        dt: A datetime object that may be naive
+
+    Returns:
+        A timezone-aware datetime object
+    """
+    if dt is None:
+        return None
+
+    if isinstance(dt, datetime.datetime) and timezone.is_naive(dt):
+        return timezone.make_aware(dt)
+    return dt

endoreg_db/utils/defaults/set_default_center.py

@@ -0,0 +1,33 @@
+import subprocess
+
+# Start process with interactive pipes
+proc = subprocess.Popen(
+    ["python3", "-i"],  # or your target program
+    stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT,
+    text=True,
+    bufsize=1,
+)
+
+proc.stdin.write("print('Trying to extract...')\n")
+proc.stdin.flush()
+
+try:
+    subprocess.run(["python", "manage.py", "load_center_data"], check=True)
+    proc.stdout.write("print('found center')")
+except subprocess.CalledProcessError:
+    proc.stdout.write(
+        "print('Didn't find center. Please add it to endoreg_db luxnix or via export DEFAULT_CENTER"
+    )
+# """
+# Future Implementation using dialogue
+# """
+# # Send commands as if from terminal
+# proc.stdin.write("print('You dont have a default center set up yet. Please enter one here.')\n\nprint('Rule: use_this_format_and_connect_words_with_underscore')")
+# proc.stdin.flush()
+
+# # Read responses
+# for _ in range(3):
+#     line = proc.stdout.readline()
+#     print("Selected >>", line.strip())

endoreg_db/utils/env.py

@@ -0,0 +1,37 @@
+import os
+
+DEBUG = os.getenv("DEBUG", "false").lower() == "true"
+
+
+def get_env_var(var_name: str, default: str = None) -> str | None:
+    """
+    Get the value of an environment variable, with an optional default value.
+    If the environment variable is set, we need to remove flanking quotation marks and spaces.
+    if the environment variable is not set, we set it to the default value.
+    :param var_name: The name of the environment variable.
+    :param default: The default value to return if the environment variable is not set.
+    :return: The value of the environment variable or the default value.
+    """
+    value = os.environ.get(var_name)
+    if value:
+        value = value.strip("\"'")  # Strip both single and double quotes
+        if DEBUG:
+            print(f"Environment variable {var_name}: {value}")
+        return value
+    return default
+
+
+def set_env_var(var_name: str, value: str) -> None:
+    """
+    Set the value of an environment variable.
+    :param var_name: The name of the environment variable.
+    :param value: The value to set.
+    """
+    os.environ[var_name] = value
+    if DEBUG:
+        print(f"Set environment variable {var_name}: {value}")
+
+
+DJANGO_SETTINGS_MODULE = (
+    get_env_var("DJANGO_SETTINGS_MODULE") or "endoreg_db.settings_dev"
+)