endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

endoreg_db/utils/dataloader.py

@@ -1,8 +1,38 @@
 import os
+from datetime import UTC, datetime
+
 import yaml
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import OperationalError, transaction
 
+from endoreg_db.utils.paths import STORAGE_DIR
+
+_WARNING_LOG_PATH = None
+
+
+def _get_warning_log_path():
+    """Return the path used for warning logs, creating it on first access."""
+    global _WARNING_LOG_PATH
+    if _WARNING_LOG_PATH is None:
+        log_dir = STORAGE_DIR / "logs"
+        log_dir.mkdir(parents=True, exist_ok=True)
+        timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+        _WARNING_LOG_PATH = log_dir / f"dataloader_warnings_{timestamp}.log"
+    return _WARNING_LOG_PATH
+
+
+def _record_warning(command, message, verbose, context):
+    """Write a warning to stdout (when verbose) and append it to the log file."""
+    prefix = f"[{context}] " if context else ""
+    full_message = f"{prefix}{message}"
+
+    if verbose:
+        command.stdout.write(command.style.WARNING(full_message))
+
+    log_path = _get_warning_log_path()
+    with open(log_path, "a", encoding="utf-8") as log_file:
+        log_file.write(f"{datetime.now(UTC).isoformat()}Z {full_message}\n")
+
 
 def load_model_data_from_yaml(command, model_name, metadata, verbose):
     """
@@ -16,10 +46,15 @@ def load_model_data_from_yaml(command, model_name, metadata, verbose):
     """
     if verbose:
         command.stdout.write(f"Start loading {model_name}")
+
+    warning_log_path = _get_warning_log_path()
+    if verbose:
+        command.stdout.write(f"Warning log file: {warning_log_path}")
     model = metadata["model"]
     dir_path = metadata["dir"]
     foreign_keys = metadata["foreign_keys"]
     foreign_key_models = metadata["foreign_key_models"]
+    validators = metadata.get("validators", [])
 
     _files = [f for f in os.listdir(dir_path) if f.endswith(".yaml")]
     # sort
@@ -29,16 +64,30 @@ def load_model_data_from_yaml(command, model_name, metadata, verbose):
             yaml_data = yaml.safe_load(file)
 
         load_data_with_foreign_keys(
-            command, model, yaml_data, foreign_keys, foreign_key_models, verbose
+            command,
+            model,
+            yaml_data,
+            foreign_keys,
+            foreign_key_models,
+            validators,
+            verbose,
+            log_context=model_name or model.__name__,
         )
 
 
 def load_data_with_foreign_keys(
-    command, model, yaml_data, foreign_keys, foreign_key_models, verbose
+    command,
+    model,
+    yaml_data,
+    foreign_keys,
+    foreign_key_models,
+    validators,
+    verbose,
+    log_context=None,
 ):
     """
     Load YAML data into Django model instances with FK and M2M support.
-    
+
     Processes each YAML entry to create or update a model instance. For each entry, the
     function extracts field data and uses the presence of a 'name' field to decide whether
     to update an existing instance or create a new one. Foreign key fields listed in
@@ -46,34 +95,49 @@ def load_data_with_foreign_keys(
     contains a list, it is treated as a many-to-many relationship and the corresponding
     objects are set after the instance is saved. Missing or unresolved foreign keys trigger
     warnings if verbose output is enabled.
-    
+
     Parameters:
         model: The Django model class representing the data.
         yaml_data: A list of dictionaries representing YAML entries.
         foreign_keys: A list of foreign key field names to process from each entry.
         foreign_key_models: The corresponding Django model classes for each foreign key.
+        validators: A sequence of callables invoked before persisting each entry. Each
+            validator receives a shallow copy of the entry's field dictionary along with
+            the original entry and model for context.
         verbose: If True, prints detailed output and warnings during processing.
+        log_context: Label that identifies the source dataset inside the warning log.
     """
+    context_label = log_context or getattr(model, "__name__", "dataloader")
+
     for entry in yaml_data:
-        fields = entry.get("fields", {})
+        raw_fields = entry.get("fields", {})
+
+        for validator in validators:
+            validator(dict(raw_fields), entry=entry, model=model)
+
+        fields = dict(raw_fields)
         name = fields.pop("name", None)
 
+        if getattr(model, "_meta", None) and model._meta.model_name == "requirement":
+            requirement_types = fields.get("requirement_types", [])
+
+            if not requirement_types:
+                raise ValueError(f"Requirement '{name}' must define at least one requirement_types entry.")
 
         ####################
-        #TODO REMOVE AFTER TRANSLATION SUPPORT IS ADDED
-        SKIP_NAMES=[
+        # TODO REMOVE AFTER TRANSLATION SUPPORT IS ADDED
+        SKIP_NAMES = [
             "name_de",  # German name, not used
             "name_en",  # English name, not used
             "description_de",  # German description
             "description_en",  # English description
         ]
 
-        
         # Remove fields that are not needed
         for skip_name in SKIP_NAMES:
             if skip_name in fields:
                 fields.pop(skip_name)
-        ########################
+        # ########################
 
         m2m_relationships = {}  # Store many-to-many relationships
         # print(entry)
@@ -88,12 +152,12 @@ def load_data_with_foreign_keys(
 
             # Ensure the foreign key exists
             if target_keys is None:
-                if verbose:
-                    command.stdout.write(
-                        command.style.WARNING(
-                            f"Foreign key {fk_field} not found in fields"
-                        )
-                    )
+                _record_warning(
+                    command,
+                    f"Foreign key {fk_field} not found in fields",
+                    verbose,
+                    context_label,
+                )
                 continue  # Skip if no foreign key provided
 
             # Process many-to-many fields or foreign keys
@@ -103,29 +167,69 @@ def load_data_with_foreign_keys(
                     try:
                         obj = fk_model.objects.get_by_natural_key(key)
                     except ObjectDoesNotExist:
-                        if verbose:
-                            command.stdout.write(
-                                command.style.WARNING(
-                                    f"{fk_model.__name__} with key {key} not found"
-                                )
-                            )
+                        _record_warning(
+                            command,
+                            f"{fk_model.__name__} with key {key} not found",
+                            verbose,
+                            context_label,
+                        )
                         continue
                     related_objects.append(obj)
                 m2m_relationships[fk_field] = related_objects
             else:  # Single foreign key relationship
-                try:
-                    obj = fk_model.objects.get_by_natural_key(target_keys)
-                except ObjectDoesNotExist:
-                    if verbose:
-                        command.stdout.write(
-                            command.style.WARNING(
-                                f"{fk_model.__name__} with key {target_keys} not found"
-                            )
+                if model.__name__ == "ModelMeta" and fk_field == "labelset":
+                    labelset_version = fields.pop("labelset_version", None)
+
+                    if isinstance(target_keys, (tuple, list)):
+                        labelset_name = target_keys[0] if target_keys else None
+                        if len(target_keys) > 1 and labelset_version in (None, ""):
+                            labelset_version = target_keys[1]
+                    else:
+                        labelset_name = target_keys
+
+                    if not labelset_name:
+                        _record_warning(
+                            command,
+                            "LabelSet name missing for ModelMeta entry",
+                            verbose,
+                            context_label,
+                        )
+                        continue
+
+                    queryset = fk_model.objects.filter(name=labelset_name)
+                    if labelset_version not in (None, "", -1):
+                        try:
+                            version_value = int(labelset_version)
+                        except (TypeError, ValueError):
+                            version_value = labelset_version
+                        queryset = queryset.filter(version=version_value)
+
+                    obj = queryset.order_by("-version").first()
+                    if obj is None:
+                        _record_warning(
+                            command,
+                            f"LabelSet '{labelset_name}' (version={labelset_version}) not found",
+                            verbose,
+                            context_label,
                         )
-                    continue
-                fields[fk_field] = obj
+                        continue
+                    fields[fk_field] = obj
+                else:
+                    try:
+                        obj = fk_model.objects.get_by_natural_key(target_keys)
+                    except ObjectDoesNotExist:
+                        _record_warning(
+                            command,
+                            f"{fk_model.__name__} with key {target_keys} not found",
+                            verbose,
+                            context_label,
+                        )
+                        continue
+                    fields[fk_field] = obj
 
         # Create or update the main object (avoid update_or_create to prevent SQLite locks)
+        version_value = fields.get("version")
+
         def _save_instance():
             if name is None:
                 # Try to find an existing object by all provided fields
@@ -136,7 +240,11 @@ def load_data_with_foreign_keys(
                 else:
                     created = False
             else:
-                obj = model.objects.filter(name=name).first()
+                lookup_kwargs = {"name": name}
+                if model.__name__ == "LabelSet" and version_value is not None:
+                    lookup_kwargs["version"] = version_value
+
+                obj = model.objects.filter(**lookup_kwargs).first()
                 if obj is None:
                     obj = model.objects.create(name=name, **fields)
                     created = True
@@ -157,9 +265,7 @@ def load_data_with_foreign_keys(
             obj, created = _save_instance()
 
         if created and verbose:
-            command.stdout.write(
-                command.style.SUCCESS(f"Created {model.__name__} {name}")
-            )
+            command.stdout.write(command.style.SUCCESS(f"Created {model.__name__} {name}"))
         elif verbose:
             pass
 
@@ -168,8 +274,4 @@ def load_data_with_foreign_keys(
             if related_objs:  # Only set if there are objects to set
                 getattr(obj, field_name).set(related_objs)
                 if verbose:
-                    command.stdout.write(
-                        command.style.SUCCESS(
-                            f"Set {len(related_objs)} {field_name} for {model.__name__} {name}"
-                        )
-                    )
+                    command.stdout.write(command.style.SUCCESS(f"Set {len(related_objs)} {field_name} for {model.__name__} {name}"))

endoreg_db/utils/defaults/set_default_center.py

@@ -0,0 +1,32 @@
+import subprocess
+from endoreg_db.models.administration.center import center
+
+# Start process with interactive pipes
+proc = subprocess.Popen(
+    ["python3", "-i"],  # or your target program
+    stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.STDOUT,
+    text=True,
+    bufsize=1
+)
+
+proc.stdin.write("print('Trying to extract...')\n")
+proc.stdin.flush()
+
+try:
+    subprocess.run(["python", "manage.py", "load_center_data"], check=True)
+    proc.stdout.write("print('found center')")
+except subprocess.CalledProcessError as e:
+    proc.stdout.write("print('Didn't find center. Please add it to endoreg_db luxnix or via export DEFAULT_CENTER")
+# """
+# Future Implementation using dialogue
+#"""
+# # Send commands as if from terminal
+# proc.stdin.write("print('You dont have a default center set up yet. Please enter one here.')\n\nprint('Rule: use_this_format_and_connect_words_with_underscore')")
+# proc.stdin.flush()
+
+# # Read responses
+# for _ in range(3):
+#     line = proc.stdout.readline()
+#     print("Selected >>", line.strip())

endoreg_db/utils/names.py

@@ -55,22 +55,28 @@ def create_mock_patient_name(gender: str) -> tuple[str, str]:
 
 
 def guess_name_gender(name: str) -> str:
-    """
-    Guesses the gender for a given name by using a gender detector and retrieving the corresponding Gender model instance.
-    Parameters:
-        name (str): The name for which the gender is to be determined.
-    Returns:
-        Gender: The Gender object corresponding to the detected gender name.
-    Raises:
-        Gender.DoesNotExist: If no Gender object matching the detected gender is found.
-        Exception: For any other exceptions that occur during gender detection or database lookup.
-    """
+    """Return a normalized gender slug (male|female|unknown) for the given name.
 
-    from endoreg_db.models import Gender
+    Uses :mod:`gender_guesser` to infer gender without touching the database. All
+    detector outputs are mapped onto our canonical slugs so callers can perform
+    their own model lookups or fall back safely.
+    """
 
     detector = gender_detector.Detector(case_sensitive=False)
-    gender_name = detector.get_gender(name)
-    gender = Gender.objects.get(name=gender_name)
-    if not gender:
-        gender = Gender.objects.get(name="unknown")
-    return gender
+    try:
+        detected = detector.get_gender(name or "")
+    except Exception:  # pragma: no cover - defensive, detector is pure-Python
+        detected = None
+
+    if not detected:
+        return "unknown"
+
+    normalized = detected.lower()
+    if normalized in {"male", "mostly_male"}:
+        return "male"
+    if normalized in {"female", "mostly_female"}:
+        return "female"
+
+    # gender-guesser returns "andy" for androgynous names and "unknown" for
+    # unrecognised inputs – both should map to our "unknown" slug.
+    return "unknown"

endoreg_db/utils/paths.py

@@ -7,89 +7,153 @@ It provides a unified dictionary 'data_paths' for accessing all path objects.
 
 from logging import getLogger
 
-from sphinx.search import no
 logger = getLogger(__name__)
 
-from pathlib import Path
-from typing import Dict
 import os
+from pathlib import Path
+from typing import Dict, Union
+
 # Alternative approach using env_path helper, deprecated since monorepo setup. Alright for single install, env is always preferred.
-#from endoreg_db.config.env import env_path
+# from endoreg_db.config.env import env_path
 
-STORAGE_DIR = Path(os.getenv("STORAGE_DIR", "storage"))
+STORAGE_DIR = Path(os.getenv("STORAGE_DIR", "internal_storage"))
 
 storage_dir_env = os.getenv("STORAGE_DIR")
 if storage_dir_env is None:
     raise RuntimeError("STORAGE_DIR environment variable is not set.")
-storage_dir_str = Path(storage_dir_env)
-STORAGE_DIR = storage_dir_str
+storage_dir = Path(storage_dir_env)
+STORAGE_DIR = storage_dir
 if not STORAGE_DIR.exists():
     STORAGE_DIR.mkdir(parents=True, exist_ok=True)
 
+
 PREFIX_RAW = "raw_"
+io_dir_env = Path(os.getenv("IO_DIR", "data"))
+io_dir = Path(io_dir_env)
+IO_DIR = io_dir
+if not STORAGE_DIR.exists():
+    STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+# Data dropoff folders - These can be external, determined by IO_DIR (Default: set to desktop root folder of OS)
+
 IMPORT_DIR_NAME = "import"
 EXPORT_DIR_NAME = "export"
 
-VIDEO_DIR_NAME = "videos"
-ANONYM_VIDEO_DIR_NAME = "anonym_videos" # Added for processed videos
+IMPORT_DIR = IO_DIR / IMPORT_DIR_NAME      # data/import
+EXPORT_DIR = IO_DIR / EXPORT_DIR_NAME      # data/export
+
+
+IMPORT_VIDEO_DIR_NAME = "video_import"
+REPORT_IMPORT_DIR_NAME = "report_import"
+
+VIDEO_EXPORT_DIR_NAME = "video_export"
+REPORT_EXPORT_DIR_NAME = "report_export"
+
+
+IMPORT_VIDEO_DIR = IMPORT_DIR / IMPORT_VIDEO_DIR_NAME
+IMPORT_REPORT_DIR = IMPORT_DIR / REPORT_IMPORT_DIR_NAME
+
+
+VIDEO_EXPORT_DIR = EXPORT_DIR / VIDEO_EXPORT_DIR_NAME
+REPORT_EXPORT_DIR = EXPORT_DIR / REPORT_EXPORT_DIR_NAME
+
+# Document Dir
+
+DOCUMENT_DIR = STORAGE_DIR / "documents"
+
+# After initial import, files will remain here.
+
+TRANSCODING_DIR = STORAGE_DIR /"temp"
+
+SENSITIVE_VIDEO_DIR_NAME = "sensitive_videos"
+SENSITIVE_REPORT_DIR_NAME = "sensitive_reports"
+ANONYM_VIDEO_DIR_NAME = "processed_videos_final"
+ANONYM_REPORT_DIR_NAME = "processed_reports_final"
+
+RAW_FRAME_DIR_NAME = f"{PREFIX_RAW}frames"
 FRAME_DIR_NAME = "frames"
-PDF_DIR_NAME = "pdfs" # Changed from reports
 WEIGHTS_DIR_NAME = "model_weights"
 EXAMINATION_DIR_NAME = "examinations"
 
-RAW_VIDEO_DIR_NAME = f"{PREFIX_RAW}videos"
-RAW_FRAME_DIR_NAME = f"{PREFIX_RAW}frames"
-RAW_PDF_DIR_NAME = f"{PREFIX_RAW}pdfs" # Changed from reports
-
 # Define data subdirectories under STORAGE_DIR
-VIDEO_DIR = STORAGE_DIR / VIDEO_DIR_NAME
-ANONYM_VIDEO_DIR = STORAGE_DIR / ANONYM_VIDEO_DIR_NAME # Added
+ANONYM_VIDEO_DIR = STORAGE_DIR / ANONYM_VIDEO_DIR_NAME
+SENSITIVE_VIDEO_DIR = STORAGE_DIR / SENSITIVE_VIDEO_DIR_NAME
+ANONYM_REPORT_DIR = STORAGE_DIR / ANONYM_REPORT_DIR_NAME
+SENSITIVE_REPORT_DIR = STORAGE_DIR / SENSITIVE_REPORT_DIR_NAME
+
 FRAME_DIR = STORAGE_DIR / FRAME_DIR_NAME
-PDF_DIR = STORAGE_DIR / PDF_DIR_NAME # Changed
+
+
 WEIGHTS_DIR = STORAGE_DIR / WEIGHTS_DIR_NAME
-RAW_VIDEO_DIR = STORAGE_DIR / RAW_VIDEO_DIR_NAME
 RAW_FRAME_DIR = STORAGE_DIR / RAW_FRAME_DIR_NAME
-RAW_PDF_DIR = STORAGE_DIR / RAW_PDF_DIR_NAME # Changed
 
-IMPORT_DIR = STORAGE_DIR / IMPORT_DIR_NAME
-VIDEO_IMPORT_DIR = IMPORT_DIR / VIDEO_DIR_NAME
-FRAME_IMPORT_DIR = IMPORT_DIR / FRAME_DIR_NAME
-PDF_IMPORT_DIR = IMPORT_DIR / PDF_DIR_NAME # Changed
 WEIGHTS_IMPORT_DIR = IMPORT_DIR / WEIGHTS_DIR_NAME
+WEIGHTS_EXPORT_DIR = EXPORT_DIR / WEIGHTS_DIR_NAME
+
+FRAME_IMPORT_DIR = IMPORT_DIR / FRAME_DIR_NAME
 
-EXPORT_DIR = STORAGE_DIR / EXPORT_DIR_NAME
+FRAME_EXPORT_DIR = EXPORT_DIR / FRAME_DIR_NAME
 
-data_paths:Dict[str,Path] = {
+
+data_paths: Dict[str, Path] = {
     "storage": STORAGE_DIR,
-    "video": VIDEO_DIR,
-    "anonym_video": ANONYM_VIDEO_DIR, # Added
-    "frame": FRAME_DIR,
-    "pdf": PDF_DIR, # Changed
     "import": IMPORT_DIR,
-    "video_import": VIDEO_IMPORT_DIR,
-    "frame_import": FRAME_IMPORT_DIR,
-    "pdf_import": PDF_IMPORT_DIR, # Changed
-    "raw_video": RAW_VIDEO_DIR,
+    "import_video": IMPORT_VIDEO_DIR,
+    "sensitive_video": SENSITIVE_VIDEO_DIR,
+    "sensitive_report": SENSITIVE_REPORT_DIR,
+    "anonym_video": ANONYM_VIDEO_DIR,
+    "anonym_report": ANONYM_REPORT_DIR,
+    "import_frame": FRAME_IMPORT_DIR,
+    "import_report": IMPORT_REPORT_DIR,
     "raw_frame": RAW_FRAME_DIR,
-    "raw_pdf": RAW_PDF_DIR, # Changed
     "weights": WEIGHTS_DIR,
     "weights_import": WEIGHTS_IMPORT_DIR,
     "export": EXPORT_DIR,
-    "video_export": EXPORT_DIR / VIDEO_DIR_NAME,
-    "anonym_video_export": EXPORT_DIR / ANONYM_VIDEO_DIR_NAME, # Added
-    "frame_export": EXPORT_DIR / FRAME_DIR_NAME,
-    "pdf_export": EXPORT_DIR / PDF_DIR_NAME, # Changed
+    "report_export": REPORT_EXPORT_DIR,
+    "video_export": VIDEO_EXPORT_DIR,
+    "frame_export": FRAME_EXPORT_DIR,
     "weights_export": EXPORT_DIR / WEIGHTS_DIR_NAME,
-    "examination_export": EXPORT_DIR / EXAMINATION_DIR_NAME,
-    "raw_video_export": EXPORT_DIR / RAW_VIDEO_DIR_NAME,
-    "raw_frame_export": EXPORT_DIR / RAW_FRAME_DIR_NAME,
-    "raw_pdf_export": EXPORT_DIR / RAW_PDF_DIR_NAME, # Changed
+    "transcoding": TRANSCODING_DIR,
+    "frame": FRAME_DIR,
+    "documents": DOCUMENT_DIR
 }
 
 logger.info(f"Storage directory: {STORAGE_DIR.resolve()}")
 logger.info(f"Export directory: {EXPORT_DIR.resolve()}")
 
-# for key, path in data_paths.items():
-#     path.mkdir(parents=True, exist_ok=True)
-
-#     logger.info(f"{key.capitalize()} directory: {path.resolve()}")
+for key, path in data_paths.items():
+    path.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"{key.capitalize()} directory: {path.resolve()}")
+    
+
+def to_storage_relative(path: Union[str, Path]) -> str:
+    """
+    Return a path string relative to STORAGE_DIR, suitable for Django FileField.name.
+
+    - If `path` is inside STORAGE_DIR (absolute or contains STORAGE_DIR as prefix),
+      we strip the STORAGE_DIR prefix and return the relative part.
+    - If `path` is outside STORAGE_DIR, we return it as a string unchanged
+      (caller can decide what to do).
+    """
+    # Normalize input to Path
+    p = Path(path)
+
+    # Resolve absolute path for comparison
+    storage_root = STORAGE_DIR.resolve()
+
+    if not p.is_absolute():
+        # Resolve relative path against current working directory
+        # (in tests, cwd should be project root, so this still lands under STORAGE_DIR)
+        p = p.resolve()
+
+    try:
+        rel = p.relative_to(storage_root)
+    except ValueError:
+        # Not under STORAGE_DIR – probably already a relative name or external.
+        # In that case, just return the string representation as-is.
+        return str(path)
+
+    # Use POSIX-style separators for Django FileField
+    return rel.as_posix()

endoreg_db/utils/permissions.py

@@ -7,7 +7,7 @@ the DEBUG setting and other environment configurations.
 
 from django.conf import settings
 from rest_framework.permissions import IsAuthenticated, AllowAny, BasePermission
-from rest_framework.decorators import permission_classes as drf_permission_classes
+#from rest_framework.decorators import permission_classes as drf_permission_classes
 from functools import wraps
 import logging
 import os
@@ -50,6 +50,7 @@ def dynamic_permission_classes(force_auth=None):
             - None: Use environment-based logic (default)
     """
     def decorator(view_func):
+        from rest_framework.decorators import permission_classes as drf_permission_classes
         @wraps(view_func)
         def wrapper(*args, **kwargs):
             return view_func(*args, **kwargs)

endoreg_db/utils/pipelines/Readme.md

@@ -154,7 +154,7 @@ At this point in the pipeline:
   `VideoFile.processed_file`
 - Video hash is calculated and saved in:  
   `VideoFile.processed_video_hash`
-- `VideoState.anonymized = True`
+- `VideoState.state.anonymization_status.mark_anonymized()`
 
 ---
 

endoreg_db/utils/pipelines/process_video_dir.py

@@ -20,7 +20,7 @@ DEFAULT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "data/study
 CENTER_NAME = os.environ.get("CENTER_NAME", "university_hospital_wuerzburg")
 ENDOSCOPY_PROCESSOR_NAME = os.environ.get("ENDOSCOPY_PROCESSOR_NAME", "olympus_cv_1500")
 MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
-MODEL_WEIGHTS_PATH = "./tests/assets/colo_segmentation_RegNetX800MF_6.ckpt"
+MODEL_WEIGHTS_PATH = "./tests/assets/colo_segmentation_RegNetX800MF_6.safetensors"
 ic(DEFAULT_DIR)