endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.9.2__py3-none-any.whl

endoreg_db/utils/dataloader.py

@@ -1,8 +1,39 @@
 import os
+from datetime import UTC, datetime
+
 import yaml
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import OperationalError, transaction
 
+from endoreg_db.utils.paths import STORAGE_DIR
+
+_WARNING_LOG_PATH = None
+
+
+def _get_warning_log_path():
+    """Return the path used for warning logs, creating it on first access."""
+    global _WARNING_LOG_PATH
+    if _WARNING_LOG_PATH is None:
+        log_dir = STORAGE_DIR / "logs"
+        log_dir.mkdir(parents=True, exist_ok=True)
+        timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+        _WARNING_LOG_PATH = log_dir / f"dataloader_warnings_{timestamp}.log"
+    return _WARNING_LOG_PATH
+
+
+def _record_warning(command, message, verbose, context):
+    """Write a warning to stdout (when verbose) and append it to the log file."""
+    prefix = f"[{context}] " if context else ""
+    full_message = f"{prefix}{message}"
+
+    if verbose:
+        command.stdout.write(command.style.WARNING(full_message))
+
+    log_path = _get_warning_log_path()
+    with open(log_path, "a", encoding="utf-8") as log_file:
+        log_file.write(f"{datetime.now(UTC).isoformat()}Z {full_message}\n")
+
+
 def load_model_data_from_yaml(command, model_name, metadata, verbose):
     """
     Load model data from YAML files.
@@ -15,6 +46,10 @@ def load_model_data_from_yaml(command, model_name, metadata, verbose):
     """
     if verbose:
         command.stdout.write(f"Start loading {model_name}")
+
+    warning_log_path = _get_warning_log_path()
+    if verbose:
+        command.stdout.write(f"Warning log file: {warning_log_path}")
     model = metadata["model"]
     dir_path = metadata["dir"]
     foreign_keys = metadata["foreign_keys"]
@@ -29,7 +64,14 @@ def load_model_data_from_yaml(command, model_name, metadata, verbose):
             yaml_data = yaml.safe_load(file)
 
         load_data_with_foreign_keys(
-            command, model, yaml_data, foreign_keys, foreign_key_models, validators, verbose
+            command,
+            model,
+            yaml_data,
+            foreign_keys,
+            foreign_key_models,
+            validators,
+            verbose,
+            log_context=model_name or model.__name__,
         )
 
 
@@ -41,10 +83,11 @@ def load_data_with_foreign_keys(
     foreign_key_models,
     validators,
     verbose,
+    log_context=None,
 ):
     """
     Load YAML data into Django model instances with FK and M2M support.
-    
+
     Processes each YAML entry to create or update a model instance. For each entry, the
     function extracts field data and uses the presence of a 'name' field to decide whether
     to update an existing instance or create a new one. Foreign key fields listed in
@@ -52,7 +95,7 @@ def load_data_with_foreign_keys(
     contains a list, it is treated as a many-to-many relationship and the corresponding
     objects are set after the instance is saved. Missing or unresolved foreign keys trigger
     warnings if verbose output is enabled.
-    
+
     Parameters:
         model: The Django model class representing the data.
         yaml_data: A list of dictionaries representing YAML entries.
@@ -62,7 +105,10 @@ def load_data_with_foreign_keys(
             validator receives a shallow copy of the entry's field dictionary along with
             the original entry and model for context.
         verbose: If True, prints detailed output and warnings during processing.
+        log_context: Label that identifies the source dataset inside the warning log.
     """
+    context_label = log_context or getattr(model, "__name__", "dataloader")
+
     for entry in yaml_data:
         raw_fields = entry.get("fields", {})
 
@@ -72,30 +118,21 @@ def load_data_with_foreign_keys(
         fields = dict(raw_fields)
         name = fields.pop("name", None)
 
-
         if getattr(model, "_meta", None) and model._meta.model_name == "requirement":
             requirement_types = fields.get("requirement_types", [])
-            operators = fields.get("operators", [])
-        
+
             if not requirement_types:
-                raise ValueError(
-                    f"Requirement '{name}' must define at least one requirement_types entry."
-                )
-            if not operators:
-                raise ValueError(
-                    f"Requirement '{name}' must define at least one operators entry."
-                )
-                      
+                raise ValueError(f"Requirement '{name}' must define at least one requirement_types entry.")
+
         ####################
-        #TODO REMOVE AFTER TRANSLATION SUPPORT IS ADDED
-        SKIP_NAMES=[
+        # TODO REMOVE AFTER TRANSLATION SUPPORT IS ADDED
+        SKIP_NAMES = [
             "name_de",  # German name, not used
             "name_en",  # English name, not used
             "description_de",  # German description
             "description_en",  # English description
         ]
 
-        
         # Remove fields that are not needed
         for skip_name in SKIP_NAMES:
             if skip_name in fields:
@@ -115,12 +152,12 @@ def load_data_with_foreign_keys(
 
             # Ensure the foreign key exists
             if target_keys is None:
-                if verbose:
-                    command.stdout.write(
-                        command.style.WARNING(
-                            f"Foreign key {fk_field} not found in fields"
-                        )
-                    )
+                _record_warning(
+                    command,
+                    f"Foreign key {fk_field} not found in fields",
+                    verbose,
+                    context_label,
+                )
                 continue  # Skip if no foreign key provided
 
             # Process many-to-many fields or foreign keys
@@ -130,20 +167,17 @@ def load_data_with_foreign_keys(
                     try:
                         obj = fk_model.objects.get_by_natural_key(key)
                     except ObjectDoesNotExist:
-                        if verbose:
-                            command.stdout.write(
-                                command.style.WARNING(
-                                    f"{fk_model.__name__} with key {key} not found"
-                                )
-                            )
+                        _record_warning(
+                            command,
+                            f"{fk_model.__name__} with key {key} not found",
+                            verbose,
+                            context_label,
+                        )
                         continue
                     related_objects.append(obj)
                 m2m_relationships[fk_field] = related_objects
             else:  # Single foreign key relationship
-                if (
-                    model.__name__ == "ModelMeta"
-                    and fk_field == "labelset"
-                ):
+                if model.__name__ == "ModelMeta" and fk_field == "labelset":
                     labelset_version = fields.pop("labelset_version", None)
 
                     if isinstance(target_keys, (tuple, list)):
@@ -154,10 +188,12 @@ def load_data_with_foreign_keys(
                         labelset_name = target_keys
 
                     if not labelset_name:
-                        if verbose:
-                            command.stdout.write(
-                                command.style.WARNING("LabelSet name missing for ModelMeta entry")
-                            )
+                        _record_warning(
+                            command,
+                            "LabelSet name missing for ModelMeta entry",
+                            verbose,
+                            context_label,
+                        )
                         continue
 
                     queryset = fk_model.objects.filter(name=labelset_name)
@@ -170,24 +206,24 @@ def load_data_with_foreign_keys(
 
                     obj = queryset.order_by("-version").first()
                     if obj is None:
-                        if verbose:
-                            command.stdout.write(
-                                command.style.WARNING(
-                                    f"LabelSet '{labelset_name}' (version={labelset_version}) not found"
-                                )
-                            )
+                        _record_warning(
+                            command,
+                            f"LabelSet '{labelset_name}' (version={labelset_version}) not found",
+                            verbose,
+                            context_label,
+                        )
                         continue
                     fields[fk_field] = obj
                 else:
                     try:
                         obj = fk_model.objects.get_by_natural_key(target_keys)
                     except ObjectDoesNotExist:
-                        if verbose:
-                            command.stdout.write(
-                                command.style.WARNING(
-                                    f"{fk_model.__name__} with key {target_keys} not found"
-                                )
-                            )
+                        _record_warning(
+                            command,
+                            f"{fk_model.__name__} with key {target_keys} not found",
+                            verbose,
+                            context_label,
+                        )
                         continue
                     fields[fk_field] = obj
 
@@ -229,9 +265,7 @@ def load_data_with_foreign_keys(
             obj, created = _save_instance()
 
         if created and verbose:
-            command.stdout.write(
-                command.style.SUCCESS(f"Created {model.__name__} {name}")
-            )
+            command.stdout.write(command.style.SUCCESS(f"Created {model.__name__} {name}"))
         elif verbose:
             pass
 
@@ -240,8 +274,4 @@ def load_data_with_foreign_keys(
             if related_objs:  # Only set if there are objects to set
                 getattr(obj, field_name).set(related_objs)
                 if verbose:
-                    command.stdout.write(
-                        command.style.SUCCESS(
-                            f"Set {len(related_objs)} {field_name} for {model.__name__} {name}"
-                        )
-                    )
+                    command.stdout.write(command.style.SUCCESS(f"Set {len(related_objs)} {field_name} for {model.__name__} {name}"))

endoreg_db/utils/paths.py

@@ -7,89 +7,153 @@ It provides a unified dictionary 'data_paths' for accessing all path objects.
 
 from logging import getLogger
 
-from sphinx.search import no
 logger = getLogger(__name__)
 
-from pathlib import Path
-from typing import Dict
 import os
+from pathlib import Path
+from typing import Dict, Union
+
 # Alternative approach using env_path helper, deprecated since monorepo setup. Alright for single install, env is always preferred.
-#from endoreg_db.config.env import env_path
+# from endoreg_db.config.env import env_path
 
-STORAGE_DIR = Path(os.getenv("STORAGE_DIR", "storage"))
+STORAGE_DIR = Path(os.getenv("STORAGE_DIR", "internal_storage"))
 
 storage_dir_env = os.getenv("STORAGE_DIR")
 if storage_dir_env is None:
     raise RuntimeError("STORAGE_DIR environment variable is not set.")
-storage_dir_str = Path(storage_dir_env)
-STORAGE_DIR = storage_dir_str
+storage_dir = Path(storage_dir_env)
+STORAGE_DIR = storage_dir
 if not STORAGE_DIR.exists():
     STORAGE_DIR.mkdir(parents=True, exist_ok=True)
 
+
 PREFIX_RAW = "raw_"
+io_dir_env = Path(os.getenv("IO_DIR", "data"))
+io_dir = Path(io_dir_env)
+IO_DIR = io_dir
+if not STORAGE_DIR.exists():
+    STORAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+# Data dropoff folders - These can be external, determined by IO_DIR (Default: set to desktop root folder of OS)
+
 IMPORT_DIR_NAME = "import"
 EXPORT_DIR_NAME = "export"
 
-VIDEO_DIR_NAME = "videos"
-ANONYM_VIDEO_DIR_NAME = "anonym_videos" # Added for processed videos
+IMPORT_DIR = IO_DIR / IMPORT_DIR_NAME      # data/import
+EXPORT_DIR = IO_DIR / EXPORT_DIR_NAME      # data/export
+
+
+IMPORT_VIDEO_DIR_NAME = "video_import"
+REPORT_IMPORT_DIR_NAME = "report_import"
+
+VIDEO_EXPORT_DIR_NAME = "video_export"
+REPORT_EXPORT_DIR_NAME = "report_export"
+
+
+IMPORT_VIDEO_DIR = IMPORT_DIR / IMPORT_VIDEO_DIR_NAME
+IMPORT_REPORT_DIR = IMPORT_DIR / REPORT_IMPORT_DIR_NAME
+
+
+VIDEO_EXPORT_DIR = EXPORT_DIR / VIDEO_EXPORT_DIR_NAME
+REPORT_EXPORT_DIR = EXPORT_DIR / REPORT_EXPORT_DIR_NAME
+
+# Document Dir
+
+DOCUMENT_DIR = STORAGE_DIR / "documents"
+
+# After initial import, files will remain here.
+
+TRANSCODING_DIR = STORAGE_DIR /"temp"
+
+SENSITIVE_VIDEO_DIR_NAME = "sensitive_videos"
+SENSITIVE_REPORT_DIR_NAME = "sensitive_reports"
+ANONYM_VIDEO_DIR_NAME = "processed_videos_final"
+ANONYM_REPORT_DIR_NAME = "processed_reports_final"
+
+RAW_FRAME_DIR_NAME = f"{PREFIX_RAW}frames"
 FRAME_DIR_NAME = "frames"
-PDF_DIR_NAME = "pdfs" # Changed from reports
 WEIGHTS_DIR_NAME = "model_weights"
 EXAMINATION_DIR_NAME = "examinations"
 
-RAW_VIDEO_DIR_NAME = f"{PREFIX_RAW}videos"
-RAW_FRAME_DIR_NAME = f"{PREFIX_RAW}frames"
-RAW_PDF_DIR_NAME = f"{PREFIX_RAW}pdfs" # Changed from reports
-
 # Define data subdirectories under STORAGE_DIR
-VIDEO_DIR = STORAGE_DIR / VIDEO_DIR_NAME
-ANONYM_VIDEO_DIR = STORAGE_DIR / ANONYM_VIDEO_DIR_NAME # Added
+ANONYM_VIDEO_DIR = STORAGE_DIR / ANONYM_VIDEO_DIR_NAME
+SENSITIVE_VIDEO_DIR = STORAGE_DIR / SENSITIVE_VIDEO_DIR_NAME
+ANONYM_REPORT_DIR = STORAGE_DIR / ANONYM_REPORT_DIR_NAME
+SENSITIVE_REPORT_DIR = STORAGE_DIR / SENSITIVE_REPORT_DIR_NAME
+
 FRAME_DIR = STORAGE_DIR / FRAME_DIR_NAME
-PDF_DIR = STORAGE_DIR / PDF_DIR_NAME # Changed
+
+
 WEIGHTS_DIR = STORAGE_DIR / WEIGHTS_DIR_NAME
-RAW_VIDEO_DIR = STORAGE_DIR / RAW_VIDEO_DIR_NAME
 RAW_FRAME_DIR = STORAGE_DIR / RAW_FRAME_DIR_NAME
-RAW_PDF_DIR = STORAGE_DIR / RAW_PDF_DIR_NAME # Changed
 
-IMPORT_DIR = STORAGE_DIR / IMPORT_DIR_NAME
-VIDEO_IMPORT_DIR = IMPORT_DIR / VIDEO_DIR_NAME
-FRAME_IMPORT_DIR = IMPORT_DIR / FRAME_DIR_NAME
-PDF_IMPORT_DIR = IMPORT_DIR / PDF_DIR_NAME # Changed
 WEIGHTS_IMPORT_DIR = IMPORT_DIR / WEIGHTS_DIR_NAME
+WEIGHTS_EXPORT_DIR = EXPORT_DIR / WEIGHTS_DIR_NAME
+
+FRAME_IMPORT_DIR = IMPORT_DIR / FRAME_DIR_NAME
 
-EXPORT_DIR = STORAGE_DIR / EXPORT_DIR_NAME
+FRAME_EXPORT_DIR = EXPORT_DIR / FRAME_DIR_NAME
 
-data_paths:Dict[str,Path] = {
+
+data_paths: Dict[str, Path] = {
     "storage": STORAGE_DIR,
-    "video": VIDEO_DIR,
-    "anonym_video": ANONYM_VIDEO_DIR, # Added
-    "frame": FRAME_DIR,
-    "pdf": PDF_DIR, # Changed
     "import": IMPORT_DIR,
-    "video_import": VIDEO_IMPORT_DIR,
-    "frame_import": FRAME_IMPORT_DIR,
-    "pdf_import": PDF_IMPORT_DIR, # Changed
-    "raw_video": RAW_VIDEO_DIR,
+    "import_video": IMPORT_VIDEO_DIR,
+    "sensitive_video": SENSITIVE_VIDEO_DIR,
+    "sensitive_report": SENSITIVE_REPORT_DIR,
+    "anonym_video": ANONYM_VIDEO_DIR,
+    "anonym_report": ANONYM_REPORT_DIR,
+    "import_frame": FRAME_IMPORT_DIR,
+    "import_report": IMPORT_REPORT_DIR,
     "raw_frame": RAW_FRAME_DIR,
-    "raw_pdf": RAW_PDF_DIR, # Changed
     "weights": WEIGHTS_DIR,
     "weights_import": WEIGHTS_IMPORT_DIR,
     "export": EXPORT_DIR,
-    "video_export": EXPORT_DIR / VIDEO_DIR_NAME,
-    "anonym_video_export": EXPORT_DIR / ANONYM_VIDEO_DIR_NAME, # Added
-    "frame_export": EXPORT_DIR / FRAME_DIR_NAME,
-    "pdf_export": EXPORT_DIR / PDF_DIR_NAME, # Changed
+    "report_export": REPORT_EXPORT_DIR,
+    "video_export": VIDEO_EXPORT_DIR,
+    "frame_export": FRAME_EXPORT_DIR,
     "weights_export": EXPORT_DIR / WEIGHTS_DIR_NAME,
-    "examination_export": EXPORT_DIR / EXAMINATION_DIR_NAME,
-    "raw_video_export": EXPORT_DIR / RAW_VIDEO_DIR_NAME,
-    "raw_frame_export": EXPORT_DIR / RAW_FRAME_DIR_NAME,
-    "raw_pdf_export": EXPORT_DIR / RAW_PDF_DIR_NAME, # Changed
+    "transcoding": TRANSCODING_DIR,
+    "frame": FRAME_DIR,
+    "documents": DOCUMENT_DIR
 }
 
 logger.info(f"Storage directory: {STORAGE_DIR.resolve()}")
 logger.info(f"Export directory: {EXPORT_DIR.resolve()}")
 
-# for key, path in data_paths.items():
-#     path.mkdir(parents=True, exist_ok=True)
-
-#     logger.info(f"{key.capitalize()} directory: {path.resolve()}")
+for key, path in data_paths.items():
+    path.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"{key.capitalize()} directory: {path.resolve()}")
+    
+
+def to_storage_relative(path: Union[str, Path]) -> str:
+    """
+    Return a path string relative to STORAGE_DIR, suitable for Django FileField.name.
+
+    - If `path` is inside STORAGE_DIR (absolute or contains STORAGE_DIR as prefix),
+      we strip the STORAGE_DIR prefix and return the relative part.
+    - If `path` is outside STORAGE_DIR, we return it as a string unchanged
+      (caller can decide what to do).
+    """
+    # Normalize input to Path
+    p = Path(path)
+
+    # Resolve absolute path for comparison
+    storage_root = STORAGE_DIR.resolve()
+
+    if not p.is_absolute():
+        # Resolve relative path against current working directory
+        # (in tests, cwd should be project root, so this still lands under STORAGE_DIR)
+        p = p.resolve()
+
+    try:
+        rel = p.relative_to(storage_root)
+    except ValueError:
+        # Not under STORAGE_DIR – probably already a relative name or external.
+        # In that case, just return the string representation as-is.
+        return str(path)
+
+    # Use POSIX-style separators for Django FileField
+    return rel.as_posix()

endoreg_db/utils/pipelines/Readme.md

@@ -154,7 +154,7 @@ At this point in the pipeline:
   `VideoFile.processed_file`
 - Video hash is calculated and saved in:  
   `VideoFile.processed_video_hash`
-- `VideoState.anonymized = True`
+- `VideoState.state.anonymization_status.mark_anonymized()`
 
 ---
 

endoreg_db/utils/requirement_operator_logic/new_operator_logic.py

@@ -0,0 +1,97 @@
+from typing import TYPE_CHECKING, Any, Dict, Tuple, Union
+
+if TYPE_CHECKING:
+    from endoreg_db.models import Patient, PatientExamination, Requirement
+    from endoreg_db.models.requirement.requirement_operator import OperatorInstructions, RequirementOperator
+
+
+def fetch_input_target(input_object: Union["Patient", "PatientExamination"], operator_instructions: "OperatorInstructions") -> Tuple[str, Any]:
+    input_target_names = operator_instructions.input_targets
+
+    # Iterate over targets and stop with first successful fetch
+    for input_target in input_target_names:
+        # separate input_target using "." for nested attributes
+        attributes = input_target.split(".")
+        current_value = input_object
+        try:
+            for attr in attributes:
+                current_value = getattr(current_value, attr)
+            return (input_target, current_value)
+        except AttributeError:
+            continue  # Try next input target
+
+    raise AttributeError(f"None of the input targets {input_target_names} could be resolved on the input object.")
+
+
+def fetch_requirement_targets(
+    requirement: "Requirement",
+    operator_instructions: "OperatorInstructions",
+):
+    attribute_target_names = operator_instructions.requirement_targets
+    target_values = {}
+
+    for target_name in attribute_target_names:
+        try:
+            target_value = getattr(requirement, target_name)
+            target_values[target_name] = target_value
+        except AttributeError:
+            raise AttributeError(f"Requirement does not have attribute '{target_name}'.")
+        target_values[target_name] = target_value
+
+    return target_values
+
+
+def model_attribute_set_any(
+    input_object: Any,
+    operator_instructions: "OperatorInstructions",
+    requirement: "Requirement",
+) -> bool:
+    input_target_name, input_value = fetch_input_target(input_object, operator_instructions)
+
+    if not input_value:
+        return False
+    else:
+        return True
+
+
+def model_attribute_numeric_in_range(
+    input_object: Any,
+    operator_instructions: "OperatorInstructions",
+    requirement: "Requirement",
+) -> bool:
+    input_target_name, input_value = fetch_input_target(input_object, operator_instructions)
+
+    # make sure, input_value is numeric
+    try:
+        numeric_value = float(input_value)
+    except (TypeError, ValueError):
+        raise ValueError(f"Input value for target '{input_target_name}' is not numeric: {input_value}")
+
+    _min = requirement.numeric_value_min
+    _max = requirement.numeric_value_max
+
+    assert _min is not None and _max is not None, "Numeric range requires both min and max values to be set."
+
+    return_value = _min <= numeric_value <= _max
+    return return_value
+
+
+def model_attribute_is_among_values(
+    input_object: Any,
+    operator_instructions: "OperatorInstructions",
+    requirement: "Requirement",
+) -> bool:
+    input_target_name, input_value = fetch_input_target(input_object, operator_instructions)
+    target_values = fetch_requirement_targets(requirement, operator_instructions)
+
+    for target_name, target_values in target_values.items():
+        if input_value in target_values:
+            return True
+    return False
+
+
+REQUIREMENT_OPERATORS = {
+    "model_attribute_set_any": model_attribute_set_any,
+    "model_attribute_numeric_in_range": model_attribute_numeric_in_range,
+    "model_attribute_is_among_values": model_attribute_is_among_values,
+}