endoreg-db 0.5.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

endoreg_db/utils/dataloader.py

@@ -2,10 +2,13 @@ import os
 import yaml
 from django.core.exceptions import ObjectDoesNotExist
 
+from icecream import ic
+
+
 def load_model_data_from_yaml(command, model_name, metadata, verbose):
     """
     Load model data from YAML files.
-    
+
     Args:
         command: Command object for stdout writing.
         model_name: Name of the model being loaded.
@@ -19,19 +22,24 @@ def load_model_data_from_yaml(command, model_name, metadata, verbose):
     foreign_keys = metadata["foreign_keys"]
     foreign_key_models = metadata["foreign_key_models"]
 
-    _files = [f for f in os.listdir(dir_path) if f.endswith('.yaml')]
+    _files = [f for f in os.listdir(dir_path) if f.endswith(".yaml")]
     # sort
     _files.sort()
     for file in _files:
-        with open(os.path.join(dir_path, file), 'r') as file:
+        with open(os.path.join(dir_path, file), "r", encoding="utf-8") as file:
             yaml_data = yaml.safe_load(file)
-        
-        load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose)
 
-def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose):
+        load_data_with_foreign_keys(
+            command, model, yaml_data, foreign_keys, foreign_key_models, verbose
+        )
+
+
+def load_data_with_foreign_keys(
+    command, model, yaml_data, foreign_keys, foreign_key_models, verbose
+):
     """
     Load data handling foreign keys and many-to-many relationships.
-    
+
     Args:
         command: Command object for stdout writing.
         model: The Django model for the data.
@@ -41,20 +49,24 @@ def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign
         verbose: Boolean indicating whether to print verbose output.
     """
     for entry in yaml_data:
-        fields = entry.get('fields', {})
-        name = fields.pop('name', None)
+        fields = entry.get("fields", {})
+        name = fields.pop("name", None)
         m2m_relationships = {}  # Store many-to-many relationships
-        print(entry)
+        # print(entry)
 
         # Handle foreign keys and many-to-many relationships
         for fk_field, fk_model in zip(foreign_keys, foreign_key_models):
-            print(fk_field, fk_model)
+            # print(fk_field, fk_model)
             target_keys = fields.pop(fk_field, None)
-            
+
             # Ensure the foreign key exists
             if target_keys is None:
                 if verbose:
-                    command.stdout.write(command.style.WARNING(f"Foreign key {fk_field} not found in fields"))
+                    command.stdout.write(
+                        command.style.WARNING(
+                            f"Foreign key {fk_field} not found in fields"
+                        )
+                    )
                 continue  # Skip if no foreign key provided
 
             # Process many-to-many fields or foreign keys
@@ -63,29 +75,43 @@ def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign
                 for key in target_keys:
                     obj, created = fk_model.objects.get_or_create(name=key)
                     if created and verbose:
-                        command.stdout.write(command.style.SUCCESS(f"Created {fk_model.__name__} {key}"))
+                        command.stdout.write(
+                            command.style.SUCCESS(f"Created {fk_model.__name__} {key}")
+                        )
                     related_objects.append(obj)
                 m2m_relationships[fk_field] = related_objects
             else:  # Single foreign key relationship
                 try:
                     if model == "endoreg_db.case_template_rule":
-                        print(fk_model, target_keys)
+                        # print(fk_model, target_keys)
+                        pass
                     obj = fk_model.objects.get_by_natural_key(target_keys)
                 except ObjectDoesNotExist:
                     if verbose:
-                        command.stdout.write(command.style.WARNING(f"{fk_model.__name__} with key {target_keys} not found"))
+                        command.stdout.write(
+                            command.style.WARNING(
+                                f"{fk_model.__name__} with key {target_keys} not found"
+                            )
+                        )
                     continue
                 fields[fk_field] = obj
-    
+
         # Create or update the main object
         if name is None:
             obj, created = model.objects.get_or_create(**fields)
         else:
             obj, created = model.objects.update_or_create(defaults=fields, name=name)
         if created and verbose:
-            command.stdout.write(command.style.SUCCESS(f'Created {model.__name__} {name}'))
+            command.stdout.write(
+                command.style.SUCCESS(f"Created {model.__name__} {name}")
+            )
         elif verbose:
-            command.stdout.write(command.style.WARNING(f'Skipped {model.__name__} {name}, already exists'))
+            pass
+            # command.stdout.write(
+            #     command.style.WARNING(
+            #         f"Skipped {model.__name__} {name}, already exists"
+            #     )
+            # )
 
         # Set many-to-many relationships
         for field_name, related_objs in m2m_relationships.items():

endoreg_db/utils/dates.py

@@ -0,0 +1,39 @@
+from datetime import datetime, date, timedelta
+from random import randint
+from calendar import monthrange
+
+# TODO replace used random_day_by_year function implementation when
+# creating pseudo patients with new function "random date by age_at_date and examination_date"
+
+
+def random_day_by_age_at_date(age_at_date: int, examination_date: date) -> date:
+    """
+    Return a random birth day for a patient with the specified age at the specified examination date.
+    """
+    examination_year = examination_date.year
+    latest_birthdate = examination_date.replace(year=examination_year - age_at_date)
+    valid_dates = [latest_birthdate - timedelta(days=i) for i in range(365)]
+
+    select = randint(0, len(valid_dates) - 1)
+    birth_date = valid_dates[select]
+
+    return birth_date
+
+
+def random_day_by_year(year: int) -> date:
+    """
+    Return a random birth day within the specified year.
+    """
+    month = randint(1, 12)
+    day = randint(1, monthrange(year, month)[1])
+
+    return date(year, month, day)
+
+
+def random_day_by_month_year(month: int, year) -> date:
+    """
+    Return a random birth day within the specified month and year.
+    """
+
+    day = randint(1, monthrange(year, month)[1])
+    return date(year, month, day)

endoreg_db/utils/hashs.py

@@ -1,12 +1,18 @@
 import hashlib
 from pathlib import Path
+from datetime import datetime, date
+
+import os
+
+SALT = os.getenv("DJANGO_SALT", "default_salt")
+
 
 def get_video_hash(video_path):
     """
     Get the hash of a video file.
     """
     # Open the video file in read-binary mode:
-    with open(video_path, 'rb') as f:
+    with open(video_path, "rb") as f:
         # Create the hash object, passing in the video contents for hashing:
         hash_object = hashlib.sha256(f.read())
         # Get the hexadecimal representation of the hash
@@ -15,14 +21,15 @@ def get_video_hash(video_path):
 
     return video_hash
 
-def get_pdf_hash(pdf_path:Path):
+
+def get_pdf_hash(pdf_path: Path):
     """
     Get the hash of a pdf file.
     """
     pdf_hash = None
 
     # Open the file in binary mode and read its contents
-    with open(pdf_path, 'rb') as f:
+    with open(pdf_path, "rb") as f:
         pdf_contents = f.read()
         # Create a hash object using SHA-256 algorithm
 
@@ -31,4 +38,115 @@ def get_pdf_hash(pdf_path:Path):
     pdf_hash = hash_object.hexdigest()
     assert len(pdf_hash) <= 255, "Hash length exceeds 255 characters"
 
-    return pdf_hash
+    return pdf_hash
+
+
+def _get_date_hash_string(date_obj: date) -> str:
+    # if date is datetime object, convert to date
+    if isinstance(date_obj, datetime):
+        # warnings.warn("Date is a datetime object. Converting to date object.")
+        date_obj = date_obj.date()
+    elif isinstance(date_obj, str):
+        # warnings.warn(f"Date is a string ({date_obj}). Converting to date object.")
+        date_obj = datetime.strptime(date_obj, "%Y-%m-%d").date()
+
+    assert isinstance(date_obj, date), "Date must be a date object"
+    # if date is 1900-01-01, make it an empty string
+    if date_obj == date(1900, 1, 1):
+        date_str = ""
+    else:
+        date_str = date_obj.strftime("%Y-%m-%d")
+
+    return date_str
+
+
+def get_hash_string(
+    first_name: str = "",
+    last_name: str = "",
+    dob: date = date(1900, 1, 1),
+    center_name: str = "",
+    examination_date: date = date(1900, 1, 1),
+    endoscope_sn: str = "",
+    salt: str = "",
+):
+    """
+    Get the string to be hashed for a patient's first name, last name, date of birth, examination date, and endoscope serial number.
+    """
+    if not salt:
+        salt = SALT
+
+    examination_date_str = _get_date_hash_string(examination_date)
+    dob_str = _get_date_hash_string(dob)
+
+    # Concatenate the patient's first name, last name, date of birth, examination date, endoscope serial number, and salt:
+    hash_str = f"{first_name}{last_name}{dob_str}{center_name}{dob_str}{examination_date_str}{endoscope_sn}{salt}"
+    return hash_str
+
+
+def get_patient_hash(
+    first_name: str, last_name: str, dob: date, center: str, salt: str = ""
+):
+    """
+    Get the hash of a patient's first name, last name, and date of birth.
+    """
+    # Concatenate the patient's first name, last name, date of birth, and salt:
+    hash_str = get_hash_string(
+        first_name=first_name,
+        last_name=last_name,
+        dob=dob,
+        center_name=center,
+        salt=salt,
+    )
+    # Create a hash object using SHA-256 algorithm
+    hash_object = hashlib.sha256(hash_str.encode())
+    # Get the hexadecimal representation of the hash
+    patient_hash = hash_object.hexdigest()
+
+    return patient_hash
+
+
+def get_patient_examination_hash(
+    first_name: str,
+    last_name: str,
+    dob: date,
+    center: str,
+    examination_date: date,
+    salt: str = "",
+):
+    """
+    Get the hash of a patient's first name, last name, date of birth, and examination date.
+    """
+    # Concatenate the patient's first name, last name, date of birth, examination date, and salt:
+    hash_str = get_hash_string(
+        first_name=first_name,
+        last_name=last_name,
+        center_name=center,
+        dob=dob,
+        examination_date=examination_date,
+        salt=salt,
+    )
+    # Create a hash object using SHA-256 algorithm
+    hash_object = hashlib.sha256(hash_str.encode())
+    # Get the hexadecimal representation of the hash
+    patient_examination_hash = hash_object.hexdigest()
+
+    return patient_examination_hash
+
+
+def get_examiner_hash(first_name, last_name, center_name, salt):
+    """
+    Get the hash of an examiner's first name, last name, and center name.
+    """
+    # Concatenate the examiner's first name, last name, center name, and salt:
+    hash_str = get_hash_string(
+        first_name=first_name,
+        last_name=last_name,
+        center_name=center_name,
+        salt=salt,
+    )
+    # Create a hash object using SHA-256 algorithm
+    hash_object = hashlib.sha256(hash_str.encode())
+    # Get the hexadecimal representation of the hash
+    examiner_hash = hash_object.hexdigest()
+
+    return examiner_hash

endoreg_db/utils/names.py

@@ -0,0 +1,74 @@
+# Use faker library to generate fake names by gender
+# Use german names by default
+
+from faker import Faker
+import gender_guesser.detector as gender_detector
+
+
+def create_mock_examiner_name() -> tuple[str, str]:
+    """
+    Generate a mock examiner's name using the Faker library.
+    This function creates a tuple with a first name and a last name for a mock examiner. It utilizes the "de_DE" locale for generating German names.
+    Returns:
+        tuple[str, str]: A tuple containing the first name and the last name.
+    """
+
+    fake = Faker("de_DE")
+    first_name = fake.first_name()
+    last_name = fake.last_name()
+    return first_name, last_name
+
+
+def create_mock_patient_name(gender: str) -> tuple[str, str]:
+    """
+    Generate a mock patient's name based on the provided gender using the Faker library.
+    This function creates a tuple with a first name and a last name for a mock patient. It utilizes the "de_DE" locale for generating German names. When the input gender string is checked:
+    - If it contains "male", a male name is generated.
+    - If it contains "female", a female name is generated.
+    - Otherwise, a generic name is generated without considering gender.
+    Parameters:
+        gender (str): A string indicating the gender to be used for generating the name.
+    Returns:
+        tuple[str, str]: A tuple containing the first name and the last name.
+    """
+
+    fake = Faker("de_DE")
+
+    if "male" in gender.lower():
+        gender = "male"
+    elif "female" in gender.lower():
+        gender = "female"
+
+    if gender == "male":
+        first_name = fake.first_name_male()
+        last_name = fake.last_name_male()
+
+    elif gender == "female":
+        first_name = fake.first_name_female()
+        last_name = fake.last_name_female()
+
+    else:
+        first_name = fake.first_name()
+        last_name = fake.last_name()
+
+    return first_name, last_name
+
+
+def guess_name_gender(name: str) -> str:
+    """
+    Guesses the gender for a given name by using a gender detector and retrieving the corresponding Gender model instance.
+    Parameters:
+        name (str): The name for which the gender is to be determined.
+    Returns:
+        Gender: The Gender object corresponding to the detected gender name.
+    Raises:
+        Gender.DoesNotExist: If no Gender object matching the detected gender is found.
+        Exception: For any other exceptions that occur during gender detection or database lookup.
+    """
+
+    from endoreg_db.models import Gender
+
+    detector = gender_detector.Detector(case_sensitive=False)
+    gender_name = detector.get_gender(name)
+    gender = Gender.objects.get(name=gender_name)
+    return gender

endoreg_db/utils/parse_and_generate_yaml.py

@@ -0,0 +1,46 @@
+import yaml
+from pathlib import Path
+
+# get this files path
+file_path = Path(__file__)
+module_root = file_path.parent.parent
+data_dir = module_root / "data"
+
+
+def collect_center_names(
+):
+    input_file_path = data_dir / "center/data.yaml"
+    fist_name_dir = data_dir / "names_first"
+    last_name_dir = data_dir / "names_last"
+    # Load the input YAML file
+    with open(input_file_path, 'r', encoding='utf-8') as file:
+        data = yaml.safe_load(file)
+    
+    # Containers for first and last names
+    first_names_set = set()
+    last_names_set = set()
+    
+    # Extract first and last names from the YAML data
+    for entry in data:
+        fields = entry.get('fields', {})
+        first_names_set.update(fields.get('first_names', []))
+        last_names_set.update(fields.get('last_names', []))
+    
+    # Create a list of dictionaries for first and last names
+    first_names_data = [
+        {"model": "endoreg_db.first_name", "fields": {"name": name}}
+        for name in sorted(first_names_set)
+    ]
+    last_names_data = [
+        {"model": "endoreg_db.last_name", "fields": {"name": name}}
+        for name in sorted(last_names_set)
+    ]
+    
+    # Write the data to separate YAML files
+    with open(fist_name_dir/"first_names.yaml", "w", encoding='utf-8') as first_file:
+        yaml.dump(first_names_data, first_file, allow_unicode=True, sort_keys=False)
+    
+    with open(last_name_dir/"last_names.yaml", "w", encoding='utf-8') as last_file:
+        yaml.dump(last_names_data, last_file, allow_unicode=True, sort_keys=False)
+
+    # print("Generated first_names.yaml and last_names.yaml successfully.")

endoreg_db/utils/pydantic_models/__init__.py

@@ -0,0 +1,6 @@
+'''Module for Pydantic models.'''
+from .db_config import DbConfig
+
+__all__ = [
+    'DbConfig'
+]

endoreg_db/utils/pydantic_models/db_config.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+from typing import Optional, Union
+import yaml
+from icecream import ic
+from pydantic import BaseModel
+
+
+class DbConfig(BaseModel):
+    """Database configuration model."""
+
+    engine: str = "django.db.backends.postgresql"
+    host: str
+    port: int
+    user: str
+    password: Optional[str] = None
+    password_file: str = "/etc/secrets/vault/SCRT_local_password_maintenance_password"
+    name: str
+
+    # class options should allow for unused excess data
+
+    @classmethod
+    def from_file(cls, path: Union[Path, str]) -> "DbConfig":
+        """Create a DbConfig instance from a YAML configuration file."""
+        if isinstance(path, str):
+            filepath = Path(path)
+        else:
+            filepath = path
+        assert filepath.exists(), f"Missing Config {filepath}"
+
+        with open(filepath, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f)
+
+        obj = cls(**cfg)
+
+        return obj
+
+    def sync_password(self):
+        """Load the password from the configured password file."""
+        with open(self.password_file, "r", encoding="utf-8") as f:
+            self.password = f.read().strip()
+
+    def validate(self):
+        """Fully validate all database configuration fields."""
+        self.sync_password()
+
+        assert self.host, "Missing Host"
+        assert self.port, "Missing Port"
+        assert self.user, "Missing User"
+        assert self.password, "Missing Password"
+        assert self.name, "Missing Database"
+
+    def to_file(self, target: str = "./conf/db.yml", ask_override: bool = True):
+        """Export the configuration to a YAML file."""
+        ic(target)
+
+        with open(target, "w") as f:
+            yaml.safe_dump(self.model_dump(), f)

endoreg_db/utils/validate_endo_roi.py

@@ -0,0 +1,19 @@
+def validate_endo_roi(endo_roi_dict:dict):
+    """
+    Validate endoscope ROI dictionary. The dictionary must have the following
+    keys: x, y, width, height. The values must be greater than or equal to 0
+    for x and y, and greater than 0 for width and height.
+    """
+    for key, value in endo_roi_dict.items():
+        if key == "x":
+            assert value >= 0, f"Endoscope ROI x value must be greater than or equal to 0. Got {value}"
+        elif key == "y":
+            assert value >= 0, f"Endoscope ROI y value must be greater than or equal to 0. Got {value}"
+        elif key == "width":
+            assert value > 0, f"Endoscope ROI width value must be greater than 0. Got {value}"
+        elif key == "height":
+            assert value > 0, f"Endoscope ROI height value must be greater than 0. Got {value}"
+        else:
+            raise ValueError(f"Endoscope ROI key {key} not recognized")
+
+    return True

endoreg_db/utils/validate_subcategory_dict.py

@@ -0,0 +1,91 @@
+from endoreg_db.models import Unit
+
+def validate_subcategory_dict(self, subcategory_dict:dict=None):
+    if subcategory_dict is None:
+        return False
+    
+    if not isinstance(subcategory_dict, dict):
+        return False
+
+    # check if key choices exists and is a list of strings
+    if "choices" not in subcategory_dict:
+        return False
+    
+    if not isinstance(subcategory_dict["choices"], list):
+        return False
+    
+    for choice in subcategory_dict["choices"]:
+        if not isinstance(choice, str):
+            return False
+        
+    # check if key default exists and is a string
+    if "default" not in subcategory_dict:
+        return False
+    
+    if not isinstance(subcategory_dict["default"], str):
+        return False
+    
+    # check if default is in choices
+    if subcategory_dict["default"] not in subcategory_dict["choices"]:
+        return False
+    
+    if not subcategory_dict["required"]:
+        return False
+    
+    return True
+
+
+def validate_numerical_descriptor(self, numerical_descriptor_dict:dict=None):
+    # Check if numerical_descriptor_dict is None
+    error_message = ""
+    if numerical_descriptor_dict is None:
+        error_message = "numerical_descriptor_dict is None"
+        return False, error_message
+    
+    # Check if numerical_descriptor_dict is a dictionary
+    if not isinstance(numerical_descriptor_dict, dict):
+        error_message = "numerical_descriptor_dict is not a dictionary"
+        return False, error_message
+    
+    # if key unit exists and is a string and Unit object with that name exists
+    if "unit" not in numerical_descriptor_dict:
+        error_message = "unit key does not exist in numerical_descriptor_dict"
+        return False, error_message
+    
+    elif not isinstance(numerical_descriptor_dict["unit"], str):
+        error_message = "unit key is not a string"
+        return False, error_message
+
+    elif not Unit.objects.filter(name=numerical_descriptor_dict["unit"]).exists():
+        error_message = "Unit object with that name does not exist"
+        return False, error_message
+    
+    
+    if not "required" in numerical_descriptor_dict:
+        error_message = "required key does not exist in numerical_descriptor_dict"
+        return False, error_message
+    
+    elif not isinstance(numerical_descriptor_dict["required"], bool):
+        error_message = "required key is not a boolean"
+        return False, error_message
+    
+    # check if min, max, mean, std exist and are either None or float
+    for key in ["min", "max", "mean", "std", "default"]:
+        if key not in numerical_descriptor_dict:
+            error_message = f"{key} key does not exist in numerical_descriptor_dict"
+            return False, error_message
+        
+        if numerical_descriptor_dict[key] is not None and not isinstance(numerical_descriptor_dict[key], float):
+            error_message = f"{key} key is not a float"
+            return False, error_message
+        
+    # check if distribution exists and is either "normal" or "uniform"
+    if "distribution" not in numerical_descriptor_dict:
+        error_message = "distribution key does not exist in numerical_descriptor_dict"
+        return False, error_message
+    
+    if numerical_descriptor_dict["distribution"] not in ["normal", "uniform"]:
+        error_message = "distribution key is not either 'normal' or 'uniform'"
+        return False, error_message
+    
+    return True, None

endoreg_db/utils/video/__init__.py

@@ -0,0 +1,13 @@
+from .transcode_videofile import (
+    transcode_videofile,
+    transcode_videofile_if_required,
+)
+
+from .extract_frames import extract_frames, initialize_frame_objects
+
+__all__ = [
+    "extract_frames",
+    "initialize_frame_objects",
+    "transcode_videofile",
+    "transcode_videofile_if_required",
+]