endoreg-db 0.6.3__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/helpers/default_objects.py

@@ -0,0 +1,359 @@
+import random
+from endoreg_db.models import (
+    Center, 
+    Gender, 
+    Patient,
+    Examination,
+    ExaminationIndication,
+    RawPdfFile,
+    EndoscopyProcessor,
+    ModelMeta,
+    InformationSource,
+    AiModel,
+)
+from logging import getLogger
+from datetime import date
+import shutil
+from pathlib import Path
+from django.conf import settings # Import settings
+from django.core.files.storage import default_storage # Import default storage
+
+from endoreg_db.utils import (
+    create_mock_patient_name,
+)
+
+logger = getLogger("default_objects")
+
+
+DEFAULT_CENTER_NAME = "university_hospital_wuerzburg"
+DEFAULT_ENDOSCOPE_NAME = "test_endoscope"
+DEFAULT_ENDOSCOPY_PROCESSOR_NAME = "olympus_cv_1500"
+
+DEFAULT_EGD_PATH = Path("tests/assets/lux-gastro-report.pdf")
+DEFAULT_GENDERS = ["male","female","unknown"]
+DEFAULT_EXAMINATIONS = ["colonoscopy"]
+DEFAULT_INDICATIONS = [
+    "colonoscopy",
+    "colonoscopy_screening",
+    "colonoscopy_lesion_removal_small",
+    "colonoscopy_lesion_removal_emr",
+    "colonoscopy_lesion_removal_large",
+    "colonoscopy_diagnostic_acute_symptomatic",
+]
+
+DEFAULT_SEGMENTATION_MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
+
+DEFAULT_GENDER = "unknown"
+
+def get_information_source_prediction():
+    """
+    Retrieves the InformationSource object with the name "prediction".
+    
+    Loads information source data if needed and returns the corresponding InformationSource instance. Raises a ValueError if the object is not found or is not an InformationSource.
+    """
+    from .data_loader import load_information_source
+    load_information_source()
+    source = InformationSource.objects.get(name="prediction")
+    if not isinstance(source, InformationSource):
+        raise ValueError("No InformationSource found in the database.")
+    return source
+
+def get_latest_segmentation_model(model_name:str=DEFAULT_SEGMENTATION_MODEL_NAME) -> ModelMeta:
+
+    """
+    Retrieves the latest metadata for a segmentation model by name.
+    
+    Loads necessary data and returns the most recent ModelMeta instance for the specified AI model. If no metadata exists, attempts to initialize it automatically; if initialization fails, raises a ValueError with instructions for manual setup.
+    
+    Args:
+        model_name: The name of the segmentation model to retrieve.
+    
+    Returns:
+        The latest ModelMeta instance for the specified model.
+    
+    Raises:
+        ValueError: If the AI model does not exist, or if model metadata cannot be found or initialized.
+    """
+    from .data_loader import (
+        load_center_data,
+        load_ai_model_label_data,
+        load_ai_model_data,
+    )
+    load_center_data()
+    load_ai_model_label_data()
+    load_ai_model_data()
+    
+    try:
+        ai_model = AiModel.objects.get(name=model_name)
+    except AiModel.DoesNotExist:
+        raise ValueError(f"AI model '{model_name}' not found. Run 'python manage.py load_ai_model_data' first.")
+    
+    try:
+        latest_meta = ai_model.get_latest_version()
+        return latest_meta
+    except ValueError as e:
+        if "No model metadata found" in str(e):
+            logger.warning(f"No ModelMeta found for {model_name}. Attempting to initialize default model metadata...")
+            
+            # Try to initialize the default model metadata
+            try:
+                from django.core.management import call_command
+                call_command('init_default_ai_model')
+                # Try again after initialization
+                latest_meta = ai_model.get_latest_version()
+                return latest_meta
+            except Exception as init_error:
+                raise ValueError(
+                    f"No model metadata found for AI model '{model_name}' and failed to auto-initialize. "
+                    f"Please run 'python manage.py init_default_ai_model' manually. "
+                    f"Original error: {e}. Initialization error: {init_error}"
+                ) from e
+        else:
+            raise
+    
+
+def get_default_gender() -> Gender:
+    """
+    Retrieves the Gender object representing the default "unknown" gender.
+    
+    Returns:
+        The Gender instance with the name "unknown".
+    """
+    return Gender.objects.get(name=DEFAULT_GENDER)
+
+def get_gender_m_or_f() -> Gender:
+    """
+    Returns a randomly selected Gender object representing either male or female.
+    """
+    gender_name = random.choice(["male", "female"])
+    return Gender.objects.get(name=gender_name)
+
+def get_random_gender() -> Gender:
+    """
+    Returns a randomly selected Gender object from the available default genders.
+    """
+    gender_name = random.choice(DEFAULT_GENDERS)
+    return Gender.objects.get(name=gender_name) # Fetch and return the Gender object
+
+def get_default_processor() -> EndoscopyProcessor:
+    """
+    Retrieves the default EndoscopyProcessor object by its predefined name.
+    
+    Raises:
+        ValueError: If no EndoscopyProcessor with the default name exists.
+        
+    Returns:
+        The EndoscopyProcessor instance with the default name.
+    """
+    processor = EndoscopyProcessor.objects.get(name=DEFAULT_ENDOSCOPY_PROCESSOR_NAME)
+    if not isinstance(processor, EndoscopyProcessor):
+        raise ValueError(f"No EndoscopyProcessor found with name {DEFAULT_ENDOSCOPY_PROCESSOR_NAME}")
+    return processor
+
+
+def get_default_center() -> Center:
+    """
+    Retrieves the default Center object with the predefined name.
+    
+    Raises:
+        ValueError: If no Center with the default name exists.
+    
+    Returns:
+        The Center instance with the default name.
+    """
+    center = Center.objects.get(
+        name=DEFAULT_CENTER_NAME,
+    )
+    if not isinstance(center, Center):
+        raise ValueError(f"No Center found with name {DEFAULT_CENTER_NAME}")
+    
+    return center
+
+def generate_patient(**kwargs) -> Patient:
+    """
+    Creates a Patient instance with randomized or specified attributes.
+    
+    Randomly generates first name, last name, date of birth, gender, and center for the patient unless overridden by keyword arguments. Raises ValueError if the provided gender is invalid.
+    
+    Args:
+        **kwargs: Optional overrides for patient attributes such as 'first_name', 'last_name', 'birth_date', 'gender', and 'center'.
+    
+    Returns:
+        A Patient instance with the specified or randomly generated attributes.
+    """
+    # Set default values
+    gender = kwargs.get("gender", get_random_gender())
+    if not isinstance(gender, Gender):
+        gender = Gender.objects.get(name=gender)
+
+    if not isinstance(gender, Gender):
+        raise ValueError("No Gender Found")
+    first_name, last_name = create_mock_patient_name(gender = gender.name)
+    first_name = kwargs.get("first_name", first_name)
+    last_name = kwargs.get("last_name", last_name)
+    birth_date = kwargs.get("birth_date", "1970-01-01")
+    dob = date.fromisoformat(birth_date)
+    center = kwargs.get("center", None)
+    if center is None:
+        center = get_default_center()
+    else:
+        center = Center.objects.get(name=center)
+
+    patient = Patient(
+        first_name=first_name,
+        last_name=last_name,
+        dob=dob,
+        center = center,
+        gender = gender,
+    )
+
+    return patient
+    
+def get_random_default_examination():
+    """
+    Retrieves a random Examination object from the default examination names.
+    
+    Returns:
+        Examination: A randomly selected Examination instance from the defaults.
+    """
+    examination_name = random.choice(DEFAULT_EXAMINATIONS)
+
+    examination = Examination.objects.get(name=examination_name)
+    return examination
+
+def get_random_default_examination_indication():
+    """
+    Returns a random ExaminationIndication object from the default indications list.
+    
+    Selects a random indication name from the predefined defaults and retrieves the corresponding ExaminationIndication instance from the database.
+    """
+    examination_indication = random.choice(DEFAULT_INDICATIONS)
+    all_examination_indications = ExaminationIndication.objects.all()
+    try:
+        examination_indication = ExaminationIndication.objects.get(name=examination_indication)
+        
+    except Exception as e:
+        logger.info(f"examination_indication: {examination_indication}")
+        logger.info(f"all_examination_indications: {all_examination_indications}")
+        raise e
+    return examination_indication
+
+def get_default_egd_pdf():
+    """
+    Creates and processes a default EGD PDF file for testing purposes.
+    
+    This function copies a default EGD PDF to a temporary location, creates a RawPdfFile instance from it, processes the file to generate associated metadata, and ensures cleanup of the temporary file. The resulting RawPdfFile instance is returned for use in tests.
+    
+    Returns:
+        RawPdfFile: The created and processed RawPdfFile instance.
+    """
+    egd_path = DEFAULT_EGD_PATH
+    center = get_default_center()
+    center_name = center.name
+
+    # Create a temporary file path within the test's media root if possible,
+    # otherwise use the source directory. Using MEDIA_ROOT is safer.
+    # Ensure MEDIA_ROOT is configured correctly in test settings.
+    temp_dir = Path(settings.MEDIA_ROOT) / "temp_test_files"
+    temp_dir.mkdir(parents=True, exist_ok=True)
+    temp_file_path = temp_dir / f"temp_{egd_path.name}"
+
+    shutil.copy(egd_path, temp_file_path)
+
+    pdf_file = None
+    try:
+        # Create the PDF record using the temporary file.
+        # delete_source=True will ensure temp_file_path is deleted by create_from_file
+        pdf_file = RawPdfFile.create_from_file(
+            file_path=temp_file_path,
+            center_name=center_name,
+            save=True, # save=True is default and handled internally now
+            delete_source=True,
+        )
+
+        if pdf_file is None:
+            raise RuntimeError("Failed to create PDF file object")
+        
+        # Use storage API to check existence
+        if not default_storage.exists(pdf_file.file.path):
+            raise RuntimeError(f"PDF file does not exist in storage at {pdf_file.file.path}")
+        
+        # Check that the source temp file was deleted
+        if temp_file_path.exists():
+            raise RuntimeError(f"Temporary source file {temp_file_path} still exists after creation")
+
+        # Prepare a minimal report_meta for SensitiveMeta creation
+        default_report_meta = {
+            "patient_first_name": "DefaultFirstName",
+            "patient_last_name": "DefaultLastName",
+            "patient_dob": date(1980, 1, 1), # Pass date object directly
+            "examination_date": date(2024, 1, 1), # Pass date object directly
+            # center_name will be added by process_file using pdf_file.center.name
+        }
+
+        # Call process_file to create SensitiveMeta and extract other info
+        pdf_file.process_file(
+            text="Default PDF text content.",
+            anonymized_text="Default anonymized PDF text content.",
+            report_meta=default_report_meta,
+            verbose=False
+        )
+        # process_file calls sensitive_meta.save() and self.save() (for RawPdfFile)
+
+    except Exception as e:
+        # Clean up temp file in case of error before deletion could occur
+        if temp_file_path.exists():
+            temp_file_path.unlink()
+        raise e # Re-raise the exception
+
+    # pdf_file.file.path might fail if storage doesn't support direct paths (like S3)
+    # Prefer using storage API for checks. Logging path if available.
+    try:
+        logger.info(f"PDF file created: {pdf_file.file.name}, Path: {pdf_file.file.path}")
+    except NotImplementedError:
+        logger.info(f"PDF file created: {pdf_file.file.name}, Path: (Not available from storage)")
+
+
+    return pdf_file
+
+def get_default_video_file():
+    """
+    Creates and returns a VideoFile instance using a randomly selected EGD examination video.
+    
+    Loads all necessary data dependencies, selects a random video file for the 'egd' examination, and initializes a VideoFile object with the default center and processor names. The original video file is retained after creation.
+    
+    Returns:
+        VideoFile: The created and initialized VideoFile instance.
+    """
+    from .test_video_helper import get_random_video_path_by_examination_alias
+    from endoreg_db.models import VideoFile
+    from .data_loader import (
+        load_disease_data,
+        load_event_data,
+        load_information_source,
+        load_examination_data,
+        load_center_data,
+        load_endoscope_data,
+        load_ai_model_label_data,
+        load_ai_model_data,
+    )
+    load_disease_data()
+    load_event_data()
+    load_information_source()
+    load_examination_data()
+    load_center_data()
+    load_endoscope_data()
+    load_ai_model_label_data()
+    load_ai_model_data()
+    video_path = get_random_video_path_by_examination_alias(
+        examination_alias='egd', is_anonymous=False
+    )
+
+    video_file = VideoFile.create_from_file_initialized(
+        file_path=video_path,
+        center_name=DEFAULT_CENTER_NAME,  # Pass center name as expected by _create_from_file
+        delete_source=False,  # Keep the original asset for other tests
+        processor_name = DEFAULT_ENDOSCOPY_PROCESSOR_NAME,
+    )
+
+    return video_file

endoreg_db/helpers/interact.py

@@ -0,0 +1,6 @@
+from endoreg_db.models import EndoscopyProcessor
+from django.core.management.base import BaseCommand
+from typing import TYPE_CHECKING
+from django.db.models import QuerySet
+
+

endoreg_db/helpers/test_video_helper.py

@@ -0,0 +1,119 @@
+from pathlib import Path
+import random
+import re
+from typing import Optional
+
+from logging import getLogger
+logger = getLogger(__name__)
+
+
+from endoreg_db.utils.video.names import (
+    get_video_key_regex_by_examination_alias
+)
+
+from django.conf import settings
+
+
+ASSET_DIR:Path = settings.ASSET_DIR
+assert ASSET_DIR.exists(), f"ASSET_DIR does not exist: {ASSET_DIR}"
+
+_TEST_VIDEOS = {
+    # "egd-instrument-non_anonymous": ASSET_DIR / "test_instrument.mp4", # No detected segments
+    "egd-endoscope-non_anonymous": ASSET_DIR / "test_endoscope.mp4",
+    "egd-nbi-non_anonymous": ASSET_DIR / "test_nbi.mp4",
+    # "egd-outside-non_anonymous": ASSET_DIR / "test_outside.mp4",
+    "egd-small_intestine-non_anonymous": ASSET_DIR / "test_small_intestine.mp4",
+}
+
+# Only keep entries that actually exist on disk to avoid returning None paths
+TEST_VIDEOS = {key: value for key, value in _TEST_VIDEOS.items() if value.exists()}
+
+
+def get_video_path(video_key:str) -> Path:
+    """
+    Returns the file path associated with the given video key.
+    
+    Raises:
+        ValueError: If the video key does not exist in the available test videos.
+    """
+    if video_key in TEST_VIDEOS:
+        return TEST_VIDEOS[video_key]
+    else:
+        raise ValueError(f"Video key '{video_key}' not found in TEST_VIDEOS.")
+
+def get_video_keys(
+    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+):
+    """
+    Returns a list of video keys matching the specified examination alias, content type, and anonymity status.
+    
+    If no direct matches are found, falls back to suffix-based filtering for anonymity. Logs warnings and errors when fallback logic is used or no matches are found.
+    
+    Args:
+        examination_alias: The examination alias to filter by, or None for any.
+        content: The content type to filter by, or None for any.
+        is_anonymous: Whether to filter for anonymous, non-anonymous, or both.
+    
+    Returns:
+        A list of matching video keys.
+    """
+    # Build a more flexible pattern locally if content is None
+    if content is None:
+        pattern_parts = ["^"]
+        if examination_alias:
+            pattern_parts.append(re.escape(examination_alias))
+            pattern_parts.append("-.*-") # Match any content
+        else:
+            pattern_parts.append(".*-") # Match any examination alias and content
+
+        if is_anonymous is True:
+            pattern_parts.append("anonymous$")
+        elif is_anonymous is False:
+            pattern_parts.append("non_anonymous$")
+        else: # is_anonymous is None
+            pattern_parts.append("(non_)?anonymous$") # Match either
+
+        pattern = "".join(pattern_parts)
+    else:
+        # Use the imported function if content is specified
+        pattern = get_video_key_regex_by_examination_alias(examination_alias, content, is_anonymous)
+        logger.warning(f"Generated pattern (from imported function): {pattern}")
+
+
+    # Only consider keys for which the file actually exists
+    keys_to_check = list(TEST_VIDEOS.keys())
+    matched_keys = [key for key in keys_to_check if re.match(pattern, key)]
+
+    # Fallback logic remains as a safety net, but ideally shouldn't be needed now for this case
+    if not matched_keys and is_anonymous is False:
+        logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=False. Falling back to suffix check '-non_anonymous'.")
+        matched_keys = [key for key in keys_to_check if key.endswith("-non_anonymous")]
+    elif not matched_keys and is_anonymous is True:
+         logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=True. Falling back to suffix check '-anonymous'.")
+         matched_keys = [key for key in keys_to_check if key.endswith("-anonymous")]
+
+    if not matched_keys:
+        logger.error(f"No keys found matching pattern '{pattern}' or fallback logic for keys: {keys_to_check}")
+
+
+    return matched_keys
+
+
+def get_random_video_path_by_examination_alias(
+    examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
+):
+    """
+    Returns the file path of a randomly selected video matching the specified examination alias, content type, and anonymity status.
+    
+    Raises:
+        ValueError: If no matching video keys are found for the given criteria.
+    """
+    keys = get_video_keys(examination_alias, content, is_anonymous)
+    if keys:
+        random_video_key = random.choice(keys)
+        video_path = get_video_path(random_video_key)
+        return video_path  # Return the first match for simplicity
+    else:
+        raise ValueError("No matching video keys found for the given criteria.")
+
+

endoreg_db/logger_conf.py

@@ -0,0 +1,140 @@
+import logging
+from pathlib import Path
+import os
+
+LOG_DIR = Path("data/logs")
+DEFAULT_FILE_LOG_LEVEL = "INFO"
+DEFAULT_CONSOLE_LOG_LEVEL = "WARNING"
+
+def clear_log_files(logger_names, log_dir=None):
+    """
+    Clears specified log files in the log directory.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        log_dir (Path, optional): The directory containing log files. Defaults to LOG_DIR.
+    """
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists (though clearing implies it might)
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    # Files to manage
+    log_files_to_clear = [log_dir / f"{logger_name}.log" for logger_name in logger_names]
+    log_files_to_clear.append(log_dir / "root.log") # Add root log file
+
+    # Clear existing log files in the directory managed by this config
+    for log_file in log_files_to_clear:
+        if log_file.exists():
+            try:
+                log_file.unlink()
+                print(f"Removed log file: {log_file}")
+            except OSError as e:
+                print(f"Error removing log file {log_file}: {e}")
+        # Optionally, create the log file to ensure it exists after clearing
+        # log_file.touch() # Removed touch to just clear
+
+def get_logging_config(logger_names, file_log_level=None, console_log_level=None, log_dir=None):
+    """
+    Generates Django LOGGING configuration dynamically.
+
+    Args:
+        logger_names (list): A list of strings, each being a logger name.
+        file_log_level (str, optional): The default log level for file handlers. Defaults to DEFAULT_FILE_LOG_LEVEL.
+        console_log_level (str, optional): The default log level for the console handler. Defaults to DEFAULT_CONSOLE_LOG_LEVEL.
+        log_dir (Path, optional): The directory to store log files. Defaults to LOG_DIR.
+
+    Returns:
+        dict: The Django LOGGING configuration dictionary.
+    """
+    file_log_level = file_log_level or os.environ.get("FILE_LOG_LEVEL", DEFAULT_FILE_LOG_LEVEL)
+    console_log_level = console_log_level or os.environ.get("CONSOLE_LOG_LEVEL", DEFAULT_CONSOLE_LOG_LEVEL)
+    log_dir = log_dir or LOG_DIR
+
+    # Ensure log directory exists
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    handlers = {
+        'console': {
+            'level': console_log_level, # Use console level
+            'class': 'logging.StreamHandler',
+            'formatter': 'standard',
+        },
+        'file_root': { # Handler for the root logger's file
+            'level': file_log_level, # Use file level
+            'class': 'logging.FileHandler',
+            'filename': log_dir / "root.log",
+            'formatter': 'standard',
+        },
+    }
+    loggers = {
+        # Root logger configuration - logs INFO+ to file, WARNING+ to console
+        '': {
+            'handlers': ['console', 'file_root'], # Use both handlers
+            'level': file_log_level, # Set to lowest level needed (INFO for file)
+            'propagate': False, # Root logger doesn't propagate further
+        },
+         # Django's default logger
+        'django': {
+            'handlers': ['console', 'file_root'], # Log to console and root file
+            'level': file_log_level, # Or your preferred level, ensure it's captured by root file
+            'propagate': False, # Don't propagate Django logs to avoid double handling by root
+        },
+    }
+
+    # Dynamically create file handlers and logger configurations
+    for name in logger_names:
+        handler_name = f"file_{name}"
+        handlers[handler_name] = {
+            'level': file_log_level, # Use file level
+            'class': 'logging.FileHandler',
+            'filename': log_dir / f"{name}.log",
+            'formatter': 'standard',
+        }
+        loggers[name] = {
+            'handlers': [handler_name], # Log only to its own file directly
+            'level': file_log_level, # Use file level
+            'propagate': True, # Propagate to root logger (which handles console and root.log)
+        }
+
+    return {
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'standard': {
+                'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            },
+        },
+        'handlers': handlers,
+        'loggers': loggers,
+    }
+
+# Example usage (optional, for testing the function itself)
+if __name__ == "__main__":
+    test_loggers = ["app1", "app2", "database"]
+    log_directory = Path("data/logs_test") # Use a test directory
+
+    # Optionally clear logs before configuring logging
+    print("Clearing logs...")
+    clear_log_files(test_loggers, log_dir=log_directory)
+    print("Log clearing complete.")
+
+    # Get logging configuration
+    logging_config = get_logging_config(test_loggers, file_log_level="DEBUG", console_log_level="WARNING", log_dir=log_directory)
+    import json
+    print("\nGenerated Logging Configuration:")
+    print(json.dumps(logging_config, indent=4))
+
+    # Example of how to use it with Python's logging
+    # import logging.config
+    # logging.config.dictConfig(logging_config)
+    # logger1 = logging.getLogger("app1")
+    # logger1.debug("This is a debug message for app1.")
+    # logger1.info("This is an info message for app1.")
+
+    # logger_db = logging.getLogger("database")
+    # logger_db.warning("This is a warning for database.")
+
+    # root_logger = logging.getLogger()
+    # root_logger.info("This is an info message from root.") # Should go to root.log and console (if level >= WARNING)
+    # root_logger.warning("This is a warning message from root.") # Should go to root.log and console

endoreg_db/management/__init__.py

@@ -0,0 +1 @@
+# Management commands package

endoreg_db/management/commands/__init__.py

@@ -0,0 +1 @@
+# Management commands