endoreg-db 0.8.9.32__py3-none-any.whl

endoreg_db/helpers/default_objects.py

@@ -0,0 +1,414 @@
+import random
+import shutil
+from datetime import date
+from logging import getLogger
+from pathlib import Path
+from typing import Optional
+
+from django.conf import settings  # Import settings
+from django.core.files.storage import default_storage  # Import default storage
+from django.db.models.fields.files import FieldFile
+from endoreg_db.utils.file_operations import sha256_file
+from endoreg_db.models import (
+    AiModel,
+    Center,
+    EndoscopyProcessor,
+    Examination,
+    ExaminationIndication,
+    Gender,
+    InformationSource,
+    ModelMeta,
+    Patient,
+    RawPdfFile,
+)
+from endoreg_db.utils import create_mock_patient_name
+
+logger = getLogger("default_objects")
+
+
+DEFAULT_CENTER_NAME = "university_hospital_wuerzburg"
+DEFAULT_ENDOSCOPE_NAME = "test_endoscope"
+DEFAULT_ENDOSCOPY_PROCESSOR_NAME = "olympus_cv_1500"
+
+DEFAULT_EGD_PATH = Path("tests/assets/lux-gastro-report.pdf")
+DEFAULT_GENDERS = ["male", "female", "unknown"]
+DEFAULT_EXAMINATIONS = ["colonoscopy"]
+DEFAULT_INDICATIONS = [
+    "colonoscopy",
+    "colonoscopy_screening",
+    "colonoscopy_lesion_removal_small",
+    "colonoscopy_lesion_removal_emr",
+    "colonoscopy_lesion_removal_large",
+    "colonoscopy_diagnostic_acute_symptomatic",
+]
+
+DEFAULT_SEGMENTATION_MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
+
+DEFAULT_GENDER = "unknown"
+DEFAULT_PATIENT_FIRST_NAME = "TestFirst"
+DEFAULT_PATIENT_LAST_NAME = "TestLast"
+DEFAULT_PATIENT_GENDER_NAME = "female"
+DEFAULT_PATIENT_BIRTH_DATE = date(1970, 1, 1)
+
+
+def get_information_source_prediction():
+    """
+    Retrieves the InformationSource object with the name "prediction".
+
+    Loads information source data if needed and returns the corresponding InformationSource instance. Raises a ValueError if the object is not found or is not an InformationSource.
+    """
+    from .data_loader import load_information_source
+
+    load_information_source()
+    source = InformationSource.objects.get(name="prediction")
+    if not isinstance(source, InformationSource):
+        raise ValueError("No InformationSource found in the database.")
+    return source
+
+
+def get_latest_segmentation_model(
+    model_name: str = DEFAULT_SEGMENTATION_MODEL_NAME,
+) -> ModelMeta:
+    """
+    Retrieves the latest metadata for a segmentation model by name.
+
+    Loads necessary data and returns the most recent ModelMeta instance for the specified AI model. If no metadata exists, attempts to initialize it automatically; if initialization fails, raises a ValueError with instructions for manual setup.
+
+    Args:
+        model_name: The name of the segmentation model to retrieve.
+
+    Returns:
+        The latest ModelMeta instance for the specified model.
+
+    Raises:
+        ValueError: If the AI model does not exist, or if model metadata cannot be found or initialized.
+    """
+    from .data_loader import (
+        load_ai_model_data,
+        load_ai_model_label_data,
+        load_center_data,
+    )
+
+    load_center_data()
+    load_ai_model_label_data()
+    load_ai_model_data()
+
+    try:
+        ai_model = AiModel.objects.get(name=model_name)
+    except AiModel.DoesNotExist:
+        raise ValueError(
+            f"AI model '{model_name}' not found. Run 'python manage.py load_ai_model_data' first."
+        )
+
+    try:
+        latest_meta = ai_model.get_latest_version()
+        return latest_meta
+    except ValueError as e:
+        if "No model metadata found" in str(e):
+            logger.warning(
+                f"No ModelMeta found for {model_name}. Attempting to initialize default model metadata..."
+            )
+
+            # Try to initialize the default model metadata
+            try:
+                from django.core.management import call_command
+
+                call_command("init_default_ai_model")
+                # Try again after initialization
+                latest_meta = ai_model.get_latest_version()
+                return latest_meta
+            except Exception as init_error:
+                raise ValueError(
+                    f"No model metadata found for AI model '{model_name}' and failed to auto-initialize. "
+                    f"Please run 'python manage.py init_default_ai_model' manually. "
+                    f"Original error: {e}. Initialization error: {init_error}"
+                ) from e
+        else:
+            raise
+
+
+def get_default_gender() -> Gender:
+    """
+    Retrieves the Gender object representing the default "unknown" gender.
+
+    Returns:
+        The Gender instance with the name "unknown".
+    """
+    return Gender.objects.get(name=DEFAULT_GENDER)
+
+
+def get_gender_m_or_f() -> Gender:
+    """
+    Returns a randomly selected Gender object representing either male or female.
+    """
+    gender_name = random.choice(["male", "female"])
+    return Gender.objects.get(name=gender_name)
+
+
+def get_random_gender() -> Gender:
+    """
+    Returns a randomly selected Gender object from the available default genders.
+    """
+    gender_name = random.choice(DEFAULT_GENDERS)
+    return Gender.objects.get(name=gender_name)  # Fetch and return the Gender object
+
+
+def get_default_processor() -> EndoscopyProcessor:
+    """
+    Retrieves the default EndoscopyProcessor object by its predefined name.
+
+    Raises:
+        ValueError: If no EndoscopyProcessor with the default name exists.
+
+    Returns:
+        The EndoscopyProcessor instance with the default name.
+    """
+    processor = EndoscopyProcessor.objects.get(name=DEFAULT_ENDOSCOPY_PROCESSOR_NAME)
+    if not isinstance(processor, EndoscopyProcessor):
+        raise ValueError(
+            f"No EndoscopyProcessor found with name {DEFAULT_ENDOSCOPY_PROCESSOR_NAME}"
+        )
+    return processor
+
+
+def get_default_center() -> Center:
+    """
+    Retrieves the default Center object with the predefined name.
+
+    Raises:
+        ValueError: If no Center with the default name exists.
+
+    Returns:
+        The Center instance with the default name.
+    """
+    center = Center.objects.get(
+        name=DEFAULT_CENTER_NAME,
+    )
+    if not isinstance(center, Center):
+        raise ValueError(f"No Center found with name {DEFAULT_CENTER_NAME}")
+
+    return center
+
+
+def generate_patient(**kwargs) -> Patient:
+    """Create a Patient with deterministic defaults unless ``randomize=True`` is supplied."""
+
+    randomize = kwargs.pop("randomize", False)
+
+    gender = kwargs.get("gender")
+    if gender is None:
+        if randomize:
+            gender = get_random_gender()
+        else:
+            gender = Gender.objects.get(name=DEFAULT_PATIENT_GENDER_NAME)
+    elif not isinstance(gender, Gender):
+        gender = Gender.objects.get(name=gender)
+
+    first_name = kwargs.get("first_name")
+    last_name = kwargs.get("last_name")
+    if first_name is None or last_name is None:
+        if randomize:
+            generated_first, generated_last = create_mock_patient_name(
+                gender=gender.name
+            )
+        else:
+            generated_first, generated_last = (
+                DEFAULT_PATIENT_FIRST_NAME,
+                DEFAULT_PATIENT_LAST_NAME,
+            )
+        first_name = first_name or generated_first
+        last_name = last_name or generated_last
+
+    dob = kwargs.get("dob")
+    if dob is None:
+        birth_date = kwargs.get("birth_date", DEFAULT_PATIENT_BIRTH_DATE)
+        if isinstance(birth_date, date):
+            dob = birth_date
+        else:
+            dob = date.fromisoformat(str(birth_date))
+
+    center = kwargs.get("center")
+    if center is None:
+        center = get_default_center()
+    elif not isinstance(center, Center):
+        center = Center.objects.get(name=center)
+
+    patient = Patient(
+        first_name=first_name,
+        last_name=last_name,
+        dob=dob,
+        center=center,
+        gender=gender,
+    )
+
+    return patient
+
+
+def get_random_default_examination():
+    """
+    Retrieves a random Examination object from the default examination names.
+
+    Returns:
+        Examination: A randomly selected Examination instance from the defaults.
+    """
+    examination_name = random.choice(DEFAULT_EXAMINATIONS)
+
+    examination = Examination.objects.get(name=examination_name)
+    return examination
+
+
+def get_random_default_examination_indication():
+    """
+    Returns a random ExaminationIndication object from the default indications list.
+
+    Selects a random indication name from the predefined defaults and retrieves the corresponding ExaminationIndication instance from the database.
+    """
+    examination_indication = random.choice(DEFAULT_INDICATIONS)
+    all_examination_indications = ExaminationIndication.objects.all()
+    try:
+        examination_indication = ExaminationIndication.objects.get(
+            name=examination_indication
+        )
+
+    except Exception as e:
+        logger.info(f"examination_indication: {examination_indication}")
+        logger.info(f"all_examination_indications: {all_examination_indications}")
+        raise e
+    return examination_indication
+
+
+def get_default_egd_pdf():
+    """
+    Creates and processes a default EGD report file for testing purposes.
+
+    This function copies a default EGD report to a temporary location, creates a RawPdfFile instance from it, processes the file to generate associated metadata, and ensures cleanup of the temporary file. The resulting RawPdfFile instance is returned for use in tests.
+
+    Returns:
+        RawPdfFile: The created and processed RawPdfFile instance.
+    """
+    egd_path = DEFAULT_EGD_PATH
+    center = get_default_center()
+    center_name = center.name
+
+    # Create a temporary file path within the test's media root if possible,
+    # otherwise use the source directory. Using MEDIA_ROOT is safer.
+    # Ensure MEDIA_ROOT is configured correctly in test settings.
+    temp_dir = Path(settings.MEDIA_ROOT) / "temp_test_files"
+    temp_dir.mkdir(parents=True, exist_ok=True)
+    temp_file_path = temp_dir / f"temp_{egd_path.name}"
+
+    shutil.copy(egd_path, temp_file_path)
+
+    pdf_file = None
+    file_field: Optional[FieldFile] = None
+    try:
+        # Create the report record using the temporary file.
+        # delete_source=True will ensure temp_file_path is deleted by create_from_file
+        pdf_file = RawPdfFile.create_from_file(
+            file_path=temp_file_path,
+            center_name=center_name,
+            save=True,  # save=True is default and handled internally now
+            delete_source=True,
+        )
+
+        if pdf_file is None:
+            raise RuntimeError("Failed to create report file object")
+
+        # Use storage API to check existence
+        file_field = pdf_file.file
+        if not isinstance(file_field, FieldFile):
+            raise RuntimeError("RawPdfFile.file did not return a FieldFile instance")
+        if not default_storage.exists(file_field.path):
+            raise RuntimeError(
+                f"report file does not exist in storage at {file_field.path}"
+            )
+
+        # Check that the source temp file was deleted
+        if temp_file_path.exists():
+            raise RuntimeError(
+                f"Temporary source file {temp_file_path} still exists after creation"
+            )
+
+        # Prepare a minimal report_meta for SensitiveMeta creation
+        default_report_meta = {
+            "patient_first_name": "DefaultFirstName",
+            "patient_last_name": "DefaultLastName",
+            "patient_dob": date(1980, 1, 1),  # Pass date object directly
+            "examination_date": date(2024, 1, 1),  # Pass date object directly
+            # center_name will be added by process_file using pdf_file.center.name
+        }
+
+        # Call process_file to create SensitiveMeta and extract other info
+        pdf_file.process_file(
+            text="Default report text content.",
+            anonymized_text="Default anonymized report text content.",
+            report_meta=default_report_meta,
+            verbose=False,
+        )
+        # process_file calls sensitive_meta.save() and self.save() (for RawPdfFile)
+
+    except Exception as e:
+        # Clean up temp file in case of error before deletion could occur
+        if temp_file_path.exists():
+            temp_file_path.unlink()
+        raise e  # Re-raise the exception
+
+    # pdf_file.file.path might fail if storage doesn't support direct paths (like S3)
+    # Prefer using storage API for checks. Logging path if available.
+    if file_field is not None:
+        try:
+            logger.info(
+                f"report file created: {file_field.name}, Path: {file_field.path}"
+            )
+        except NotImplementedError:
+            logger.info(
+                f"report file created: {file_field.name}, Path: (Not available from storage)"
+            )
+
+    return pdf_file
+
+
+def get_default_video_file():
+    """
+    Creates and returns a VideoFile instance using a randomly selected EGD examination video.
+
+    Loads all necessary data dependencies, selects a random video file for the 'egd' examination, and initializes a VideoFile object with the default center and processor names. The original video file is retained after creation.
+
+    Returns:
+        VideoFile: The created and initialized VideoFile instance.
+    """
+    from endoreg_db.models import VideoFile
+
+    from .data_loader import (
+        load_ai_model_data,
+        load_ai_model_label_data,
+        load_center_data,
+        load_disease_data,
+        load_endoscope_data,
+        load_event_data,
+        load_examination_data,
+        load_information_source,
+    )
+    from .test_video_helper import get_random_video_path_by_examination_alias
+
+    load_disease_data()
+    load_event_data()
+    load_information_source()
+    load_examination_data()
+    load_center_data()
+    load_endoscope_data()
+    load_ai_model_label_data()
+    load_ai_model_data()
+    video_path = get_random_video_path_by_examination_alias(
+        examination_alias="egd", is_anonymous=False
+    )
+
+    video_file = VideoFile.create_from_file_initialized(
+        file_path=video_path,
+        center_name=DEFAULT_CENTER_NAME,  # Pass center name as expected by _create_from_file
+        delete_source=False,  # Keep the original asset for other tests
+        processor_name=DEFAULT_ENDOSCOPY_PROCESSOR_NAME,
+        video_hash=sha256_file(video_path),
+    )
+
+    return video_file

endoreg_db/helpers/download_segmentation_model.py

@@ -0,0 +1,32 @@
+import huggingface_hub
+from typing import Optional
+
+
+def download_segmentation_model(
+    repo_id: str = "wg-lux/colo_segmentation_RegNetX800MF_base",
+    filename: str = "model.safetensors",
+    cache_dir: Optional[str] = None,
+) -> str:
+    """
+    Downloads a segmentation model from Hugging Face and caches it locally.
+
+    Args:
+        repo_id (str): The Hugging Face repository ID (default: wg-lux model).
+        filename (str): The specific file to download from the repo (default: model.safetensors).
+        cache_dir (str): The directory to cache the downloaded model. If None, uses HF default cache.
+
+    Returns:
+        str: The local path to the downloaded model.
+
+    Example:
+        >>> model_path = download_segmentation_model()
+        >>> # Downloads from wg-lux/colo_segmentation_RegNetX800MF_base
+    """
+    local_path = huggingface_hub.hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        cache_dir=cache_dir,
+        force_download=False,
+        resume_download=True,
+    )
+    return local_path

endoreg_db/helpers/interact.py

@@ -0,0 +1 @@
+

endoreg_db/helpers/test_video_helper.py

@@ -0,0 +1,127 @@
+from pathlib import Path
+import random
+import re
+from typing import Optional
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+
+from endoreg_db.utils.video.names import get_video_key_regex_by_examination_alias
+
+from django.conf import settings
+
+
+ASSET_DIR: Path = settings.ASSET_DIR
+assert ASSET_DIR.exists(), f"ASSET_DIR does not exist: {ASSET_DIR}"
+
+_TEST_VIDEOS = {
+    # "egd-instrument-non_anonymous": ASSET_DIR / "test_instrument.mp4", # No detected segments
+    "egd-endoscope-non_anonymous": ASSET_DIR / "test_endoscope.mp4",
+    "egd-nbi-non_anonymous": ASSET_DIR / "test_nbi.mp4",
+    # "egd-outside-non_anonymous": ASSET_DIR / "test_outside.mp4",
+    "egd-small_intestine-non_anonymous": ASSET_DIR / "test_small_intestine.mp4",
+}
+
+# Only keep entries that actually exist on disk to avoid returning None paths
+TEST_VIDEOS = {key: value for key, value in _TEST_VIDEOS.items() if value.exists()}
+
+
+def get_video_path(video_key: str) -> Path:
+    """
+    Returns the file path associated with the given video key.
+
+    Raises:
+        ValueError: If the video key does not exist in the available test videos.
+    """
+    if video_key in TEST_VIDEOS:
+        return TEST_VIDEOS[video_key]
+    else:
+        raise ValueError(f"Video key '{video_key}' not found in TEST_VIDEOS.")
+
+
+def get_video_keys(
+    examination_alias: Optional[str] = None,
+    content: Optional[str] = None,
+    is_anonymous: Optional[bool] = None,
+):
+    """
+    Returns a list of video keys matching the specified examination alias, content type, and anonymity status.
+
+    If no direct matches are found, falls back to suffix-based filtering for anonymity. Logs warnings and errors when fallback logic is used or no matches are found.
+
+    Args:
+        examination_alias: The examination alias to filter by, or None for any.
+        content: The content type to filter by, or None for any.
+        is_anonymous: Whether to filter for anonymous, non-anonymous, or both.
+
+    Returns:
+        A list of matching video keys.
+    """
+    # Build a more flexible pattern locally if content is None
+    if content is None:
+        pattern_parts = ["^"]
+        if examination_alias:
+            pattern_parts.append(re.escape(examination_alias))
+            pattern_parts.append("-.*-")  # Match any content
+        else:
+            pattern_parts.append(".*-")  # Match any examination alias and content
+
+        if is_anonymous is True:
+            pattern_parts.append("anonymous$")
+        elif is_anonymous is False:
+            pattern_parts.append("non_anonymous$")
+        else:  # is_anonymous is None
+            pattern_parts.append("(non_)?anonymous$")  # Match either
+
+        pattern = "".join(pattern_parts)
+    else:
+        # Use the imported function if content is specified
+        pattern = get_video_key_regex_by_examination_alias(
+            examination_alias, content, is_anonymous
+        )
+        logger.warning(f"Generated pattern (from imported function): {pattern}")
+
+    # Only consider keys for which the file actually exists
+    keys_to_check = list(TEST_VIDEOS.keys())
+    matched_keys = [key for key in keys_to_check if re.match(pattern, key)]
+
+    # Fallback logic remains as a safety net, but ideally shouldn't be needed now for this case
+    if not matched_keys and is_anonymous is False:
+        logger.warning(
+            f"Pattern '{pattern}' yielded no results for is_anonymous=False. Falling back to suffix check '-non_anonymous'."
+        )
+        matched_keys = [key for key in keys_to_check if key.endswith("-non_anonymous")]
+    elif not matched_keys and is_anonymous is True:
+        logger.warning(
+            f"Pattern '{pattern}' yielded no results for is_anonymous=True. Falling back to suffix check '-anonymous'."
+        )
+        matched_keys = [key for key in keys_to_check if key.endswith("-anonymous")]
+
+    if not matched_keys:
+        logger.error(
+            f"No keys found matching pattern '{pattern}' or fallback logic for keys: {keys_to_check}"
+        )
+
+    return matched_keys
+
+
+def get_random_video_path_by_examination_alias(
+    examination_alias: Optional[str] = None,
+    content: Optional[str] = None,
+    is_anonymous: Optional[bool] = None,
+):
+    """
+    Returns the file path of a randomly selected video matching the specified examination alias, content type, and anonymity status.
+
+    Raises:
+        ValueError: If no matching video keys are found for the given criteria.
+    """
+    keys = get_video_keys(examination_alias, content, is_anonymous)
+    if keys:
+        random_video_key = random.choice(keys)
+        video_path = get_video_path(random_video_key)
+        return video_path  # Return the first match for simplicity
+    else:
+        raise ValueError("No matching video keys found for the given criteria.")

endoreg_db/import_files/__init__.py

@@ -0,0 +1,27 @@
+# Re-export context helpers
+from .context.file_lock import file_lock
+from .context.default_sensitive_meta import default_sensitive_meta  # if needed
+from .context.import_context import ImportContext  # if needed
+from .context.validate_directories import validate_directories  # if needed
+
+# Re-export storage helpers
+from .file_storage import create_report_file
+from .file_storage import create_video_file
+from .file_storage import sensitive_meta_storage
+
+# Re-export import services
+from .report_import_service import ReportImportService
+from .video_import_service import VideoImportService
+
+# Public API
+__all__ = [
+    "file_lock",
+    "create_report_file",
+    "create_video_file",
+    "sensitive_meta_storage",
+    "ReportImportService",
+    "VideoImportService",
+    "ImportContext",
+    "validate_directories",
+    "default_sensitive_meta",
+]

endoreg_db/import_files/context/__init__.py

@@ -0,0 +1,7 @@
+from .file_lock import file_lock
+from .import_context import ImportContext
+
+__all__ = [
+    "file_lock",
+    "ImportContext",
+]

endoreg_db/import_files/context/default_sensitive_meta.py

@@ -0,0 +1,83 @@
+# endoreg_db/import_files/processing/create_sensitive_meta.py
+
+import os
+import logging
+from datetime import date
+from typing import Union
+
+from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta  # adjust path
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PATIENT_FIRST_NAME = "unknown"
+DEFAULT_PATIENT_LAST_NAME = "unknown"
+DEFAULT_CENTER_NAME = "endoreg_db_demo"
+# DEFAULT_PATIENT_DOB can be a fixed date or None to let logic.generate_random_dob handle it
+DEFAULT_PATIENT_DOB = date(1970, 1, 1)
+
+
+def default_sensitive_meta(
+    instance: Union[RawPdfFile, VideoFile],
+) -> SensitiveMeta | None:
+    """
+    Ensure the given instance has a minimal SensitiveMeta attached.
+
+    Called after text extraction + merging; only creates meta if none exists.
+    """
+    if instance is None:
+        logger.warning("No instance available for ensuring default patient data")
+        return
+
+    if instance.sensitive_meta:
+        # Already has meta; nothing to do
+        return
+
+    logger.info(
+        "No SensitiveMeta found for report %s, creating default",
+        getattr(instance, "pdf_hash", instance.pk),
+    )
+    if not isinstance(instance.center.name, str):
+        try:
+            center_name = os.environ.get("DEFAULT_CENTER_NAME")
+            assert center_name is not None
+            instance.center.name = center_name
+        except AssertionError as e:
+            logger.debug(
+                f"{e}Center name is not set! You can set it in .env under DEFAULT_CENTER_NAME using default from default_sensitive_meta"
+            )
+            instance.center.name = DEFAULT_CENTER_NAME
+            instance.center.get_by_name(DEFAULT_CENTER_NAME)
+
+    default_data = {
+        "patient_first_name": DEFAULT_PATIENT_FIRST_NAME,
+        "patient_last_name": DEFAULT_PATIENT_LAST_NAME,
+        "patient_dob": DEFAULT_PATIENT_DOB,
+        "examination_date": date.today(),
+        "center_name": (
+            instance.center.name
+            if getattr(instance, "center", None) is not None
+            else DEFAULT_CENTER_NAME
+        ),
+        # optional: link file_path for debugging/tracing
+        "file_path": str(instance.file_path)
+        if getattr(instance, "file_path", None)
+        else None,
+    }
+
+    try:
+        meta = SensitiveMeta.create_from_dict(default_data)
+        instance.sensitive_meta = meta
+        instance.save(update_fields=["sensitive_meta"])
+        logger.info(
+            "Created default SensitiveMeta for report %s",
+            getattr(instance, "pdf_hash", instance.pk),
+        )
+        return meta
+    except Exception as e:
+        logger.error(
+            "Failed to create default SensitiveMeta for report %s: %s",
+            getattr(instance, "pdf_hash", instance.pk),
+            e,
+        )
+        return None