endoreg-db 0.6.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

endoreg_db/forms/questionnaires/tto_questionnaire.py

@@ -1,23 +1,23 @@
-# Path: endoreg_db/forms/questionnaires/tto_questionnaire.py
-from endoreg_db.models import TtoQuestionnaire
-from django import forms
+# # Path: endoreg_db/forms/questionnaires/tto_questionnaire.py
+# from endoreg_db.models import TtoQuestionnaire
+# from django import forms
 
-class TtoQuestionnaireForm(forms.ModelForm):
-    class Meta:
-        model = TtoQuestionnaire
-        fields = '__all__'
-        widgets = {
-            # Specify Bootstrap classes for other fields as needed
-            'patient_name': forms.TextInput(attrs={'class': 'form-control'}),
-            'birth_date': forms.DateInput(attrs={'class': 'form-control', 'type': 'date'}),
-            # You can continue to specify widgets for other fields
-        }
+# class TtoQuestionnaireForm(forms.ModelForm):
+#     class Meta:
+#         model = TtoQuestionnaire
+#         fields = '__all__'
+#         widgets = {
+#             # Specify Bootstrap classes for other fields as needed
+#             'patient_name': forms.TextInput(attrs={'class': 'form-control'}),
+#             'birth_date': forms.DateInput(attrs={'class': 'form-control', 'type': 'date'}),
+#             # You can continue to specify widgets for other fields
+#         }
 
 
-from django.views.generic.edit import CreateView
-from django.urls import reverse_lazy
+# from django.views.generic.edit import CreateView
+# from django.urls import reverse_lazy
 
-class TtoQuestionnaireCreate(CreateView):
-    model = TtoQuestionnaire
-    form_class = TtoQuestionnaireForm
-    success_url = reverse_lazy('questionnaire_success')  # Redirect to a success page after submission
+# class TtoQuestionnaireCreate(CreateView):
+#     model = TtoQuestionnaire
+#     form_class = TtoQuestionnaireForm
+#     success_url = reverse_lazy('questionnaire_success')  # Redirect to a success page after submission

endoreg_db/helpers/count_db.py

@@ -0,0 +1,45 @@
+# endoreg_db/helpers/count_db.py  
+from endoreg_db.models.state.audit_ledger import AuditLedger
+
+def _distinct(object_type: str, action: str):
+    """
+    Returns the count of distinct primary keys for records in AuditLedger filtered by object type and action.
+    
+    Args:
+        object_type: The type of object to filter by.
+        action: The action to filter by.
+    
+    Returns:
+        The number of unique object primary keys matching the specified type and action.
+    """
+    return (
+        AuditLedger.objects
+        .filter(object_type=object_type, action=action)
+        .values('object_pk')
+        .distinct()
+        .count()
+    )
+
+def collect_counters():
+    """
+    Returns a dictionary with statistical counts for cases, videos, annotations, anonymizations, images, and video statuses.
+    
+    The returned dictionary includes:
+    - "totalCases": Count of distinct created video files.
+    - "totalVideos": Count of distinct created video files.
+    - "totalAnnotations": Count of audit log entries where an annotation was added.
+    - "totalAnonymizations": Count of distinct video files that have been anonymized.
+    - "totalImages": Count of distinct created images.
+    - "videosCompleted": Count of distinct video files that have been validated.
+    - "videosAnonym": Count of distinct video files that have been anonymized.
+    """
+    return {
+        #TODO @maxhild can we remove the totalCases key?
+        # "totalCases":           _distinct("VideoFile", "created"),
+        "totalVideos":          _distinct("VideoFile", "created"),
+        "totalAnnotations":     AuditLedger.objects.filter(action="annotation_added").count(),
+        "totalAnonymizations":  _distinct("VideoFile", "anonymized"),
+        "totalImages":          _distinct("Image",     "created"),
+        "videosCompleted":      _distinct("VideoFile", "validated"),
+        "videosAnonym":         _distinct("VideoFile", "anonymized"),
+    }

endoreg_db/helpers/data_loader.py

@@ -0,0 +1,208 @@
+"""
+Data Loader Helpers and others
+"""
+from django.core.management import call_command
+from pathlib import Path
+ASSET_DIR = Path(__file__).resolve().parent.parent / "assets"
+
+def load_default_ai_model(): 
+    """
+    Loads the default AI model into the database using a predefined checkpoint file.
+    
+    This function constructs the path to the default AI model checkpoint and invokes the
+    Django management command `create_multilabel_model_meta` to register the model.
+    """
+    model_path = f"{ASSET_DIR.as_posix()}/colo_segmentation_RegNetX800MF_6.ckpt"
+    # Pass arguments individually to call_command
+    call_command(
+        "create_multilabel_model_meta",
+        "--model_path",
+        model_path
+    )
+
+def load_qualification_data():
+    """
+    Loads qualification data into the database using the corresponding Django management command.
+    """
+    call_command("load_qualification_data", )
+
+def load_shift_data():
+    """
+    Loads shift data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_shift_data", )
+
+def load_base_db_data():
+    """
+    Loads the base database data by invoking the corresponding Django management command.
+    """
+    call_command("load_base_db_data", )
+
+def load_information_source():
+    """
+    Loads information source data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_information_source", )
+
+def load_risk_data():
+    """
+    Loads risk data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_risk_data", )
+
+def load_center_data():
+    """
+    Loads center data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_center_data", )
+
+def load_endoscope_data():
+    """
+    Loads endoscope data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_endoscope_data", )
+
+def load_distribution_data():
+    """
+    Loads distribution data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_distribution_data", )
+
+def load_gender_data():
+    """
+    Loads gender data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_gender_data", )
+
+def load_report_reader_flag_data():
+    """
+    Loads the report reader flag data into the database using the corresponding management command.
+    """
+    call_command("load_report_reader_flag_data", )
+    
+def load_pdf_type_data():
+    """
+    Loads PDF type data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_pdf_type_data", )
+    
+def load_unit_data():
+    """
+    Loads unit data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_unit_data", )
+    
+def load_disease_data():
+    """
+    Loads disease data into the database using the corresponding Django management command.
+    """
+    call_command("load_disease_data", )
+    
+def load_event_data():
+    """
+    Loads event data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_event_data", )
+    
+def load_organ_data():
+    """
+    Loads organ data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_organ_data", )
+    
+def load_contraindication_data():
+    """
+    Loads contraindication data into the database using the corresponding management command.
+    """
+    call_command("load_contraindication_data", )
+    
+def load_examination_data():
+    """
+    Loads examination data into the database using the corresponding Django management command.
+    """
+    call_command("load_examination_data", )
+    
+def load_lab_value_data():
+    """
+    Loads laboratory value data into the database using the corresponding Django management command.
+    """
+    call_command("load_lab_value_data", )
+    
+def load_finding_data():
+    """
+    Loads finding data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_finding_data", )
+    
+def load_examination_indication_data():
+    """
+    Loads examination indication data into the database using the corresponding Django management command.
+    """
+    call_command("load_examination_indication_data", )
+    
+def load_medication_data():
+    """
+    Loads medication data into the database using the corresponding Django management command.
+    """
+    call_command("load_medication_data", )
+
+def load_requirement_data():
+
+    """
+    Loads requirement data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_requirement_data", )
+
+def load_ai_model_label_data():
+    """
+    Loads AI model label data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_ai_model_label_data", )
+
+def load_ai_model_data():    
+    """
+    Loads AI model data into the database by invoking the corresponding Django management command.
+    """
+    call_command("load_ai_model_data", )
+
+def load_green_endoscopy_wuerzburg_data():
+    """
+    Loads the Green Endoscopy Wuerzburg dataset into the database using the corresponding Django management command.
+    """
+    call_command("load_green_endoscopy_wuerzburg_data", )
+
+def load_data():
+    """
+    Loads all predefined data sets into the database in the required sequence.
+    
+    Calls individual data loading functions in a specific order to ensure that dependencies between data sets are respected.
+    """
+    
+    load_information_source()
+    load_risk_data()
+    load_center_data()
+    load_endoscope_data()
+    load_distribution_data()
+
+    load_gender_data()
+    load_report_reader_flag_data()
+    load_pdf_type_data()
+    load_unit_data()
+    load_disease_data()
+    load_event_data()
+    load_organ_data()
+    load_contraindication_data()
+    load_examination_data()
+    load_lab_value_data()
+    load_finding_data()
+    load_examination_indication_data()
+    load_medication_data()
+    load_requirement_data()
+
+    load_ai_model_label_data()
+    load_ai_model_data()
+
+    load_green_endoscopy_wuerzburg_data()
+
+    
+    

endoreg_db/helpers/default_objects.py

@@ -0,0 +1,359 @@
+import random
+from endoreg_db.models import (
+    Center, 
+    Gender, 
+    Patient,
+    Examination,
+    ExaminationIndication,
+    RawPdfFile,
+    EndoscopyProcessor,
+    ModelMeta,
+    InformationSource,
+    AiModel,
+)
+from logging import getLogger
+from datetime import date
+import shutil
+from pathlib import Path
+from django.conf import settings # Import settings
+from django.core.files.storage import default_storage # Import default storage
+
+from endoreg_db.utils import (
+    create_mock_patient_name,
+)
+
+logger = getLogger("default_objects")
+
+
+DEFAULT_CENTER_NAME = "university_hospital_wuerzburg"
+DEFAULT_ENDOSCOPE_NAME = "test_endoscope"
+DEFAULT_ENDOSCOPY_PROCESSOR_NAME = "olympus_cv_1500"
+
+DEFAULT_EGD_PATH = Path("tests/assets/lux-gastro-report.pdf")
+DEFAULT_GENDERS = ["male","female","unknown"]
+DEFAULT_EXAMINATIONS = ["colonoscopy"]
+DEFAULT_INDICATIONS = [
+    "colonoscopy",
+    "colonoscopy_screening",
+    "colonoscopy_lesion_removal_small",
+    "colonoscopy_lesion_removal_emr",
+    "colonoscopy_lesion_removal_large",
+    "colonoscopy_diagnostic_acute_symptomatic",
+]
+
+DEFAULT_SEGMENTATION_MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
+
+DEFAULT_GENDER = "unknown"
+
+def get_information_source_prediction():
+    """
+    Retrieves the InformationSource object with the name "prediction".
+    
+    Loads information source data if needed and returns the corresponding InformationSource instance. Raises a ValueError if the object is not found or is not an InformationSource.
+    """
+    from .data_loader import load_information_source
+    load_information_source()
+    source = InformationSource.objects.get(name="prediction")
+    if not isinstance(source, InformationSource):
+        raise ValueError("No InformationSource found in the database.")
+    return source
+
+def get_latest_segmentation_model(model_name:str=DEFAULT_SEGMENTATION_MODEL_NAME) -> ModelMeta:
+
+    """
+    Retrieves the latest metadata for a segmentation model by name.
+    
+    Loads necessary data and returns the most recent ModelMeta instance for the specified AI model. If no metadata exists, attempts to initialize it automatically; if initialization fails, raises a ValueError with instructions for manual setup.
+    
+    Args:
+        model_name: The name of the segmentation model to retrieve.
+    
+    Returns:
+        The latest ModelMeta instance for the specified model.
+    
+    Raises:
+        ValueError: If the AI model does not exist, or if model metadata cannot be found or initialized.
+    """
+    from .data_loader import (
+        load_center_data,
+        load_ai_model_label_data,
+        load_ai_model_data,
+    )
+    load_center_data()
+    load_ai_model_label_data()
+    load_ai_model_data()
+    
+    try:
+        ai_model = AiModel.objects.get(name=model_name)
+    except AiModel.DoesNotExist:
+        raise ValueError(f"AI model '{model_name}' not found. Run 'python manage.py load_ai_model_data' first.")
+    
+    try:
+        latest_meta = ai_model.get_latest_version()
+        return latest_meta
+    except ValueError as e:
+        if "No model metadata found" in str(e):
+            logger.warning(f"No ModelMeta found for {model_name}. Attempting to initialize default model metadata...")
+            
+            # Try to initialize the default model metadata
+            try:
+                from django.core.management import call_command
+                call_command('init_default_ai_model')
+                # Try again after initialization
+                latest_meta = ai_model.get_latest_version()
+                return latest_meta
+            except Exception as init_error:
+                raise ValueError(
+                    f"No model metadata found for AI model '{model_name}' and failed to auto-initialize. "
+                    f"Please run 'python manage.py init_default_ai_model' manually. "
+                    f"Original error: {e}. Initialization error: {init_error}"
+                ) from e
+        else:
+            raise
+    
+
+def get_default_gender() -> Gender:
+    """
+    Retrieves the Gender object representing the default "unknown" gender.
+    
+    Returns:
+        The Gender instance with the name "unknown".
+    """
+    return Gender.objects.get(name=DEFAULT_GENDER)
+
+def get_gender_m_or_f() -> Gender:
+    """
+    Returns a randomly selected Gender object representing either male or female.
+    """
+    gender_name = random.choice(["male", "female"])
+    return Gender.objects.get(name=gender_name)
+
+def get_random_gender() -> Gender:
+    """
+    Returns a randomly selected Gender object from the available default genders.
+    """
+    gender_name = random.choice(DEFAULT_GENDERS)
+    return Gender.objects.get(name=gender_name) # Fetch and return the Gender object
+
+def get_default_processor() -> EndoscopyProcessor:
+    """
+    Retrieves the default EndoscopyProcessor object by its predefined name.
+    
+    Raises:
+        ValueError: If no EndoscopyProcessor with the default name exists.
+        
+    Returns:
+        The EndoscopyProcessor instance with the default name.
+    """
+    processor = EndoscopyProcessor.objects.get(name=DEFAULT_ENDOSCOPY_PROCESSOR_NAME)
+    if not isinstance(processor, EndoscopyProcessor):
+        raise ValueError(f"No EndoscopyProcessor found with name {DEFAULT_ENDOSCOPY_PROCESSOR_NAME}")
+    return processor
+
+
+def get_default_center() -> Center:
+    """
+    Retrieves the default Center object with the predefined name.
+    
+    Raises:
+        ValueError: If no Center with the default name exists.
+    
+    Returns:
+        The Center instance with the default name.
+    """
+    center = Center.objects.get(
+        name=DEFAULT_CENTER_NAME,
+    )
+    if not isinstance(center, Center):
+        raise ValueError(f"No Center found with name {DEFAULT_CENTER_NAME}")
+    
+    return center
+
+def generate_patient(**kwargs) -> Patient:
+    """
+    Creates a Patient instance with randomized or specified attributes.
+    
+    Randomly generates first name, last name, date of birth, gender, and center for the patient unless overridden by keyword arguments. Raises ValueError if the provided gender is invalid.
+    
+    Args:
+        **kwargs: Optional overrides for patient attributes such as 'first_name', 'last_name', 'birth_date', 'gender', and 'center'.
+    
+    Returns:
+        A Patient instance with the specified or randomly generated attributes.
+    """
+    # Set default values
+    gender = kwargs.get("gender", get_random_gender())
+    if not isinstance(gender, Gender):
+        gender = Gender.objects.get(name=gender)
+
+    if not isinstance(gender, Gender):
+        raise ValueError("No Gender Found")
+    first_name, last_name = create_mock_patient_name(gender = gender.name)
+    first_name = kwargs.get("first_name", first_name)
+    last_name = kwargs.get("last_name", last_name)
+    birth_date = kwargs.get("birth_date", "1970-01-01")
+    dob = date.fromisoformat(birth_date)
+    center = kwargs.get("center", None)
+    if center is None:
+        center = get_default_center()
+    else:
+        center = Center.objects.get(name=center)
+
+    patient = Patient(
+        first_name=first_name,
+        last_name=last_name,
+        dob=dob,
+        center = center,
+        gender = gender,
+    )
+
+    return patient
+    
+def get_random_default_examination():
+    """
+    Retrieves a random Examination object from the default examination names.
+    
+    Returns:
+        Examination: A randomly selected Examination instance from the defaults.
+    """
+    examination_name = random.choice(DEFAULT_EXAMINATIONS)
+
+    examination = Examination.objects.get(name=examination_name)
+    return examination
+
+def get_random_default_examination_indication():
+    """
+    Returns a random ExaminationIndication object from the default indications list.
+    
+    Selects a random indication name from the predefined defaults and retrieves the corresponding ExaminationIndication instance from the database.
+    """
+    examination_indication = random.choice(DEFAULT_INDICATIONS)
+    all_examination_indications = ExaminationIndication.objects.all()
+    try:
+        examination_indication = ExaminationIndication.objects.get(name=examination_indication)
+        
+    except Exception as e:
+        logger.info(f"examination_indication: {examination_indication}")
+        logger.info(f"all_examination_indications: {all_examination_indications}")
+        raise e
+    return examination_indication
+
+def get_default_egd_pdf():
+    """
+    Creates and processes a default EGD PDF file for testing purposes.
+    
+    This function copies a default EGD PDF to a temporary location, creates a RawPdfFile instance from it, processes the file to generate associated metadata, and ensures cleanup of the temporary file. The resulting RawPdfFile instance is returned for use in tests.
+    
+    Returns:
+        RawPdfFile: The created and processed RawPdfFile instance.
+    """
+    egd_path = DEFAULT_EGD_PATH
+    center = get_default_center()
+    center_name = center.name
+
+    # Create a temporary file path within the test's media root if possible,
+    # otherwise use the source directory. Using MEDIA_ROOT is safer.
+    # Ensure MEDIA_ROOT is configured correctly in test settings.
+    temp_dir = Path(settings.MEDIA_ROOT) / "temp_test_files"
+    temp_dir.mkdir(parents=True, exist_ok=True)
+    temp_file_path = temp_dir / f"temp_{egd_path.name}"
+
+    shutil.copy(egd_path, temp_file_path)
+
+    pdf_file = None
+    try:
+        # Create the PDF record using the temporary file.
+        # delete_source=True will ensure temp_file_path is deleted by create_from_file
+        pdf_file = RawPdfFile.create_from_file(
+            file_path=temp_file_path,
+            center_name=center_name,
+            save=True, # save=True is default and handled internally now
+            delete_source=True,
+        )
+
+        if pdf_file is None:
+            raise RuntimeError("Failed to create PDF file object")
+        
+        # Use storage API to check existence
+        if not default_storage.exists(pdf_file.file.path):
+            raise RuntimeError(f"PDF file does not exist in storage at {pdf_file.file.path}")
+        
+        # Check that the source temp file was deleted
+        if temp_file_path.exists():
+            raise RuntimeError(f"Temporary source file {temp_file_path} still exists after creation")
+
+        # Prepare a minimal report_meta for SensitiveMeta creation
+        default_report_meta = {
+            "patient_first_name": "DefaultFirstName",
+            "patient_last_name": "DefaultLastName",
+            "patient_dob": date(1980, 1, 1), # Pass date object directly
+            "examination_date": date(2024, 1, 1), # Pass date object directly
+            # center_name will be added by process_file using pdf_file.center.name
+        }
+
+        # Call process_file to create SensitiveMeta and extract other info
+        pdf_file.process_file(
+            text="Default PDF text content.",
+            anonymized_text="Default anonymized PDF text content.",
+            report_meta=default_report_meta,
+            verbose=False
+        )
+        # process_file calls sensitive_meta.save() and self.save() (for RawPdfFile)
+
+    except Exception as e:
+        # Clean up temp file in case of error before deletion could occur
+        if temp_file_path.exists():
+            temp_file_path.unlink()
+        raise e # Re-raise the exception
+
+    # pdf_file.file.path might fail if storage doesn't support direct paths (like S3)
+    # Prefer using storage API for checks. Logging path if available.
+    try:
+        logger.info(f"PDF file created: {pdf_file.file.name}, Path: {pdf_file.file.path}")
+    except NotImplementedError:
+        logger.info(f"PDF file created: {pdf_file.file.name}, Path: (Not available from storage)")
+
+
+    return pdf_file
+
+def get_default_video_file():
+    """
+    Creates and returns a VideoFile instance using a randomly selected EGD examination video.
+    
+    Loads all necessary data dependencies, selects a random video file for the 'egd' examination, and initializes a VideoFile object with the default center and processor names. The original video file is retained after creation.
+    
+    Returns:
+        VideoFile: The created and initialized VideoFile instance.
+    """
+    from .test_video_helper import get_random_video_path_by_examination_alias
+    from endoreg_db.models import VideoFile
+    from .data_loader import (
+        load_disease_data,
+        load_event_data,
+        load_information_source,
+        load_examination_data,
+        load_center_data,
+        load_endoscope_data,
+        load_ai_model_label_data,
+        load_ai_model_data,
+    )
+    load_disease_data()
+    load_event_data()
+    load_information_source()
+    load_examination_data()
+    load_center_data()
+    load_endoscope_data()
+    load_ai_model_label_data()
+    load_ai_model_data()
+    video_path = get_random_video_path_by_examination_alias(
+        examination_alias='egd', is_anonymous=False
+    )
+
+    video_file = VideoFile.create_from_file_initialized(
+        file_path=video_path,
+        center_name=DEFAULT_CENTER_NAME,  # Pass center name as expected by _create_from_file
+        delete_source=False,  # Keep the original asset for other tests
+        processor_name = DEFAULT_ENDOSCOPY_PROCESSOR_NAME,
+    )
+
+    return video_file

endoreg_db/helpers/interact.py

@@ -0,0 +1,6 @@
+from endoreg_db.models import EndoscopyProcessor
+from django.core.management.base import BaseCommand
+from typing import TYPE_CHECKING
+from django.db.models import QuerySet
+
+