endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.9.2__py3-none-any.whl

endoreg_db/helpers/default_objects.py

@@ -1,28 +1,27 @@
 import random
+import shutil
+from datetime import date
+from logging import getLogger
+from pathlib import Path
 from typing import Optional
+
+from django.conf import settings  # Import settings
+from django.core.files.storage import default_storage  # Import default storage
+from django.db.models.fields.files import FieldFile
+
 from endoreg_db.models import (
-    Center, 
-    Gender, 
-    Patient,
+    AiModel,
+    Center,
+    EndoscopyProcessor,
     Examination,
     ExaminationIndication,
-    RawPdfFile,
-    EndoscopyProcessor,
-    ModelMeta,
+    Gender,
     InformationSource,
-    AiModel,
-)
-from logging import getLogger
-from datetime import date
-import shutil
-from pathlib import Path
-from django.conf import settings # Import settings
-from django.core.files.storage import default_storage # Import default storage
-from django.db.models.fields.files import FieldFile
-
-from endoreg_db.utils import (
-    create_mock_patient_name,
+    ModelMeta,
+    Patient,
+    RawPdfFile,
 )
+from endoreg_db.utils import create_mock_patient_name
 
 logger = getLogger("default_objects")
 
@@ -32,7 +31,7 @@ DEFAULT_ENDOSCOPE_NAME = "test_endoscope"
 DEFAULT_ENDOSCOPY_PROCESSOR_NAME = "olympus_cv_1500"
 
 DEFAULT_EGD_PATH = Path("tests/assets/lux-gastro-report.pdf")
-DEFAULT_GENDERS = ["male","female","unknown"]
+DEFAULT_GENDERS = ["male", "female", "unknown"]
 DEFAULT_EXAMINATIONS = ["colonoscopy"]
 DEFAULT_INDICATIONS = [
     "colonoscopy",
@@ -51,60 +50,70 @@ DEFAULT_PATIENT_LAST_NAME = "TestLast"
 DEFAULT_PATIENT_GENDER_NAME = "female"
 DEFAULT_PATIENT_BIRTH_DATE = date(1970, 1, 1)
 
+
 def get_information_source_prediction():
     """
     Retrieves the InformationSource object with the name "prediction".
-    
+
     Loads information source data if needed and returns the corresponding InformationSource instance. Raises a ValueError if the object is not found or is not an InformationSource.
     """
     from .data_loader import load_information_source
+
     load_information_source()
     source = InformationSource.objects.get(name="prediction")
     if not isinstance(source, InformationSource):
         raise ValueError("No InformationSource found in the database.")
     return source
 
-def get_latest_segmentation_model(model_name:str=DEFAULT_SEGMENTATION_MODEL_NAME) -> ModelMeta:
 
+def get_latest_segmentation_model(
+    model_name: str = DEFAULT_SEGMENTATION_MODEL_NAME,
+) -> ModelMeta:
     """
     Retrieves the latest metadata for a segmentation model by name.
-    
+
     Loads necessary data and returns the most recent ModelMeta instance for the specified AI model. If no metadata exists, attempts to initialize it automatically; if initialization fails, raises a ValueError with instructions for manual setup.
-    
+
     Args:
         model_name: The name of the segmentation model to retrieve.
-    
+
     Returns:
         The latest ModelMeta instance for the specified model.
-    
+
     Raises:
         ValueError: If the AI model does not exist, or if model metadata cannot be found or initialized.
     """
     from .data_loader import (
-        load_center_data,
-        load_ai_model_label_data,
         load_ai_model_data,
+        load_ai_model_label_data,
+        load_center_data,
     )
+
     load_center_data()
     load_ai_model_label_data()
     load_ai_model_data()
-    
+
     try:
         ai_model = AiModel.objects.get(name=model_name)
     except AiModel.DoesNotExist:
-        raise ValueError(f"AI model '{model_name}' not found. Run 'python manage.py load_ai_model_data' first.")
-    
+        raise ValueError(
+            f"AI model '{model_name}' not found. Run 'python manage.py load_ai_model_data' first."
+        )
+
     try:
         latest_meta = ai_model.get_latest_version()
         return latest_meta
     except ValueError as e:
         if "No model metadata found" in str(e):
-            logger.warning(f"No ModelMeta found for {model_name}. Attempting to initialize default model metadata...")
-            
+            logger.warning(
+                f"No ModelMeta found for {model_name}. Attempting to initialize default model metadata..."
+            )
+
             # Try to initialize the default model metadata
             try:
                 from django.core.management import call_command
-                call_command('init_default_ai_model')
+
+                call_command("init_default_ai_model")
                 # Try again after initialization
                 latest_meta = ai_model.get_latest_version()
                 return latest_meta
@@ -116,17 +125,18 @@ def get_latest_segmentation_model(model_name:str=DEFAULT_SEGMENTATION_MODEL_NAME
                 ) from e
         else:
             raise
-    
+
 
 def get_default_gender() -> Gender:
     """
     Retrieves the Gender object representing the default "unknown" gender.
-    
+
     Returns:
         The Gender instance with the name "unknown".
     """
     return Gender.objects.get(name=DEFAULT_GENDER)
 
+
 def get_gender_m_or_f() -> Gender:
     """
     Returns a randomly selected Gender object representing either male or female.
@@ -134,36 +144,40 @@ def get_gender_m_or_f() -> Gender:
     gender_name = random.choice(["male", "female"])
     return Gender.objects.get(name=gender_name)
 
+
 def get_random_gender() -> Gender:
     """
     Returns a randomly selected Gender object from the available default genders.
     """
     gender_name = random.choice(DEFAULT_GENDERS)
-    return Gender.objects.get(name=gender_name) # Fetch and return the Gender object
+    return Gender.objects.get(name=gender_name)  # Fetch and return the Gender object
+
 
 def get_default_processor() -> EndoscopyProcessor:
     """
     Retrieves the default EndoscopyProcessor object by its predefined name.
-    
+
     Raises:
         ValueError: If no EndoscopyProcessor with the default name exists.
-        
+
     Returns:
         The EndoscopyProcessor instance with the default name.
     """
     processor = EndoscopyProcessor.objects.get(name=DEFAULT_ENDOSCOPY_PROCESSOR_NAME)
     if not isinstance(processor, EndoscopyProcessor):
-        raise ValueError(f"No EndoscopyProcessor found with name {DEFAULT_ENDOSCOPY_PROCESSOR_NAME}")
+        raise ValueError(
+            f"No EndoscopyProcessor found with name {DEFAULT_ENDOSCOPY_PROCESSOR_NAME}"
+        )
     return processor
 
 
 def get_default_center() -> Center:
     """
     Retrieves the default Center object with the predefined name.
-    
+
     Raises:
         ValueError: If no Center with the default name exists.
-    
+
     Returns:
         The Center instance with the default name.
     """
@@ -172,9 +186,10 @@ def get_default_center() -> Center:
     )
     if not isinstance(center, Center):
         raise ValueError(f"No Center found with name {DEFAULT_CENTER_NAME}")
-    
+
     return center
 
+
 def generate_patient(**kwargs) -> Patient:
     """Create a Patient with deterministic defaults unless ``randomize=True`` is supplied."""
 
@@ -193,9 +208,14 @@ def generate_patient(**kwargs) -> Patient:
     last_name = kwargs.get("last_name")
     if first_name is None or last_name is None:
         if randomize:
-            generated_first, generated_last = create_mock_patient_name(gender=gender.name)
+            generated_first, generated_last = create_mock_patient_name(
+                gender=gender.name
+            )
         else:
-            generated_first, generated_last = DEFAULT_PATIENT_FIRST_NAME, DEFAULT_PATIENT_LAST_NAME
+            generated_first, generated_last = (
+                DEFAULT_PATIENT_FIRST_NAME,
+                DEFAULT_PATIENT_LAST_NAME,
+            )
         first_name = first_name or generated_first
         last_name = last_name or generated_last
 
@@ -217,16 +237,17 @@ def generate_patient(**kwargs) -> Patient:
         first_name=first_name,
         last_name=last_name,
         dob=dob,
-        center = center,
-        gender = gender,
+        center=center,
+        gender=gender,
     )
 
     return patient
-    
+
+
 def get_random_default_examination():
     """
     Retrieves a random Examination object from the default examination names.
-    
+
     Returns:
         Examination: A randomly selected Examination instance from the defaults.
     """
@@ -235,29 +256,33 @@ def get_random_default_examination():
     examination = Examination.objects.get(name=examination_name)
     return examination
 
+
 def get_random_default_examination_indication():
     """
     Returns a random ExaminationIndication object from the default indications list.
-    
+
     Selects a random indication name from the predefined defaults and retrieves the corresponding ExaminationIndication instance from the database.
     """
     examination_indication = random.choice(DEFAULT_INDICATIONS)
     all_examination_indications = ExaminationIndication.objects.all()
     try:
-        examination_indication = ExaminationIndication.objects.get(name=examination_indication)
-        
+        examination_indication = ExaminationIndication.objects.get(
+            name=examination_indication
+        )
+
     except Exception as e:
         logger.info(f"examination_indication: {examination_indication}")
         logger.info(f"all_examination_indications: {all_examination_indications}")
         raise e
     return examination_indication
 
+
 def get_default_egd_pdf():
     """
-    Creates and processes a default EGD PDF file for testing purposes.
-    
-    This function copies a default EGD PDF to a temporary location, creates a RawPdfFile instance from it, processes the file to generate associated metadata, and ensures cleanup of the temporary file. The resulting RawPdfFile instance is returned for use in tests.
-    
+    Creates and processes a default EGD report file for testing purposes.
+
+    This function copies a default EGD report to a temporary location, creates a RawPdfFile instance from it, processes the file to generate associated metadata, and ensures cleanup of the temporary file. The resulting RawPdfFile instance is returned for use in tests.
+
     Returns:
         RawPdfFile: The created and processed RawPdfFile instance.
     """
@@ -277,44 +302,48 @@ def get_default_egd_pdf():
     pdf_file = None
     file_field: Optional[FieldFile] = None
     try:
-        # Create the PDF record using the temporary file.
+        # Create the report record using the temporary file.
         # delete_source=True will ensure temp_file_path is deleted by create_from_file
         pdf_file = RawPdfFile.create_from_file(
             file_path=temp_file_path,
             center_name=center_name,
-            save=True, # save=True is default and handled internally now
+            save=True,  # save=True is default and handled internally now
             delete_source=True,
         )
 
         if pdf_file is None:
-            raise RuntimeError("Failed to create PDF file object")
-        
+            raise RuntimeError("Failed to create report file object")
+
         # Use storage API to check existence
         file_field = pdf_file.file
         if not isinstance(file_field, FieldFile):
             raise RuntimeError("RawPdfFile.file did not return a FieldFile instance")
         if not default_storage.exists(file_field.path):
-            raise RuntimeError(f"PDF file does not exist in storage at {file_field.path}")
-        
+            raise RuntimeError(
+                f"report file does not exist in storage at {file_field.path}"
+            )
+
         # Check that the source temp file was deleted
         if temp_file_path.exists():
-            raise RuntimeError(f"Temporary source file {temp_file_path} still exists after creation")
+            raise RuntimeError(
+                f"Temporary source file {temp_file_path} still exists after creation"
+            )
 
         # Prepare a minimal report_meta for SensitiveMeta creation
         default_report_meta = {
             "patient_first_name": "DefaultFirstName",
             "patient_last_name": "DefaultLastName",
-            "patient_dob": date(1980, 1, 1), # Pass date object directly
-            "examination_date": date(2024, 1, 1), # Pass date object directly
+            "patient_dob": date(1980, 1, 1),  # Pass date object directly
+            "examination_date": date(2024, 1, 1),  # Pass date object directly
             # center_name will be added by process_file using pdf_file.center.name
         }
 
         # Call process_file to create SensitiveMeta and extract other info
         pdf_file.process_file(
-            text="Default PDF text content.",
-            anonymized_text="Default anonymized PDF text content.",
+            text="Default report text content.",
+            anonymized_text="Default anonymized report text content.",
             report_meta=default_report_meta,
-            verbose=False
+            verbose=False,
         )
         # process_file calls sensitive_meta.save() and self.save() (for RawPdfFile)
 
@@ -322,40 +351,46 @@ def get_default_egd_pdf():
         # Clean up temp file in case of error before deletion could occur
         if temp_file_path.exists():
             temp_file_path.unlink()
-        raise e # Re-raise the exception
+        raise e  # Re-raise the exception
 
     # pdf_file.file.path might fail if storage doesn't support direct paths (like S3)
     # Prefer using storage API for checks. Logging path if available.
     if file_field is not None:
         try:
-            logger.info(f"PDF file created: {file_field.name}, Path: {file_field.path}")
+            logger.info(
+                f"report file created: {file_field.name}, Path: {file_field.path}"
+            )
         except NotImplementedError:
-            logger.info(f"PDF file created: {file_field.name}, Path: (Not available from storage)")
-
+            logger.info(
+                f"report file created: {file_field.name}, Path: (Not available from storage)"
+            )
 
     return pdf_file
 
+
 def get_default_video_file():
     """
     Creates and returns a VideoFile instance using a randomly selected EGD examination video.
-    
+
     Loads all necessary data dependencies, selects a random video file for the 'egd' examination, and initializes a VideoFile object with the default center and processor names. The original video file is retained after creation.
-    
+
     Returns:
         VideoFile: The created and initialized VideoFile instance.
     """
-    from .test_video_helper import get_random_video_path_by_examination_alias
     from endoreg_db.models import VideoFile
+
     from .data_loader import (
+        load_ai_model_data,
+        load_ai_model_label_data,
+        load_center_data,
         load_disease_data,
+        load_endoscope_data,
         load_event_data,
-        load_information_source,
         load_examination_data,
-        load_center_data,
-        load_endoscope_data,
-        load_ai_model_label_data,
-        load_ai_model_data,
+        load_information_source,
     )
+    from .test_video_helper import get_random_video_path_by_examination_alias
+
     load_disease_data()
     load_event_data()
     load_information_source()
@@ -365,14 +400,14 @@ def get_default_video_file():
     load_ai_model_label_data()
     load_ai_model_data()
     video_path = get_random_video_path_by_examination_alias(
-        examination_alias='egd', is_anonymous=False
+        examination_alias="egd", is_anonymous=False
     )
 
     video_file = VideoFile.create_from_file_initialized(
         file_path=video_path,
         center_name=DEFAULT_CENTER_NAME,  # Pass center name as expected by _create_from_file
         delete_source=False,  # Keep the original asset for other tests
-        processor_name = DEFAULT_ENDOSCOPY_PROCESSOR_NAME,
+        processor_name=DEFAULT_ENDOSCOPY_PROCESSOR_NAME,
     )
 
     return video_file

endoreg_db/import_files/__init__.py

@@ -0,0 +1,27 @@
+# Re-export context helpers
+from .context.file_lock import file_lock
+from .context.default_sensitive_meta import default_sensitive_meta  # if needed
+from .context.import_context import ImportContext  # if needed
+from .context.validate_directories import validate_directories  # if needed
+
+# Re-export storage helpers
+from .file_storage import create_report_file
+from .file_storage import create_video_file
+from .file_storage import sensitive_meta_storage
+
+# Re-export import services
+from .report_import_service import ReportImportService
+from .video_import_service import VideoImportService
+
+# Public API
+__all__ = [
+    "file_lock",
+    "create_report_file",
+    "create_video_file",
+    "sensitive_meta_storage",
+    "ReportImportService",
+    "VideoImportService",
+    "ImportContext",
+    "validate_directories",
+    "default_sensitive_meta"
+]

endoreg_db/import_files/context/__init__.py

@@ -0,0 +1,7 @@
+from .file_lock import file_lock
+from .import_context import ImportContext
+
+__all__ = [
+    "file_lock",
+    "ImportContext",
+]

endoreg_db/import_files/context/default_sensitive_meta.py

@@ -0,0 +1,81 @@
+# endoreg_db/import_files/processing/create_sensitive_meta.py
+
+import os
+import logging
+from datetime import date
+from typing import Optional, Union
+
+from endoreg_db.models.media import RawPdfFile, VideoFile
+from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta  # adjust path
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PATIENT_FIRST_NAME = "unknown"
+DEFAULT_PATIENT_LAST_NAME = "unknown"
+DEFAULT_CENTER_NAME = "endoreg_db_demo"
+# DEFAULT_PATIENT_DOB can be a fixed date or None to let logic.generate_random_dob handle it
+DEFAULT_PATIENT_DOB = date(1970, 1, 1)
+
+
+def default_sensitive_meta(instance: Union[RawPdfFile, VideoFile]) -> SensitiveMeta | None:
+    """
+    Ensure the given instance has a minimal SensitiveMeta attached.
+
+    Called after text extraction + merging; only creates meta if none exists.
+    """
+    if instance is None:
+        logger.warning(
+            "No instance available for ensuring default patient data"
+        )
+        return
+    
+
+    if instance.sensitive_meta:
+        # Already has meta; nothing to do
+        return
+
+    logger.info(
+        "No SensitiveMeta found for report %s, creating default",
+        getattr(instance, "pdf_hash", instance.pk),
+    )
+    if not isinstance(instance.center.name, str):
+        try:
+            center_name = os.environ.get("DEFAULT_CENTER_NAME")
+            assert center_name is not None
+            instance.center.name = center_name
+        except AssertionError as e:
+            logger.debug(f"{e}Center name is not set! You can set it in .env under DEFAULT_CENTER_NAME using default from default_sensitive_meta")
+            instance.center.name = DEFAULT_CENTER_NAME
+            instance.center.get_by_name(DEFAULT_CENTER_NAME)
+
+    default_data = {
+        "patient_first_name": DEFAULT_PATIENT_FIRST_NAME,
+        "patient_last_name": DEFAULT_PATIENT_LAST_NAME,
+        "patient_dob": DEFAULT_PATIENT_DOB,
+        "examination_date": date.today(),
+        "center_name": (
+            instance.center.name
+            if getattr(instance, "center", None) is not None
+            else DEFAULT_CENTER_NAME
+        ),
+        # optional: link file_path for debugging/tracing
+        "file_path": str(instance.file_path) if getattr(instance, "file_path", None) else None,
+    }
+
+    try:
+        meta = SensitiveMeta.create_from_dict(default_data)
+        instance.sensitive_meta = meta
+        instance.save(update_fields=["sensitive_meta"])
+        logger.info(
+            "Created default SensitiveMeta for report %s",
+            getattr(instance, "pdf_hash", instance.pk),
+        )
+        return meta
+    except Exception as e:
+        logger.error(
+            "Failed to create default SensitiveMeta for report %s: %s",
+            getattr(instance, "pdf_hash", instance.pk),
+            e,
+        )
+        return None
+

endoreg_db/import_files/context/ensure_center.py

@@ -0,0 +1,17 @@
+from typing import Union, Optional
+import os
+
+from endoreg_db.models import Center
+from endoreg_db.models.media import RawPdfFile, VideoFile
+
+
+def ensure_center(instance: Union[RawPdfFile, VideoFile], center: Optional[str]) -> Center:
+    if not isinstance(instance.center, Center):
+        raise AssertionError
+    if not isinstance(instance.center.name, str):
+        raise AssertionError
+    assert isinstance(instance.center.get_by_name(center), Center)
+    if not instance.center.get_by_name(center).name ==instance.center.name:
+        raise AssertionError
+    return instance.center
+

endoreg_db/import_files/context/file_lock.py

@@ -0,0 +1,66 @@
+from contextlib import contextmanager
+from pathlib import Path
+import os
+import time
+from logging import getLogger
+import errno
+import shutil
+from typing import Generator, Any
+
+logger = getLogger(__name__)
+
+STALE_LOCK_SECONDS = 6000
+MAX_LOCK_WAIT_SECONDS = 90
+
+
+@contextmanager
+def file_lock(path: Path) -> Generator[None, Any, None]:
+    """
+    Create a file lock to prevent duplicate processing of the same file.
+
+    Lock is created *next to* the source file: "<path>.lock".
+    """
+    lock_path = Path(str(path) + ".lock")
+    fd = None
+    try:
+        deadline = time.time() + MAX_LOCK_WAIT_SECONDS
+        while True:
+            try:
+                fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
+                break
+            except FileExistsError:
+                age = None
+                try:
+                    st = os.stat(lock_path)
+                    age = time.time() - st.st_mtime
+                except FileNotFoundError:
+                    age = None
+
+                if age is not None and age > STALE_LOCK_SECONDS:
+                    try:
+                        logger.warning(
+                            "Stale lock detected for %s (age %.0fs). Reclaiming lock...",
+                            path,
+                            age,
+                        )
+                        lock_path.unlink()
+                    except Exception as e:
+                        logger.warning("Failed to remove stale lock %s: %s", lock_path, e)
+                    continue
+
+                if time.time() >= deadline:
+                    raise ValueError(f"File already being processed: {path}")
+                time.sleep(1.0)
+
+        os.write(fd, b"lock")
+        os.close(fd)
+        fd = None
+        yield
+    finally:
+        try:
+            if fd is not None:
+                os.close(fd)
+            if lock_path.exists():
+                lock_path.unlink()
+        except OSError:
+            pass

endoreg_db/import_files/context/import_context.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional, Dict, Any, Union
+
+from lx_anonymizer.sensitive_meta_interface import SensitiveMeta
+from endoreg_db.models.media import RawPdfFile, VideoFile
+
+
+@dataclass
+class ImportContext:
+    """
+    Tracking the import success and reasons of failure.
+    """
+
+    # core import parameters
+    file_path: Path
+    center_name: str
+    processor_name: str = "olympus-cv-500"
+    delete_source: bool = True
+    retry: bool = False
+    import_completed: bool = False
+    error_reason: str = ""
+
+    # paths
+    original_path: Optional[Path] = None
+    quarantine_path: Optional[Path] = None
+    sensitive_path: Optional[Path] = None
+    anonymized_path: Optional[Path] = None
+
+    # associated objects
+    current_report: Optional[RawPdfFile] = None
+    current_video: Optional[VideoFile] = None
+    current_meta: Optional[SensitiveMeta] = None
+    instance: Optional[Union[RawPdfFile, VideoFile]] = None
+    
+    file_type: str = "undefined"
+    
+    
+    # processing metadata
+    file_hash: str = ""
+    original_text: Optional[str] = None
+    anonymized_text: Optional[str] = None
+    extracted_metadata: Dict[str, Any] = field(default_factory=dict)