endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

endoreg_db/models/media/pdf/raw_pdf.py

@@ -4,33 +4,27 @@
 # Class contains classmethod to create object from pdf file
 # objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
 # ------------------------------------------------------------------------------
-import os
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Optional, cast
 
 from django.core.exceptions import ValidationError
 from django.core.files import File
 from django.core.validators import FileExtensionValidator
 from django.db import models
-from numpy import isin  # Import Django File
 
 from endoreg_db.utils.file_operations import get_uuid_filename
 from endoreg_db.utils.hashs import get_pdf_hash
-from endoreg_db.utils.paths import PDF_DIR, RAW_PDF_DIR
-
-# Use the specific paths from the centralized paths module
-from ...utils import PDF_DIR
+from endoreg_db.utils.paths import PDF_DIR
+from endoreg_db.utils.storage import (
+    delete_field_file,
+    ensure_local_file,
+    file_exists,
+    save_local_file,
+)
 
 if TYPE_CHECKING:
-    from endoreg_db.models.administration.person import (
-        Examiner,
-        Patient,
-    )
+    from django.db.models.fields.files import FieldFile
 
-    from ...administration import Center
-    from ...medical.patient import PatientExamination
-    from ...metadata.pdf_meta import PdfType
-    from ...state import RawPdfState
-    from .report_file import AnonymExaminationReport
+    from endoreg_db.models.state import RawPdfState
 
 # setup logging to pdf_import.log
 import logging
@@ -42,6 +36,7 @@ logger = logging.getLogger("raw_pdf")
 
 
 class RawPdfFile(models.Model):
+    objects = models.Manager()
     # Fields from AbstractPdfFile
     pdf_hash = models.CharField(max_length=255, unique=True)
     pdf_type = models.ForeignKey(
@@ -55,50 +50,89 @@ class RawPdfFile(models.Model):
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
-    )  # type: ignore
+    )
     examination = models.ForeignKey(
         "PatientExamination",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
         related_name="raw_pdf_files",
-    )  # type: ignore
+    )
     examiner = models.ForeignKey(
         "Examiner",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
-    )  # type: ignore
+    )
     text = models.TextField(blank=True, null=True)
     date_created = models.DateTimeField(auto_now_add=True)
     date_modified = models.DateTimeField(auto_now=True)
-    anonymized = models.BooleanField(
-        default=False, help_text="True if the PDF has been anonymized."
-    )
-
-    # Fields specific to RawPdfFile (keeping existing related_names)
+    anonymized = models.BooleanField(default=False, help_text="True if the PDF has been anonymized.")
     file = models.FileField(
         # Use the relative path from the specific PDF_DIR
         upload_to=PDF_DIR.name,
         validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
-    )  # type: ignore
-
+    )
     anonymized_file = models.FileField(
         upload_to=PDF_DIR.name,
         validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
         null=True,
         blank=True,
-    )  # type: ignore
-
+    )
     state = models.OneToOneField(
         "RawPdfState",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
         related_name="raw_pdf_file",
-    )  # type: ignore
+    )
+    patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="raw_pdf_files",
+    )
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.SET_NULL,
+        related_name="raw_pdf_files",
+        null=True,
+        blank=True,
+    )
+    state_report_processing_required = models.BooleanField(default=True)
+    state_report_processed = models.BooleanField(default=False)
+    raw_meta = models.JSONField(blank=True, null=True)
+    anonym_examination_report = models.OneToOneField(
+        "AnonymExaminationReport",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="raw_pdf_file",
+    )
+    anonymized_text = models.TextField(blank=True, null=True)
 
-    objects = models.Manager()
+    # Type hinting is needed, improve and use correct django types
+    if TYPE_CHECKING:
+        from endoreg_db.models import (
+            AnonymExaminationReport,
+            Center,
+            Examiner,
+            Patient,
+            PatientExamination,
+            RawPdfState,
+            SensitiveMeta,
+        )
+
+        center: models.ForeignKey["Center | None"]
+        examination: models.ForeignKey["PatientExamination | None"]
+        examiner: models.ForeignKey["Examiner | None"]
+        state: models.ForeignKey["RawPdfState | None"]
+        patient: models.ForeignKey["Patient | None"]
+        sensitive_meta: models.ForeignKey["SensitiveMeta | None"]
+        anonym_examination_report: models.OneToOneField["AnonymExaminationReport | None"]
+        file = cast(FieldFile, file)
+        anonymized_file = cast(FieldFile, anonymized_file)
 
     @property
     def uuid(self):
@@ -130,7 +164,10 @@ class RawPdfFile(models.Model):
         """
         Sets the file path of the stored PDF file.
         """
-        self.file = File(file_path)  # type: ignore
+        if not file_path.exists():
+            raise FileNotFoundError(f"File path does not exist: {file_path}")
+
+        save_local_file(self.file, file_path, name=file_path.name, save=False)
         self.save(update_fields=["file"])
 
     @property
@@ -149,7 +186,10 @@ class RawPdfFile(models.Model):
         """
         Sets the file path of the anonymized PDF file.
         """
-        self.anonymized_file = File(file_path)  # type: ignore
+        if not file_path.exists():
+            raise FileNotFoundError(f"File path does not exist: {file_path}")
+
+        save_local_file(self.anonymized_file, file_path, name=file_path.name, save=False)
         self.save(update_fields=["anonymized_file"])
 
     def get_raw_file_path(self) -> Optional[Path]:
@@ -228,55 +268,10 @@ class RawPdfFile(models.Model):
         Returns the URL of the stored PDF file if available; otherwise, returns None.
         """
         try:
-            return (
-                self.anonymized_file.url
-                if self.anonymized_file and self.anonymized_file.name
-                else None
-            )
+            return self.anonymized_file.url if self.anonymized_file and self.anonymized_file.name else None
         except (ValueError, AttributeError):
             return None
 
-    patient = models.ForeignKey(
-        "Patient",
-        on_delete=models.SET_NULL,
-        blank=True,
-        null=True,
-        related_name="raw_pdf_files",
-    )  # type: ignore
-    sensitive_meta = models.ForeignKey(
-        "SensitiveMeta",
-        on_delete=models.SET_NULL,
-        related_name="raw_pdf_files",
-        null=True,
-        blank=True,
-    )  # type: ignore
-    state_report_processing_required = models.BooleanField(default=True)
-    state_report_processed = models.BooleanField(default=False)
-    raw_meta = models.JSONField(blank=True, null=True)
-    anonym_examination_report = models.OneToOneField(
-        "AnonymExaminationReport",
-        on_delete=models.SET_NULL,
-        blank=True,
-        null=True,
-        related_name="raw_pdf_file",
-    )  # type: ignore
-    anonymized_text = models.TextField(blank=True, null=True)
-
-    # Type hinting is needed, improve and use correct django types
-    if TYPE_CHECKING:
-        file: Optional[Union[models.FieldFile, models.FileField]]
-        anonymized_file: Optional[Union[models.FieldFile, models.FileField]]
-        pdf_type: Optional[models.ForeignKey]
-        examination: Optional[models.ForeignKey["PatientExamination"]]
-        examiner: Optional[models.ForeignKey["Examiner"]]
-        patient: Optional[models.ForeignKey["Patient"]]
-        center: Optional[models.ForeignKey["Center"]]
-        anonym_examination_report: Optional[
-            models.OneToOneField["AnonymExaminationReport"]
-        ]
-        sensitive_meta: Optional[models.ForeignKey["SensitiveMeta"]]
-        state: Optional[models.ForeignKey["RawPdfState"]]
-
     def __str__(self):
         """
         Return a string representation of the RawPdfFile, including its PDF hash, type, and center.
@@ -290,35 +285,17 @@ class RawPdfFile(models.Model):
 
         This method ensures that the physical PDF file is deleted from the file system after the database record is removed. Logs warnings or errors if the file cannot be found or deleted.
         """
-        # Call the original delete method first to remove DB record
-        if self.file:
-            try:
-                if self.file_path:
-                    os.remove(Path(self.file_path))
-                    logger.info("Original file removed: %s", self.file)
-            except Exception as e:
-                logger.warning(
-                    f"Could not get file path for {self.file.name} before deletion: {e}"
-                )
-        if self.anonymized_file:
-            try:
-                if self.anonymized_file_path:
-                    os.remove(Path(self.anonymized_file_path))
-                    logger.info(
-                        "Anonymized file removed: %s", self.anonymized_file.name
-                    )
-            except OSError as e:
-                logger.error(
-                    "Error removing anonymized file %s: %s",
-                    self.anonymized_file.name,
-                    e,
-                )
+        primary_name = self.file.name if self.file and self.file.name else None
+        anonymized_name = self.anonymized_file.name if self.anonymized_file and self.anonymized_file.name else None
+
+        if delete_field_file(self.file, missing_ok=True, save=False):
+            logger.info("Original file removed from storage: %s", primary_name)
+        if delete_field_file(self.anonymized_file, missing_ok=True, save=False):
+            logger.info("Anonymized file removed from storage: %s", anonymized_name)
 
         super().delete(*args, **kwargs)
 
-    def validate_metadata_annotation(
-        self, extracted_data_dict: Optional[dict] = None
-    ) -> bool:
+    def validate_metadata_annotation(self, extracted_data_dict: Optional[dict] = None) -> bool:
         """
         Validate the metadata of the RawPdf instance.
 
@@ -334,8 +311,10 @@ class RawPdfFile(models.Model):
             logger.error("No extracted data provided for validation.")
             return False
 
-        # Update sensitive meta with the provided data
-        self.sensitive_meta.update_from_dict(extracted_data_dict)
+        if extracted_data_dict:
+            self.sensitive_meta.update_from_dict(extracted_data_dict)
+        else:
+            return False
 
         # Save the sensitive meta to ensure changes are persisted
         self.sensitive_meta.save()
@@ -345,21 +324,12 @@ class RawPdfFile(models.Model):
 
         logger.info(f"Metadata for PDF {self.pk} validated and updated successfully.")
 
-        if self.file_path:
-            try:
-                os.unlink(self.file_path)  # Delete the original file if it exists
-            except OSError as e:
-                logger.error(f"Error removing original file {self.file_path}: {e}")
-
-        if self.anonymized_file_path:
-            try:
-                os.unlink(self.anonymized_file_path)
-            except OSError as e:
-                logger.error(
-                    f"Error removing anonymized file {self.anonymized_file_path}: {e}"
-                )
+        deleted_original = delete_field_file(self.file, missing_ok=True, save=False)
+        deleted_anonymized = delete_field_file(self.anonymized_file, missing_ok=True, save=False)
+        self.get_or_create_state().mark_anonymization_validated()
 
-        self.save()  # Save the model to persist the cleared file fields
+        if deleted_original or deleted_anonymized:
+            self.save(update_fields=["file", "anonymized_file"])  # Persist cleared fields
 
         logger.info(f"Files for PDF {self.pk} deleted successfully.")
         return True
@@ -457,13 +427,9 @@ class RawPdfFile(models.Model):
                         )
                         # Re-save the file from the source to potentially fix it
                         with file_path.open("rb") as f:
-                            django_file = File(
-                                f, name=Path(_file.name).name
-                            )  # Use existing name if possible
-                            existing_pdf_file.file = django_file  # type: ignore
-                            existing_pdf_file.save(
-                                update_fields=["file"]
-                            )  # Only update file field
+                            django_file = File(f, name=Path(_file.name).name)  # Use existing name if possible
+                            existing_pdf_file.file = django_file
+                            existing_pdf_file.save(update_fields=["file"])  # Only update file field
                     else:
                         pass
                         # logger.debug("File for existing RawPdfFile %s already exists in storage.", pdf_hash)
@@ -519,9 +485,7 @@ class RawPdfFile(models.Model):
                     "File was not saved correctly to storage path %s after model save.",
                     _file.name,
                 )
-                raise IOError(
-                    f"File not found at expected storage path after save: {_file.name}"
-                )
+                raise IOError(f"File not found at expected storage path after save: {_file.name}")
 
             try:
                 logger.info("File saved to absolute path: %s", _file.path)
@@ -532,18 +496,14 @@ class RawPdfFile(models.Model):
                 )
 
         except Exception as e:
-            logger.error(
-                "Error processing or saving file %s for new record: %s", file_path, e
-            )
+            logger.error("Error processing or saving file %s for new record: %s", file_path, e)
             raise
 
         # Delete source file *after* successful save and verification
         if delete_source:
             try:
                 file_path.unlink()
-                logger.info(
-                    "Deleted source file %s after creating new record.", file_path
-                )
+                logger.info("Deleted source file %s after creating new record.", file_path)
             except OSError as e:
                 logger.error("Error deleting source file %s: %s", file_path, e)
 
@@ -560,27 +520,15 @@ class RawPdfFile(models.Model):
         """
         if not self.pk and not self.pdf_hash and self.file:
             try:
-                file_path = Path(self.file.path).resolve()
-                if not file_path.exists():
-                    raise FileNotFoundError(f"File path does not exist: {file_path}")
-                # Read from the file object before it's saved by storage
-                self.file.open("rb")  # Ensure file is open
-                self.file.seek(0)  # Go to beginning
-                self.pdf_hash = get_pdf_hash(
-                    file_path
-                )  # Assuming get_pdf_hash can handle file obj
-                self.file.seek(0)  # Reset position
-                self.file.close()  # Close after reading
-                logger.info(f"Calculated hash during pre-save for {self.file.name}")
-            except Exception as e:
+                with ensure_local_file(self.file) as local_path:
+                    self.pdf_hash = get_pdf_hash(local_path)
+                    logger.info("Calculated hash during pre-save for %s", self.file.name)
+            except Exception as exc:
                 logger.warning(
                     "Could not calculate hash before initial save for %s: %s",
                     self.file.name,
-                    e,
+                    exc,
                 )
-                # Ensure file is closed if opened
-                if hasattr(self.file, "closed") and not self.file.closed:
-                    self.file.close()
 
         if self.file and not self.file.name.endswith(".pdf"):
             raise ValidationError("Only PDF files are allowed")
@@ -588,30 +536,16 @@ class RawPdfFile(models.Model):
         # If hash is still missing after potential creation logic (e.g., direct instantiation)
         # and the file exists in storage, try calculating it from storage path.
         # This is less ideal as it requires the file to be saved first.
-        if (
-            not self.pdf_hash
-            and self.pk
-            and self.file
-            and self.file.storage.exists(self.file.name)
-        ):
+        if not self.pdf_hash and self.pk and self.file and file_exists(self.file):
             try:
-                file_path = Path(self.file.path).resolve()
-                if not file_path.exists():
-                    raise FileNotFoundError(f"File path does not exist: {file_path}")
-                logger.warning(
-                    f"Hash missing for saved file {self.file.name}. Recalculating."
-                )
-                with self.file.storage.open(self.file.name, "rb") as f:
-                    self.pdf_hash = get_pdf_hash(
-                        file_path
-                    )  # Assuming get_pdf_hash handles file obj
-                # No need to save again just for hash unless update_fields is used carefully
-                # Let the main super().save() handle saving the hash if it changed
-            except Exception as e:
+                with ensure_local_file(self.file) as local_path:
+                    logger.warning("Hash missing for saved file %s. Recalculating.", self.file.name)
+                    self.pdf_hash = get_pdf_hash(local_path)
+            except Exception as exc:
                 logger.error(
                     "Could not calculate hash during save for existing file %s: %s",
                     self.file.name,
-                    e,
+                    exc,
                 )
 
         # Derive related fields from sensitive_meta if available
@@ -663,18 +597,12 @@ class RawPdfFile(models.Model):
         assert _file is not None
         try:
             if not _file.field.storage.exists(_file.name):
-                logger.warning(
-                    f"File missing at storage path {_file.name}. Attempting copy from fallback {fallback_file}"
-                )
+                logger.warning(f"File missing at storage path {_file.name}. Attempting copy from fallback {fallback_file}")
                 if fallback_file.exists():
                     with fallback_file.open("rb") as f:
                         # Use save method which handles storage backend
-                        _file.save(
-                            Path(_file.name).name, File(f), save=True
-                        )  # Re-save the file content
-                    logger.info(
-                        f"Successfully restored file from fallback {fallback_file} to {_file.name}"
-                    )
+                        _file.save(Path(_file.name).name, File(f), save=True)  # Re-save the file content
+                    logger.info(f"Successfully restored file from fallback {fallback_file} to {_file.name}")
                 else:
                     logger.error(f"Fallback file {fallback_file} does not exist.")
         except Exception as e:

endoreg_db/models/media/pdf/report_file.py

@@ -1,7 +1,9 @@
-from ...utils import DOCUMENT_DIR, STORAGE_DIR
-from django.db import models
 from typing import TYPE_CHECKING
 
+from django.db import models
+
+from ...utils import DOCUMENT_DIR, STORAGE_DIR
+
 if TYPE_CHECKING:
     from ...administration import (
         Center,
@@ -12,17 +14,21 @@ if TYPE_CHECKING:
     )
     from ...metadata import SensitiveMeta
 
+
 class DocumentTypeManager(models.Manager):
     """
     Custom manager for DocumentType.
     """
+
     def get_by_natural_key(self, name):
         return self.get(name=name)
 
+
 class DocumentType(models.Model):
     """
     Represents the type of a document.
     """
+
     name = models.CharField(max_length=255, unique=True)
     description = models.TextField(blank=True, null=True)
 
@@ -33,21 +39,23 @@ class DocumentType(models.Model):
 
     def __str__(self):
         return str(self.name)
-    
+
     class Meta:
         verbose_name = "Document Type"
         verbose_name_plural = "Document Types"
 
+
 class AbstractDocument(models.Model):
     """
     Abstract base class for documents.
     """
+
     meta = models.JSONField(blank=True, null=True)
     text = models.TextField(blank=True, null=True)
     date = models.DateField(blank=True, null=True)
     time = models.TimeField(blank=True, null=True)
     file = models.FileField(
-        upload_to=DOCUMENT_DIR.relative_to(STORAGE_DIR),
+        upload_to=DOCUMENT_DIR.relative_to(STORAGE_DIR).as_posix(),
         blank=True,
         null=True,
     )
@@ -67,22 +75,19 @@ class AbstractDocument(models.Model):
     )
 
     if TYPE_CHECKING:
-        center: "Center"
-        type: "DocumentType"
+        center: models.ForeignKey["Center|None"]
+        type: models.ForeignKey["DocumentType|None"]
 
     class Meta:
         abstract = True
 
 
-    
-
 class AbstractExaminationReport(AbstractDocument):
     """
     Abstract base class for examination reports.
     """
-    patient = models.ForeignKey(
-        "Patient", on_delete=models.DO_NOTHING, blank=True, null=True
-    )
+
+    patient = models.ForeignKey("Patient", on_delete=models.DO_NOTHING, blank=True, null=True)
 
     patient_examination = models.ForeignKey(
         "PatientExamination",
@@ -96,25 +101,18 @@ class AbstractExaminationReport(AbstractDocument):
         blank=True,
     )
 
-    sensitive_meta = models.ForeignKey(
-        "SensitiveMeta",
-        on_delete=models.SET_NULL,
-        null=True,
-        blank=True
-    )
+    sensitive_meta = models.ForeignKey("SensitiveMeta", on_delete=models.SET_NULL, null=True, blank=True)
 
     if TYPE_CHECKING:
-        center: "Center"
-        type: "DocumentType"
-        patient: "Patient"
-        patient_examination: "PatientExamination"
-        sensitive_meta: "SensitiveMeta"
-
+        center: models.ForeignKey["Center|None"]
+        type: models.ForeignKey["DocumentType|None"]
+        patient: models.ForeignKey["Patient|None"]
+        patient_examination: models.ForeignKey["PatientExamination|None"]
+        sensitive_meta: models.ForeignKey["SensitiveMeta|None"]
 
     class Meta:
         abstract = True
 
-    
     def get_or_create_examiner(self, examiner_first_name, examiner_last_name):
         raise NotImplementedError("Subclasses must implement this method.")
 
@@ -122,10 +120,8 @@ class AbstractExaminationReport(AbstractDocument):
         raise NotImplementedError("Subclasses must implement this method.")
 
 
-
 class AnonymExaminationReport(AbstractExaminationReport):
-
-    def get_or_create_examiner(self, examiner_first_name:str, examiner_last_name:str):
+    def get_or_create_examiner(self, examiner_first_name: str, examiner_last_name: str):
         from ...administration.person import Examiner
 
         examiner_center = self.center
@@ -139,7 +135,7 @@ class AnonymExaminationReport(AbstractExaminationReport):
         return examiner, created
 
     def set_examination_date_and_time(self, report_meta=None):
-        #TODO
+        # TODO
         if not report_meta:
             report_meta = self.meta
         # examination_date_str = report_meta["examination_date"]
@@ -152,11 +148,11 @@ class AnonymExaminationReport(AbstractExaminationReport):
         #     # TODO: get django TimeField compatible time from string (e.g. "12:00")
         #     self.time = time.fromisoformat(examination_time_str)
 
+
 class AnonymHistologyReport(AbstractExaminationReport):
     """
     Represents a histology report.
     """
 
-
     def get_or_create_examiner(self, examiner_first_name, examiner_last_name):
         raise NotImplementedError("Subclasses must implement this method.")