endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

endoreg_db/models/media/pdf/raw_pdf.py

@@ -4,35 +4,32 @@
 # Class contains classmethod to create object from pdf file
 # objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
 # ------------------------------------------------------------------------------
-import os
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Optional, cast
 
 from django.core.exceptions import ValidationError
 from django.core.files import File
 from django.core.validators import FileExtensionValidator
 from django.db import models
-from numpy import isin  # Import Django File
 
 from endoreg_db.utils.file_operations import get_uuid_filename
 from endoreg_db.utils.hashs import get_pdf_hash
-from endoreg_db.utils.paths import PDF_DIR, RAW_PDF_DIR
-
-# Use the specific paths from the centralized paths module
-from ...utils import PDF_DIR
+from endoreg_db.utils.paths import (
+    ANONYM_REPORT_DIR,
+    IMPORT_REPORT_DIR,
+    SENSITIVE_REPORT_DIR,
+)
+from endoreg_db.utils.storage import (
+    delete_field_file,
+    ensure_local_file,
+    file_exists,
+    save_local_file,
+)
 
 if TYPE_CHECKING:
-    from endoreg_db.models.administration.person import (
-        Examiner,
-        Patient,
-    )
+    from django.db.models.fields.files import FieldFile
 
-    from ...administration import Center
-    from ...medical.patient import PatientExamination
-    from ...metadata.pdf_meta import PdfType
-    from ...state import RawPdfState
-    from .report_file import AnonymExaminationReport
+    from endoreg_db.models.state import RawPdfState
 
-# setup logging to pdf_import.log
 import logging
 from pathlib import Path
 
@@ -42,6 +39,7 @@ logger = logging.getLogger("raw_pdf")
 
 
 class RawPdfFile(models.Model):
+    objects = models.Manager()
     # Fields from AbstractPdfFile
     pdf_hash = models.CharField(max_length=255, unique=True)
     pdf_type = models.ForeignKey(
@@ -55,50 +53,91 @@ class RawPdfFile(models.Model):
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
-    )  # type: ignore
+    )
     examination = models.ForeignKey(
         "PatientExamination",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
         related_name="raw_pdf_files",
-    )  # type: ignore
+    )
     examiner = models.ForeignKey(
         "Examiner",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
-    )  # type: ignore
+    )
     text = models.TextField(blank=True, null=True)
     date_created = models.DateTimeField(auto_now_add=True)
     date_modified = models.DateTimeField(auto_now=True)
-    anonymized = models.BooleanField(
-        default=False, help_text="True if the PDF has been anonymized."
-    )
 
-    # Fields specific to RawPdfFile (keeping existing related_names)
     file = models.FileField(
-        # Use the relative path from the specific PDF_DIR
-        upload_to=PDF_DIR.name,
+        # Use the relative path from the specific REPORT_DIR
+        upload_to=SENSITIVE_REPORT_DIR.name,
         validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
-    )  # type: ignore
-
-    anonymized_file = models.FileField(
-        upload_to=PDF_DIR.name,
+    )
+    processed_file = models.FileField(
+        upload_to=ANONYM_REPORT_DIR.name,
         validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
         null=True,
         blank=True,
-    )  # type: ignore
-
+    )
     state = models.OneToOneField(
         "RawPdfState",
         on_delete=models.SET_NULL,
         blank=True,
         null=True,
         related_name="raw_pdf_file",
-    )  # type: ignore
+    )
+    patient = models.ForeignKey(
+        "Patient",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="raw_pdf_files",
+    )
+    sensitive_meta = models.ForeignKey(
+        "SensitiveMeta",
+        on_delete=models.SET_NULL,
+        related_name="raw_pdf_files",
+        null=True,
+        blank=True,
+    )
+    state_report_processing_required = models.BooleanField(default=True)
+    state_report_processed = models.BooleanField(default=False)
+    raw_meta = models.JSONField(blank=True, null=True)
+    anonym_examination_report = models.OneToOneField(
+        "AnonymExaminationReport",
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name="raw_pdf_file",
+    )
+    anonymized_text = models.TextField(blank=True, null=True)
 
-    objects = models.Manager()
+    # Type hinting is needed, improve and use correct django types
+    if TYPE_CHECKING:
+        from endoreg_db.models import (
+            AnonymExaminationReport,
+            Center,
+            Examiner,
+            Patient,
+            PatientExamination,
+            RawPdfState,
+            SensitiveMeta,
+        )
+
+        center: models.ForeignKey["Center | None"]
+        examination: models.ForeignKey["PatientExamination | None"]
+        examiner: models.ForeignKey["Examiner | None"]
+        state: models.ForeignKey["RawPdfState | None"]
+        patient: models.ForeignKey["Patient | None"]
+        sensitive_meta: models.ForeignKey["SensitiveMeta | None"]
+        anonym_examination_report: models.OneToOneField[
+            "AnonymExaminationReport | None"
+        ]
+        file = cast(FieldFile, file)
+        processed_file = cast(FieldFile, processed_file)
 
     @property
     def uuid(self):
@@ -113,7 +152,7 @@ class RawPdfFile(models.Model):
     @property
     def file_path(self) -> Path | None:
         """
-        Returns the file path of the stored PDF file if available; otherwise, returns None.
+        Returns the file path of the stored report file if available; otherwise, returns None.
         """
         from django.db.models.fields.files import FieldFile
 
@@ -128,35 +167,41 @@ class RawPdfFile(models.Model):
 
     def set_file_path(self, file_path: Path):
         """
-        Sets the file path of the stored PDF file.
+        Sets the file path of the stored report file.
         """
-        self.file = File(file_path)  # type: ignore
+        if not file_path.exists():
+            raise FileNotFoundError(f"File path does not exist: {file_path}")
+
+        save_local_file(self.file, file_path, name=file_path.name, save=False)
         self.save(update_fields=["file"])
 
     @property
     def anonymized_file_path(self) -> Path | None:
         """
-        Returns the file path of the anonymized PDF file if available; otherwise, returns None.
+        Returns the file path of the anonymized report file if available; otherwise, returns None.
         """
-        if self.anonymized_file and self.anonymized_file.name:
+        if self.processed_file and self.processed_file.name:
             try:
-                return Path(self.anonymized_file.path)
+                return Path(self.processed_file.path)
             except (ValueError, AttributeError, NotImplementedError):
                 return None
         return None
 
     def set_anonymized_file_path(self, file_path: Path):
         """
-        Sets the file path of the anonymized PDF file.
+        Sets the file path of the anonymized report file.
         """
-        self.anonymized_file = File(file_path)  # type: ignore
-        self.save(update_fields=["anonymized_file"])
+        if not file_path.exists():
+            raise FileNotFoundError(f"File path does not exist: {file_path}")
+
+        save_local_file(self.processed_file, file_path, name=file_path.name, save=False)
+        self.save(update_fields=["processed_file"])
 
     def get_raw_file_path(self) -> Optional[Path]:
         """
-        Get the path to the raw PDF file, searching common locations.
+        Get the path to the raw report file, searching common locations.
 
-        This method attempts to find the original raw PDF file by checking:
+        This method attempts to find the original raw report file by checking:
         1. Direct hash-based path in raw_pdfs/
         2. Scanning raw_pdfs/ directory for files matching the hash
         3. Checking the file field if it exists
@@ -171,17 +216,17 @@ class RawPdfFile(models.Model):
             try:
                 file_path = Path(self.file.path)
                 if file_path.exists():
-                    logger.debug(f"Found raw PDF via file field: {file_path}")
+                    logger.debug(f"Found raw report via file field: {file_path}")
                     return file_path
             except (ValueError, AttributeError, NotImplementedError):
                 pass
 
         # Define potential raw directories
         raw_dirs = [
-            PDF_DIR / "sensitive",  # Files might be in sensitive dir
-            Path(settings.BASE_DIR) / "data" / "raw_pdfs",
+            SENSITIVE_REPORT_DIR,  # Files might be in sensitive dir
+            Path(settings.BASE_DIR) / "data" / "temporary_reports",
             Path(settings.BASE_DIR) / "data" / "pdfs" / "raw",
-            PDF_DIR,  # General PDF directory
+            IMPORT_REPORT_DIR,  # General report directory
         ]
 
         # Check direct hash-based name in each directory
@@ -191,7 +236,7 @@ class RawPdfFile(models.Model):
 
             hash_path = raw_dir / f"{self.pdf_hash}.pdf"
             if hash_path.exists():
-                logger.debug(f"Found raw PDF at: {hash_path}")
+                logger.debug(f"Found raw report at: {hash_path}")
                 return hash_path
 
         # Scan directories for matching hash
@@ -203,19 +248,19 @@ class RawPdfFile(models.Model):
                 try:
                     file_hash = get_pdf_hash(file_path)
                     if file_hash == self.pdf_hash:
-                        logger.debug(f"Found matching PDF by hash: {file_path}")
+                        logger.debug(f"Found matching report by hash: {file_path}")
                         return file_path
                 except Exception as e:
                     logger.debug(f"Error checking {file_path}: {e}")
                     continue
 
-        logger.warning(f"No raw file found for PDF hash: {self.pdf_hash}")
+        logger.warning(f"No raw file found for report hash: {self.pdf_hash}")
         return None
 
     @property
     def file_url(self):
         """
-        Returns the URL of the stored PDF file if available; otherwise, returns None.
+        Returns the URL of the stored report file if available; otherwise, returns None.
         """
         try:
             return self.file.url if self.file and self.file.name else None
@@ -225,61 +270,20 @@ class RawPdfFile(models.Model):
     @property
     def anonymized_file_url(self):
         """
-        Returns the URL of the stored PDF file if available; otherwise, returns None.
+        Returns the URL of the stored report file if available; otherwise, returns None.
         """
         try:
             return (
-                self.anonymized_file.url
-                if self.anonymized_file and self.anonymized_file.name
+                self.processed_file.url
+                if self.processed_file and self.processed_file.name
                 else None
             )
         except (ValueError, AttributeError):
             return None
 
-    patient = models.ForeignKey(
-        "Patient",
-        on_delete=models.SET_NULL,
-        blank=True,
-        null=True,
-        related_name="raw_pdf_files",
-    )  # type: ignore
-    sensitive_meta = models.ForeignKey(
-        "SensitiveMeta",
-        on_delete=models.SET_NULL,
-        related_name="raw_pdf_files",
-        null=True,
-        blank=True,
-    )  # type: ignore
-    state_report_processing_required = models.BooleanField(default=True)
-    state_report_processed = models.BooleanField(default=False)
-    raw_meta = models.JSONField(blank=True, null=True)
-    anonym_examination_report = models.OneToOneField(
-        "AnonymExaminationReport",
-        on_delete=models.SET_NULL,
-        blank=True,
-        null=True,
-        related_name="raw_pdf_file",
-    )  # type: ignore
-    anonymized_text = models.TextField(blank=True, null=True)
-
-    # Type hinting is needed, improve and use correct django types
-    if TYPE_CHECKING:
-        file: Optional[Union[models.FieldFile, models.FileField]]
-        anonymized_file: Optional[Union[models.FieldFile, models.FileField]]
-        pdf_type: Optional[models.ForeignKey]
-        examination: Optional[models.ForeignKey["PatientExamination"]]
-        examiner: Optional[models.ForeignKey["Examiner"]]
-        patient: Optional[models.ForeignKey["Patient"]]
-        center: Optional[models.ForeignKey["Center"]]
-        anonym_examination_report: Optional[
-            models.OneToOneField["AnonymExaminationReport"]
-        ]
-        sensitive_meta: Optional[models.ForeignKey["SensitiveMeta"]]
-        state: Optional[models.ForeignKey["RawPdfState"]]
-
     def __str__(self):
         """
-        Return a string representation of the RawPdfFile, including its PDF hash, type, and center.
+        Return a string representation of the RawPdfFile, including its report hash, type, and center.
         """
         str_repr = f"{self.pdf_hash} ({self.pdf_type}, {self.center})"
         return str_repr
@@ -288,34 +292,54 @@ class RawPdfFile(models.Model):
         """
         Deletes the RawPdfFile instance from the database and removes the associated file from storage if it exists.
 
-        This method ensures that the physical PDF file is deleted from the file system after the database record is removed. Logs warnings or errors if the file cannot be found or deleted.
+        This method ensures that the physical report file is deleted from the file system after the database record is removed. Logs warnings or errors if the file cannot be found or deleted.
         """
-        # Call the original delete method first to remove DB record
-        if self.file:
-            try:
-                if self.file_path:
-                    os.remove(Path(self.file_path))
-                    logger.info("Original file removed: %s", self.file)
-            except Exception as e:
-                logger.warning(
-                    f"Could not get file path for {self.file.name} before deletion: {e}"
-                )
-        if self.anonymized_file:
-            try:
-                if self.anonymized_file_path:
-                    os.remove(Path(self.anonymized_file_path))
-                    logger.info(
-                        "Anonymized file removed: %s", self.anonymized_file.name
-                    )
-            except OSError as e:
-                logger.error(
-                    "Error removing anonymized file %s: %s",
-                    self.anonymized_file.name,
-                    e,
-                )
+        primary_name = self.file.name if self.file and self.file.name else None
+        anonymized_name = (
+            self.processed_file.name
+            if self.processed_file and self.processed_file.name
+            else None
+        )
+
+        if delete_field_file(self.file, missing_ok=True, save=False):
+            logger.info("Original file removed from storage: %s", primary_name)
+        if delete_field_file(self.processed_file, missing_ok=True, save=False):
+            logger.info("Anonymized file removed from storage: %s", anonymized_name)
 
         super().delete(*args, **kwargs)
 
+        # --- Convenience state/meta helpers used in tests and admin workflows ---
+
+    def mark_sensitive_meta_processed(self, *, save: bool = True) -> "RawPdfFile":
+        """
+        Mark this video's processing state as having its sensitive meta fully processed.
+        This proxies to the related VideoState and persists by default.
+        """
+        sm = self.sensitive_meta
+        from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
+
+        if not isinstance(sm, SensitiveMeta):
+            raise AttributeError()
+        state = self.get_or_create_state()
+        state.mark_sensitive_meta_processed(save=save)
+        return self
+
+    def mark_sensitive_meta_verified(self) -> "RawPdfFile":
+        """
+        Mark the associated SensitiveMeta as verified by setting both DOB and names as verified.
+        Ensures the SensitiveMeta and its state exist.
+        """
+        sm = self.sensitive_meta
+        # Use SensitiveMeta methods to update underlying SensitiveMetaState
+        from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
+
+        if not isinstance(sm, SensitiveMeta):
+            raise AttributeError()
+
+        sm.mark_dob_verified()
+        sm.mark_names_verified()
+        return self
+
     def validate_metadata_annotation(
         self, extracted_data_dict: Optional[dict] = None
     ) -> bool:
@@ -326,16 +350,17 @@ class RawPdfFile(models.Model):
         It also ensures the video file is properly saved after the metadata update.
         """
 
-        if not self.sensitive_meta:
-            logger.error("No sensitive meta data associated with this PDF file.")
-            return False
+        self.mark_sensitive_meta_processed()
+        self.mark_sensitive_meta_verified()
 
         if not extracted_data_dict:
             logger.error("No extracted data provided for validation.")
             return False
 
-        # Update sensitive meta with the provided data
-        self.sensitive_meta.update_from_dict(extracted_data_dict)
+        if extracted_data_dict:
+            self.sensitive_meta.update_from_dict(extracted_data_dict)
+        else:
+            return False
 
         # Save the sensitive meta to ensure changes are persisted
         self.sensitive_meta.save()
@@ -343,25 +368,22 @@ class RawPdfFile(models.Model):
         # Save the RawPdfFile instance to ensure all changes are saved
         self.save()
 
-        logger.info(f"Metadata for PDF {self.pk} validated and updated successfully.")
-
-        if self.file_path:
-            try:
-                os.unlink(self.file_path)  # Delete the original file if it exists
-            except OSError as e:
-                logger.error(f"Error removing original file {self.file_path}: {e}")
+        logger.info(
+            f"Metadata for report {self.pk} validated and updated successfully."
+        )
 
-        if self.anonymized_file_path:
-            try:
-                os.unlink(self.anonymized_file_path)
-            except OSError as e:
-                logger.error(
-                    f"Error removing anonymized file {self.anonymized_file_path}: {e}"
-                )
+        deleted_original = delete_field_file(self.file, missing_ok=True, save=False)
+        deleted_anonymized = delete_field_file(
+            self.processed_file, missing_ok=True, save=False
+        )
+        self.get_or_create_state().mark_anonymization_validated()
 
-        self.save()  # Save the model to persist the cleared file fields
+        if deleted_original or deleted_anonymized:
+            self.save(
+                update_fields=["file", "processed_file"]
+            )  # Persist cleared fields
 
-        logger.info(f"Files for PDF {self.pk} deleted successfully.")
+        logger.info(f"Files for report {self.pk} deleted successfully.")
         return True
 
     @classmethod
@@ -375,8 +397,8 @@ class RawPdfFile(models.Model):
         Creates a RawPdfFile instance from a file and center name, ensuring an associated RawPdfState exists.
 
         Parameters:
-            file_path (Path): Path to the source PDF file.
-            center_name (str): Name of the center to associate with the PDF.
+            file_path (Path): Path to the source report file.
+            center_name (str): Name of the center to associate with the report.
             delete_source (bool): Whether to delete the source file after processing. Defaults to True.
 
         Returns:
@@ -400,12 +422,12 @@ class RawPdfFile(models.Model):
         delete_source=True,
     ):
         """
-        Creates or retrieves a RawPdfFile instance from a given PDF file path and center name.
+        Creates or retrieves a RawPdfFile instance from a given report file path and center name.
 
-        If a RawPdfFile with the same PDF hash already exists, verifies the file exists in storage and restores it if missing. Otherwise, creates a new RawPdfFile, assigns the file, and saves it to storage. Optionally deletes the source file after processing.
+        If a RawPdfFile with the same report hash already exists, verifies the file exists in storage and restores it if missing. Otherwise, creates a new RawPdfFile, assigns the file, and saves it to storage. Optionally deletes the source file after processing.
 
         Parameters:
-            file_path (Path): Path to the source PDF file.
+            file_path (Path): Path to the source report file.
             center_name (str): Name of the center to associate with the file.
             save (bool, optional): Deprecated; saving occurs internally.
             delete_source (bool, optional): Whether to delete the source file after processing (default True).
@@ -416,7 +438,7 @@ class RawPdfFile(models.Model):
         Raises:
             FileNotFoundError: If the source file does not exist.
             Center.DoesNotExist: If the specified center is not found.
-            ValueError: If the PDF hash cannot be calculated.
+            ValueError: If the report hash cannot be calculated.
             IOError: If the file fails to save to storage.
         """
         from endoreg_db.models.administration import Center
@@ -460,7 +482,7 @@ class RawPdfFile(models.Model):
                             django_file = File(
                                 f, name=Path(_file.name).name
                             )  # Use existing name if possible
-                            existing_pdf_file.file = django_file  # type: ignore
+                            existing_pdf_file.file = django_file
                             existing_pdf_file.save(
                                 update_fields=["file"]
                             )  # Only update file field
@@ -554,64 +576,42 @@ class RawPdfFile(models.Model):
         # Ensure hash is calculated before the first save if possible and not already set
         # This is primarily a fallback if instance created manually without using create_from_file
         """
-        Saves the RawPdfFile instance, ensuring the PDF hash is set and related fields are derived from metadata.
+        Saves the RawPdfFile instance, ensuring the report hash is set and related fields are derived from metadata.
 
-        If the PDF hash is missing, attempts to calculate it from the file before saving. Validates that the file has a `.pdf` extension. If related fields such as patient, examination, center, or examiner are unset but available in the associated sensitive metadata, they are populated accordingly before saving.
+        If the report hash is missing, attempts to calculate it from the file before saving. Validates that the file has a `.pdf` extension. If related fields such as patient, examination, center, or examiner are unset but available in the associated sensitive metadata, they are populated accordingly before saving.
         """
         if not self.pk and not self.pdf_hash and self.file:
             try:
-                file_path = Path(self.file.path).resolve()
-                if not file_path.exists():
-                    raise FileNotFoundError(f"File path does not exist: {file_path}")
-                # Read from the file object before it's saved by storage
-                self.file.open("rb")  # Ensure file is open
-                self.file.seek(0)  # Go to beginning
-                self.pdf_hash = get_pdf_hash(
-                    file_path
-                )  # Assuming get_pdf_hash can handle file obj
-                self.file.seek(0)  # Reset position
-                self.file.close()  # Close after reading
-                logger.info(f"Calculated hash during pre-save for {self.file.name}")
-            except Exception as e:
+                with ensure_local_file(self.file) as local_path:
+                    self.pdf_hash = get_pdf_hash(local_path)
+                    logger.info(
+                        "Calculated hash during pre-save for %s", self.file.name
+                    )
+            except Exception as exc:
                 logger.warning(
                     "Could not calculate hash before initial save for %s: %s",
                     self.file.name,
-                    e,
+                    exc,
                 )
-                # Ensure file is closed if opened
-                if hasattr(self.file, "closed") and not self.file.closed:
-                    self.file.close()
 
         if self.file and not self.file.name.endswith(".pdf"):
-            raise ValidationError("Only PDF files are allowed")
+            raise ValidationError("Only report files are allowed")
 
         # If hash is still missing after potential creation logic (e.g., direct instantiation)
         # and the file exists in storage, try calculating it from storage path.
         # This is less ideal as it requires the file to be saved first.
-        if (
-            not self.pdf_hash
-            and self.pk
-            and self.file
-            and self.file.storage.exists(self.file.name)
-        ):
+        if not self.pdf_hash and self.pk and self.file and file_exists(self.file):
             try:
-                file_path = Path(self.file.path).resolve()
-                if not file_path.exists():
-                    raise FileNotFoundError(f"File path does not exist: {file_path}")
-                logger.warning(
-                    f"Hash missing for saved file {self.file.name}. Recalculating."
-                )
-                with self.file.storage.open(self.file.name, "rb") as f:
-                    self.pdf_hash = get_pdf_hash(
-                        file_path
-                    )  # Assuming get_pdf_hash handles file obj
-                # No need to save again just for hash unless update_fields is used carefully
-                # Let the main super().save() handle saving the hash if it changed
-            except Exception as e:
+                with ensure_local_file(self.file) as local_path:
+                    logger.warning(
+                        "Hash missing for saved file %s. Recalculating.", self.file.name
+                    )
+                    self.pdf_hash = get_pdf_hash(local_path)
+            except Exception as exc:
                 logger.error(
                     "Could not calculate hash during save for existing file %s: %s",
                     self.file.name,
-                    e,
+                    exc,
                 )
 
         # Derive related fields from sensitive_meta if available
@@ -651,7 +651,7 @@ class RawPdfFile(models.Model):
         # This method might still be useful if called explicitly, but create_from_file now handles restoration
         # Ensure fallback_file is a Path object.
         """
-        Checks if the stored PDF file exists in storage and attempts to restore it from a fallback file path if missing.
+        Checks if the stored report file exists in storage and attempts to restore it from a fallback file path if missing.
 
         Parameters:
             fallback_file: Path or string representing the fallback file location to restore from if the stored file is missing.
@@ -750,8 +750,8 @@ class RawPdfFile(models.Model):
         return settings_dict
 
     @staticmethod
-    def get_pdf_by_id(pdf_id: int) -> "RawPdfFile":
+    def get_pdf_by_pk(pk: int) -> "RawPdfFile":
         try:
-            return RawPdfFile.objects.get(pk=pdf_id)
+            return RawPdfFile.objects.get(pk=pk)
         except RawPdfFile.DoesNotExist:
-            raise ValueError(f"PDF with ID {pdf_id} does not exist.")
+            raise ValueError(f"report with ID {pdf_id} does not exist.")