endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

endoreg_db/services/pdf_import.py

@@ -2,7 +2,9 @@
 PDF import service module.
 
 Provides high-level functions for importing and anonymizing PDF files,
-combining RawPdfFile creation with text extraction and anonymization.
+combining RawPdfFile creation with text extraction and anonymization using lx anonymizer.
+
+All Fields should be overwritten from anonymizer defaults except for the center which is given.
 """
 
 import errno
@@ -16,8 +18,10 @@ from contextlib import contextmanager
 from datetime import date, datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Union
-
+import subprocess
 from django.db import transaction
+from django.core.exceptions import ObjectDoesNotExist
+import lx_anonymizer
 
 from endoreg_db.models import SensitiveMeta
 from endoreg_db.models.media.pdf.raw_pdf import RawPdfFile
@@ -44,7 +48,7 @@ class PdfImportService:
     """
 
     def __init__(
-        self, allow_meta_overwrite: bool = False, processing_mode: str = "blackening"
+        self, allow_meta_overwrite: bool = True, processing_mode: str = "blackening"
     ):
         """
         Initialize the PDF import service.
@@ -68,7 +72,14 @@ class PdfImportService:
 
         # Central PDF instance management
         self.current_pdf = None
+        self.current_pdf_state = None
         self.processing_context = {}
+        self.original_path = None
+        
+        self.DEFAULT_PATIENT_FIRST_NAME = "Patient"
+        self.DEFAULT_PATIENT_LAST_NAME = "Unknown"
+        self.DEFAULT_PATIENT_DOB = date(1990, 1, 1)
+        self.DEFAULT_CENTER_NAME = "university_hospital_wuerzburg"
 
     @classmethod
     def with_blackening(cls, allow_meta_overwrite: bool = False) -> "PdfImportService":
@@ -137,6 +148,7 @@ class PdfImportService:
                     fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
                 else:
                     # Another worker is processing this file
+                    
                     raise ValueError(f"File already being processed: {path}")
 
             os.write(fd, b"lock")
@@ -200,6 +212,10 @@ class PdfImportService:
                 shutil.move(str(source), str(target))
             else:
                 raise
+        lock_path = Path(str(source) + ".lock")
+        if lock_path.exists():
+            lock_path.unlink()
+
         return target
 
     def _ensure_state(self, pdf_file: "RawPdfFile"):
@@ -209,14 +225,10 @@ class PdfImportService:
         if hasattr(pdf_file, "get_or_create_state"):
             state = pdf_file.get_or_create_state()
             pdf_file.state = state
+            self.current_pdf_state = state
+            assert isinstance(self.current_pdf_state, RawPdfState)
             return state
-        # Very defensive fallback
-        try:
-            state, _ = pdf_file.get_or_create_state(raw_pdf_file=pdf_file)
-            pdf_file.state = state
-            return state
-        except Exception:
-            return None
+
 
     def _ensure_report_reading_available(self):
         """
@@ -266,7 +278,7 @@ class PdfImportService:
         self._report_reader_class = None
         return False, None
 
-    def _ensure_default_patient_data(self, pdf_instance: "RawPdfFile" = None) -> None:
+    def _ensure_default_patient_data(self, pdf_instance: "RawPdfFile") -> None:
         """
         Ensure PDF has minimum required patient data in SensitiveMeta.
         Creates default values if data is missing after text processing.
@@ -289,15 +301,18 @@ class PdfImportService:
 
             # Create default SensitiveMeta with placeholder data
             default_data = {
-                "patient_first_name": "Patient",
-                "patient_last_name": "Unknown",
-                "patient_dob": date(1990, 1, 1),  # Default DOB
-                "examination_date": date.today(),
-                "center_name": pdf_file.center.name
-                if pdf_file.center
-                else "university_hospital_wuerzburg",
+                "patient_first_name": self.DEFAULT_PATIENT_FIRST_NAME,
+                "patient_last_name": self.DEFAULT_PATIENT_LAST_NAME,
+                "patient_dob": self.DEFAULT_PATIENT_DOB,
+                "examination_date": date.today(),  # today is intentionally *not* a constant
+                "center_name": (
+                    pdf_file.center.name
+                    if pdf_file.center
+                    else self.DEFAULT_CENTER_NAME
+                ),
             }
 
+
             try:
                 sensitive_meta = SensitiveMeta.create_from_dict(default_data)
                 pdf_file.sensitive_meta = sensitive_meta
@@ -316,7 +331,7 @@ class PdfImportService:
         center_name: str,
         delete_source: bool = False,
         retry: bool = False,
-    ) -> "RawPdfFile":
+    ) -> "RawPdfFile | None":
         """
         Import a PDF file and anonymize it using ReportReader.
         Uses centralized PDF instance management pattern.
@@ -354,8 +369,7 @@ class PdfImportService:
                 logger.warning(
                     f"No PDF instance created for {file_path}, returning None"
                 )
-                return None
-
+                raise ObjectDoesNotExist
             # Step 3: Setup processing environment
             self._setup_processing_environment()
 
@@ -371,7 +385,7 @@ class PdfImportService:
             # Handle "File already being processed" case specifically
             if "already being processed" in str(e):
                 logger.info(f"Skipping file {file_path}: {e}")
-                return None
+                return
             else:
                 logger.error(f"PDF import failed for {file_path}: {e}")
                 self._cleanup_on_error()
@@ -405,6 +419,7 @@ class PdfImportService:
             "metadata_processed": False,
             "anonymization_completed": False,
         }
+        self.original_path = Path(file_path)
 
         # Check if already processed (only during current session to prevent race conditions)
         if str(file_path) in self.processed_files:
@@ -497,8 +512,17 @@ class PdfImportService:
     def _setup_processing_environment(self):
         """Setup processing environment and state."""
         original_path = self.processing_context.get("file_path")
-
+        if not original_path or not self.current_pdf:
+            try:
+                self.current_pdf = RawPdfFile.objects.get(pdf_hash=self.processing_context["file_hash"])
+                self.original_path = Path(str(self.current_pdf.file.path))
+                    
+            except RawPdfFile.DoesNotExist:
+                raise RuntimeError("Processing environment setup failed")
         # Create sensitive file copy
+        if original_path is None or not isinstance(original_path, (str, Path)):
+            logger.error(f"No original path: {original_path!r}")
+            return
         self.create_sensitive_file(self.current_pdf, original_path)
 
         # Update file path to point to sensitive copy
@@ -525,13 +549,17 @@ class PdfImportService:
 
     def _process_text_and_metadata(self):
         """Process text extraction and metadata using ReportReader."""
-        report_reading_available, ReportReader = self._ensure_report_reading_available()
-
+        report_reading_available, ReportReaderCls = self._ensure_report_reading_available()
+        try:
+            assert ReportReaderCls is not None and report_reading_available
+            assert self.current_pdf is not None 
+        except AssertionError as e:
+            logger.error(f"PDF Import failed on Error:{e} Ensure the pdf was passed correctly and report reading is available in function _process_text_and_metadata() ")
         if not report_reading_available:
             logger.warning("Report reading not available (lx_anonymizer not found)")
             self._mark_processing_incomplete("no_report_reader")
-            return
-
+            return 
+        assert self.current_pdf is not None
         if not self.current_pdf.file:
             logger.warning("No file available for text processing")
             self._mark_processing_incomplete("no_file")
@@ -541,9 +569,10 @@ class PdfImportService:
             logger.info(
                 f"Starting text extraction and metadata processing with ReportReader (mode: {self.processing_mode})..."
             )
+            ReportReaderCls = lx_anonymizer.ReportReader            
 
             # Initialize ReportReader
-            report_reader = ReportReader(
+            report_reader = ReportReaderCls(
                 report_root_path=str(path_utils.STORAGE_DIR),
                 locale="de_DE",
                 text_date_format="%d.%m.%Y",
@@ -567,7 +596,7 @@ class PdfImportService:
         # Setup anonymized directory
         anonymized_dir = path_utils.PDF_DIR / "anonymized"
         anonymized_dir.mkdir(parents=True, exist_ok=True)
-
+        assert self.current_pdf is not None
         # Generate output path for anonymized PDF
         pdf_hash = self.current_pdf.pdf_hash
         anonymized_output_path = anonymized_dir / f"{pdf_hash}_anonymized.pdf"
@@ -702,7 +731,8 @@ class PdfImportService:
             "examiner_first_name": "examiner_first_name",
             "examiner_last_name": "examiner_last_name",
             "endoscope_type": "endoscope_type",
-            "casenumber": "case_number",
+            "casenumber": "casenumber",
+            "center_name": "center_name",
         }
 
         # Update fields with extracted information
@@ -727,9 +757,9 @@ class PdfImportService:
                 # Configurable overwrite policy
                 should_overwrite = (
                     self.allow_meta_overwrite
-                    or not old_value
-                    or old_value in ["Patient", "Unknown"]
+                    or self._is_placeholder_value(sm_field, old_value)
                 )
+
                 if new_value and should_overwrite:
                     setattr(sm, sm_field, new_value)
                     updated_fields.append(sm_field)
@@ -835,8 +865,9 @@ class PdfImportService:
 
             # Mark state as anonymized immediately; this keeps downstream flows working
             state = self._ensure_state(self.current_pdf)
-            if state and not state.anonymized:
-                state.mark_anonymized(save=True)
+            
+            if state and not state.processing_started:
+                state.mark_processing_started()
 
             logger.info(
                 "Updated anonymized_file reference to: %s",
@@ -944,15 +975,66 @@ class PdfImportService:
 
     def _cleanup_on_error(self):
         """Cleanup processing context on error."""
+        original_path = self.original_path
         try:
             if self.current_pdf and hasattr(self.current_pdf, "state"):
                 state = self._ensure_state(self.current_pdf)
+                raw_file_path = self.current_pdf.get_raw_file_path()
+                if raw_file_path is not None and original_path is not None:
+                    # Ensure reprocessing for next attempt by restoring original file
+                    shutil.copy2(str(raw_file_path), str(original_path))
+                    
+                # Ensure no two files can remain
+                if raw_file_path == original_path and raw_file_path is not None and original_path is not None:
+                    os.remove(str(raw_file_path))
+                    
+                    
+                # Remove Lock file also
+                lock_path = Path(str(path_utils.PDF_DIR) + ".lock")
+                try:
+                    if lock_path.exists():
+                        lock_path.unlink()
+                        logger.info("Removed lock file during quarantine: %s", lock_path)
+                except Exception as e:
+                    logger.warning("Could not remove lock file during quarantine: %s", e)
+
+                
                 if state and self.processing_context.get("processing_started"):
                     state.text_meta_extracted = False
                     state.pdf_meta_extracted = False
                     state.sensitive_meta_processed = False
+                    state.anonymized = False
                     state.save()
                     logger.debug("Updated PDF state to indicate processing failure")
+            else:
+                # 🔧 Early failure: no current_pdf (or no state).
+                # In this case we want to make sure we don't leave stray files
+                # under PDF_DIR or PDF_DIR/sensitive.
+
+                pdf_dir = self._get_pdf_dir()
+                if pdf_dir and pdf_dir.exists():
+                    for candidate_dir in (pdf_dir, pdf_dir / "sensitive"):
+                        if candidate_dir.exists():
+                            for candidate in candidate_dir.glob("*.pdf"):
+                                # Don't delete the original ingress file
+                                if (
+                                    original_path is not None
+                                    and candidate.resolve() == Path(original_path).resolve()
+                                ):
+                                    continue
+                                try:
+                                    candidate.unlink()
+                                    logger.debug(
+                                        "Removed stray PDF during early error cleanup: %s",
+                                        candidate,
+                                    )
+                                except Exception as e:
+                                    logger.warning(
+                                        "Failed to remove stray PDF %s: %s",
+                                        candidate,
+                                        e,
+                                    )
+
         except Exception as e:
             logger.warning(f"Error during cleanup: {e}")
         finally:
@@ -975,6 +1057,25 @@ class PdfImportService:
                         "Failed to remove sensitive copy during error cleanup: %s",
                         cleanup_exc,
                     )
+                pdf_dir = self._get_pdf_dir()
+                if original_path and pdf_dir:
+                    # Try to remove any extra file that was created during import
+                    # Simplest heuristic: same basename as original, but in pdf dir or pdf/sensitive dir
+                    for candidate_dir in (pdf_dir, pdf_dir / "sensitive"):
+                        candidate = candidate_dir / original_path.name
+                        if candidate.exists() and candidate != original_path:
+                            try:
+                                candidate.unlink()
+                                logger.debug(
+                                    "Removed stray PDF copy during early error cleanup: %s",
+                                    candidate,
+                                )
+                            except Exception as e:
+                                logger.warning(
+                                    "Failed to remove stray PDF copy %s: %s",
+                                    candidate,
+                                    e,
+                                )
 
             # Always clean up processed files set to prevent blocks
             file_path = self.processing_context.get("file_path")
@@ -985,32 +1086,10 @@ class PdfImportService:
                 )
 
             try:
-                original_path = self.processing_context.get("original_file_path")
-                logger.debug(
-                    "PDF cleanup original path: %s (%s)",
-                    original_path,
-                    type(original_path),
-                )
                 raw_dir = (
                     original_path.parent if isinstance(original_path, Path) else None
                 )
-                if (
-                    isinstance(original_path, Path)
-                    and original_path.exists()
-                    and not self.processing_context.get("sensitive_copy_created")
-                ):
-                    try:
-                        original_path.unlink()
-                        logger.info(
-                            "Removed original file %s during error cleanup",
-                            original_path,
-                        )
-                    except Exception as remove_exc:
-                        logger.warning(
-                            "Could not remove original file %s during error cleanup: %s",
-                            original_path,
-                            remove_exc,
-                        )
+
                 pdf_dir = self._get_pdf_dir()
                 if not pdf_dir and raw_dir:
                     base_dir = raw_dir.parent
@@ -1181,7 +1260,7 @@ class PdfImportService:
 
     def check_storage_capacity(
         self, file_path: Union[Path, str], storage_root, min_required_space
-    ) -> None:
+    ) -> bool:
         """
         Check if there is sufficient storage capacity for the PDF file.
 
@@ -1221,7 +1300,7 @@ class PdfImportService:
         return True
 
     def create_sensitive_file(
-        self, pdf_instance: "RawPdfFile" = None, file_path: Union[Path, str] = None
+        self, pdf_instance: "RawPdfFile", file_path: Union[Path, str]
     ) -> None:
         """
         Create a copy of the PDF file in the sensitive directory and update the file reference.
@@ -1304,10 +1383,10 @@ class PdfImportService:
 
     def archive_or_quarantine_file(
         self,
-        pdf_instance: "RawPdfFile" = None,
-        source_file_path: Union[Path, str] = None,
-        quarantine_reason: str = None,
-        is_pdf_problematic: bool = None,
+        pdf_instance: "RawPdfFile",
+        source_file_path: Union[Path, str],
+        quarantine_reason: str,
+        is_pdf_problematic: bool,
     ) -> bool:
         """
         Archive or quarantine file based on the state of the PDF processing.
@@ -1328,9 +1407,9 @@ class PdfImportService:
             if source_file_path
             else self.processing_context.get("file_path")
         )
-        quarantine_reason = quarantine_reason or self.processing_context.get(
+        quarantine_reason = str(quarantine_reason or self.processing_context.get(
             "error_reason"
-        )
+        ))
 
         if not pdf_file:
             raise ValueError("No PDF instance available for archiving/quarantine")
@@ -1355,9 +1434,6 @@ class PdfImportService:
             quarantine_path = quarantine_dir / f"{pdf_file.pdf_hash}.pdf"
             try:
                 shutil.move(file_path, quarantine_path)
-                pdf_file.quarantine_reason = (
-                    quarantine_reason or "File processing failed"
-                )
                 pdf_file.save(update_fields=["quarantine_reason"])
                 logger.info(f"Moved problematic PDF to quarantine: {quarantine_path}")
                 return True
@@ -1380,3 +1456,25 @@ class PdfImportService:
             except Exception as e:
                 logger.error(f"Failed to archive PDF {pdf_file.pdf_hash}: {e}")
                 return False
+    
+    def _is_placeholder_value(self, field_name: str, value) -> bool:
+        """Return True if a SensitiveMeta field still has a dummy/default value."""
+        if value is None:
+            return True
+
+        # String placeholders
+        if isinstance(value, str):
+            if value in {self.DEFAULT_PATIENT_FIRST_NAME, self.DEFAULT_PATIENT_LAST_NAME}:
+                return True
+
+        # Date placeholders
+        if isinstance(value, date):
+            # Default DOB
+            if field_name == "patient_dob" and value == self.DEFAULT_PATIENT_DOB:
+                return True
+            # "Today" exam date created as fallback – allow anonymizer to override
+            if field_name == "examination_date" and value == date.today():
+                return True
+
+        return False
+