endoreg-db 0.8.3.0__py3-none-any.whl → 0.8.3.2__py3-none-any.whl

endoreg_db/services/video_import.py

@@ -19,16 +19,12 @@ from pathlib import Path
 from typing import Union, Dict, Any, Optional, List, Tuple
 from django.db import transaction
 from endoreg_db.models import VideoFile, SensitiveMeta
-from endoreg_db.utils.paths import STORAGE_DIR, RAW_FRAME_DIR, VIDEO_DIR, ANONYM_VIDEO_DIR
+from endoreg_db.utils.paths import STORAGE_DIR, VIDEO_DIR, ANONYM_VIDEO_DIR
 import random
-from lx_anonymizer.ocr import trocr_full_image_ocr
 from endoreg_db.utils.hashs import get_video_hash
-from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets, _anonymize
-from typing import TYPE_CHECKING
+from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
 from django.db.models.fields.files import FieldFile
-
-if TYPE_CHECKING:
-    from endoreg_db.models import EndoscopyProcessor
+from endoreg_db.models import EndoscopyProcessor
 
 # File lock configuration (matches PDF import)
 STALE_LOCK_SECONDS = 6000  # 100 minutes - reclaim locks older than this
@@ -58,15 +54,13 @@ class VideoImportService():
             self.project_root = Path(__file__).parent.parent.parent.parent
         
         # Track processed files to prevent duplicates
-        self.processed_files = set(str(file) for file in os.listdir(ANONYM_VIDEO_DIR))
-            
-        self.STORAGE_DIR = STORAGE_DIR
-        
+        self.processed_files = set(str(Path(ANONYM_VIDEO_DIR) / file) for file in os.listdir(ANONYM_VIDEO_DIR))
+                    
         # Central video instance and processing context
         self.current_video: Optional[VideoFile] = None
         self.processing_context: Dict[str, Any] = {}
         
-        self.delete_source = False
+        self.delete_source = True
         
         self.logger = logging.getLogger(__name__)
 
@@ -166,12 +160,12 @@ class VideoImportService():
                     return None
                 raise
             
-            # Create sensitive meta file, ensure raw is moved out of processing folder watched by file watcher.
-            self._create_sensitive_file()
-            
             # Create or retrieve video instance
             self._create_or_retrieve_video_instance()
             
+            # Create sensitive meta file, ensure raw is moved out of processing folder watched by file watcher.
+            self._create_sensitive_file()
+            
             # Setup processing environment
             self._setup_processing_environment()
             
@@ -225,8 +219,12 @@ class VideoImportService():
         
         # Acquire file lock to prevent concurrent processing
         # Lock will be held until finally block in import_and_anonymize()
-        self.processing_context['_lock_context'] = self._file_lock(file_path)
-        self.processing_context['_lock_context'].__enter__()
+        try:
+            self.processing_context['_lock_context'] = self._file_lock(file_path)
+            self.processing_context['_lock_context'].__enter__()
+        except Exception:
+            self._cleanup_processing_context()
+            raise
         
         self.logger.info("Acquired file lock for: %s", file_path)
         
@@ -244,7 +242,6 @@ class VideoImportService():
 
     def _create_or_retrieve_video_instance(self):
         """Create or retrieve the VideoFile instance and move to final storage."""
-        # Removed duplicate import of VideoFile (already imported at module level)
         
         self.logger.info("Creating VideoFile instance...")
         
@@ -275,96 +272,78 @@ class VideoImportService():
     def _move_to_final_storage(self):
         """
         Move video from raw_videos to final storage locations.
-        - Raw video → /data/videos (raw_file_path) 
+        - Raw video → /data/videos (raw_file_path)
         - Processed video will later → /data/anonym_videos (file_path)
         """
         from endoreg_db.utils import data_paths
-        
-        source_path = self.processing_context['file_path']
 
-        videos_dir = data_paths["video"]
-        videos_dir.mkdir(parents=True, exist_ok=True)
+        source_path = Path(self.processing_context["file_path"])
+        _current_video = self._require_current_video()
+        videos_dir = Path(data_paths["video"])
+        storage_root = Path(data_paths["storage"])
 
-        _current_video = self.current_video
-        assert _current_video is not None, "Current video instance is None during storage move"
+        videos_dir.mkdir(parents=True, exist_ok=True)
 
+        # --- Derive stored_raw_path safely ---
         stored_raw_path = None
-        if hasattr(_current_video, "get_raw_file_path"):
-            possible_path = _current_video.get_raw_file_path()
-            if possible_path:
-                try:
-                    stored_raw_path = Path(possible_path)
-                except (TypeError, ValueError):
-                    stored_raw_path = None
-
-        if stored_raw_path:
-            try:
-                storage_root = data_paths["storage"]
-                if stored_raw_path.is_absolute():
-                    if not stored_raw_path.is_relative_to(storage_root):
+        try:
+            if hasattr(_current_video, "get_raw_file_path"):
+                candidate = _current_video.get_raw_file_path()
+                if candidate:
+                    candidate_path = Path(candidate)
+                    # Accept only if under storage_root
+                    try:
+                        candidate_path.relative_to(storage_root)
+                        stored_raw_path = candidate_path
+                    except ValueError:
+                        # outside storage_root, reset
                         stored_raw_path = None
-                else:
-                    if stored_raw_path.parts and stored_raw_path.parts[0] == videos_dir.name:
-                        stored_raw_path = storage_root / stored_raw_path
-                    else:
-                        stored_raw_path = videos_dir / stored_raw_path.name
-            except Exception:
-                stored_raw_path = None
-
-        if stored_raw_path and not stored_raw_path.suffix:
+        except Exception:
             stored_raw_path = None
 
+        # Fallback: derive from UUID + suffix
         if not stored_raw_path:
+            suffix = source_path.suffix or ".mp4"
             uuid_str = getattr(_current_video, "uuid", None)
-            source_suffix = Path(source_path).suffix or ".mp4"
-            filename = f"{uuid_str}{source_suffix}" if uuid_str else Path(source_path).name
+            filename = f"{uuid_str}{suffix}" if uuid_str else source_path.name
             stored_raw_path = videos_dir / filename
 
-        delete_source = bool(self.processing_context.get('delete_source'))
+        delete_source = bool(self.processing_context.get("delete_source", True))
         stored_raw_path.parent.mkdir(parents=True, exist_ok=True)
 
-        if not stored_raw_path.exists():
-            try:
-                if source_path.exists():
-                    if delete_source:
-                        shutil.move(str(source_path), str(stored_raw_path))
-                        self.logger.info("Moved raw video to: %s", stored_raw_path)
-                    else:
-                        shutil.copy2(str(source_path), str(stored_raw_path))
-                        self.logger.info("Copied raw video to: %s", stored_raw_path)
-                else:
-                    raise FileNotFoundError(f"Neither stored raw path nor source path exists for {self.processing_context['file_path']}")
-            except Exception as e:
-                self.logger.error("Failed to place video in final storage: %s", e)
-                raise
-        else:
-            # If we already have the stored copy, respect delete_source flag without touching assets unnecessarily
-            if delete_source and source_path.exists():
+        # --- Move or copy raw video ---
+        try:
+            if delete_source:
+                # Try atomic move first, fallback to copy+unlink
                 try:
+                    os.replace(source_path, stored_raw_path)
+                    self.logger.info("Moved raw video to: %s", stored_raw_path)
+                except Exception:
+                    shutil.copy2(source_path, stored_raw_path)
                     os.remove(source_path)
-                    self.logger.info("Removed original source file after storing copy: %s", source_path)
-                except OSError as e:
-                    self.logger.warning("Failed to remove source file %s: %s", source_path, e)
+                    self.logger.info("Copied & removed raw video to: %s", stored_raw_path)
+            else:
+                shutil.copy2(source_path, stored_raw_path)
+                self.logger.info("Copied raw video to: %s", stored_raw_path)
+        except Exception as e:
+            self.logger.error("Failed to move/copy video to final storage: %s", e)
+            raise
 
-        # Ensure database path points to stored location (relative to storage root)
+        # --- Ensure DB raw_file is relative to storage root ---
         try:
-            storage_root = data_paths["storage"]
-            relative_path = Path(stored_raw_path).relative_to(storage_root)
-            if _current_video.raw_file.name != str(relative_path):
-                _current_video.raw_file.name = str(relative_path)
-                _current_video.save(update_fields=['raw_file'])
-                self.logger.info("Updated raw_file path to: %s", relative_path)
-        except Exception as e:
-            self.logger.error("Failed to ensure raw_file path is relative: %s", e)
-            fallback_relative = Path("videos") / Path(stored_raw_path).name
-            if _current_video.raw_file.name != fallback_relative.as_posix():
-                _current_video.raw_file.name = fallback_relative.as_posix()
-                _current_video.save(update_fields=['raw_file'])
-                self.logger.info("Updated raw_file path using fallback: %s", fallback_relative.as_posix())
+            rel_path = stored_raw_path.relative_to(storage_root)
+        except Exception:
+            rel_path = Path("videos") / stored_raw_path.name
+
+        if _current_video.raw_file.name != rel_path.as_posix():
+            _current_video.raw_file.name = rel_path.as_posix()
+            _current_video.save(update_fields=["raw_file"])
+            self.logger.info("Updated raw_file path to: %s", rel_path.as_posix())
+
+        # --- Store for later stages ---
+        self.processing_context["raw_video_path"] = stored_raw_path
+        self.processing_context["video_filename"] = stored_raw_path.name
 
-        # Store paths for later processing
-        self.processing_context['raw_video_path'] = Path(stored_raw_path)
-        self.processing_context['video_filename'] = Path(stored_raw_path).name
 
     def _setup_processing_environment(self):
         """Setup the processing environment without file movement."""
@@ -406,7 +385,7 @@ class VideoImportService():
     def _process_frames_and_metadata(self):
         """Process frames and extract metadata with anonymization."""
         # Check frame cleaning availability
-        frame_cleaning_available, FrameCleaner, ReportReader = self._ensure_frame_cleaning_available()
+        frame_cleaning_available, frame_cleaner  = self._ensure_frame_cleaning_available()
         video = self._require_current_video()
 
         raw_file_field = video.raw_file
@@ -427,7 +406,7 @@ class VideoImportService():
             from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError
             
             with ThreadPoolExecutor(max_workers=1) as executor:
-                future = executor.submit(self._perform_frame_cleaning, FrameCleaner, endoscope_data_roi_nested, endoscope_image_roi)
+                future = executor.submit(self._perform_frame_cleaning, endoscope_data_roi_nested, endoscope_image_roi)
                 try:
                     # Increased timeout to better accommodate ffmpeg + OCR
                     future.result(timeout=300)
@@ -473,6 +452,9 @@ class VideoImportService():
                 self.processing_context['error_reason'] = f"Frame cleaning failed: {e}, Fallback failed: {fallback_error}"
 
     def _save_anonymized_video(self):
+        
+        original_raw_file_path_to_delete = None
+        original_raw_frame_dir_to_delete = None
         video = self._require_current_video()
         anonymized_video_path = video.get_target_anonymized_video_path()
 
@@ -760,6 +742,17 @@ class VideoImportService():
         except Exception as exc:
             self.logger.error("Failed to retrieve processor ROI information: %s", exc)
 
+        # Convert dict to nested list if necessary to match return type
+        if isinstance(endoscope_data_roi_nested, dict):
+            # Convert dict[str, dict[str, int | None] | None] to List[List[Dict[str, Any]]]
+            converted_roi = []
+            for key, value in endoscope_data_roi_nested.items():
+                if isinstance(value, dict):
+                    converted_roi.append([value])
+                elif value is None:
+                    converted_roi.append([])
+            endoscope_data_roi_nested = converted_roi
+
         return endoscope_data_roi_nested, endoscope_image_roi
 
     def _ensure_default_patient_data(self, video_instance: VideoFile | None = None) -> None:
@@ -781,8 +774,6 @@ class VideoImportService():
                 sensitive_meta = SensitiveMeta.create_from_dict(default_data)
                 video.sensitive_meta = sensitive_meta
                 video.save(update_fields=["sensitive_meta"])
-                state = video.get_or_create_state()
-                state.mark_sensitive_meta_processed(save=True)
                 self.logger.info("Created default SensitiveMeta for video %s", video.uuid)
             except Exception as exc:
                 self.logger.error("Failed to create default SensitiveMeta for video %s: %s", video.uuid, exc)
@@ -821,67 +812,43 @@ class VideoImportService():
             Tuple of (availability_flag, FrameCleaner_class, ReportReader_class)
         """
         try:
-            # Check if we can find the lx-anonymizer directory
-            from importlib import resources
-            lx_anonymizer_path = resources.files("lx_anonymizer")
+            # Check if we can find lx-anonymizer
+            from lx_anonymizer import FrameCleaner  # type: ignore[import]
 
-            # make sure lx_anonymizer_path is a Path object
-            lx_anonymizer_path = Path(str(lx_anonymizer_path))
-            
-            if lx_anonymizer_path.exists():
-                # Add to Python path temporarily
-                if str(lx_anonymizer_path) not in sys.path:
-                    sys.path.insert(0, str(lx_anonymizer_path))
-                
-                # Try simple import
-                from lx_anonymizer import FrameCleaner, ReportReader
-                
-                self.logger.info("Successfully imported lx_anonymizer modules")
-                
-                # Remove from path to avoid conflicts
-                if str(lx_anonymizer_path) in sys.path:
-                    sys.path.remove(str(lx_anonymizer_path))
-                    
-                return True, FrameCleaner, ReportReader
-            
-            else:
-                self.logger.warning(f"lx-anonymizer path not found: {lx_anonymizer_path}") 
-                
+            if FrameCleaner:
+                return True, FrameCleaner
+                        
         except Exception as e:
-            self.logger.warning(f"Frame cleaning not available: {e}")
+            self.logger.warning(f"Frame cleaning not available: {e} Please install or update lx_anonymizer.")
         
-        return False, None, None
+        return False, None
 
     
 
-    def _perform_frame_cleaning(self, FrameCleaner, endoscope_data_roi_nested, endoscope_image_roi):
+    def _perform_frame_cleaning(self, endoscope_data_roi_nested, endoscope_image_roi):
         """Perform frame cleaning and anonymization."""
         # Instantiate frame cleaner
-        frame_cleaner = FrameCleaner()
-        
+        is_available, frame_cleaner = self._ensure_frame_cleaning_available()
+
+        if not is_available:
+            raise RuntimeError("Frame cleaning not available")
+
         # Prepare parameters for frame cleaning
         raw_video_path = self.processing_context.get('raw_video_path')
         
         if not raw_video_path or not Path(raw_video_path).exists():
             raise RuntimeError(f"Raw video path not found: {raw_video_path}")
-        
-        # Get processor name safely
-        video = self._require_current_video()
-        video_meta = getattr(video, "video_meta", None)
-        processor = getattr(video_meta, "processor", None) if video_meta else None
-        device_name = processor.name if processor else self.processing_context['processor_name']
+
                 
         # Create temporary output path for cleaned video
         video_filename = self.processing_context.get('video_filename', Path(raw_video_path).name)
         cleaned_filename = f"cleaned_{video_filename}"
         cleaned_video_path = Path(raw_video_path).parent / cleaned_filename
                 
-        # Processor roi is used later to OCR preknown regions.
         
         # Clean video with ROI masking (heavy I/O operation)
         actual_cleaned_path, extracted_metadata = frame_cleaner.clean_video(
             video_path=Path(raw_video_path),
-            video_file_obj=video,
             endoscope_image_roi=endoscope_image_roi,
             endoscope_data_roi_nested=endoscope_data_roi_nested,
             output_path=cleaned_video_path,
@@ -1024,7 +991,7 @@ def import_and_anonymize(
     center_name: str,
     processor_name: str,
     save_video: bool = True,
-    delete_source: bool = False,
+    delete_source: bool = True,
 ) -> VideoFile | None:
     """Module-level helper that instantiates VideoImportService and runs import_and_anonymize.
     Kept for backward compatibility with callers that import this function directly.

endoreg_db/tasks/video_processing_tasks.py

@@ -53,7 +53,7 @@ def apply_video_mask_task(self, video_id: int, mask_type: str = 'device_default'
         self.update_state(state='PROGRESS', meta={'progress': 10, 'message': 'Setting up FrameCleaner...'})
         
         # Initialize FrameCleaner
-        cleaner = FrameCleaner(use_minicpm=True)
+        cleaner = FrameCleaner()
         
         # Determine mask configuration
         if mask_type == 'custom' and custom_mask:
@@ -110,14 +110,14 @@ def _setup_frame_removal(video_id: int, detection_engine: str):
     from lx_anonymizer.frame_cleaner import FrameCleaner
     from django.shortcuts import get_object_or_404
     video = get_object_or_404(VideoFile, pk=video_id)
-    video_path = Path(video.file.path)
+    video_path = Path(video.raw_file.path)
     if not video_path.exists():
         raise FileNotFoundError(f"Video file not found: {video_path}")
     output_dir = video_path.parent / "processed"
     output_dir.mkdir(exist_ok=True)
     output_path = output_dir / f"{video_path.stem}_cleaned{video_path.suffix}"
     use_minicpm = detection_engine == 'minicpm'
-    cleaner = FrameCleaner(use_minicpm=use_minicpm)
+    cleaner = FrameCleaner()
     return video, video_path, output_path, cleaner
 
 def _detect_sensitive_frames(self, cleaner, video_path, selection_method, manual_frames, total_frames):
@@ -257,7 +257,7 @@ def reprocess_video_task(self, video_id: int):
         self.update_state(state='PROGRESS', meta={'progress': 20, 'message': 'Initializing FrameCleaner...'})
         
         # Initialize FrameCleaner with optimal settings
-        cleaner = FrameCleaner(use_minicpm=True)
+        cleaner = FrameCleaner()
         
         # Create output path
         output_dir = video_path.parent / "processed"

{endoreg_db-0.8.3.0.dist-info → endoreg_db-0.8.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: endoreg-db
-Version: 0.8.3.0
+Version: 0.8.3.2
 Summary: EndoReg Db Django App
 Project-URL: Homepage, https://info.coloreg.de
 Project-URL: Repository, https://github.com/wg-lux/endoreg-db
@@ -33,7 +33,7 @@ Requires-Dist: huggingface-hub>=0.35.3
 Requires-Dist: icecream>=2.1.4
 Requires-Dist: librosa==0.11.0
 Requires-Dist: llvmlite>=0.44.0
-Requires-Dist: lx-anonymizer[llm,ocr]>=0.8.7
+Requires-Dist: lx-anonymizer[llm,ocr]>=0.8.8
 Requires-Dist: moviepy==2.2.1
 Requires-Dist: mypy>=1.16.0
 Requires-Dist: numpy>=2.2.3

{endoreg_db-0.8.3.0.dist-info → endoreg_db-0.8.3.2.dist-info}/RECORD

@@ -600,10 +600,10 @@ endoreg_db/services/pseudonym_service.py,sha256=CJhbtRa6K6SPbphgCZgEMi8AFQtB18CU
 endoreg_db/services/requirements_object.py,sha256=290zf8AEbVtCoHhW4Jr7_ud-RvrqYmb1Nz9UBHtTnc0,6164
 endoreg_db/services/segment_sync.py,sha256=YgHvIHkbW4mqCu0ACf3zjRSZnNfxWwt4gh5syUVXuE0,6400
 endoreg_db/services/storage_aware_video_processor.py,sha256=kKFK64vXLeBSVkp1YJonU3gFDTeXZ8C4qb9QZZB99SE,13420
-endoreg_db/services/video_import.py,sha256=PhcOgxU5M4uSEklBXEWHpIaNX-yIYv1rJy-T-fCU8cs,47830
+endoreg_db/services/video_import.py,sha256=gDuVTW5WUYGSc0m5ly67cc10YpnTpBkxO7uOEcRa3Ok,45663
 endoreg_db/tasks/upload_tasks.py,sha256=OJq7DhNwcbWdXzHY8jz5c51BCVkPN5gSWOz-6Fx6W5M,7799
 endoreg_db/tasks/video_ingest.py,sha256=kxFuYkHijINV0VabQKCFVpJRv6eCAw07tviONurDgg8,5265
-endoreg_db/tasks/video_processing_tasks.py,sha256=KjcERRJ1TZzmavBpvr6OsvSTUViU0PR1ECWnEdzu2Js,14140
+endoreg_db/tasks/video_processing_tasks.py,sha256=rZ7Kr49bAR4Q-vALO2SURebrhcJ5hSFGwjF4aULrOao,14089
 endoreg_db/templates/timeline.html,sha256=H9VXKOecCzqcWWkpNIZXFI29ztg-oxV5uvxMglgoClk,6167
 endoreg_db/templates/admin/patient_finding_intervention.html,sha256=F3JUKm3HhWIf_xoZZ-SET5d5ZDlm2jMM8g909w1dnYc,10164
 endoreg_db/templates/admin/start_examination.html,sha256=3K4wirul9KNyB5mN9cpfCSCAyAD6ro19GwxFOY5sZ3A,267
@@ -784,7 +784,7 @@ endoreg_db/views/video/video_meta.py,sha256=C1wBMTtQb_yzEUrhFGAy2UHEWMk_CbU75WXX
 endoreg_db/views/video/video_processing_history.py,sha256=mhFuS8RG5GV8E-lTtuD0qrq-bIpnUFp8vy9aERfC-J8,770
 endoreg_db/views/video/video_remove_frames.py,sha256=2FmvNrSPM0fUXiBxINN6vBUUDCqDlBkNcGR3WsLDgKo,1696
 endoreg_db/views/video/video_stream.py,sha256=kLyuf0ORTmsLeYUQkTQ6iRYqlIQozWhMMR3Lhfe_trk,12148
-endoreg_db-0.8.3.0.dist-info/METADATA,sha256=q7jvhqzrBQmwSOuzXARYftJxbQ5vBUL_zmJG9U338dA,14758
-endoreg_db-0.8.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-endoreg_db-0.8.3.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-endoreg_db-0.8.3.0.dist-info/RECORD,,
+endoreg_db-0.8.3.2.dist-info/METADATA,sha256=Hdg0xL9WKegEgoyGOY0vgwAX1UVB87Ph86WNsYgcSms,14758
+endoreg_db-0.8.3.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+endoreg_db-0.8.3.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+endoreg_db-0.8.3.2.dist-info/RECORD,,