endoreg-db 0.8.5.0__py3-none-any.whl → 0.8.5.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

@@ -9,7 +9,6 @@ Changelog:
9
9
  during concurrent video imports (matches PDF import pattern)
10
10
  """
11
11
 
12
- from datetime import date
13
12
  import logging
14
13
  import os
15
14
  import random
@@ -27,14 +26,15 @@ from lx_anonymizer import FrameCleaner
27
26
  from moviepy import video
28
27
 
29
28
  from endoreg_db.models import EndoscopyProcessor, SensitiveMeta, VideoFile
30
- from endoreg_db.models.media.video.video_file_anonymize import \
31
- _cleanup_raw_assets
29
+ from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
32
30
  from endoreg_db.utils.hashs import get_video_hash
33
31
  from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, STORAGE_DIR, VIDEO_DIR
34
32
 
35
33
  # File lock configuration (matches PDF import)
36
34
  STALE_LOCK_SECONDS = 6000 # 100 minutes - reclaim locks older than this
37
- MAX_LOCK_WAIT_SECONDS = 90 # New: wait up to 90s for a non-stale lock to clear before skipping
35
+ MAX_LOCK_WAIT_SECONDS = (
36
+ 90 # New: wait up to 90s for a non-stale lock to clear before skipping
37
+ )
38
38
 
39
39
  logger = logging.getLogger(__name__)
40
40
 
@@ -63,7 +63,10 @@ class VideoImportService:
63
63
  # Ensure anonym_video directory exists before listing files
64
64
  anonym_video_dir = Path(ANONYM_VIDEO_DIR)
65
65
  if anonym_video_dir.exists():
66
- self.processed_files = set(str(anonym_video_dir / file) for file in os.listdir(ANONYM_VIDEO_DIR))
66
+ self.processed_files = set(
67
+ str(anonym_video_dir / file)
68
+ for file in os.listdir(ANONYM_VIDEO_DIR)
69
+ )
67
70
  else:
68
71
  logger.info(f"Creating anonym_videos directory: {anonym_video_dir}")
69
72
  anonym_video_dir.mkdir(parents=True, exist_ok=True)
@@ -80,7 +83,9 @@ class VideoImportService:
80
83
 
81
84
  self.logger = logging.getLogger(__name__)
82
85
 
83
- self.cleaner = None # This gets instantiated in the perform_frame_cleaning method
86
+ self.cleaner = (
87
+ None # This gets instantiated in the perform_frame_cleaning method
88
+ )
84
89
 
85
90
  def _require_current_video(self) -> VideoFile:
86
91
  """Return the current VideoFile or raise if it has not been initialized."""
@@ -119,10 +124,16 @@ class VideoImportService:
119
124
 
120
125
  if age is not None and age > STALE_LOCK_SECONDS:
121
126
  try:
122
- logger.warning("Stale lock detected for %s (age %.0fs). Reclaiming lock...", path, age)
127
+ logger.warning(
128
+ "Stale lock detected for %s (age %.0fs). Reclaiming lock...",
129
+ path,
130
+ age,
131
+ )
123
132
  lock_path.unlink()
124
133
  except Exception as e:
125
- logger.warning("Failed to remove stale lock %s: %s", lock_path, e)
134
+ logger.warning(
135
+ "Failed to remove stale lock %s: %s", lock_path, e
136
+ )
126
137
  # Loop continues and retries acquire immediately
127
138
  continue
128
139
 
@@ -165,7 +176,9 @@ class VideoImportService:
165
176
 
166
177
  try:
167
178
  # Initialize processing context
168
- self._initialize_processing_context(file_path, center_name, processor_name, save_video, delete_source)
179
+ self._initialize_processing_context(
180
+ file_path, center_name, processor_name, save_video, delete_source
181
+ )
169
182
 
170
183
  # Validate and prepare file (may raise ValueError if another worker holds a non-stale lock)
171
184
  try:
@@ -199,17 +212,28 @@ class VideoImportService:
199
212
 
200
213
  except Exception as e:
201
214
  # Safe file path access - handles cases where processing_context wasn't initialized
202
- safe_file_path = getattr(self, "processing_context", {}).get("file_path", file_path)
215
+ safe_file_path = getattr(self, "processing_context", {}).get(
216
+ "file_path", file_path
217
+ )
203
218
  # Debug: Log context state for troubleshooting
204
219
  context_keys = list(getattr(self, "processing_context", {}).keys())
205
220
  self.logger.debug(f"Context keys during error: {context_keys}")
206
- self.logger.error(f"Video import and anonymization failed for {safe_file_path}: {e}")
221
+ self.logger.error(
222
+ f"Video import and anonymization failed for {safe_file_path}: {e}"
223
+ )
207
224
  self._cleanup_on_error()
208
225
  raise
209
226
  finally:
210
227
  self._cleanup_processing_context()
211
228
 
212
- def _initialize_processing_context(self, file_path: Union[Path, str], center_name: str, processor_name: str, save_video: bool, delete_source: bool):
229
+ def _initialize_processing_context(
230
+ self,
231
+ file_path: Union[Path, str],
232
+ center_name: str,
233
+ processor_name: str,
234
+ save_video: bool,
235
+ delete_source: bool,
236
+ ):
213
237
  """Initialize the processing context for the current video import."""
214
238
  self.processing_context = {
215
239
  "file_path": Path(file_path),
@@ -322,12 +346,23 @@ class VideoImportService:
322
346
  except Exception:
323
347
  stored_raw_path = None
324
348
 
325
- # Fallback: derive from UUID + suffix
349
+ # Fallback: derive from UUID + suffix - ALWAYS use UUID for consistency
326
350
  if not stored_raw_path:
327
351
  suffix = source_path.suffix or ".mp4"
328
352
  uuid_str = getattr(_current_video, "uuid", None)
329
- filename = f"{uuid_str}{suffix}" if uuid_str else source_path.name
353
+ if uuid_str:
354
+ filename = f"{uuid_str}{suffix}"
355
+ else:
356
+ # Emergency fallback with timestamp to avoid conflicts
357
+ import time
358
+
359
+ timestamp = int(time.time())
360
+ filename = f"video_{timestamp}{suffix}"
361
+ self.logger.warning(
362
+ "No UUID available, using timestamp-based filename: %s", filename
363
+ )
330
364
  stored_raw_path = videos_dir / filename
365
+ self.logger.debug("Using UUID-based raw filename: %s", filename)
331
366
 
332
367
  delete_source = bool(self.processing_context.get("delete_source", True))
333
368
  stored_raw_path.parent.mkdir(parents=True, exist_ok=True)
@@ -342,7 +377,9 @@ class VideoImportService:
342
377
  except Exception:
343
378
  shutil.copy2(source_path, stored_raw_path)
344
379
  os.remove(source_path)
345
- self.logger.info("Copied & removed raw video to: %s", stored_raw_path)
380
+ self.logger.info(
381
+ "Copied & removed raw video to: %s", stored_raw_path
382
+ )
346
383
  else:
347
384
  shutil.copy2(source_path, stored_raw_path)
348
385
  self.logger.info("Copied raw video to: %s", stored_raw_path)
@@ -372,8 +409,7 @@ class VideoImportService:
372
409
  # Initialize video specifications
373
410
  video.initialize_video_specs()
374
411
 
375
- # Initialize frame objects in database
376
- video.initialize_frames()
412
+
377
413
 
378
414
  # Extract frames BEFORE processing to prevent pipeline 1 conflicts
379
415
  self.logger.info("Pre-extracting frames to avoid pipeline conflicts...")
@@ -382,6 +418,8 @@ class VideoImportService:
382
418
  if frames_extracted:
383
419
  self.processing_context["frames_extracted"] = True
384
420
  self.logger.info("Frame extraction completed successfully")
421
+ # Initialize frame objects in database
422
+ video.initialize_frames(video.get_frame_paths())
385
423
 
386
424
  # CRITICAL: Immediately save the frames_extracted state to database
387
425
  # to prevent refresh_from_db() in pipeline 1 from overriding it
@@ -394,7 +432,9 @@ class VideoImportService:
394
432
  self.logger.warning("Frame extraction failed, but continuing...")
395
433
  self.processing_context["frames_extracted"] = False
396
434
  except Exception as e:
397
- self.logger.warning(f"Frame extraction failed during setup: {e}, but continuing...")
435
+ self.logger.warning(
436
+ f"Frame extraction failed during setup: {e}, but continuing..."
437
+ )
398
438
  self.processing_context["frames_extracted"] = False
399
439
 
400
440
  # Ensure default patient data
@@ -405,38 +445,60 @@ class VideoImportService:
405
445
  def _process_frames_and_metadata(self):
406
446
  """Process frames and extract metadata with anonymization."""
407
447
  # Check frame cleaning availability
408
- frame_cleaning_available, frame_cleaner = self._ensure_frame_cleaning_available()
448
+ frame_cleaning_available, frame_cleaner = (
449
+ self._ensure_frame_cleaning_available()
450
+ )
409
451
  video = self._require_current_video()
410
452
 
411
453
  raw_file_field = video.raw_file
412
- has_raw_file = isinstance(raw_file_field, FieldFile) and bool(raw_file_field.name)
454
+ has_raw_file = isinstance(raw_file_field, FieldFile) and bool(
455
+ raw_file_field.name
456
+ )
413
457
 
414
458
  if not (frame_cleaning_available and has_raw_file):
415
- self.logger.warning("Frame cleaning not available or conditions not met, using fallback anonymization.")
459
+ self.logger.warning(
460
+ "Frame cleaning not available or conditions not met, using fallback anonymization."
461
+ )
416
462
  self._fallback_anonymize_video()
417
463
  return
418
464
 
419
465
  try:
420
- self.logger.info("Starting frame-level anonymization with processor ROI masking...")
466
+ self.logger.info(
467
+ "Starting frame-level anonymization with processor ROI masking..."
468
+ )
421
469
 
422
470
  # Get processor ROI information
423
- endoscope_data_roi_nested, endoscope_image_roi = self._get_processor_roi_info()
471
+ endoscope_data_roi_nested, endoscope_image_roi = (
472
+ self._get_processor_roi_info()
473
+ )
424
474
 
425
475
  # Perform frame cleaning with timeout to prevent blocking
426
- from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError
476
+ from concurrent.futures import ThreadPoolExecutor
477
+ from concurrent.futures import TimeoutError as FutureTimeoutError
427
478
 
428
479
  with ThreadPoolExecutor(max_workers=1) as executor:
429
- future = executor.submit(self._perform_frame_cleaning, endoscope_data_roi_nested, endoscope_image_roi)
480
+ future = executor.submit(
481
+ self._perform_frame_cleaning,
482
+ endoscope_data_roi_nested,
483
+ endoscope_image_roi,
484
+ )
430
485
  try:
431
486
  # Increased timeout to better accommodate ffmpeg + OCR
432
487
  future.result(timeout=50000)
433
488
  self.processing_context["anonymization_completed"] = True
434
- self.logger.info("Frame cleaning completed successfully within timeout")
489
+ self.logger.info(
490
+ "Frame cleaning completed successfully within timeout"
491
+ )
435
492
  except FutureTimeoutError:
436
- self.logger.warning("Frame cleaning timed out; entering grace period check for cleaned output")
493
+ self.logger.warning(
494
+ "Frame cleaning timed out; entering grace period check for cleaned output"
495
+ )
437
496
  # Grace period: detect if cleaned file appears shortly after timeout
438
497
  raw_video_path = self.processing_context.get("raw_video_path")
439
- video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name if raw_video_path else "video.mp4")
498
+ video_filename = self.processing_context.get(
499
+ "video_filename",
500
+ Path(raw_video_path).name if raw_video_path else "video.mp4",
501
+ )
440
502
  grace_seconds = 60
441
503
  expected_cleaned_path: Optional[Path] = None
442
504
  processed_field = video.processed_file
@@ -449,27 +511,42 @@ class VideoImportService:
449
511
  if expected_cleaned_path is not None:
450
512
  for _ in range(grace_seconds):
451
513
  if expected_cleaned_path.exists():
452
- self.processing_context["cleaned_video_path"] = expected_cleaned_path
453
- self.processing_context["anonymization_completed"] = True
454
- self.logger.info("Detected cleaned video during grace period: %s", expected_cleaned_path)
514
+ self.processing_context["cleaned_video_path"] = (
515
+ expected_cleaned_path
516
+ )
517
+ self.processing_context["anonymization_completed"] = (
518
+ True
519
+ )
520
+ self.logger.info(
521
+ "Detected cleaned video during grace period: %s",
522
+ expected_cleaned_path,
523
+ )
455
524
  found = True
456
525
  break
457
526
  time.sleep(1)
458
527
  else:
459
528
  self._fallback_anonymize_video()
460
529
  if not found:
461
- raise TimeoutError("Frame cleaning operation timed out - likely Ollama connection issue")
530
+ raise TimeoutError(
531
+ "Frame cleaning operation timed out - likely Ollama connection issue"
532
+ )
462
533
 
463
534
  except Exception as e:
464
- self.logger.warning("Frame cleaning failed (reason: %s), falling back to simple copy", e)
535
+ self.logger.warning(
536
+ "Frame cleaning failed (reason: %s), falling back to simple copy", e
537
+ )
465
538
  # Try fallback anonymization when frame cleaning fails
466
539
  try:
467
540
  self._fallback_anonymize_video()
468
541
  except Exception as fallback_error:
469
- self.logger.error("Fallback anonymization also failed: %s", fallback_error)
542
+ self.logger.error(
543
+ "Fallback anonymization also failed: %s", fallback_error
544
+ )
470
545
  # If even fallback fails, mark as not anonymized but continue import
471
546
  self.processing_context["anonymization_completed"] = False
472
- self.processing_context["error_reason"] = f"Frame cleaning failed: {e}, Fallback failed: {fallback_error}"
547
+ self.processing_context["error_reason"] = (
548
+ f"Frame cleaning failed: {e}, Fallback failed: {fallback_error}"
549
+ )
473
550
 
474
551
  def _save_anonymized_video(self):
475
552
  original_raw_file_path_to_delete = None
@@ -478,14 +555,24 @@ class VideoImportService:
478
555
  anonymized_video_path = video.get_target_anonymized_video_path()
479
556
 
480
557
  if not anonymized_video_path.exists():
481
- raise RuntimeError(f"Processed video file not found after assembly for {video.uuid}: {anonymized_video_path}")
558
+ raise RuntimeError(
559
+ f"Processed video file not found after assembly for {video.uuid}: {anonymized_video_path}"
560
+ )
482
561
 
483
562
  new_processed_hash = get_video_hash(anonymized_video_path)
484
- if video.__class__.objects.filter(processed_video_hash=new_processed_hash).exclude(pk=video.pk).exists():
485
- raise ValueError(f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid}).")
563
+ if (
564
+ video.__class__.objects.filter(processed_video_hash=new_processed_hash)
565
+ .exclude(pk=video.pk)
566
+ .exists()
567
+ ):
568
+ raise ValueError(
569
+ f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid})."
570
+ )
486
571
 
487
572
  video.processed_video_hash = new_processed_hash
488
- video.processed_file.name = anonymized_video_path.relative_to(STORAGE_DIR).as_posix()
573
+ video.processed_file.name = anonymized_video_path.relative_to(
574
+ STORAGE_DIR
575
+ ).as_posix()
489
576
 
490
577
  update_fields = [
491
578
  "processed_video_hash",
@@ -503,7 +590,9 @@ class VideoImportService:
503
590
 
504
591
  transaction.on_commit(
505
592
  lambda: _cleanup_raw_assets(
506
- video_uuid=video.uuid, raw_file_path=original_raw_file_path_to_delete, raw_frame_dir=original_raw_frame_dir_to_delete
593
+ video_uuid=video.uuid,
594
+ raw_file_path=original_raw_file_path_to_delete,
595
+ raw_frame_dir=original_raw_frame_dir_to_delete,
507
596
  )
508
597
  )
509
598
 
@@ -521,15 +610,23 @@ class VideoImportService:
521
610
  self.logger.info("Attempting fallback video anonymization...")
522
611
  video = self.current_video
523
612
  if video is None:
524
- self.logger.warning("No VideoFile instance available for fallback anonymization")
613
+ self.logger.warning(
614
+ "No VideoFile instance available for fallback anonymization"
615
+ )
525
616
 
526
617
  # Strategy 2: Simple copy (no processing, just copy raw to processed)
527
- self.logger.info("Using simple copy fallback (raw video will be used as 'processed' video)")
618
+ self.logger.info(
619
+ "Using simple copy fallback (raw video will be used as 'processed' video)"
620
+ )
528
621
  self.processing_context["anonymization_completed"] = False
529
622
  self.processing_context["use_raw_as_processed"] = True
530
- self.logger.warning("Fallback: Video will be imported without anonymization (raw copy used)")
623
+ self.logger.warning(
624
+ "Fallback: Video will be imported without anonymization (raw copy used)"
625
+ )
531
626
  except Exception as e:
532
- self.logger.error(f"Error during fallback anonymization: {e}", exc_info=True)
627
+ self.logger.error(
628
+ f"Error during fallback anonymization: {e}", exc_info=True
629
+ )
533
630
  self.processing_context["anonymization_completed"] = False
534
631
  self.processing_context["error_reason"] = str(e)
535
632
 
@@ -542,7 +639,11 @@ class VideoImportService:
542
639
  try:
543
640
  video.refresh_from_db()
544
641
  except Exception as refresh_error:
545
- self.logger.warning("Could not refresh VideoFile %s from DB: %s", video.uuid, refresh_error)
642
+ self.logger.warning(
643
+ "Could not refresh VideoFile %s from DB: %s",
644
+ video.uuid,
645
+ refresh_error,
646
+ )
546
647
 
547
648
  state = video.get_or_create_state()
548
649
 
@@ -559,12 +660,18 @@ class VideoImportService:
559
660
  state.text_meta_extracted = True
560
661
 
561
662
  # ✅ FIX: Only mark as processed if anonymization actually completed
562
- anonymization_completed = self.processing_context.get("anonymization_completed", False)
663
+ anonymization_completed = self.processing_context.get(
664
+ "anonymization_completed", False
665
+ )
563
666
  if anonymization_completed:
564
667
  state.mark_sensitive_meta_processed(save=False)
565
- self.logger.info("Anonymization completed - marking sensitive meta as processed")
668
+ self.logger.info(
669
+ "Anonymization completed - marking sensitive meta as processed"
670
+ )
566
671
  else:
567
- self.logger.warning(f"Anonymization NOT completed - NOT marking as processed. Reason: {self.processing_context.get('error_reason', 'Unknown')}")
672
+ self.logger.warning(
673
+ f"Anonymization NOT completed - NOT marking as processed. Reason: {self.processing_context.get('error_reason', 'Unknown')}"
674
+ )
568
675
  # Explicitly mark as NOT processed
569
676
  state.sensitive_meta_processed = False
570
677
 
@@ -590,12 +697,15 @@ class VideoImportService:
590
697
  else:
591
698
  raw_video_path = self.processing_context.get("raw_video_path")
592
699
  if raw_video_path and Path(raw_video_path).exists():
593
- video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name)
594
- processed_filename = f"processed_{video_filename}"
700
+ # Use UUID-based naming to avoid conflicts
701
+ suffix = Path(raw_video_path).suffix or ".mp4"
702
+ processed_filename = f"processed_{video.uuid}{suffix}"
595
703
  processed_video_path = Path(raw_video_path).parent / processed_filename
596
704
  try:
597
705
  shutil.copy2(str(raw_video_path), str(processed_video_path))
598
- self.logger.info("Copied raw video for processing: %s", processed_video_path)
706
+ self.logger.info(
707
+ "Copied raw video for processing: %s", processed_video_path
708
+ )
599
709
  except Exception as exc:
600
710
  self.logger.error("Failed to copy raw video: %s", exc)
601
711
  processed_video_path = None
@@ -615,10 +725,16 @@ class VideoImportService:
615
725
  relative_path = anonym_target_path.relative_to(storage_root)
616
726
  video.processed_file.name = str(relative_path)
617
727
  video.save(update_fields=["processed_file"])
618
- self.logger.info("Updated processed_file path to: %s", relative_path)
728
+ self.logger.info(
729
+ "Updated processed_file path to: %s", relative_path
730
+ )
619
731
  except Exception as exc:
620
- self.logger.error("Failed to update processed_file path: %s", exc)
621
- video.processed_file.name = f"anonym_videos/{anonym_video_filename}"
732
+ self.logger.error(
733
+ "Failed to update processed_file path: %s", exc
734
+ )
735
+ video.processed_file.name = (
736
+ f"anonym_videos/{anonym_video_filename}"
737
+ )
622
738
  video.save(update_fields=["processed_file"])
623
739
  self.logger.info(
624
740
  "Updated processed_file path using fallback: %s",
@@ -627,17 +743,26 @@ class VideoImportService:
627
743
 
628
744
  self.processing_context["anonymization_completed"] = True
629
745
  else:
630
- self.logger.warning("Processed video file not found after move: %s", anonym_target_path)
746
+ self.logger.warning(
747
+ "Processed video file not found after move: %s",
748
+ anonym_target_path,
749
+ )
631
750
  except Exception as exc:
632
- self.logger.error("Failed to move processed video to anonym_videos: %s", exc)
751
+ self.logger.error(
752
+ "Failed to move processed video to anonym_videos: %s", exc
753
+ )
633
754
  else:
634
- self.logger.warning("No processed video available - processed_file will remain empty")
755
+ self.logger.warning(
756
+ "No processed video available - processed_file will remain empty"
757
+ )
635
758
 
636
759
  try:
637
760
  from endoreg_db.utils.paths import RAW_FRAME_DIR
638
761
 
639
762
  shutil.rmtree(RAW_FRAME_DIR, ignore_errors=True)
640
- self.logger.debug("Cleaned up temporary frames directory: %s", RAW_FRAME_DIR)
763
+ self.logger.debug(
764
+ "Cleaned up temporary frames directory: %s", RAW_FRAME_DIR
765
+ )
641
766
  except Exception as exc:
642
767
  self.logger.warning("Failed to remove directory %s: %s", RAW_FRAME_DIR, exc)
643
768
 
@@ -647,10 +772,14 @@ class VideoImportService:
647
772
  os.remove(source_path)
648
773
  self.logger.info("Removed remaining source file: %s", source_path)
649
774
  except Exception as exc:
650
- self.logger.warning("Failed to remove source file %s: %s", source_path, exc)
775
+ self.logger.warning(
776
+ "Failed to remove source file %s: %s", source_path, exc
777
+ )
651
778
 
652
779
  if not video.processed_file or not Path(video.processed_file.path).exists():
653
- self.logger.warning("No processed_file found after cleanup - video will be unprocessed")
780
+ self.logger.warning(
781
+ "No processed_file found after cleanup - video will be unprocessed"
782
+ )
654
783
  try:
655
784
  video.anonymize(delete_original_raw=self.delete_source)
656
785
  video.save(update_fields=["processed_file"])
@@ -665,10 +794,14 @@ class VideoImportService:
665
794
 
666
795
  with transaction.atomic():
667
796
  video.refresh_from_db()
668
- if hasattr(video, "state") and self.processing_context.get("anonymization_completed"):
797
+ if hasattr(video, "state") and self.processing_context.get(
798
+ "anonymization_completed"
799
+ ):
669
800
  video.state.mark_sensitive_meta_processed(save=True)
670
801
 
671
- self.logger.info("Import and anonymization completed for VideoFile UUID: %s", video.uuid)
802
+ self.logger.info(
803
+ "Import and anonymization completed for VideoFile UUID: %s", video.uuid
804
+ )
672
805
  self.logger.info("Raw video stored in: /data/videos")
673
806
  self.logger.info("Processed video stored in: /data/anonym_videos")
674
807
 
@@ -695,7 +828,9 @@ class VideoImportService:
695
828
  if source_path is None:
696
829
  raise ValueError("No file path available for creating sensitive file")
697
830
  if not raw_field:
698
- raise ValueError("VideoFile must have a raw_file to create a sensitive file")
831
+ raise ValueError(
832
+ "VideoFile must have a raw_file to create a sensitive file"
833
+ )
699
834
 
700
835
  target_dir = VIDEO_DIR / "sensitive"
701
836
  if not target_dir.exists():
@@ -705,9 +840,13 @@ class VideoImportService:
705
840
  target_file_path = target_dir / source_path.name
706
841
  try:
707
842
  shutil.move(str(source_path), str(target_file_path))
708
- self.logger.info("Moved raw file to sensitive directory: %s", target_file_path)
843
+ self.logger.info(
844
+ "Moved raw file to sensitive directory: %s", target_file_path
845
+ )
709
846
  except Exception as exc:
710
- self.logger.warning("Failed to move raw file to sensitive dir, copying instead: %s", exc)
847
+ self.logger.warning(
848
+ "Failed to move raw file to sensitive dir, copying instead: %s", exc
849
+ )
711
850
  shutil.copy(str(source_path), str(target_file_path))
712
851
  try:
713
852
  os.remove(source_path)
@@ -721,7 +860,10 @@ class VideoImportService:
721
860
  relative_path = target_file_path.relative_to(storage_root)
722
861
  video.raw_file.name = str(relative_path)
723
862
  video.save(update_fields=["raw_file"])
724
- self.logger.info("Updated video.raw_file to point to sensitive location: %s", relative_path)
863
+ self.logger.info(
864
+ "Updated video.raw_file to point to sensitive location: %s",
865
+ relative_path,
866
+ )
725
867
  except Exception as exc:
726
868
  self.logger.warning("Failed to set relative path, using fallback: %s", exc)
727
869
  video.raw_file.name = f"videos/sensitive/{target_file_path.name}"
@@ -734,10 +876,14 @@ class VideoImportService:
734
876
  self.processing_context["raw_video_path"] = target_file_path
735
877
  self.processing_context["video_filename"] = target_file_path.name
736
878
 
737
- self.logger.info("Created sensitive file for %s at %s", video.uuid, target_file_path)
879
+ self.logger.info(
880
+ "Created sensitive file for %s at %s", video.uuid, target_file_path
881
+ )
738
882
  return target_file_path
739
883
 
740
- def _get_processor_roi_info(self) -> Tuple[Optional[List[List[Dict[str, Any]]]], Optional[Dict[str, Any]]]:
884
+ def _get_processor_roi_info(
885
+ self,
886
+ ) -> Tuple[Optional[List[List[Dict[str, Any]]]], Optional[Dict[str, Any]]]:
741
887
  """Get processor ROI information for masking."""
742
888
  endoscope_data_roi_nested = None
743
889
  endoscope_image_roi = None
@@ -748,10 +894,15 @@ class VideoImportService:
748
894
  video_meta = getattr(video, "video_meta", None)
749
895
  processor = getattr(video_meta, "processor", None) if video_meta else None
750
896
  if processor:
751
- assert isinstance(processor, EndoscopyProcessor), "Processor is not of type EndoscopyProcessor"
897
+ assert isinstance(processor, EndoscopyProcessor), (
898
+ "Processor is not of type EndoscopyProcessor"
899
+ )
752
900
  endoscope_image_roi = processor.get_roi_endoscope_image()
753
901
  endoscope_data_roi_nested = processor.get_sensitive_rois()
754
- self.logger.info("Retrieved processor ROI information: endoscope_image_roi=%s", endoscope_image_roi)
902
+ self.logger.info(
903
+ "Retrieved processor ROI information: endoscope_image_roi=%s",
904
+ endoscope_image_roi,
905
+ )
755
906
  else:
756
907
  self.logger.warning(
757
908
  "No processor found for video %s, proceeding without ROI masking",
@@ -773,28 +924,40 @@ class VideoImportService:
773
924
 
774
925
  return endoscope_data_roi_nested, endoscope_image_roi
775
926
 
776
- def _ensure_default_patient_data(self, video_instance: VideoFile | None = None) -> None:
927
+ def _ensure_default_patient_data(
928
+ self, video_instance: VideoFile | None = None
929
+ ) -> None:
777
930
  """Ensure minimum patient data is present on the video's SensitiveMeta."""
778
931
 
779
932
  video = video_instance or self._require_current_video()
780
933
 
781
934
  sensitive_meta = getattr(video, "sensitive_meta", None)
782
935
  if not sensitive_meta:
783
- self.logger.info("No SensitiveMeta found for video %s, creating default", video.uuid)
936
+ self.logger.info(
937
+ "No SensitiveMeta found for video %s, creating default", video.uuid
938
+ )
784
939
  default_data = {
785
940
  "patient_first_name": "Patient",
786
941
  "patient_last_name": "Unknown",
787
942
  "patient_dob": date(1990, 1, 1),
788
943
  "examination_date": date.today(),
789
- "center_name": video.center.name if video.center else "university_hospital_wuerzburg",
944
+ "center_name": video.center.name
945
+ if video.center
946
+ else "university_hospital_wuerzburg",
790
947
  }
791
948
  try:
792
949
  sensitive_meta = SensitiveMeta.create_from_dict(default_data)
793
950
  video.sensitive_meta = sensitive_meta
794
951
  video.save(update_fields=["sensitive_meta"])
795
- self.logger.info("Created default SensitiveMeta for video %s", video.uuid)
952
+ self.logger.info(
953
+ "Created default SensitiveMeta for video %s", video.uuid
954
+ )
796
955
  except Exception as exc:
797
- self.logger.error("Failed to create default SensitiveMeta for video %s: %s", video.uuid, exc)
956
+ self.logger.error(
957
+ "Failed to create default SensitiveMeta for video %s: %s",
958
+ video.uuid,
959
+ exc,
960
+ )
798
961
  return
799
962
  else:
800
963
  update_data: Dict[str, Any] = {}
@@ -818,7 +981,11 @@ class VideoImportService:
818
981
  list(update_data.keys()),
819
982
  )
820
983
  except Exception as exc:
821
- self.logger.error("Failed to update SensitiveMeta for video %s: %s", video.uuid, exc)
984
+ self.logger.error(
985
+ "Failed to update SensitiveMeta for video %s: %s",
986
+ video.uuid,
987
+ exc,
988
+ )
822
989
 
823
990
  def _ensure_frame_cleaning_available(self):
824
991
  """
@@ -835,7 +1002,9 @@ class VideoImportService:
835
1002
  return True, FrameCleaner()
836
1003
 
837
1004
  except Exception as e:
838
- self.logger.warning(f"Frame cleaning not available: {e} Please install or update lx_anonymizer.")
1005
+ self.logger.warning(
1006
+ f"Frame cleaning not available: {e} Please install or update lx_anonymizer."
1007
+ )
839
1008
 
840
1009
  return False, None
841
1010
 
@@ -857,12 +1026,17 @@ class VideoImportService:
857
1026
  except Exception:
858
1027
  raise RuntimeError(f"Raw video path not found: {raw_video_path}")
859
1028
 
860
- # Create temporary output path for cleaned video
861
- video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name)
862
- cleaned_filename = f"cleaned_{video_filename}"
1029
+ # Create temporary output path for cleaned video using UUID to avoid naming conflicts
1030
+ video = self._require_current_video()
1031
+ # Ensure raw_video_path is not None
863
1032
  if not raw_video_path:
864
- raise RuntimeError("raw_video_path is None after fallback, cannot construct cleaned_video_path")
1033
+ raise RuntimeError(
1034
+ "raw_video_path is None, cannot construct cleaned_video_path"
1035
+ )
1036
+ suffix = Path(raw_video_path).suffix or ".mp4"
1037
+ cleaned_filename = f"cleaned_{video.uuid}{suffix}"
865
1038
  cleaned_video_path = Path(raw_video_path).parent / cleaned_filename
1039
+ self.logger.debug("Using UUID-based cleaned filename: %s", cleaned_filename)
866
1040
 
867
1041
  # Clean video with ROI masking (heavy I/O operation)
868
1042
  actual_cleaned_path, extracted_metadata = frame_cleaner.clean_video(
@@ -879,9 +1053,13 @@ class VideoImportService:
879
1053
 
880
1054
  # Update sensitive metadata with extracted information
881
1055
  self._update_sensitive_metadata(extracted_metadata)
882
- self.logger.info(f"Extracted metadata from frame cleaning: {extracted_metadata}")
1056
+ self.logger.info(
1057
+ f"Extracted metadata from frame cleaning: {extracted_metadata}"
1058
+ )
883
1059
 
884
- self.logger.info(f"Frame cleaning with ROI masking completed: {actual_cleaned_path}")
1060
+ self.logger.info(
1061
+ f"Frame cleaning with ROI masking completed: {actual_cleaned_path}"
1062
+ )
885
1063
  self.logger.info("Cleaned video will be moved to anonym_videos during cleanup")
886
1064
 
887
1065
  def _update_sensitive_metadata(self, extracted_metadata: Dict[str, Any]):
@@ -898,31 +1076,41 @@ class VideoImportService:
898
1076
 
899
1077
  sm = sensitive_meta
900
1078
  updated_fields = []
901
-
1079
+
902
1080
  # Ensure center is set from video.center if not in extracted_metadata
903
1081
  metadata_to_update = extracted_metadata.copy()
904
-
1082
+
905
1083
  # FIX: Set center object instead of center_name string
906
- if not hasattr(sm, 'center') or not sm.center:
1084
+ if not hasattr(sm, "center") or not sm.center:
907
1085
  if video.center:
908
- metadata_to_update['center'] = video.center
909
- self.logger.debug("Added center object '%s' to metadata for SensitiveMeta update", video.center.name)
1086
+ metadata_to_update["center"] = video.center
1087
+ self.logger.debug(
1088
+ "Added center object '%s' to metadata for SensitiveMeta update",
1089
+ video.center.name,
1090
+ )
910
1091
  else:
911
- center_name = metadata_to_update.get('center_name')
1092
+ center_name = metadata_to_update.get("center_name")
912
1093
  if center_name:
913
1094
  try:
914
1095
  from ..models.administration import Center
1096
+
915
1097
  center_obj = Center.objects.get(name=center_name)
916
- metadata_to_update['center'] = center_obj
917
- self.logger.debug("Loaded center object '%s' from center_name", center_name)
918
- metadata_to_update.pop('center_name', None)
1098
+ metadata_to_update["center"] = center_obj
1099
+ self.logger.debug(
1100
+ "Loaded center object '%s' from center_name", center_name
1101
+ )
1102
+ metadata_to_update.pop("center_name", None)
919
1103
  except Center.DoesNotExist:
920
- self.logger.error("Center '%s' not found in database", center_name)
1104
+ self.logger.error(
1105
+ "Center '%s' not found in database", center_name
1106
+ )
921
1107
  return
922
-
1108
+
923
1109
  try:
924
1110
  sm.update_from_dict(metadata_to_update)
925
- updated_fields = list(extracted_metadata.keys()) # Only log originally extracted fields
1111
+ updated_fields = list(
1112
+ extracted_metadata.keys()
1113
+ ) # Only log originally extracted fields
926
1114
  except KeyError as e:
927
1115
  self.logger.warning(f"Failed to update SensitiveMeta field {e}")
928
1116
  return
@@ -930,16 +1118,25 @@ class VideoImportService:
930
1118
  if updated_fields:
931
1119
  try:
932
1120
  sm.save() # Remove update_fields to allow all necessary fields to be saved
933
- self.logger.info("Updated SensitiveMeta fields for video %s: %s", video.uuid, updated_fields)
1121
+ self.logger.info(
1122
+ "Updated SensitiveMeta fields for video %s: %s",
1123
+ video.uuid,
1124
+ updated_fields,
1125
+ )
934
1126
 
935
1127
  state = video.get_or_create_state()
936
1128
  state.mark_sensitive_meta_processed(save=True)
937
- self.logger.info("Marked sensitive metadata as processed for video %s", video.uuid)
1129
+ self.logger.info(
1130
+ "Marked sensitive metadata as processed for video %s", video.uuid
1131
+ )
938
1132
  except Exception as e:
939
1133
  self.logger.error(f"Failed to save SensitiveMeta: {e}")
940
1134
  raise # Re-raise to trigger fallback in calling method
941
1135
  else:
942
- self.logger.info("No SensitiveMeta fields updated for video %s - all existing values preserved", video.uuid)
1136
+ self.logger.info(
1137
+ "No SensitiveMeta fields updated for video %s - all existing values preserved",
1138
+ video.uuid,
1139
+ )
943
1140
 
944
1141
  def _signal_completion(self):
945
1142
  """Signal completion to the tracking system."""
@@ -954,14 +1151,25 @@ class VideoImportService:
954
1151
  except (ValueError, OSError):
955
1152
  raw_exists = False
956
1153
 
957
- video_processing_complete = video.sensitive_meta is not None and video.video_meta is not None and raw_exists
1154
+ video_processing_complete = (
1155
+ video.sensitive_meta is not None
1156
+ and video.video_meta is not None
1157
+ and raw_exists
1158
+ )
958
1159
 
959
1160
  if video_processing_complete:
960
- self.logger.info("Video %s processing completed successfully - ready for validation", video.uuid)
1161
+ self.logger.info(
1162
+ "Video %s processing completed successfully - ready for validation",
1163
+ video.uuid,
1164
+ )
961
1165
 
962
1166
  # Update completion flags if they exist
963
1167
  completion_fields = []
964
- for field_name in ["import_completed", "processing_complete", "ready_for_validation"]:
1168
+ for field_name in [
1169
+ "import_completed",
1170
+ "processing_complete",
1171
+ "ready_for_validation",
1172
+ ]:
965
1173
  if hasattr(video, field_name):
966
1174
  setattr(video, field_name, True)
967
1175
  completion_fields.append(field_name)
@@ -1018,7 +1226,9 @@ class VideoImportService:
1018
1226
  file_path_str = str(file_path)
1019
1227
  if file_path_str in self.processed_files:
1020
1228
  self.processed_files.remove(file_path_str)
1021
- self.logger.info(f"Removed {file_path_str} from processed files (failed processing)")
1229
+ self.logger.info(
1230
+ f"Removed {file_path_str} from processed files (failed processing)"
1231
+ )
1022
1232
 
1023
1233
  except Exception as e:
1024
1234
  self.logger.warning(f"Error during context cleanup: {e}")