endoreg-db 0.8.5.0__py3-none-any.whl → 0.8.5.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/models/media/video/video_file.py +255 -147
- endoreg_db/models/metadata/sensitive_meta_logic.py +292 -56
- endoreg_db/services/video_import.py +312 -102
- endoreg_db/views/media/video_media.py +1 -1
- {endoreg_db-0.8.5.0.dist-info → endoreg_db-0.8.5.2.dist-info}/METADATA +1 -1
- {endoreg_db-0.8.5.0.dist-info → endoreg_db-0.8.5.2.dist-info}/RECORD +8 -8
- {endoreg_db-0.8.5.0.dist-info → endoreg_db-0.8.5.2.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.5.0.dist-info → endoreg_db-0.8.5.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -9,7 +9,6 @@ Changelog:
|
|
|
9
9
|
during concurrent video imports (matches PDF import pattern)
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
|
-
from datetime import date
|
|
13
12
|
import logging
|
|
14
13
|
import os
|
|
15
14
|
import random
|
|
@@ -27,14 +26,15 @@ from lx_anonymizer import FrameCleaner
|
|
|
27
26
|
from moviepy import video
|
|
28
27
|
|
|
29
28
|
from endoreg_db.models import EndoscopyProcessor, SensitiveMeta, VideoFile
|
|
30
|
-
from endoreg_db.models.media.video.video_file_anonymize import
|
|
31
|
-
_cleanup_raw_assets
|
|
29
|
+
from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
|
|
32
30
|
from endoreg_db.utils.hashs import get_video_hash
|
|
33
31
|
from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, STORAGE_DIR, VIDEO_DIR
|
|
34
32
|
|
|
35
33
|
# File lock configuration (matches PDF import)
|
|
36
34
|
STALE_LOCK_SECONDS = 6000 # 100 minutes - reclaim locks older than this
|
|
37
|
-
MAX_LOCK_WAIT_SECONDS =
|
|
35
|
+
MAX_LOCK_WAIT_SECONDS = (
|
|
36
|
+
90 # New: wait up to 90s for a non-stale lock to clear before skipping
|
|
37
|
+
)
|
|
38
38
|
|
|
39
39
|
logger = logging.getLogger(__name__)
|
|
40
40
|
|
|
@@ -63,7 +63,10 @@ class VideoImportService:
|
|
|
63
63
|
# Ensure anonym_video directory exists before listing files
|
|
64
64
|
anonym_video_dir = Path(ANONYM_VIDEO_DIR)
|
|
65
65
|
if anonym_video_dir.exists():
|
|
66
|
-
self.processed_files = set(
|
|
66
|
+
self.processed_files = set(
|
|
67
|
+
str(anonym_video_dir / file)
|
|
68
|
+
for file in os.listdir(ANONYM_VIDEO_DIR)
|
|
69
|
+
)
|
|
67
70
|
else:
|
|
68
71
|
logger.info(f"Creating anonym_videos directory: {anonym_video_dir}")
|
|
69
72
|
anonym_video_dir.mkdir(parents=True, exist_ok=True)
|
|
@@ -80,7 +83,9 @@ class VideoImportService:
|
|
|
80
83
|
|
|
81
84
|
self.logger = logging.getLogger(__name__)
|
|
82
85
|
|
|
83
|
-
self.cleaner =
|
|
86
|
+
self.cleaner = (
|
|
87
|
+
None # This gets instantiated in the perform_frame_cleaning method
|
|
88
|
+
)
|
|
84
89
|
|
|
85
90
|
def _require_current_video(self) -> VideoFile:
|
|
86
91
|
"""Return the current VideoFile or raise if it has not been initialized."""
|
|
@@ -119,10 +124,16 @@ class VideoImportService:
|
|
|
119
124
|
|
|
120
125
|
if age is not None and age > STALE_LOCK_SECONDS:
|
|
121
126
|
try:
|
|
122
|
-
logger.warning(
|
|
127
|
+
logger.warning(
|
|
128
|
+
"Stale lock detected for %s (age %.0fs). Reclaiming lock...",
|
|
129
|
+
path,
|
|
130
|
+
age,
|
|
131
|
+
)
|
|
123
132
|
lock_path.unlink()
|
|
124
133
|
except Exception as e:
|
|
125
|
-
logger.warning(
|
|
134
|
+
logger.warning(
|
|
135
|
+
"Failed to remove stale lock %s: %s", lock_path, e
|
|
136
|
+
)
|
|
126
137
|
# Loop continues and retries acquire immediately
|
|
127
138
|
continue
|
|
128
139
|
|
|
@@ -165,7 +176,9 @@ class VideoImportService:
|
|
|
165
176
|
|
|
166
177
|
try:
|
|
167
178
|
# Initialize processing context
|
|
168
|
-
self._initialize_processing_context(
|
|
179
|
+
self._initialize_processing_context(
|
|
180
|
+
file_path, center_name, processor_name, save_video, delete_source
|
|
181
|
+
)
|
|
169
182
|
|
|
170
183
|
# Validate and prepare file (may raise ValueError if another worker holds a non-stale lock)
|
|
171
184
|
try:
|
|
@@ -199,17 +212,28 @@ class VideoImportService:
|
|
|
199
212
|
|
|
200
213
|
except Exception as e:
|
|
201
214
|
# Safe file path access - handles cases where processing_context wasn't initialized
|
|
202
|
-
safe_file_path = getattr(self, "processing_context", {}).get(
|
|
215
|
+
safe_file_path = getattr(self, "processing_context", {}).get(
|
|
216
|
+
"file_path", file_path
|
|
217
|
+
)
|
|
203
218
|
# Debug: Log context state for troubleshooting
|
|
204
219
|
context_keys = list(getattr(self, "processing_context", {}).keys())
|
|
205
220
|
self.logger.debug(f"Context keys during error: {context_keys}")
|
|
206
|
-
self.logger.error(
|
|
221
|
+
self.logger.error(
|
|
222
|
+
f"Video import and anonymization failed for {safe_file_path}: {e}"
|
|
223
|
+
)
|
|
207
224
|
self._cleanup_on_error()
|
|
208
225
|
raise
|
|
209
226
|
finally:
|
|
210
227
|
self._cleanup_processing_context()
|
|
211
228
|
|
|
212
|
-
def _initialize_processing_context(
|
|
229
|
+
def _initialize_processing_context(
|
|
230
|
+
self,
|
|
231
|
+
file_path: Union[Path, str],
|
|
232
|
+
center_name: str,
|
|
233
|
+
processor_name: str,
|
|
234
|
+
save_video: bool,
|
|
235
|
+
delete_source: bool,
|
|
236
|
+
):
|
|
213
237
|
"""Initialize the processing context for the current video import."""
|
|
214
238
|
self.processing_context = {
|
|
215
239
|
"file_path": Path(file_path),
|
|
@@ -322,12 +346,23 @@ class VideoImportService:
|
|
|
322
346
|
except Exception:
|
|
323
347
|
stored_raw_path = None
|
|
324
348
|
|
|
325
|
-
# Fallback: derive from UUID + suffix
|
|
349
|
+
# Fallback: derive from UUID + suffix - ALWAYS use UUID for consistency
|
|
326
350
|
if not stored_raw_path:
|
|
327
351
|
suffix = source_path.suffix or ".mp4"
|
|
328
352
|
uuid_str = getattr(_current_video, "uuid", None)
|
|
329
|
-
|
|
353
|
+
if uuid_str:
|
|
354
|
+
filename = f"{uuid_str}{suffix}"
|
|
355
|
+
else:
|
|
356
|
+
# Emergency fallback with timestamp to avoid conflicts
|
|
357
|
+
import time
|
|
358
|
+
|
|
359
|
+
timestamp = int(time.time())
|
|
360
|
+
filename = f"video_{timestamp}{suffix}"
|
|
361
|
+
self.logger.warning(
|
|
362
|
+
"No UUID available, using timestamp-based filename: %s", filename
|
|
363
|
+
)
|
|
330
364
|
stored_raw_path = videos_dir / filename
|
|
365
|
+
self.logger.debug("Using UUID-based raw filename: %s", filename)
|
|
331
366
|
|
|
332
367
|
delete_source = bool(self.processing_context.get("delete_source", True))
|
|
333
368
|
stored_raw_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -342,7 +377,9 @@ class VideoImportService:
|
|
|
342
377
|
except Exception:
|
|
343
378
|
shutil.copy2(source_path, stored_raw_path)
|
|
344
379
|
os.remove(source_path)
|
|
345
|
-
self.logger.info(
|
|
380
|
+
self.logger.info(
|
|
381
|
+
"Copied & removed raw video to: %s", stored_raw_path
|
|
382
|
+
)
|
|
346
383
|
else:
|
|
347
384
|
shutil.copy2(source_path, stored_raw_path)
|
|
348
385
|
self.logger.info("Copied raw video to: %s", stored_raw_path)
|
|
@@ -372,8 +409,7 @@ class VideoImportService:
|
|
|
372
409
|
# Initialize video specifications
|
|
373
410
|
video.initialize_video_specs()
|
|
374
411
|
|
|
375
|
-
|
|
376
|
-
video.initialize_frames()
|
|
412
|
+
|
|
377
413
|
|
|
378
414
|
# Extract frames BEFORE processing to prevent pipeline 1 conflicts
|
|
379
415
|
self.logger.info("Pre-extracting frames to avoid pipeline conflicts...")
|
|
@@ -382,6 +418,8 @@ class VideoImportService:
|
|
|
382
418
|
if frames_extracted:
|
|
383
419
|
self.processing_context["frames_extracted"] = True
|
|
384
420
|
self.logger.info("Frame extraction completed successfully")
|
|
421
|
+
# Initialize frame objects in database
|
|
422
|
+
video.initialize_frames(video.get_frame_paths())
|
|
385
423
|
|
|
386
424
|
# CRITICAL: Immediately save the frames_extracted state to database
|
|
387
425
|
# to prevent refresh_from_db() in pipeline 1 from overriding it
|
|
@@ -394,7 +432,9 @@ class VideoImportService:
|
|
|
394
432
|
self.logger.warning("Frame extraction failed, but continuing...")
|
|
395
433
|
self.processing_context["frames_extracted"] = False
|
|
396
434
|
except Exception as e:
|
|
397
|
-
self.logger.warning(
|
|
435
|
+
self.logger.warning(
|
|
436
|
+
f"Frame extraction failed during setup: {e}, but continuing..."
|
|
437
|
+
)
|
|
398
438
|
self.processing_context["frames_extracted"] = False
|
|
399
439
|
|
|
400
440
|
# Ensure default patient data
|
|
@@ -405,38 +445,60 @@ class VideoImportService:
|
|
|
405
445
|
def _process_frames_and_metadata(self):
|
|
406
446
|
"""Process frames and extract metadata with anonymization."""
|
|
407
447
|
# Check frame cleaning availability
|
|
408
|
-
frame_cleaning_available, frame_cleaner =
|
|
448
|
+
frame_cleaning_available, frame_cleaner = (
|
|
449
|
+
self._ensure_frame_cleaning_available()
|
|
450
|
+
)
|
|
409
451
|
video = self._require_current_video()
|
|
410
452
|
|
|
411
453
|
raw_file_field = video.raw_file
|
|
412
|
-
has_raw_file = isinstance(raw_file_field, FieldFile) and bool(
|
|
454
|
+
has_raw_file = isinstance(raw_file_field, FieldFile) and bool(
|
|
455
|
+
raw_file_field.name
|
|
456
|
+
)
|
|
413
457
|
|
|
414
458
|
if not (frame_cleaning_available and has_raw_file):
|
|
415
|
-
self.logger.warning(
|
|
459
|
+
self.logger.warning(
|
|
460
|
+
"Frame cleaning not available or conditions not met, using fallback anonymization."
|
|
461
|
+
)
|
|
416
462
|
self._fallback_anonymize_video()
|
|
417
463
|
return
|
|
418
464
|
|
|
419
465
|
try:
|
|
420
|
-
self.logger.info(
|
|
466
|
+
self.logger.info(
|
|
467
|
+
"Starting frame-level anonymization with processor ROI masking..."
|
|
468
|
+
)
|
|
421
469
|
|
|
422
470
|
# Get processor ROI information
|
|
423
|
-
endoscope_data_roi_nested, endoscope_image_roi =
|
|
471
|
+
endoscope_data_roi_nested, endoscope_image_roi = (
|
|
472
|
+
self._get_processor_roi_info()
|
|
473
|
+
)
|
|
424
474
|
|
|
425
475
|
# Perform frame cleaning with timeout to prevent blocking
|
|
426
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
476
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
477
|
+
from concurrent.futures import TimeoutError as FutureTimeoutError
|
|
427
478
|
|
|
428
479
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
429
|
-
future = executor.submit(
|
|
480
|
+
future = executor.submit(
|
|
481
|
+
self._perform_frame_cleaning,
|
|
482
|
+
endoscope_data_roi_nested,
|
|
483
|
+
endoscope_image_roi,
|
|
484
|
+
)
|
|
430
485
|
try:
|
|
431
486
|
# Increased timeout to better accommodate ffmpeg + OCR
|
|
432
487
|
future.result(timeout=50000)
|
|
433
488
|
self.processing_context["anonymization_completed"] = True
|
|
434
|
-
self.logger.info(
|
|
489
|
+
self.logger.info(
|
|
490
|
+
"Frame cleaning completed successfully within timeout"
|
|
491
|
+
)
|
|
435
492
|
except FutureTimeoutError:
|
|
436
|
-
self.logger.warning(
|
|
493
|
+
self.logger.warning(
|
|
494
|
+
"Frame cleaning timed out; entering grace period check for cleaned output"
|
|
495
|
+
)
|
|
437
496
|
# Grace period: detect if cleaned file appears shortly after timeout
|
|
438
497
|
raw_video_path = self.processing_context.get("raw_video_path")
|
|
439
|
-
video_filename = self.processing_context.get(
|
|
498
|
+
video_filename = self.processing_context.get(
|
|
499
|
+
"video_filename",
|
|
500
|
+
Path(raw_video_path).name if raw_video_path else "video.mp4",
|
|
501
|
+
)
|
|
440
502
|
grace_seconds = 60
|
|
441
503
|
expected_cleaned_path: Optional[Path] = None
|
|
442
504
|
processed_field = video.processed_file
|
|
@@ -449,27 +511,42 @@ class VideoImportService:
|
|
|
449
511
|
if expected_cleaned_path is not None:
|
|
450
512
|
for _ in range(grace_seconds):
|
|
451
513
|
if expected_cleaned_path.exists():
|
|
452
|
-
self.processing_context["cleaned_video_path"] =
|
|
453
|
-
|
|
454
|
-
|
|
514
|
+
self.processing_context["cleaned_video_path"] = (
|
|
515
|
+
expected_cleaned_path
|
|
516
|
+
)
|
|
517
|
+
self.processing_context["anonymization_completed"] = (
|
|
518
|
+
True
|
|
519
|
+
)
|
|
520
|
+
self.logger.info(
|
|
521
|
+
"Detected cleaned video during grace period: %s",
|
|
522
|
+
expected_cleaned_path,
|
|
523
|
+
)
|
|
455
524
|
found = True
|
|
456
525
|
break
|
|
457
526
|
time.sleep(1)
|
|
458
527
|
else:
|
|
459
528
|
self._fallback_anonymize_video()
|
|
460
529
|
if not found:
|
|
461
|
-
raise TimeoutError(
|
|
530
|
+
raise TimeoutError(
|
|
531
|
+
"Frame cleaning operation timed out - likely Ollama connection issue"
|
|
532
|
+
)
|
|
462
533
|
|
|
463
534
|
except Exception as e:
|
|
464
|
-
self.logger.warning(
|
|
535
|
+
self.logger.warning(
|
|
536
|
+
"Frame cleaning failed (reason: %s), falling back to simple copy", e
|
|
537
|
+
)
|
|
465
538
|
# Try fallback anonymization when frame cleaning fails
|
|
466
539
|
try:
|
|
467
540
|
self._fallback_anonymize_video()
|
|
468
541
|
except Exception as fallback_error:
|
|
469
|
-
self.logger.error(
|
|
542
|
+
self.logger.error(
|
|
543
|
+
"Fallback anonymization also failed: %s", fallback_error
|
|
544
|
+
)
|
|
470
545
|
# If even fallback fails, mark as not anonymized but continue import
|
|
471
546
|
self.processing_context["anonymization_completed"] = False
|
|
472
|
-
self.processing_context["error_reason"] =
|
|
547
|
+
self.processing_context["error_reason"] = (
|
|
548
|
+
f"Frame cleaning failed: {e}, Fallback failed: {fallback_error}"
|
|
549
|
+
)
|
|
473
550
|
|
|
474
551
|
def _save_anonymized_video(self):
|
|
475
552
|
original_raw_file_path_to_delete = None
|
|
@@ -478,14 +555,24 @@ class VideoImportService:
|
|
|
478
555
|
anonymized_video_path = video.get_target_anonymized_video_path()
|
|
479
556
|
|
|
480
557
|
if not anonymized_video_path.exists():
|
|
481
|
-
raise RuntimeError(
|
|
558
|
+
raise RuntimeError(
|
|
559
|
+
f"Processed video file not found after assembly for {video.uuid}: {anonymized_video_path}"
|
|
560
|
+
)
|
|
482
561
|
|
|
483
562
|
new_processed_hash = get_video_hash(anonymized_video_path)
|
|
484
|
-
if
|
|
485
|
-
|
|
563
|
+
if (
|
|
564
|
+
video.__class__.objects.filter(processed_video_hash=new_processed_hash)
|
|
565
|
+
.exclude(pk=video.pk)
|
|
566
|
+
.exists()
|
|
567
|
+
):
|
|
568
|
+
raise ValueError(
|
|
569
|
+
f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid})."
|
|
570
|
+
)
|
|
486
571
|
|
|
487
572
|
video.processed_video_hash = new_processed_hash
|
|
488
|
-
video.processed_file.name = anonymized_video_path.relative_to(
|
|
573
|
+
video.processed_file.name = anonymized_video_path.relative_to(
|
|
574
|
+
STORAGE_DIR
|
|
575
|
+
).as_posix()
|
|
489
576
|
|
|
490
577
|
update_fields = [
|
|
491
578
|
"processed_video_hash",
|
|
@@ -503,7 +590,9 @@ class VideoImportService:
|
|
|
503
590
|
|
|
504
591
|
transaction.on_commit(
|
|
505
592
|
lambda: _cleanup_raw_assets(
|
|
506
|
-
video_uuid=video.uuid,
|
|
593
|
+
video_uuid=video.uuid,
|
|
594
|
+
raw_file_path=original_raw_file_path_to_delete,
|
|
595
|
+
raw_frame_dir=original_raw_frame_dir_to_delete,
|
|
507
596
|
)
|
|
508
597
|
)
|
|
509
598
|
|
|
@@ -521,15 +610,23 @@ class VideoImportService:
|
|
|
521
610
|
self.logger.info("Attempting fallback video anonymization...")
|
|
522
611
|
video = self.current_video
|
|
523
612
|
if video is None:
|
|
524
|
-
self.logger.warning(
|
|
613
|
+
self.logger.warning(
|
|
614
|
+
"No VideoFile instance available for fallback anonymization"
|
|
615
|
+
)
|
|
525
616
|
|
|
526
617
|
# Strategy 2: Simple copy (no processing, just copy raw to processed)
|
|
527
|
-
self.logger.info(
|
|
618
|
+
self.logger.info(
|
|
619
|
+
"Using simple copy fallback (raw video will be used as 'processed' video)"
|
|
620
|
+
)
|
|
528
621
|
self.processing_context["anonymization_completed"] = False
|
|
529
622
|
self.processing_context["use_raw_as_processed"] = True
|
|
530
|
-
self.logger.warning(
|
|
623
|
+
self.logger.warning(
|
|
624
|
+
"Fallback: Video will be imported without anonymization (raw copy used)"
|
|
625
|
+
)
|
|
531
626
|
except Exception as e:
|
|
532
|
-
self.logger.error(
|
|
627
|
+
self.logger.error(
|
|
628
|
+
f"Error during fallback anonymization: {e}", exc_info=True
|
|
629
|
+
)
|
|
533
630
|
self.processing_context["anonymization_completed"] = False
|
|
534
631
|
self.processing_context["error_reason"] = str(e)
|
|
535
632
|
|
|
@@ -542,7 +639,11 @@ class VideoImportService:
|
|
|
542
639
|
try:
|
|
543
640
|
video.refresh_from_db()
|
|
544
641
|
except Exception as refresh_error:
|
|
545
|
-
self.logger.warning(
|
|
642
|
+
self.logger.warning(
|
|
643
|
+
"Could not refresh VideoFile %s from DB: %s",
|
|
644
|
+
video.uuid,
|
|
645
|
+
refresh_error,
|
|
646
|
+
)
|
|
546
647
|
|
|
547
648
|
state = video.get_or_create_state()
|
|
548
649
|
|
|
@@ -559,12 +660,18 @@ class VideoImportService:
|
|
|
559
660
|
state.text_meta_extracted = True
|
|
560
661
|
|
|
561
662
|
# ✅ FIX: Only mark as processed if anonymization actually completed
|
|
562
|
-
anonymization_completed = self.processing_context.get(
|
|
663
|
+
anonymization_completed = self.processing_context.get(
|
|
664
|
+
"anonymization_completed", False
|
|
665
|
+
)
|
|
563
666
|
if anonymization_completed:
|
|
564
667
|
state.mark_sensitive_meta_processed(save=False)
|
|
565
|
-
self.logger.info(
|
|
668
|
+
self.logger.info(
|
|
669
|
+
"Anonymization completed - marking sensitive meta as processed"
|
|
670
|
+
)
|
|
566
671
|
else:
|
|
567
|
-
self.logger.warning(
|
|
672
|
+
self.logger.warning(
|
|
673
|
+
f"Anonymization NOT completed - NOT marking as processed. Reason: {self.processing_context.get('error_reason', 'Unknown')}"
|
|
674
|
+
)
|
|
568
675
|
# Explicitly mark as NOT processed
|
|
569
676
|
state.sensitive_meta_processed = False
|
|
570
677
|
|
|
@@ -590,12 +697,15 @@ class VideoImportService:
|
|
|
590
697
|
else:
|
|
591
698
|
raw_video_path = self.processing_context.get("raw_video_path")
|
|
592
699
|
if raw_video_path and Path(raw_video_path).exists():
|
|
593
|
-
|
|
594
|
-
|
|
700
|
+
# Use UUID-based naming to avoid conflicts
|
|
701
|
+
suffix = Path(raw_video_path).suffix or ".mp4"
|
|
702
|
+
processed_filename = f"processed_{video.uuid}{suffix}"
|
|
595
703
|
processed_video_path = Path(raw_video_path).parent / processed_filename
|
|
596
704
|
try:
|
|
597
705
|
shutil.copy2(str(raw_video_path), str(processed_video_path))
|
|
598
|
-
self.logger.info(
|
|
706
|
+
self.logger.info(
|
|
707
|
+
"Copied raw video for processing: %s", processed_video_path
|
|
708
|
+
)
|
|
599
709
|
except Exception as exc:
|
|
600
710
|
self.logger.error("Failed to copy raw video: %s", exc)
|
|
601
711
|
processed_video_path = None
|
|
@@ -615,10 +725,16 @@ class VideoImportService:
|
|
|
615
725
|
relative_path = anonym_target_path.relative_to(storage_root)
|
|
616
726
|
video.processed_file.name = str(relative_path)
|
|
617
727
|
video.save(update_fields=["processed_file"])
|
|
618
|
-
self.logger.info(
|
|
728
|
+
self.logger.info(
|
|
729
|
+
"Updated processed_file path to: %s", relative_path
|
|
730
|
+
)
|
|
619
731
|
except Exception as exc:
|
|
620
|
-
self.logger.error(
|
|
621
|
-
|
|
732
|
+
self.logger.error(
|
|
733
|
+
"Failed to update processed_file path: %s", exc
|
|
734
|
+
)
|
|
735
|
+
video.processed_file.name = (
|
|
736
|
+
f"anonym_videos/{anonym_video_filename}"
|
|
737
|
+
)
|
|
622
738
|
video.save(update_fields=["processed_file"])
|
|
623
739
|
self.logger.info(
|
|
624
740
|
"Updated processed_file path using fallback: %s",
|
|
@@ -627,17 +743,26 @@ class VideoImportService:
|
|
|
627
743
|
|
|
628
744
|
self.processing_context["anonymization_completed"] = True
|
|
629
745
|
else:
|
|
630
|
-
self.logger.warning(
|
|
746
|
+
self.logger.warning(
|
|
747
|
+
"Processed video file not found after move: %s",
|
|
748
|
+
anonym_target_path,
|
|
749
|
+
)
|
|
631
750
|
except Exception as exc:
|
|
632
|
-
self.logger.error(
|
|
751
|
+
self.logger.error(
|
|
752
|
+
"Failed to move processed video to anonym_videos: %s", exc
|
|
753
|
+
)
|
|
633
754
|
else:
|
|
634
|
-
self.logger.warning(
|
|
755
|
+
self.logger.warning(
|
|
756
|
+
"No processed video available - processed_file will remain empty"
|
|
757
|
+
)
|
|
635
758
|
|
|
636
759
|
try:
|
|
637
760
|
from endoreg_db.utils.paths import RAW_FRAME_DIR
|
|
638
761
|
|
|
639
762
|
shutil.rmtree(RAW_FRAME_DIR, ignore_errors=True)
|
|
640
|
-
self.logger.debug(
|
|
763
|
+
self.logger.debug(
|
|
764
|
+
"Cleaned up temporary frames directory: %s", RAW_FRAME_DIR
|
|
765
|
+
)
|
|
641
766
|
except Exception as exc:
|
|
642
767
|
self.logger.warning("Failed to remove directory %s: %s", RAW_FRAME_DIR, exc)
|
|
643
768
|
|
|
@@ -647,10 +772,14 @@ class VideoImportService:
|
|
|
647
772
|
os.remove(source_path)
|
|
648
773
|
self.logger.info("Removed remaining source file: %s", source_path)
|
|
649
774
|
except Exception as exc:
|
|
650
|
-
self.logger.warning(
|
|
775
|
+
self.logger.warning(
|
|
776
|
+
"Failed to remove source file %s: %s", source_path, exc
|
|
777
|
+
)
|
|
651
778
|
|
|
652
779
|
if not video.processed_file or not Path(video.processed_file.path).exists():
|
|
653
|
-
self.logger.warning(
|
|
780
|
+
self.logger.warning(
|
|
781
|
+
"No processed_file found after cleanup - video will be unprocessed"
|
|
782
|
+
)
|
|
654
783
|
try:
|
|
655
784
|
video.anonymize(delete_original_raw=self.delete_source)
|
|
656
785
|
video.save(update_fields=["processed_file"])
|
|
@@ -665,10 +794,14 @@ class VideoImportService:
|
|
|
665
794
|
|
|
666
795
|
with transaction.atomic():
|
|
667
796
|
video.refresh_from_db()
|
|
668
|
-
if hasattr(video, "state") and self.processing_context.get(
|
|
797
|
+
if hasattr(video, "state") and self.processing_context.get(
|
|
798
|
+
"anonymization_completed"
|
|
799
|
+
):
|
|
669
800
|
video.state.mark_sensitive_meta_processed(save=True)
|
|
670
801
|
|
|
671
|
-
self.logger.info(
|
|
802
|
+
self.logger.info(
|
|
803
|
+
"Import and anonymization completed for VideoFile UUID: %s", video.uuid
|
|
804
|
+
)
|
|
672
805
|
self.logger.info("Raw video stored in: /data/videos")
|
|
673
806
|
self.logger.info("Processed video stored in: /data/anonym_videos")
|
|
674
807
|
|
|
@@ -695,7 +828,9 @@ class VideoImportService:
|
|
|
695
828
|
if source_path is None:
|
|
696
829
|
raise ValueError("No file path available for creating sensitive file")
|
|
697
830
|
if not raw_field:
|
|
698
|
-
raise ValueError(
|
|
831
|
+
raise ValueError(
|
|
832
|
+
"VideoFile must have a raw_file to create a sensitive file"
|
|
833
|
+
)
|
|
699
834
|
|
|
700
835
|
target_dir = VIDEO_DIR / "sensitive"
|
|
701
836
|
if not target_dir.exists():
|
|
@@ -705,9 +840,13 @@ class VideoImportService:
|
|
|
705
840
|
target_file_path = target_dir / source_path.name
|
|
706
841
|
try:
|
|
707
842
|
shutil.move(str(source_path), str(target_file_path))
|
|
708
|
-
self.logger.info(
|
|
843
|
+
self.logger.info(
|
|
844
|
+
"Moved raw file to sensitive directory: %s", target_file_path
|
|
845
|
+
)
|
|
709
846
|
except Exception as exc:
|
|
710
|
-
self.logger.warning(
|
|
847
|
+
self.logger.warning(
|
|
848
|
+
"Failed to move raw file to sensitive dir, copying instead: %s", exc
|
|
849
|
+
)
|
|
711
850
|
shutil.copy(str(source_path), str(target_file_path))
|
|
712
851
|
try:
|
|
713
852
|
os.remove(source_path)
|
|
@@ -721,7 +860,10 @@ class VideoImportService:
|
|
|
721
860
|
relative_path = target_file_path.relative_to(storage_root)
|
|
722
861
|
video.raw_file.name = str(relative_path)
|
|
723
862
|
video.save(update_fields=["raw_file"])
|
|
724
|
-
self.logger.info(
|
|
863
|
+
self.logger.info(
|
|
864
|
+
"Updated video.raw_file to point to sensitive location: %s",
|
|
865
|
+
relative_path,
|
|
866
|
+
)
|
|
725
867
|
except Exception as exc:
|
|
726
868
|
self.logger.warning("Failed to set relative path, using fallback: %s", exc)
|
|
727
869
|
video.raw_file.name = f"videos/sensitive/{target_file_path.name}"
|
|
@@ -734,10 +876,14 @@ class VideoImportService:
|
|
|
734
876
|
self.processing_context["raw_video_path"] = target_file_path
|
|
735
877
|
self.processing_context["video_filename"] = target_file_path.name
|
|
736
878
|
|
|
737
|
-
self.logger.info(
|
|
879
|
+
self.logger.info(
|
|
880
|
+
"Created sensitive file for %s at %s", video.uuid, target_file_path
|
|
881
|
+
)
|
|
738
882
|
return target_file_path
|
|
739
883
|
|
|
740
|
-
def _get_processor_roi_info(
|
|
884
|
+
def _get_processor_roi_info(
|
|
885
|
+
self,
|
|
886
|
+
) -> Tuple[Optional[List[List[Dict[str, Any]]]], Optional[Dict[str, Any]]]:
|
|
741
887
|
"""Get processor ROI information for masking."""
|
|
742
888
|
endoscope_data_roi_nested = None
|
|
743
889
|
endoscope_image_roi = None
|
|
@@ -748,10 +894,15 @@ class VideoImportService:
|
|
|
748
894
|
video_meta = getattr(video, "video_meta", None)
|
|
749
895
|
processor = getattr(video_meta, "processor", None) if video_meta else None
|
|
750
896
|
if processor:
|
|
751
|
-
assert isinstance(processor, EndoscopyProcessor),
|
|
897
|
+
assert isinstance(processor, EndoscopyProcessor), (
|
|
898
|
+
"Processor is not of type EndoscopyProcessor"
|
|
899
|
+
)
|
|
752
900
|
endoscope_image_roi = processor.get_roi_endoscope_image()
|
|
753
901
|
endoscope_data_roi_nested = processor.get_sensitive_rois()
|
|
754
|
-
self.logger.info(
|
|
902
|
+
self.logger.info(
|
|
903
|
+
"Retrieved processor ROI information: endoscope_image_roi=%s",
|
|
904
|
+
endoscope_image_roi,
|
|
905
|
+
)
|
|
755
906
|
else:
|
|
756
907
|
self.logger.warning(
|
|
757
908
|
"No processor found for video %s, proceeding without ROI masking",
|
|
@@ -773,28 +924,40 @@ class VideoImportService:
|
|
|
773
924
|
|
|
774
925
|
return endoscope_data_roi_nested, endoscope_image_roi
|
|
775
926
|
|
|
776
|
-
def _ensure_default_patient_data(
|
|
927
|
+
def _ensure_default_patient_data(
|
|
928
|
+
self, video_instance: VideoFile | None = None
|
|
929
|
+
) -> None:
|
|
777
930
|
"""Ensure minimum patient data is present on the video's SensitiveMeta."""
|
|
778
931
|
|
|
779
932
|
video = video_instance or self._require_current_video()
|
|
780
933
|
|
|
781
934
|
sensitive_meta = getattr(video, "sensitive_meta", None)
|
|
782
935
|
if not sensitive_meta:
|
|
783
|
-
self.logger.info(
|
|
936
|
+
self.logger.info(
|
|
937
|
+
"No SensitiveMeta found for video %s, creating default", video.uuid
|
|
938
|
+
)
|
|
784
939
|
default_data = {
|
|
785
940
|
"patient_first_name": "Patient",
|
|
786
941
|
"patient_last_name": "Unknown",
|
|
787
942
|
"patient_dob": date(1990, 1, 1),
|
|
788
943
|
"examination_date": date.today(),
|
|
789
|
-
"center_name": video.center.name
|
|
944
|
+
"center_name": video.center.name
|
|
945
|
+
if video.center
|
|
946
|
+
else "university_hospital_wuerzburg",
|
|
790
947
|
}
|
|
791
948
|
try:
|
|
792
949
|
sensitive_meta = SensitiveMeta.create_from_dict(default_data)
|
|
793
950
|
video.sensitive_meta = sensitive_meta
|
|
794
951
|
video.save(update_fields=["sensitive_meta"])
|
|
795
|
-
self.logger.info(
|
|
952
|
+
self.logger.info(
|
|
953
|
+
"Created default SensitiveMeta for video %s", video.uuid
|
|
954
|
+
)
|
|
796
955
|
except Exception as exc:
|
|
797
|
-
self.logger.error(
|
|
956
|
+
self.logger.error(
|
|
957
|
+
"Failed to create default SensitiveMeta for video %s: %s",
|
|
958
|
+
video.uuid,
|
|
959
|
+
exc,
|
|
960
|
+
)
|
|
798
961
|
return
|
|
799
962
|
else:
|
|
800
963
|
update_data: Dict[str, Any] = {}
|
|
@@ -818,7 +981,11 @@ class VideoImportService:
|
|
|
818
981
|
list(update_data.keys()),
|
|
819
982
|
)
|
|
820
983
|
except Exception as exc:
|
|
821
|
-
self.logger.error(
|
|
984
|
+
self.logger.error(
|
|
985
|
+
"Failed to update SensitiveMeta for video %s: %s",
|
|
986
|
+
video.uuid,
|
|
987
|
+
exc,
|
|
988
|
+
)
|
|
822
989
|
|
|
823
990
|
def _ensure_frame_cleaning_available(self):
|
|
824
991
|
"""
|
|
@@ -835,7 +1002,9 @@ class VideoImportService:
|
|
|
835
1002
|
return True, FrameCleaner()
|
|
836
1003
|
|
|
837
1004
|
except Exception as e:
|
|
838
|
-
self.logger.warning(
|
|
1005
|
+
self.logger.warning(
|
|
1006
|
+
f"Frame cleaning not available: {e} Please install or update lx_anonymizer."
|
|
1007
|
+
)
|
|
839
1008
|
|
|
840
1009
|
return False, None
|
|
841
1010
|
|
|
@@ -857,12 +1026,17 @@ class VideoImportService:
|
|
|
857
1026
|
except Exception:
|
|
858
1027
|
raise RuntimeError(f"Raw video path not found: {raw_video_path}")
|
|
859
1028
|
|
|
860
|
-
# Create temporary output path for cleaned video
|
|
861
|
-
|
|
862
|
-
|
|
1029
|
+
# Create temporary output path for cleaned video using UUID to avoid naming conflicts
|
|
1030
|
+
video = self._require_current_video()
|
|
1031
|
+
# Ensure raw_video_path is not None
|
|
863
1032
|
if not raw_video_path:
|
|
864
|
-
raise RuntimeError(
|
|
1033
|
+
raise RuntimeError(
|
|
1034
|
+
"raw_video_path is None, cannot construct cleaned_video_path"
|
|
1035
|
+
)
|
|
1036
|
+
suffix = Path(raw_video_path).suffix or ".mp4"
|
|
1037
|
+
cleaned_filename = f"cleaned_{video.uuid}{suffix}"
|
|
865
1038
|
cleaned_video_path = Path(raw_video_path).parent / cleaned_filename
|
|
1039
|
+
self.logger.debug("Using UUID-based cleaned filename: %s", cleaned_filename)
|
|
866
1040
|
|
|
867
1041
|
# Clean video with ROI masking (heavy I/O operation)
|
|
868
1042
|
actual_cleaned_path, extracted_metadata = frame_cleaner.clean_video(
|
|
@@ -879,9 +1053,13 @@ class VideoImportService:
|
|
|
879
1053
|
|
|
880
1054
|
# Update sensitive metadata with extracted information
|
|
881
1055
|
self._update_sensitive_metadata(extracted_metadata)
|
|
882
|
-
self.logger.info(
|
|
1056
|
+
self.logger.info(
|
|
1057
|
+
f"Extracted metadata from frame cleaning: {extracted_metadata}"
|
|
1058
|
+
)
|
|
883
1059
|
|
|
884
|
-
self.logger.info(
|
|
1060
|
+
self.logger.info(
|
|
1061
|
+
f"Frame cleaning with ROI masking completed: {actual_cleaned_path}"
|
|
1062
|
+
)
|
|
885
1063
|
self.logger.info("Cleaned video will be moved to anonym_videos during cleanup")
|
|
886
1064
|
|
|
887
1065
|
def _update_sensitive_metadata(self, extracted_metadata: Dict[str, Any]):
|
|
@@ -898,31 +1076,41 @@ class VideoImportService:
|
|
|
898
1076
|
|
|
899
1077
|
sm = sensitive_meta
|
|
900
1078
|
updated_fields = []
|
|
901
|
-
|
|
1079
|
+
|
|
902
1080
|
# Ensure center is set from video.center if not in extracted_metadata
|
|
903
1081
|
metadata_to_update = extracted_metadata.copy()
|
|
904
|
-
|
|
1082
|
+
|
|
905
1083
|
# FIX: Set center object instead of center_name string
|
|
906
|
-
if not hasattr(sm,
|
|
1084
|
+
if not hasattr(sm, "center") or not sm.center:
|
|
907
1085
|
if video.center:
|
|
908
|
-
metadata_to_update[
|
|
909
|
-
self.logger.debug(
|
|
1086
|
+
metadata_to_update["center"] = video.center
|
|
1087
|
+
self.logger.debug(
|
|
1088
|
+
"Added center object '%s' to metadata for SensitiveMeta update",
|
|
1089
|
+
video.center.name,
|
|
1090
|
+
)
|
|
910
1091
|
else:
|
|
911
|
-
center_name = metadata_to_update.get(
|
|
1092
|
+
center_name = metadata_to_update.get("center_name")
|
|
912
1093
|
if center_name:
|
|
913
1094
|
try:
|
|
914
1095
|
from ..models.administration import Center
|
|
1096
|
+
|
|
915
1097
|
center_obj = Center.objects.get(name=center_name)
|
|
916
|
-
metadata_to_update[
|
|
917
|
-
self.logger.debug(
|
|
918
|
-
|
|
1098
|
+
metadata_to_update["center"] = center_obj
|
|
1099
|
+
self.logger.debug(
|
|
1100
|
+
"Loaded center object '%s' from center_name", center_name
|
|
1101
|
+
)
|
|
1102
|
+
metadata_to_update.pop("center_name", None)
|
|
919
1103
|
except Center.DoesNotExist:
|
|
920
|
-
self.logger.error(
|
|
1104
|
+
self.logger.error(
|
|
1105
|
+
"Center '%s' not found in database", center_name
|
|
1106
|
+
)
|
|
921
1107
|
return
|
|
922
|
-
|
|
1108
|
+
|
|
923
1109
|
try:
|
|
924
1110
|
sm.update_from_dict(metadata_to_update)
|
|
925
|
-
updated_fields = list(
|
|
1111
|
+
updated_fields = list(
|
|
1112
|
+
extracted_metadata.keys()
|
|
1113
|
+
) # Only log originally extracted fields
|
|
926
1114
|
except KeyError as e:
|
|
927
1115
|
self.logger.warning(f"Failed to update SensitiveMeta field {e}")
|
|
928
1116
|
return
|
|
@@ -930,16 +1118,25 @@ class VideoImportService:
|
|
|
930
1118
|
if updated_fields:
|
|
931
1119
|
try:
|
|
932
1120
|
sm.save() # Remove update_fields to allow all necessary fields to be saved
|
|
933
|
-
self.logger.info(
|
|
1121
|
+
self.logger.info(
|
|
1122
|
+
"Updated SensitiveMeta fields for video %s: %s",
|
|
1123
|
+
video.uuid,
|
|
1124
|
+
updated_fields,
|
|
1125
|
+
)
|
|
934
1126
|
|
|
935
1127
|
state = video.get_or_create_state()
|
|
936
1128
|
state.mark_sensitive_meta_processed(save=True)
|
|
937
|
-
self.logger.info(
|
|
1129
|
+
self.logger.info(
|
|
1130
|
+
"Marked sensitive metadata as processed for video %s", video.uuid
|
|
1131
|
+
)
|
|
938
1132
|
except Exception as e:
|
|
939
1133
|
self.logger.error(f"Failed to save SensitiveMeta: {e}")
|
|
940
1134
|
raise # Re-raise to trigger fallback in calling method
|
|
941
1135
|
else:
|
|
942
|
-
self.logger.info(
|
|
1136
|
+
self.logger.info(
|
|
1137
|
+
"No SensitiveMeta fields updated for video %s - all existing values preserved",
|
|
1138
|
+
video.uuid,
|
|
1139
|
+
)
|
|
943
1140
|
|
|
944
1141
|
def _signal_completion(self):
|
|
945
1142
|
"""Signal completion to the tracking system."""
|
|
@@ -954,14 +1151,25 @@ class VideoImportService:
|
|
|
954
1151
|
except (ValueError, OSError):
|
|
955
1152
|
raw_exists = False
|
|
956
1153
|
|
|
957
|
-
video_processing_complete =
|
|
1154
|
+
video_processing_complete = (
|
|
1155
|
+
video.sensitive_meta is not None
|
|
1156
|
+
and video.video_meta is not None
|
|
1157
|
+
and raw_exists
|
|
1158
|
+
)
|
|
958
1159
|
|
|
959
1160
|
if video_processing_complete:
|
|
960
|
-
self.logger.info(
|
|
1161
|
+
self.logger.info(
|
|
1162
|
+
"Video %s processing completed successfully - ready for validation",
|
|
1163
|
+
video.uuid,
|
|
1164
|
+
)
|
|
961
1165
|
|
|
962
1166
|
# Update completion flags if they exist
|
|
963
1167
|
completion_fields = []
|
|
964
|
-
for field_name in [
|
|
1168
|
+
for field_name in [
|
|
1169
|
+
"import_completed",
|
|
1170
|
+
"processing_complete",
|
|
1171
|
+
"ready_for_validation",
|
|
1172
|
+
]:
|
|
965
1173
|
if hasattr(video, field_name):
|
|
966
1174
|
setattr(video, field_name, True)
|
|
967
1175
|
completion_fields.append(field_name)
|
|
@@ -1018,7 +1226,9 @@ class VideoImportService:
|
|
|
1018
1226
|
file_path_str = str(file_path)
|
|
1019
1227
|
if file_path_str in self.processed_files:
|
|
1020
1228
|
self.processed_files.remove(file_path_str)
|
|
1021
|
-
self.logger.info(
|
|
1229
|
+
self.logger.info(
|
|
1230
|
+
f"Removed {file_path_str} from processed files (failed processing)"
|
|
1231
|
+
)
|
|
1022
1232
|
|
|
1023
1233
|
except Exception as e:
|
|
1024
1234
|
self.logger.warning(f"Error during context cleanup: {e}")
|