endoreg-db 0.8.1__py3-none-any.whl → 0.8.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/helpers/download_segmentation_model.py +31 -0
- endoreg_db/models/media/video/pipe_1.py +13 -1
- endoreg_db/serializers/anonymization.py +3 -0
- endoreg_db/services/pdf_import.py +0 -30
- endoreg_db/services/video_import.py +301 -98
- endoreg_db/urls/__init__.py +0 -2
- endoreg_db/urls/media.py +201 -4
- endoreg_db/urls/report.py +0 -30
- endoreg_db/urls/video.py +30 -88
- endoreg_db/views/anonymization/validate.py +5 -2
- endoreg_db/views/media/__init__.py +38 -2
- endoreg_db/views/media/pdf_media.py +1 -1
- endoreg_db/views/media/segments.py +71 -0
- endoreg_db/views/media/sensitive_metadata.py +314 -0
- endoreg_db/views/media/video_segments.py +596 -0
- endoreg_db/views/pdf/reimport.py +18 -8
- endoreg_db/views/video/__init__.py +0 -8
- endoreg_db/views/video/correction.py +26 -26
- endoreg_db/views/video/reimport.py +15 -12
- endoreg_db/views/video/video_stream.py +168 -50
- {endoreg_db-0.8.1.dist-info → endoreg_db-0.8.2.dist-info}/METADATA +2 -2
- {endoreg_db-0.8.1.dist-info → endoreg_db-0.8.2.dist-info}/RECORD +34 -33
- endoreg_db/urls/pdf.py +0 -0
- endoreg_db/urls/sensitive_meta.py +0 -36
- endoreg_db/views/video/media/__init__.py +0 -23
- /endoreg_db/views/video/{media/task_status.py → task_status.py} +0 -0
- /endoreg_db/views/video/{media/video_analyze.py → video_analyze.py} +0 -0
- /endoreg_db/views/video/{media/video_apply_mask.py → video_apply_mask.py} +0 -0
- /endoreg_db/views/video/{media/video_correction.py → video_correction.py} +0 -0
- /endoreg_db/views/video/{media/video_download_processed.py → video_download_processed.py} +0 -0
- /endoreg_db/views/video/{media/video_media.py → video_media.py} +0 -0
- /endoreg_db/views/video/{media/video_meta.py → video_meta.py} +0 -0
- /endoreg_db/views/video/{media/video_processing_history.py → video_processing_history.py} +0 -0
- /endoreg_db/views/video/{media/video_remove_frames.py → video_remove_frames.py} +0 -0
- /endoreg_db/views/video/{media/video_reprocess.py → video_reprocess.py} +0 -0
- {endoreg_db-0.8.1.dist-info → endoreg_db-0.8.2.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.1.dist-info → endoreg_db-0.8.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -3,26 +3,44 @@ Video import service module.
|
|
|
3
3
|
|
|
4
4
|
Provides high-level functions for importing and anonymizing video files,
|
|
5
5
|
combining VideoFile creation with frame-level anonymization.
|
|
6
|
+
|
|
7
|
+
Changelog:
|
|
8
|
+
October 14, 2025: Added file locking mechanism to prevent race conditions
|
|
9
|
+
during concurrent video imports (matches PDF import pattern)
|
|
6
10
|
"""
|
|
7
11
|
from datetime import date
|
|
8
12
|
import logging
|
|
9
13
|
import sys
|
|
10
14
|
import os
|
|
11
15
|
import shutil
|
|
16
|
+
import time
|
|
17
|
+
from contextlib import contextmanager
|
|
12
18
|
from pathlib import Path
|
|
13
|
-
from typing import
|
|
19
|
+
from typing import Union, Dict, Any, Optional
|
|
14
20
|
from django.db import transaction
|
|
15
|
-
from django.core.exceptions import FieldError
|
|
16
21
|
from endoreg_db.models import VideoFile, SensitiveMeta
|
|
17
22
|
from endoreg_db.utils.paths import STORAGE_DIR, RAW_FRAME_DIR, VIDEO_DIR, ANONYM_VIDEO_DIR
|
|
18
23
|
import random
|
|
19
24
|
from lx_anonymizer.ocr import trocr_full_image_ocr
|
|
25
|
+
from numpy import ma
|
|
26
|
+
|
|
27
|
+
# File lock configuration (matches PDF import)
|
|
28
|
+
STALE_LOCK_SECONDS = 600 # 10 minutes - reclaim locks older than this
|
|
29
|
+
MAX_LOCK_WAIT_SECONDS = 90 # New: wait up to 90s for a non-stale lock to clear before skipping
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
20
32
|
|
|
21
33
|
|
|
22
34
|
class VideoImportService():
|
|
23
35
|
"""
|
|
24
36
|
Service for importing and anonymizing video files.
|
|
25
37
|
Uses a central video instance pattern for cleaner state management.
|
|
38
|
+
|
|
39
|
+
Features (October 14, 2025):
|
|
40
|
+
- File locking to prevent concurrent processing of the same video
|
|
41
|
+
- Stale lock detection and reclamation (600s timeout)
|
|
42
|
+
- Hash-based duplicate detection
|
|
43
|
+
- Graceful fallback processing without lx_anonymizer
|
|
26
44
|
"""
|
|
27
45
|
|
|
28
46
|
def __init__(self, project_root: Path = None):
|
|
@@ -42,11 +60,67 @@ class VideoImportService():
|
|
|
42
60
|
self.current_video = None
|
|
43
61
|
self.processing_context: Dict[str, Any] = {}
|
|
44
62
|
|
|
45
|
-
if TYPE_CHECKING:
|
|
46
|
-
from endoreg_db.models import VideoFile
|
|
47
|
-
|
|
48
63
|
self.logger = logging.getLogger(__name__)
|
|
49
64
|
|
|
65
|
+
@contextmanager
|
|
66
|
+
def _file_lock(self, path: Path):
|
|
67
|
+
"""
|
|
68
|
+
Create a file lock to prevent duplicate processing of the same video.
|
|
69
|
+
|
|
70
|
+
This context manager creates a .lock file alongside the video file.
|
|
71
|
+
If the lock file already exists, it checks if it's stale (older than
|
|
72
|
+
STALE_LOCK_SECONDS) and reclaims it if necessary. If it's not stale,
|
|
73
|
+
we now WAIT (up to MAX_LOCK_WAIT_SECONDS) instead of failing immediately.
|
|
74
|
+
"""
|
|
75
|
+
lock_path = Path(str(path) + ".lock")
|
|
76
|
+
fd = None
|
|
77
|
+
try:
|
|
78
|
+
deadline = time.time() + MAX_LOCK_WAIT_SECONDS
|
|
79
|
+
while True:
|
|
80
|
+
try:
|
|
81
|
+
# Atomic create; fail if exists
|
|
82
|
+
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
|
|
83
|
+
break # acquired
|
|
84
|
+
except FileExistsError:
|
|
85
|
+
# Check for stale lock
|
|
86
|
+
age = None
|
|
87
|
+
try:
|
|
88
|
+
st = os.stat(lock_path)
|
|
89
|
+
age = time.time() - st.st_mtime
|
|
90
|
+
except FileNotFoundError:
|
|
91
|
+
# Race: lock removed between exists and stat; retry acquire in next loop
|
|
92
|
+
age = None
|
|
93
|
+
|
|
94
|
+
if age is not None and age > STALE_LOCK_SECONDS:
|
|
95
|
+
try:
|
|
96
|
+
logger.warning(
|
|
97
|
+
"Stale lock detected for %s (age %.0fs). Reclaiming lock...",
|
|
98
|
+
path, age
|
|
99
|
+
)
|
|
100
|
+
lock_path.unlink()
|
|
101
|
+
except Exception as e:
|
|
102
|
+
logger.warning("Failed to remove stale lock %s: %s", lock_path, e)
|
|
103
|
+
# Loop continues and retries acquire immediately
|
|
104
|
+
continue
|
|
105
|
+
|
|
106
|
+
# Not stale: wait until deadline, then give up gracefully
|
|
107
|
+
if time.time() >= deadline:
|
|
108
|
+
raise ValueError(f"File already being processed: {path}")
|
|
109
|
+
time.sleep(1.0)
|
|
110
|
+
|
|
111
|
+
os.write(fd, b"lock")
|
|
112
|
+
os.close(fd)
|
|
113
|
+
fd = None
|
|
114
|
+
yield
|
|
115
|
+
finally:
|
|
116
|
+
try:
|
|
117
|
+
if fd is not None:
|
|
118
|
+
os.close(fd)
|
|
119
|
+
if lock_path.exists():
|
|
120
|
+
lock_path.unlink()
|
|
121
|
+
except OSError:
|
|
122
|
+
pass
|
|
123
|
+
|
|
50
124
|
def processed(self) -> bool:
|
|
51
125
|
"""Indicates if the current file has already been processed."""
|
|
52
126
|
return getattr(self, '_processed', False)
|
|
@@ -62,27 +136,21 @@ class VideoImportService():
|
|
|
62
136
|
"""
|
|
63
137
|
High-level helper that orchestrates the complete video import and anonymization process.
|
|
64
138
|
Uses the central video instance pattern for improved state management.
|
|
65
|
-
|
|
66
|
-
Args:
|
|
67
|
-
file_path: Path to the video file to import
|
|
68
|
-
center_name: Name of the center to associate with video
|
|
69
|
-
processor_name: Name of the processor to associate with video
|
|
70
|
-
save_video: Whether to save the video file
|
|
71
|
-
delete_source: Whether to delete the source file after import
|
|
72
|
-
|
|
73
|
-
Returns:
|
|
74
|
-
VideoFile instance after import and anonymization
|
|
75
|
-
|
|
76
|
-
Raises:
|
|
77
|
-
Exception: On any failure during import or anonymization
|
|
78
139
|
"""
|
|
79
140
|
try:
|
|
80
141
|
# Initialize processing context
|
|
81
142
|
self._initialize_processing_context(file_path, center_name, processor_name,
|
|
82
143
|
save_video, delete_source)
|
|
83
144
|
|
|
84
|
-
# Validate and prepare file
|
|
85
|
-
|
|
145
|
+
# Validate and prepare file (may raise ValueError if another worker holds a non-stale lock)
|
|
146
|
+
try:
|
|
147
|
+
self._validate_and_prepare_file()
|
|
148
|
+
except ValueError as ve:
|
|
149
|
+
# Relaxed behavior: if another process is working on this file, skip cleanly
|
|
150
|
+
if "already being processed" in str(ve):
|
|
151
|
+
self.logger.info(f"Skipping {file_path}: {ve}")
|
|
152
|
+
return None
|
|
153
|
+
raise
|
|
86
154
|
|
|
87
155
|
# Create or retrieve video instance
|
|
88
156
|
self._create_or_retrieve_video_instance()
|
|
@@ -126,24 +194,40 @@ class VideoImportService():
|
|
|
126
194
|
self.logger.info(f"Initialized processing context for: {file_path}")
|
|
127
195
|
|
|
128
196
|
def _validate_and_prepare_file(self):
|
|
129
|
-
"""
|
|
197
|
+
"""
|
|
198
|
+
Validate the video file and prepare for processing.
|
|
199
|
+
|
|
200
|
+
Uses file locking to prevent concurrent processing of the same video file.
|
|
201
|
+
This prevents race conditions where multiple workers might try to process
|
|
202
|
+
the same video simultaneously.
|
|
203
|
+
|
|
204
|
+
The lock is acquired here and held for the entire import process.
|
|
205
|
+
See _file_lock() for lock reclamation logic.
|
|
206
|
+
"""
|
|
130
207
|
file_path = self.processing_context['file_path']
|
|
131
208
|
|
|
132
|
-
#
|
|
209
|
+
# Acquire file lock to prevent concurrent processing
|
|
210
|
+
# Lock will be held until finally block in import_and_anonymize()
|
|
211
|
+
self.processing_context['_lock_context'] = self._file_lock(file_path)
|
|
212
|
+
self.processing_context['_lock_context'].__enter__()
|
|
213
|
+
|
|
214
|
+
self.logger.info("Acquired file lock for: %s", file_path)
|
|
215
|
+
|
|
216
|
+
# Check if already processed (memory-based check)
|
|
133
217
|
if str(file_path) in self.processed_files:
|
|
134
|
-
self.logger.info(
|
|
135
|
-
self.
|
|
218
|
+
self.logger.info("File %s already processed, skipping", file_path)
|
|
219
|
+
self._processed = True
|
|
136
220
|
raise ValueError(f"File already processed: {file_path}")
|
|
137
221
|
|
|
138
222
|
# Check file exists
|
|
139
223
|
if not file_path.exists():
|
|
140
224
|
raise FileNotFoundError(f"Video file not found: {file_path}")
|
|
141
225
|
|
|
142
|
-
self.logger.info(
|
|
226
|
+
self.logger.info("File validation completed for: %s", file_path)
|
|
143
227
|
|
|
144
228
|
def _create_or_retrieve_video_instance(self):
|
|
145
229
|
"""Create or retrieve the VideoFile instance and move to final storage."""
|
|
146
|
-
|
|
230
|
+
# Removed duplicate import of VideoFile (already imported at module level)
|
|
147
231
|
|
|
148
232
|
self.logger.info("Creating VideoFile instance...")
|
|
149
233
|
|
|
@@ -161,7 +245,7 @@ class VideoImportService():
|
|
|
161
245
|
# Immediately move to final storage locations
|
|
162
246
|
self._move_to_final_storage()
|
|
163
247
|
|
|
164
|
-
self.logger.info(
|
|
248
|
+
self.logger.info("Created VideoFile with UUID: %s", self.current_video.uuid)
|
|
165
249
|
|
|
166
250
|
# Get and mark processing state
|
|
167
251
|
state = VideoFile.get_or_create_state(self.current_video)
|
|
@@ -186,15 +270,16 @@ class VideoImportService():
|
|
|
186
270
|
videos_dir.mkdir(parents=True, exist_ok=True)
|
|
187
271
|
|
|
188
272
|
# Create target path for raw video in /data/videos
|
|
189
|
-
|
|
273
|
+
ext = Path(self.current_video.active_file_path).suffix or ".mp4"
|
|
274
|
+
video_filename = f"{self.current_video.uuid}{ext}"
|
|
190
275
|
raw_target_path = videos_dir / video_filename
|
|
191
276
|
|
|
192
277
|
# Move source file to raw video storage
|
|
193
278
|
try:
|
|
194
279
|
shutil.move(str(source_path), str(raw_target_path))
|
|
195
|
-
self.logger.info(
|
|
280
|
+
self.logger.info("Moved raw video to: %s", raw_target_path)
|
|
196
281
|
except Exception as e:
|
|
197
|
-
self.logger.error(
|
|
282
|
+
self.logger.error("Failed to move video to final storage: %s", e)
|
|
198
283
|
raise
|
|
199
284
|
|
|
200
285
|
# Update the raw_file path in database (relative to storage root)
|
|
@@ -203,13 +288,13 @@ class VideoImportService():
|
|
|
203
288
|
relative_path = raw_target_path.relative_to(storage_root)
|
|
204
289
|
self.current_video.raw_file.name = str(relative_path)
|
|
205
290
|
self.current_video.save(update_fields=['raw_file'])
|
|
206
|
-
self.logger.info(
|
|
291
|
+
self.logger.info("Updated raw_file path to: %s", relative_path)
|
|
207
292
|
except Exception as e:
|
|
208
|
-
self.logger.error(
|
|
293
|
+
self.logger.error("Failed to update raw_file path: %s", e)
|
|
209
294
|
# Fallback to simple relative path
|
|
210
295
|
self.current_video.raw_file.name = f"videos/{video_filename}"
|
|
211
296
|
self.current_video.save(update_fields=['raw_file'])
|
|
212
|
-
self.logger.info(
|
|
297
|
+
self.logger.info("Updated raw_file path using fallback: %s", f"videos/{video_filename}")
|
|
213
298
|
|
|
214
299
|
|
|
215
300
|
# Store paths for later processing
|
|
@@ -271,33 +356,87 @@ class VideoImportService():
|
|
|
271
356
|
# Get processor ROI information
|
|
272
357
|
processor_roi, endoscope_roi = self._get_processor_roi_info()
|
|
273
358
|
|
|
274
|
-
# Perform frame cleaning
|
|
275
|
-
|
|
359
|
+
# Perform frame cleaning with timeout to prevent blocking
|
|
360
|
+
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError
|
|
276
361
|
|
|
277
|
-
|
|
362
|
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
363
|
+
future = executor.submit(self._perform_frame_cleaning, FrameCleaner, processor_roi, endoscope_roi)
|
|
364
|
+
try:
|
|
365
|
+
# Increased timeout to better accommodate ffmpeg + OCR
|
|
366
|
+
future.result(timeout=300)
|
|
367
|
+
self.processing_context['anonymization_completed'] = True
|
|
368
|
+
self.logger.info("Frame cleaning completed successfully within timeout")
|
|
369
|
+
except FutureTimeoutError:
|
|
370
|
+
self.logger.warning("Frame cleaning timed out; entering grace period check for cleaned output")
|
|
371
|
+
# Grace period: detect if cleaned file appears shortly after timeout
|
|
372
|
+
raw_video_path = self.processing_context.get('raw_video_path')
|
|
373
|
+
video_filename = self.processing_context.get('video_filename', Path(raw_video_path).name if raw_video_path else "video.mp4")
|
|
374
|
+
grace_seconds = 60
|
|
375
|
+
expected_cleaned = self.current_video.processed_file
|
|
376
|
+
found = False
|
|
377
|
+
if expected_cleaned is not None:
|
|
378
|
+
for _ in range(grace_seconds):
|
|
379
|
+
if expected_cleaned.exists():
|
|
380
|
+
self.processing_context['cleaned_video_path'] = expected_cleaned
|
|
381
|
+
self.processing_context['anonymization_completed'] = True
|
|
382
|
+
self.logger.info("Detected cleaned video during grace period: %s", expected_cleaned)
|
|
383
|
+
found = True
|
|
384
|
+
break
|
|
385
|
+
time.sleep(1)
|
|
386
|
+
else:
|
|
387
|
+
self._fallback_anonymize_video()
|
|
388
|
+
if not found:
|
|
389
|
+
raise TimeoutError("Frame cleaning operation timed out - likely Ollama connection issue")
|
|
278
390
|
|
|
279
391
|
except Exception as e:
|
|
280
|
-
self.logger.warning(
|
|
281
|
-
|
|
282
|
-
|
|
392
|
+
self.logger.warning("Frame cleaning failed (reason: %s), falling back to simple copy", e)
|
|
393
|
+
# Try fallback anonymization when frame cleaning fails
|
|
394
|
+
try:
|
|
395
|
+
self._fallback_anonymize_video()
|
|
396
|
+
except Exception as fallback_error:
|
|
397
|
+
self.logger.error("Fallback anonymization also failed: %s", fallback_error)
|
|
398
|
+
# If even fallback fails, mark as not anonymized but continue import
|
|
399
|
+
self.processing_context['anonymization_completed'] = False
|
|
400
|
+
self.processing_context['error_reason'] = f"Frame cleaning failed: {e}, Fallback failed: {fallback_error}"
|
|
283
401
|
|
|
284
402
|
|
|
285
403
|
def _fallback_anonymize_video(self):
|
|
286
|
-
"""
|
|
404
|
+
"""
|
|
405
|
+
Fallback to create anonymized video if lx_anonymizer is not available.
|
|
406
|
+
|
|
407
|
+
This method tries multiple fallback strategies:
|
|
408
|
+
1. Use VideoFile.anonymize_video() method if available
|
|
409
|
+
2. Simple copy of raw video to anonym_videos (no processing)
|
|
410
|
+
|
|
411
|
+
The processed video will be marked in processing_context for _cleanup_and_archive().
|
|
412
|
+
"""
|
|
287
413
|
try:
|
|
288
|
-
self.logger.info("Attempting
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
self.
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
self.
|
|
296
|
-
|
|
414
|
+
self.logger.info("Attempting fallback video anonymization...")
|
|
415
|
+
|
|
416
|
+
# Strategy 1: Try VideoFile.pipe_2() method
|
|
417
|
+
if hasattr(self.current_video, 'pipe_2'):
|
|
418
|
+
self.logger.info("Trying VideoFile.pipe_2() method...")
|
|
419
|
+
|
|
420
|
+
# Try to anonymize
|
|
421
|
+
if self.current_video.pipe_2:
|
|
422
|
+
self.logger.info("VideoFile.pipe_2() succeeded")
|
|
423
|
+
self.processing_context['anonymization_completed'] = True
|
|
424
|
+
return
|
|
425
|
+
else:
|
|
426
|
+
self.logger.warning("VideoFile.pipe_2() returned False, trying simple copy fallback")
|
|
297
427
|
else:
|
|
298
|
-
self.logger.warning("
|
|
299
|
-
|
|
300
|
-
|
|
428
|
+
self.logger.warning("VideoFile.pipe_2() method not available")
|
|
429
|
+
|
|
430
|
+
# Strategy 2: Simple copy (no processing, just copy raw to processed)
|
|
431
|
+
self.logger.info("Using simple copy fallback (raw video will be used as 'processed' video)")
|
|
432
|
+
|
|
433
|
+
# The _cleanup_and_archive() method will handle the copy
|
|
434
|
+
# We just need to mark that no real anonymization happened
|
|
435
|
+
self.processing_context['anonymization_completed'] = False
|
|
436
|
+
self.processing_context['use_raw_as_processed'] = True # Signal for cleanup
|
|
437
|
+
|
|
438
|
+
self.logger.warning("Fallback: Video will be imported without anonymization (raw copy used)")
|
|
439
|
+
|
|
301
440
|
except Exception as e:
|
|
302
441
|
self.logger.error(f"Error during fallback anonymization: {e}", exc_info=True)
|
|
303
442
|
self.processing_context['anonymization_completed'] = False
|
|
@@ -309,26 +448,55 @@ class VideoImportService():
|
|
|
309
448
|
|
|
310
449
|
with transaction.atomic():
|
|
311
450
|
# Update basic processing states
|
|
451
|
+
# Ensure state exists before accessing it
|
|
452
|
+
|
|
453
|
+
if not self.current_video:
|
|
454
|
+
try:
|
|
455
|
+
self.current_video.refresh_from_db()
|
|
456
|
+
except Exception as e:
|
|
457
|
+
self.logger.error(f"Failed to refresh current_video from DB: {e}")
|
|
458
|
+
if not self.current_video:
|
|
459
|
+
raise RuntimeError("No current video instance available for finalization")
|
|
460
|
+
|
|
461
|
+
if not self.current_video.processed_file:
|
|
462
|
+
self.logger.warning("No processed file available for current video")
|
|
463
|
+
self.current_video.processed_file = None # Ensure field is not None
|
|
464
|
+
self.current_video.mark_sensitive_meta_processed = False
|
|
465
|
+
else:
|
|
466
|
+
self.current_video.mark_sensitive_meta_processed = True
|
|
467
|
+
|
|
468
|
+
state = self.current_video.get_or_create_state()
|
|
469
|
+
if not state:
|
|
470
|
+
raise RuntimeError("Failed to get or create video state")
|
|
471
|
+
|
|
312
472
|
# Only mark frames as extracted if they were successfully extracted
|
|
313
473
|
if self.processing_context.get('frames_extracted', False):
|
|
314
|
-
|
|
474
|
+
state.frames_extracted = True
|
|
315
475
|
self.logger.info("Marked frames as extracted in state")
|
|
316
476
|
else:
|
|
317
477
|
self.logger.warning("Frames were not extracted, not updating state")
|
|
318
478
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
479
|
+
# Always mark these as true (metadata extraction attempts were made)
|
|
480
|
+
state.frames_initialized = True
|
|
481
|
+
state.video_meta_extracted = True
|
|
482
|
+
state.text_meta_extracted = True
|
|
322
483
|
|
|
323
|
-
#
|
|
324
|
-
self.
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
self.logger.info(f"Video {self.current_video.uuid} successfully anonymized")
|
|
484
|
+
# ✅ FIX: Only mark as processed if anonymization actually completed
|
|
485
|
+
anonymization_completed = self.processing_context.get('anonymization_completed', False)
|
|
486
|
+
if anonymization_completed:
|
|
487
|
+
state.mark_sensitive_meta_processed(save=False)
|
|
488
|
+
self.logger.info("Anonymization completed - marking sensitive meta as processed")
|
|
329
489
|
else:
|
|
330
|
-
self.logger.warning(
|
|
490
|
+
self.logger.warning(
|
|
491
|
+
"Anonymization NOT completed - NOT marking as processed. "
|
|
492
|
+
f"Reason: {self.processing_context.get('error_reason', 'Unknown')}"
|
|
493
|
+
)
|
|
494
|
+
# Explicitly mark as NOT processed
|
|
495
|
+
state.sensitive_meta_processed = False
|
|
331
496
|
|
|
497
|
+
# Save all state changes
|
|
498
|
+
state.save()
|
|
499
|
+
self.logger.info("Video processing state updated")
|
|
332
500
|
# Save all state changes
|
|
333
501
|
self.current_video.state.save()
|
|
334
502
|
self.current_video.save()
|
|
@@ -361,61 +529,78 @@ class VideoImportService():
|
|
|
361
529
|
# Copy raw to processed location (will be moved to anonym_videos)
|
|
362
530
|
try:
|
|
363
531
|
shutil.copy2(str(raw_video_path), str(processed_video_path))
|
|
364
|
-
self.logger.info(
|
|
532
|
+
self.logger.info("Copied raw video for processing: %s", processed_video_path)
|
|
365
533
|
except Exception as e:
|
|
366
|
-
self.logger.error(
|
|
367
|
-
processed_video_path =
|
|
534
|
+
self.logger.error("Failed to copy raw video: %s", e)
|
|
535
|
+
processed_video_path = None # FIXED: Don't use raw as fallback
|
|
368
536
|
|
|
369
|
-
# Move processed video to anonym_videos
|
|
537
|
+
# Move processed video to anonym_videos ONLY if it exists
|
|
370
538
|
if processed_video_path and Path(processed_video_path).exists():
|
|
371
539
|
try:
|
|
372
|
-
|
|
540
|
+
# ✅ Clean filename: no original filename leakage
|
|
541
|
+
ext = Path(processed_video_path).suffix or ".mp4"
|
|
542
|
+
anonym_video_filename = f"anonym_{self.current_video.uuid}{ext}"
|
|
373
543
|
anonym_target_path = anonym_videos_dir / anonym_video_filename
|
|
374
|
-
|
|
544
|
+
|
|
545
|
+
# Move processed video to anonym_videos/
|
|
375
546
|
shutil.move(str(processed_video_path), str(anonym_target_path))
|
|
376
|
-
self.logger.info(
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
547
|
+
self.logger.info("Moved processed video to: %s", anonym_target_path)
|
|
548
|
+
|
|
549
|
+
# Verify the file actually exists before updating database
|
|
550
|
+
if anonym_target_path.exists():
|
|
551
|
+
try:
|
|
552
|
+
storage_root = data_paths["storage"]
|
|
553
|
+
relative_path = anonym_target_path.relative_to(storage_root)
|
|
554
|
+
# Save relative path (e.g. anonym_videos/anonym_<uuid>.mp4)
|
|
555
|
+
self.current_video.processed_file.name = str(relative_path)
|
|
556
|
+
self.current_video.save(update_fields=["processed_file"])
|
|
557
|
+
self.logger.info("Updated processed_file path to: %s", relative_path)
|
|
558
|
+
except Exception as e:
|
|
559
|
+
self.logger.error("Failed to update processed_file path: %s", e)
|
|
560
|
+
# Fallback to simple relative path
|
|
561
|
+
self.current_video.processed_file.name = f"anonym_videos/{anonym_video_filename}"
|
|
562
|
+
self.current_video.save(update_fields=['processed_file'])
|
|
563
|
+
self.logger.info(
|
|
564
|
+
"Updated processed_file path using fallback: %s",
|
|
565
|
+
f"anonym_videos/{anonym_video_filename}",
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
self.processing_context['anonymization_completed'] = True
|
|
569
|
+
else:
|
|
570
|
+
self.logger.warning("Processed video file not found after move: %s", anonym_target_path)
|
|
389
571
|
except Exception as e:
|
|
390
|
-
self.logger.error(
|
|
572
|
+
self.logger.error("Failed to move processed video to anonym_videos: %s", e)
|
|
573
|
+
else:
|
|
574
|
+
self.logger.warning("No processed video available - processed_file will remain empty")
|
|
575
|
+
# Leave processed_file empty/null - frontend should fall back to raw_file
|
|
391
576
|
|
|
392
577
|
# Cleanup temporary directories
|
|
393
578
|
try:
|
|
394
579
|
from endoreg_db.utils.paths import RAW_FRAME_DIR
|
|
395
580
|
shutil.rmtree(RAW_FRAME_DIR, ignore_errors=True)
|
|
396
|
-
self.logger.debug(
|
|
581
|
+
self.logger.debug("Cleaned up temporary frames directory: %s", RAW_FRAME_DIR)
|
|
397
582
|
except Exception as e:
|
|
398
|
-
self.logger.warning(
|
|
583
|
+
self.logger.warning("Failed to remove directory %s: %s", RAW_FRAME_DIR, e)
|
|
399
584
|
|
|
400
585
|
# Handle source file deletion - this should already be moved, but check raw_videos
|
|
401
586
|
source_path = self.processing_context['file_path']
|
|
402
587
|
if self.processing_context['delete_source'] and Path(source_path).exists():
|
|
403
588
|
try:
|
|
404
589
|
os.remove(source_path)
|
|
405
|
-
self.logger.info(
|
|
590
|
+
self.logger.info("Removed remaining source file: %s", source_path)
|
|
406
591
|
except Exception as e:
|
|
407
|
-
self.logger.warning(
|
|
592
|
+
self.logger.warning("Failed to remove source file %s: %s", source_path, e)
|
|
408
593
|
|
|
409
|
-
# Mark as processed
|
|
594
|
+
# Mark as processed (in-memory tracking)
|
|
410
595
|
self.processed_files.add(str(self.processing_context['file_path']))
|
|
411
596
|
|
|
412
597
|
# Refresh from database and finalize state
|
|
413
598
|
with transaction.atomic():
|
|
414
599
|
self.current_video.refresh_from_db()
|
|
415
|
-
if hasattr(self.current_video, 'state'):
|
|
600
|
+
if hasattr(self.current_video, 'state') and self.processing_context.get('anonymization_completed'):
|
|
416
601
|
self.current_video.state.mark_sensitive_meta_processed(save=True)
|
|
417
602
|
|
|
418
|
-
self.logger.info(
|
|
603
|
+
self.logger.info("Import and anonymization completed for VideoFile UUID: %s", self.current_video.uuid)
|
|
419
604
|
self.logger.info("Raw video stored in: /data/videos")
|
|
420
605
|
self.logger.info("Processed video stored in: /data/anonym_videos")
|
|
421
606
|
|
|
@@ -805,13 +990,13 @@ class VideoImportService():
|
|
|
805
990
|
|
|
806
991
|
# Define default/placeholder values that are safe to overwrite
|
|
807
992
|
SAFE_TO_OVERWRITE_VALUES = [
|
|
808
|
-
'
|
|
809
|
-
'
|
|
993
|
+
'Vorname unbekannt', # Default first name
|
|
994
|
+
'Nachname unbekannt', # Default last name
|
|
810
995
|
date(1990, 1, 1), # Default DOB
|
|
811
996
|
None, # Empty values
|
|
812
997
|
'', # Empty strings
|
|
813
998
|
'N/A', # Placeholder values
|
|
814
|
-
'
|
|
999
|
+
'Unbekanntes Gerät', # Default device name
|
|
815
1000
|
]
|
|
816
1001
|
|
|
817
1002
|
for meta_key, sm_field in metadata_mapping.items():
|
|
@@ -881,12 +1066,30 @@ class VideoImportService():
|
|
|
881
1066
|
self.logger.warning(f"Error during cleanup: {e}")
|
|
882
1067
|
|
|
883
1068
|
def _cleanup_processing_context(self):
|
|
884
|
-
"""
|
|
1069
|
+
"""
|
|
1070
|
+
Cleanup processing context and release file lock.
|
|
1071
|
+
|
|
1072
|
+
This method is always called in the finally block of import_and_anonymize()
|
|
1073
|
+
to ensure the file lock is released even if processing fails.
|
|
1074
|
+
"""
|
|
885
1075
|
try:
|
|
886
|
-
#
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
1076
|
+
# Release file lock if it was acquired
|
|
1077
|
+
lock_context = self.processing_context.get('_lock_context')
|
|
1078
|
+
if lock_context is not None:
|
|
1079
|
+
try:
|
|
1080
|
+
lock_context.__exit__(None, None, None)
|
|
1081
|
+
self.logger.info("Released file lock")
|
|
1082
|
+
except Exception as e:
|
|
1083
|
+
self.logger.warning(f"Error releasing file lock: {e}")
|
|
1084
|
+
|
|
1085
|
+
# Remove file from processed set if processing failed
|
|
1086
|
+
file_path = self.processing_context.get('file_path')
|
|
1087
|
+
if file_path and not self.processing_context.get('anonymization_completed'):
|
|
1088
|
+
file_path_str = str(file_path)
|
|
1089
|
+
if file_path_str in self.processed_files:
|
|
1090
|
+
self.processed_files.remove(file_path_str)
|
|
1091
|
+
self.logger.info(f"Removed {file_path_str} from processed files (failed processing)")
|
|
1092
|
+
|
|
890
1093
|
except Exception as e:
|
|
891
1094
|
self.logger.warning(f"Error during context cleanup: {e}")
|
|
892
1095
|
finally:
|
endoreg_db/urls/__init__.py
CHANGED
|
@@ -25,7 +25,6 @@ from .label_video_segments import url_patterns as label_video_segments_url_patte
|
|
|
25
25
|
from .label_video_segment_validate import url_patterns as label_video_segment_validate_url_patterns
|
|
26
26
|
# TODO Phase 1.2: Implement VideoMediaView and PDFMediaView before enabling
|
|
27
27
|
# from .media import urlpatterns as media_url_patterns
|
|
28
|
-
from .sensitive_meta import urlpatterns as pdf_url_patterns
|
|
29
28
|
from .report import url_patterns as report_url_patterns
|
|
30
29
|
from .upload import urlpatterns as upload_url_patterns
|
|
31
30
|
from .video import url_patterns as video_url_patterns
|
|
@@ -43,7 +42,6 @@ api_urls += label_video_segments_url_patterns
|
|
|
43
42
|
api_urls += label_video_segment_validate_url_patterns # Neue Validierungs-Endpunkte
|
|
44
43
|
# Phase 1.2: Enable media_url_patterns ✅ IMPLEMENTED
|
|
45
44
|
api_urls += media_url_patterns
|
|
46
|
-
api_urls += pdf_url_patterns
|
|
47
45
|
api_urls += report_url_patterns
|
|
48
46
|
api_urls += upload_url_patterns
|
|
49
47
|
api_urls += video_url_patterns
|