endoreg-db 0.8.2__py3-none-any.whl → 0.8.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/config/__init__.py +0 -0
- endoreg_db/migrations/0003_add_center_display_name.py +30 -0
- endoreg_db/models/administration/center/center.py +7 -1
- endoreg_db/models/media/pdf/raw_pdf.py +31 -26
- endoreg_db/models/media/video/create_from_file.py +26 -4
- endoreg_db/models/media/video/video_file.py +36 -13
- endoreg_db/models/media/video/video_file_anonymize.py +2 -1
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +12 -0
- endoreg_db/models/media/video/video_file_io.py +4 -2
- endoreg_db/models/metadata/video_meta.py +2 -2
- endoreg_db/services/pdf_import.py +131 -15
- endoreg_db/services/video_import.py +158 -62
- endoreg_db/urls/sensitive_meta.py +0 -0
- endoreg_db/utils/paths.py +2 -10
- endoreg_db/utils/video/ffmpeg_wrapper.py +67 -4
- endoreg_db/views/anonymization/validate.py +75 -34
- endoreg_db/views/video/correction.py +8 -6
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.1.dist-info}/METADATA +2 -2
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.1.dist-info}/RECORD +21 -18
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.1.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -22,10 +22,12 @@ from endoreg_db.models import VideoFile, SensitiveMeta
|
|
|
22
22
|
from endoreg_db.utils.paths import STORAGE_DIR, RAW_FRAME_DIR, VIDEO_DIR, ANONYM_VIDEO_DIR
|
|
23
23
|
import random
|
|
24
24
|
from lx_anonymizer.ocr import trocr_full_image_ocr
|
|
25
|
-
from
|
|
25
|
+
from endoreg_db.utils.hashs import get_video_hash
|
|
26
|
+
from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
|
|
27
|
+
|
|
26
28
|
|
|
27
29
|
# File lock configuration (matches PDF import)
|
|
28
|
-
STALE_LOCK_SECONDS =
|
|
30
|
+
STALE_LOCK_SECONDS = 6000 # 100 minutes - reclaim locks older than this
|
|
29
31
|
MAX_LOCK_WAIT_SECONDS = 90 # New: wait up to 90s for a non-stale lock to clear before skipping
|
|
30
32
|
|
|
31
33
|
logger = logging.getLogger(__name__)
|
|
@@ -43,7 +45,7 @@ class VideoImportService():
|
|
|
43
45
|
- Graceful fallback processing without lx_anonymizer
|
|
44
46
|
"""
|
|
45
47
|
|
|
46
|
-
def __init__(self, project_root: Path = None):
|
|
48
|
+
def __init__(self, project_root: Optional[Path] = None):
|
|
47
49
|
|
|
48
50
|
# Set up project root path
|
|
49
51
|
if project_root:
|
|
@@ -60,6 +62,8 @@ class VideoImportService():
|
|
|
60
62
|
self.current_video = None
|
|
61
63
|
self.processing_context: Dict[str, Any] = {}
|
|
62
64
|
|
|
65
|
+
self.delete_source = False
|
|
66
|
+
|
|
63
67
|
self.logger = logging.getLogger(__name__)
|
|
64
68
|
|
|
65
69
|
@contextmanager
|
|
@@ -132,7 +136,7 @@ class VideoImportService():
|
|
|
132
136
|
processor_name: str,
|
|
133
137
|
save_video: bool = True,
|
|
134
138
|
delete_source: bool = True,
|
|
135
|
-
) -> "VideoFile":
|
|
139
|
+
) -> "VideoFile|None":
|
|
136
140
|
"""
|
|
137
141
|
High-level helper that orchestrates the complete video import and anonymization process.
|
|
138
142
|
Uses the central video instance pattern for improved state management.
|
|
@@ -264,42 +268,90 @@ class VideoImportService():
|
|
|
264
268
|
from endoreg_db.utils import data_paths
|
|
265
269
|
|
|
266
270
|
source_path = self.processing_context['file_path']
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
videos_dir = data_paths["video"] # /data/videos for raw files
|
|
271
|
+
|
|
272
|
+
videos_dir = data_paths["video"]
|
|
270
273
|
videos_dir.mkdir(parents=True, exist_ok=True)
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
274
|
+
|
|
275
|
+
_current_video = self.current_video
|
|
276
|
+
assert _current_video is not None, "Current video instance is None during storage move"
|
|
277
|
+
|
|
278
|
+
stored_raw_path = None
|
|
279
|
+
if hasattr(_current_video, "get_raw_file_path"):
|
|
280
|
+
possible_path = _current_video.get_raw_file_path()
|
|
281
|
+
if possible_path:
|
|
282
|
+
try:
|
|
283
|
+
stored_raw_path = Path(possible_path)
|
|
284
|
+
except (TypeError, ValueError):
|
|
285
|
+
stored_raw_path = None
|
|
286
|
+
|
|
287
|
+
if stored_raw_path:
|
|
288
|
+
try:
|
|
289
|
+
storage_root = data_paths["storage"]
|
|
290
|
+
if stored_raw_path.is_absolute():
|
|
291
|
+
if not stored_raw_path.is_relative_to(storage_root):
|
|
292
|
+
stored_raw_path = None
|
|
293
|
+
else:
|
|
294
|
+
if stored_raw_path.parts and stored_raw_path.parts[0] == videos_dir.name:
|
|
295
|
+
stored_raw_path = storage_root / stored_raw_path
|
|
296
|
+
else:
|
|
297
|
+
stored_raw_path = videos_dir / stored_raw_path.name
|
|
298
|
+
except Exception:
|
|
299
|
+
stored_raw_path = None
|
|
300
|
+
|
|
301
|
+
if stored_raw_path and not stored_raw_path.suffix:
|
|
302
|
+
stored_raw_path = None
|
|
303
|
+
|
|
304
|
+
if not stored_raw_path:
|
|
305
|
+
uuid_str = getattr(_current_video, "uuid", None)
|
|
306
|
+
source_suffix = Path(source_path).suffix or ".mp4"
|
|
307
|
+
filename = f"{uuid_str}{source_suffix}" if uuid_str else Path(source_path).name
|
|
308
|
+
stored_raw_path = videos_dir / filename
|
|
309
|
+
|
|
310
|
+
delete_source = bool(self.processing_context.get('delete_source'))
|
|
311
|
+
stored_raw_path.parent.mkdir(parents=True, exist_ok=True)
|
|
312
|
+
|
|
313
|
+
if not stored_raw_path.exists():
|
|
314
|
+
try:
|
|
315
|
+
if source_path.exists():
|
|
316
|
+
if delete_source:
|
|
317
|
+
shutil.move(str(source_path), str(stored_raw_path))
|
|
318
|
+
self.logger.info("Moved raw video to: %s", stored_raw_path)
|
|
319
|
+
else:
|
|
320
|
+
shutil.copy2(str(source_path), str(stored_raw_path))
|
|
321
|
+
self.logger.info("Copied raw video to: %s", stored_raw_path)
|
|
322
|
+
else:
|
|
323
|
+
raise FileNotFoundError(f"Neither stored raw path nor source path exists for {self.processing_context['file_path']}")
|
|
324
|
+
except Exception as e:
|
|
325
|
+
self.logger.error("Failed to place video in final storage: %s", e)
|
|
326
|
+
raise
|
|
327
|
+
else:
|
|
328
|
+
# If we already have the stored copy, respect delete_source flag without touching assets unnecessarily
|
|
329
|
+
if delete_source and source_path.exists():
|
|
330
|
+
try:
|
|
331
|
+
os.remove(source_path)
|
|
332
|
+
self.logger.info("Removed original source file after storing copy: %s", source_path)
|
|
333
|
+
except OSError as e:
|
|
334
|
+
self.logger.warning("Failed to remove source file %s: %s", source_path, e)
|
|
335
|
+
|
|
336
|
+
# Ensure database path points to stored location (relative to storage root)
|
|
286
337
|
try:
|
|
287
338
|
storage_root = data_paths["storage"]
|
|
288
|
-
relative_path =
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
339
|
+
relative_path = Path(stored_raw_path).relative_to(storage_root)
|
|
340
|
+
if _current_video.raw_file.name != str(relative_path):
|
|
341
|
+
_current_video.raw_file.name = str(relative_path)
|
|
342
|
+
_current_video.save(update_fields=['raw_file'])
|
|
343
|
+
self.logger.info("Updated raw_file path to: %s", relative_path)
|
|
292
344
|
except Exception as e:
|
|
293
|
-
self.logger.error("Failed to
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
345
|
+
self.logger.error("Failed to ensure raw_file path is relative: %s", e)
|
|
346
|
+
fallback_relative = Path("videos") / Path(stored_raw_path).name
|
|
347
|
+
if _current_video.raw_file.name != fallback_relative.as_posix():
|
|
348
|
+
_current_video.raw_file.name = fallback_relative.as_posix()
|
|
349
|
+
_current_video.save(update_fields=['raw_file'])
|
|
350
|
+
self.logger.info("Updated raw_file path using fallback: %s", fallback_relative.as_posix())
|
|
351
|
+
|
|
300
352
|
# Store paths for later processing
|
|
301
|
-
self.processing_context['raw_video_path'] =
|
|
302
|
-
self.processing_context['video_filename'] =
|
|
353
|
+
self.processing_context['raw_video_path'] = Path(stored_raw_path)
|
|
354
|
+
self.processing_context['video_filename'] = Path(stored_raw_path).name
|
|
303
355
|
|
|
304
356
|
def _setup_processing_environment(self):
|
|
305
357
|
"""Setup the processing environment without file movement."""
|
|
@@ -345,7 +397,10 @@ class VideoImportService():
|
|
|
345
397
|
# Check frame cleaning availability
|
|
346
398
|
frame_cleaning_available, FrameCleaner, ReportReader = self._ensure_frame_cleaning_available()
|
|
347
399
|
|
|
348
|
-
|
|
400
|
+
_current_video = self.current_video
|
|
401
|
+
assert _current_video is not None, "Current video instance is None during frame processing"
|
|
402
|
+
|
|
403
|
+
if not (frame_cleaning_available and _current_video.raw_file):
|
|
349
404
|
self.logger.warning("Frame cleaning not available or conditions not met, using fallback anonymization.")
|
|
350
405
|
self._fallback_anonymize_video()
|
|
351
406
|
return
|
|
@@ -398,50 +453,78 @@ class VideoImportService():
|
|
|
398
453
|
# If even fallback fails, mark as not anonymized but continue import
|
|
399
454
|
self.processing_context['anonymization_completed'] = False
|
|
400
455
|
self.processing_context['error_reason'] = f"Frame cleaning failed: {e}, Fallback failed: {fallback_error}"
|
|
401
|
-
|
|
456
|
+
|
|
457
|
+
def _save_anonymized_video(self):
|
|
458
|
+
anonymized_video_path = self.current_video.get_target_anonymized_video_path()
|
|
459
|
+
|
|
460
|
+
if not anonymized_video_path.exists():
|
|
461
|
+
raise RuntimeError(f"Processed video file not found after assembly for {self.current_video.uuid}: {anonymized_video_path}")
|
|
462
|
+
|
|
463
|
+
new_processed_hash = get_video_hash(anonymized_video_path)
|
|
464
|
+
if type(self.current_video).objects.filter(processed_video_hash=new_processed_hash).exclude(pk=self.current_video.pk).exists():
|
|
465
|
+
raise ValueError(f"Processed video hash {new_processed_hash} already exists for another video (Video: {self.current_video.uuid}).")
|
|
466
|
+
|
|
467
|
+
self.current_video.processed_video_hash = new_processed_hash
|
|
468
|
+
self.current_video.processed_file.name = anonymized_video_path.relative_to(STORAGE_DIR).as_posix()
|
|
469
|
+
|
|
470
|
+
update_fields = [
|
|
471
|
+
"processed_video_hash",
|
|
472
|
+
"processed_file",
|
|
473
|
+
"frame_dir",
|
|
474
|
+
]
|
|
475
|
+
|
|
476
|
+
if self.delete_source:
|
|
477
|
+
original_raw_file_path_to_delete = self.current_video.get_raw_file_path()
|
|
478
|
+
original_raw_frame_dir_to_delete = self.current_video.get_frame_dir_path()
|
|
479
|
+
|
|
480
|
+
self.current_video.raw_file.name = None
|
|
481
|
+
|
|
482
|
+
update_fields.extend(["raw_file", "video_hash"])
|
|
483
|
+
|
|
484
|
+
transaction.on_commit(lambda: _cleanup_raw_assets(
|
|
485
|
+
video_uuid=self.current_video.uuid,
|
|
486
|
+
raw_file_path=original_raw_file_path_to_delete,
|
|
487
|
+
raw_frame_dir=original_raw_frame_dir_to_delete
|
|
488
|
+
))
|
|
489
|
+
|
|
490
|
+
self.current_video.save(update_fields=update_fields)
|
|
491
|
+
self.current_video.state.mark_anonymized(save=True)
|
|
492
|
+
self.current_video.refresh_from_db()
|
|
493
|
+
return True
|
|
402
494
|
|
|
403
495
|
def _fallback_anonymize_video(self):
|
|
404
496
|
"""
|
|
405
497
|
Fallback to create anonymized video if lx_anonymizer is not available.
|
|
406
|
-
|
|
407
|
-
This method tries multiple fallback strategies:
|
|
408
|
-
1. Use VideoFile.anonymize_video() method if available
|
|
409
|
-
2. Simple copy of raw video to anonym_videos (no processing)
|
|
410
|
-
|
|
411
|
-
The processed video will be marked in processing_context for _cleanup_and_archive().
|
|
412
498
|
"""
|
|
413
499
|
try:
|
|
414
500
|
self.logger.info("Attempting fallback video anonymization...")
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
501
|
+
if self.current_video:
|
|
502
|
+
# Try VideoFile.pipe_2() method if available
|
|
503
|
+
if hasattr(self.current_video, 'pipe_2'):
|
|
504
|
+
self.logger.info("Trying VideoFile.pipe_2() method...")
|
|
505
|
+
if self.current_video.pipe_2():
|
|
506
|
+
self.logger.info("VideoFile.pipe_2() succeeded")
|
|
507
|
+
self.processing_context['anonymization_completed'] = True
|
|
508
|
+
return
|
|
509
|
+
else:
|
|
510
|
+
self.logger.warning("VideoFile.pipe_2() returned False")
|
|
511
|
+
# Try direct anonymization via _anonymize
|
|
512
|
+
if _anonymize(self.current_video, delete_original_raw=self.delete_source):
|
|
513
|
+
self.logger.info("VideoFile._anonymize() succeeded")
|
|
423
514
|
self.processing_context['anonymization_completed'] = True
|
|
424
515
|
return
|
|
425
|
-
else:
|
|
426
|
-
self.logger.warning("VideoFile.pipe_2() returned False, trying simple copy fallback")
|
|
427
516
|
else:
|
|
428
|
-
self.logger.warning("VideoFile
|
|
517
|
+
self.logger.warning("No VideoFile instance available for fallback anonymization")
|
|
429
518
|
|
|
430
519
|
# Strategy 2: Simple copy (no processing, just copy raw to processed)
|
|
431
520
|
self.logger.info("Using simple copy fallback (raw video will be used as 'processed' video)")
|
|
432
|
-
|
|
433
|
-
# The _cleanup_and_archive() method will handle the copy
|
|
434
|
-
# We just need to mark that no real anonymization happened
|
|
435
521
|
self.processing_context['anonymization_completed'] = False
|
|
436
|
-
self.processing_context['use_raw_as_processed'] = True
|
|
437
|
-
|
|
522
|
+
self.processing_context['use_raw_as_processed'] = True
|
|
438
523
|
self.logger.warning("Fallback: Video will be imported without anonymization (raw copy used)")
|
|
439
|
-
|
|
440
524
|
except Exception as e:
|
|
441
525
|
self.logger.error(f"Error during fallback anonymization: {e}", exc_info=True)
|
|
442
526
|
self.processing_context['anonymization_completed'] = False
|
|
443
|
-
self.processing_context['error_reason']
|
|
444
|
-
|
|
527
|
+
self.processing_context['error_reason']
|
|
445
528
|
def _finalize_processing(self):
|
|
446
529
|
"""Finalize processing and update video state."""
|
|
447
530
|
self.logger.info("Updating video processing state...")
|
|
@@ -590,6 +673,18 @@ class VideoImportService():
|
|
|
590
673
|
self.logger.info("Removed remaining source file: %s", source_path)
|
|
591
674
|
except Exception as e:
|
|
592
675
|
self.logger.warning("Failed to remove source file %s: %s", source_path, e)
|
|
676
|
+
|
|
677
|
+
# Check if processed video exists and otherwise call anonymize
|
|
678
|
+
|
|
679
|
+
if not self.current_video.processed_file or not Path(self.current_video.processed_file.path).exists():
|
|
680
|
+
self.logger.warning("No processed_file found after cleanup - video will be unprocessed")
|
|
681
|
+
self.current_video.anonymize(delete_original_raw=self.delete_source)
|
|
682
|
+
self.current_video.save(update_fields=['processed_file'])
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
self.logger.info("Cleanup and archiving completed")
|
|
686
|
+
|
|
687
|
+
|
|
593
688
|
|
|
594
689
|
# Mark as processed (in-memory tracking)
|
|
595
690
|
self.processed_files.add(str(self.processing_context['file_path']))
|
|
@@ -599,6 +694,7 @@ class VideoImportService():
|
|
|
599
694
|
self.current_video.refresh_from_db()
|
|
600
695
|
if hasattr(self.current_video, 'state') and self.processing_context.get('anonymization_completed'):
|
|
601
696
|
self.current_video.state.mark_sensitive_meta_processed(save=True)
|
|
697
|
+
|
|
602
698
|
|
|
603
699
|
self.logger.info("Import and anonymization completed for VideoFile UUID: %s", self.current_video.uuid)
|
|
604
700
|
self.logger.info("Raw video stored in: /data/videos")
|
|
File without changes
|
endoreg_db/utils/paths.py
CHANGED
|
@@ -8,19 +8,12 @@ It provides a unified dictionary 'data_paths' for accessing all path objects.
|
|
|
8
8
|
from logging import getLogger
|
|
9
9
|
logger = getLogger(__name__)
|
|
10
10
|
|
|
11
|
-
import os
|
|
12
11
|
from pathlib import Path
|
|
13
12
|
from typing import Dict
|
|
14
|
-
import dotenv
|
|
15
13
|
|
|
16
|
-
|
|
17
|
-
if not os.environ.get("PYTEST_CURRENT_TEST"):
|
|
18
|
-
dotenv.load_dotenv()
|
|
19
|
-
else:
|
|
20
|
-
logger.debug("Skipping .env load under pytest")
|
|
14
|
+
from endoreg_db.config.env import env_path
|
|
21
15
|
|
|
22
|
-
|
|
23
|
-
STORAGE_DIR = Path(os.getenv("STORAGE_DIR"))
|
|
16
|
+
STORAGE_DIR = env_path("STORAGE_DIR", "storage")
|
|
24
17
|
|
|
25
18
|
# Resolve STORAGE_DIR from env or default under BASE_DIR
|
|
26
19
|
#def _resolve_storage_dir() -> Path:
|
|
@@ -36,7 +29,6 @@ STORAGE_DIR = Path(os.getenv("STORAGE_DIR"))
|
|
|
36
29
|
STORAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
37
30
|
|
|
38
31
|
PREFIX_RAW = "raw_"
|
|
39
|
-
STORAGE_DIR_NAME = "data"
|
|
40
32
|
IMPORT_DIR_NAME = "import"
|
|
41
33
|
EXPORT_DIR_NAME = "export"
|
|
42
34
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import subprocess
|
|
2
3
|
import json
|
|
3
4
|
import logging
|
|
5
|
+
from functools import lru_cache
|
|
4
6
|
from pathlib import Path
|
|
5
7
|
from typing import List, Dict, Optional, Tuple
|
|
6
8
|
import cv2
|
|
@@ -13,6 +15,67 @@ logger = logging.getLogger("ffmpeg_wrapper")
|
|
|
13
15
|
_nvenc_available = None
|
|
14
16
|
_preferred_encoder = None
|
|
15
17
|
|
|
18
|
+
|
|
19
|
+
@lru_cache(maxsize=1)
|
|
20
|
+
def _resolve_ffmpeg_executable() -> Optional[str]:
|
|
21
|
+
"""Locate the ffmpeg executable using multiple discovery strategies."""
|
|
22
|
+
# 1) Explicit overrides via env vars
|
|
23
|
+
env_candidates = [
|
|
24
|
+
os.environ.get("FFMPEG_EXECUTABLE"),
|
|
25
|
+
os.environ.get("FFMPEG_BINARY"),
|
|
26
|
+
os.environ.get("FFMPEG_PATH"),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
# 2) Django settings overrides (if Django is configured)
|
|
30
|
+
try:
|
|
31
|
+
from django.conf import settings # type: ignore
|
|
32
|
+
|
|
33
|
+
env_candidates.extend(
|
|
34
|
+
getattr(settings, attr)
|
|
35
|
+
for attr in ("FFMPEG_EXECUTABLE", "FFMPEG_BINARY", "FFMPEG_PATH")
|
|
36
|
+
if hasattr(settings, attr)
|
|
37
|
+
)
|
|
38
|
+
except Exception:
|
|
39
|
+
# Django might not be configured for every consumer
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
# Normalize and verify explicit candidates
|
|
43
|
+
for candidate in env_candidates:
|
|
44
|
+
if not candidate:
|
|
45
|
+
continue
|
|
46
|
+
candidate_path = Path(candidate)
|
|
47
|
+
if candidate_path.is_dir():
|
|
48
|
+
candidate_path = candidate_path / "ffmpeg"
|
|
49
|
+
if candidate_path.exists() and os.access(candidate_path, os.X_OK):
|
|
50
|
+
logger.debug("Using ffmpeg executable override at %s", candidate_path)
|
|
51
|
+
return str(candidate_path)
|
|
52
|
+
|
|
53
|
+
# 3) PATH lookup (shutil.which)
|
|
54
|
+
via_path = shutil.which("ffmpeg")
|
|
55
|
+
if via_path:
|
|
56
|
+
return via_path
|
|
57
|
+
|
|
58
|
+
# 4) Common fallback locations (useful for Nix-based environments)
|
|
59
|
+
nix_store = Path("/nix/store")
|
|
60
|
+
if nix_store.exists():
|
|
61
|
+
patterns = (
|
|
62
|
+
"*-ffmpeg-*/bin/ffmpeg",
|
|
63
|
+
"*-ffmpeg-headless-*/bin/ffmpeg",
|
|
64
|
+
"*-ffmpeg-headless*/bin/ffmpeg",
|
|
65
|
+
)
|
|
66
|
+
for pattern in patterns:
|
|
67
|
+
matches = sorted(nix_store.glob(pattern))
|
|
68
|
+
if matches:
|
|
69
|
+
logger.debug("Discovered ffmpeg in nix store at %s", matches[-1])
|
|
70
|
+
return str(matches[-1])
|
|
71
|
+
|
|
72
|
+
# 5) Final fallback to standard Unix locations
|
|
73
|
+
for fallback in (Path("/usr/bin/ffmpeg"), Path("/usr/local/bin/ffmpeg")):
|
|
74
|
+
if fallback.exists() and os.access(fallback, os.X_OK):
|
|
75
|
+
return str(fallback)
|
|
76
|
+
|
|
77
|
+
return None
|
|
78
|
+
|
|
16
79
|
def _detect_nvenc_support() -> bool:
|
|
17
80
|
"""
|
|
18
81
|
Detect if NVIDIA NVENC hardware acceleration is available.
|
|
@@ -163,7 +226,7 @@ def is_ffmpeg_available() -> bool:
|
|
|
163
226
|
Returns:
|
|
164
227
|
True if FFmpeg is found in the PATH; otherwise, False.
|
|
165
228
|
"""
|
|
166
|
-
return
|
|
229
|
+
return _resolve_ffmpeg_executable() is not None
|
|
167
230
|
|
|
168
231
|
def check_ffmpeg_availability():
|
|
169
232
|
"""
|
|
@@ -607,8 +670,8 @@ def extract_frames(
|
|
|
607
670
|
Returns:
|
|
608
671
|
A list of Path objects for the extracted frames.
|
|
609
672
|
"""
|
|
610
|
-
#
|
|
611
|
-
ffmpeg_executable =
|
|
673
|
+
# Resolve ffmpeg executable with multiple fallbacks
|
|
674
|
+
ffmpeg_executable = _resolve_ffmpeg_executable()
|
|
612
675
|
if not ffmpeg_executable:
|
|
613
676
|
error_msg = "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
|
|
614
677
|
logger.error(error_msg)
|
|
@@ -691,7 +754,7 @@ def extract_frame_range(
|
|
|
691
754
|
logger.warning("extract_frame_range called with start_frame (%d) >= end_frame (%d). No frames to extract.", start_frame, end_frame)
|
|
692
755
|
return []
|
|
693
756
|
|
|
694
|
-
ffmpeg_executable =
|
|
757
|
+
ffmpeg_executable = _resolve_ffmpeg_executable()
|
|
695
758
|
if not ffmpeg_executable:
|
|
696
759
|
error_msg = "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
|
|
697
760
|
logger.error(error_msg)
|
|
@@ -1,11 +1,18 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any, Dict, cast
|
|
3
|
+
|
|
4
|
+
from django.db import transaction
|
|
1
5
|
from rest_framework import status
|
|
2
6
|
from rest_framework.response import Response
|
|
3
7
|
from rest_framework.views import APIView
|
|
4
|
-
|
|
5
|
-
from endoreg_db.models import
|
|
8
|
+
|
|
9
|
+
from endoreg_db.models import RawPdfFile, VideoFile
|
|
6
10
|
from endoreg_db.serializers.anonymization import SensitiveMetaValidateSerializer
|
|
7
11
|
|
|
8
12
|
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
9
16
|
class AnonymizationValidateView(APIView):
|
|
10
17
|
"""
|
|
11
18
|
POST /api/anonymization/<int:file_id>/validate/
|
|
@@ -32,35 +39,69 @@ class AnonymizationValidateView(APIView):
|
|
|
32
39
|
# Serializer-Validierung mit deutscher Datums-Priorität
|
|
33
40
|
serializer = SensitiveMetaValidateSerializer(data=request.data or {})
|
|
34
41
|
serializer.is_valid(raise_exception=True)
|
|
35
|
-
|
|
36
|
-
payload
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
42
|
+
validated_data = cast(Dict[str, Any], serializer.validated_data)
|
|
43
|
+
payload: Dict[str, Any] = dict(validated_data)
|
|
44
|
+
if "is_verified" not in payload:
|
|
45
|
+
payload["is_verified"] = True
|
|
46
|
+
|
|
47
|
+
file_type = payload.get("file_type")
|
|
48
|
+
|
|
49
|
+
# Try Video first (unless explicitly requesting PDF)
|
|
50
|
+
if file_type in (None, "video"):
|
|
51
|
+
video = VideoFile.objects.select_related("center").filter(pk=file_id).first()
|
|
52
|
+
if video is not None:
|
|
53
|
+
prepared_payload = self._prepare_payload(payload, video)
|
|
54
|
+
try:
|
|
55
|
+
ok = video.validate_metadata_annotation(prepared_payload)
|
|
56
|
+
except Exception: # pragma: no cover - defensive safety net
|
|
57
|
+
logger.exception("Video validation crashed for id=%s", file_id)
|
|
58
|
+
return Response(
|
|
59
|
+
{"error": "Video validation encountered an unexpected error."},
|
|
60
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
if not ok:
|
|
64
|
+
return Response({"error": "Video validation failed."}, status=status.HTTP_400_BAD_REQUEST)
|
|
65
|
+
|
|
66
|
+
return Response({"message": "Video validated."}, status=status.HTTP_200_OK)
|
|
67
|
+
|
|
68
|
+
if file_type == "video":
|
|
69
|
+
return Response({"error": f"Video {file_id} not found."}, status=status.HTTP_404_NOT_FOUND)
|
|
70
|
+
|
|
71
|
+
# Then PDF (unless explicitly requesting Video)
|
|
72
|
+
if file_type in (None, "pdf"):
|
|
73
|
+
pdf = RawPdfFile.objects.select_related("center").filter(pk=file_id).first()
|
|
74
|
+
if pdf is not None:
|
|
75
|
+
prepared_payload = self._prepare_payload(payload, pdf)
|
|
76
|
+
try:
|
|
77
|
+
ok = pdf.validate_metadata_annotation(prepared_payload)
|
|
78
|
+
except Exception: # pragma: no cover - defensive safety net
|
|
79
|
+
logger.exception("PDF validation crashed for id=%s", file_id)
|
|
80
|
+
return Response(
|
|
81
|
+
{"error": "PDF validation encountered an unexpected error."},
|
|
82
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
if not ok:
|
|
86
|
+
return Response({"error": "PDF validation failed."}, status=status.HTTP_400_BAD_REQUEST)
|
|
87
|
+
|
|
88
|
+
return Response({"message": "PDF validated."}, status=status.HTTP_200_OK)
|
|
89
|
+
|
|
90
|
+
if file_type == "pdf":
|
|
91
|
+
return Response({"error": f"PDF {file_id} not found."}, status=status.HTTP_404_NOT_FOUND)
|
|
92
|
+
|
|
93
|
+
return Response({"error": f"Item {file_id} not found as video or pdf."}, status=status.HTTP_404_NOT_FOUND)
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def _prepare_payload(base_payload: Dict[str, Any], file_obj: Any) -> Dict[str, Any]:
|
|
97
|
+
"""Return a fresh payload tailored for the given file object."""
|
|
98
|
+
|
|
99
|
+
prepared = dict(base_payload)
|
|
100
|
+
prepared.pop("file_type", None)
|
|
101
|
+
|
|
102
|
+
center = getattr(file_obj, "center", None)
|
|
103
|
+
center_name = getattr(center, "name", None)
|
|
104
|
+
if center_name and not prepared.get("center_name"):
|
|
105
|
+
prepared["center_name"] = center_name
|
|
106
|
+
|
|
107
|
+
return prepared
|
|
@@ -64,15 +64,17 @@ def update_segments_after_frame_removal(video: VideoFile, removed_frames: list)
|
|
|
64
64
|
return {'segments_updated': 0, 'segments_deleted': 0, 'segments_unchanged': 0}
|
|
65
65
|
|
|
66
66
|
removed_frames = sorted(set(removed_frames)) # Ensure sorted and unique
|
|
67
|
-
segments = LabelVideoSegment.objects.filter(
|
|
67
|
+
segments = LabelVideoSegment.objects.filter(
|
|
68
|
+
video_file=video
|
|
69
|
+
).order_by('start_frame_number')
|
|
68
70
|
|
|
69
71
|
segments_updated = 0
|
|
70
72
|
segments_deleted = 0
|
|
71
73
|
segments_unchanged = 0
|
|
72
74
|
|
|
73
75
|
for segment in segments:
|
|
74
|
-
original_start = segment.
|
|
75
|
-
original_end = segment.
|
|
76
|
+
original_start = segment.start_frame_number
|
|
77
|
+
original_end = segment.end_frame_number
|
|
76
78
|
|
|
77
79
|
# Count frames removed before this segment
|
|
78
80
|
frames_before = sum(1 for f in removed_frames if f < original_start)
|
|
@@ -99,9 +101,9 @@ def update_segments_after_frame_removal(video: VideoFile, removed_frames: list)
|
|
|
99
101
|
f"{original_start}-{original_end} → {new_start}-{new_end} "
|
|
100
102
|
f"(before: {frames_before}, within: {frames_within})"
|
|
101
103
|
)
|
|
102
|
-
segment.
|
|
103
|
-
segment.
|
|
104
|
-
segment.save()
|
|
104
|
+
segment.start_frame_number = new_start
|
|
105
|
+
segment.end_frame_number = new_end
|
|
106
|
+
segment.save(update_fields=["start_frame_number", "end_frame_number"])
|
|
105
107
|
segments_updated += 1
|
|
106
108
|
else:
|
|
107
109
|
# No change needed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: endoreg-db
|
|
3
|
-
Version: 0.8.2
|
|
3
|
+
Version: 0.8.2.1
|
|
4
4
|
Summary: EndoReg Db Django App
|
|
5
5
|
Project-URL: Homepage, https://info.coloreg.de
|
|
6
6
|
Project-URL: Repository, https://github.com/wg-lux/endoreg-db
|
|
@@ -33,7 +33,7 @@ Requires-Dist: gunicorn>=23.0.0
|
|
|
33
33
|
Requires-Dist: icecream>=2.1.4
|
|
34
34
|
Requires-Dist: librosa==0.11.0
|
|
35
35
|
Requires-Dist: llvmlite>=0.44.0
|
|
36
|
-
Requires-Dist: lx-anonymizer[llm,ocr]>=0.8.2
|
|
36
|
+
Requires-Dist: lx-anonymizer[llm,ocr]>=0.8.2.1
|
|
37
37
|
Requires-Dist: moviepy==2.2.1
|
|
38
38
|
Requires-Dist: mypy>=1.16.0
|
|
39
39
|
Requires-Dist: numpy>=2.2.3
|