endoreg-db 0.8.2__py3-none-any.whl → 0.8.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/config/__init__.py +0 -0
- endoreg_db/helpers/default_objects.py +48 -29
- endoreg_db/management/commands/import_video.py +5 -3
- endoreg_db/migrations/0003_add_center_display_name.py +30 -0
- endoreg_db/models/administration/center/center.py +7 -1
- endoreg_db/models/media/pdf/raw_pdf.py +31 -26
- endoreg_db/models/media/video/create_from_file.py +26 -4
- endoreg_db/models/media/video/video_file.py +89 -57
- endoreg_db/models/media/video/video_file_anonymize.py +2 -1
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +12 -0
- endoreg_db/models/media/video/video_file_io.py +4 -2
- endoreg_db/models/metadata/sensitive_meta.py +6 -6
- endoreg_db/models/metadata/video_meta.py +2 -2
- endoreg_db/services/pdf_import.py +131 -15
- endoreg_db/services/pseudonym_service.py +1 -1
- endoreg_db/services/video_import.py +400 -387
- endoreg_db/urls/sensitive_meta.py +0 -0
- endoreg_db/utils/paths.py +2 -10
- endoreg_db/utils/video/ffmpeg_wrapper.py +67 -4
- endoreg_db/views/anonymization/validate.py +75 -34
- endoreg_db/views/video/correction.py +8 -6
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.2.dist-info}/METADATA +2 -2
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.2.dist-info}/RECORD +25 -23
- endoreg_db/services/ollama_api_docs.py +0 -1528
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.2.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.2.dist-info → endoreg_db-0.8.2.2.dist-info}/licenses/LICENSE +0 -0
|
File without changes
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import random
|
|
2
|
+
from typing import Optional
|
|
2
3
|
from endoreg_db.models import (
|
|
3
4
|
Center,
|
|
4
5
|
Gender,
|
|
@@ -17,6 +18,7 @@ import shutil
|
|
|
17
18
|
from pathlib import Path
|
|
18
19
|
from django.conf import settings # Import settings
|
|
19
20
|
from django.core.files.storage import default_storage # Import default storage
|
|
21
|
+
from django.db.models.fields.files import FieldFile
|
|
20
22
|
|
|
21
23
|
from endoreg_db.utils import (
|
|
22
24
|
create_mock_patient_name,
|
|
@@ -44,6 +46,10 @@ DEFAULT_INDICATIONS = [
|
|
|
44
46
|
DEFAULT_SEGMENTATION_MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
|
|
45
47
|
|
|
46
48
|
DEFAULT_GENDER = "unknown"
|
|
49
|
+
DEFAULT_PATIENT_FIRST_NAME = "TestFirst"
|
|
50
|
+
DEFAULT_PATIENT_LAST_NAME = "TestLast"
|
|
51
|
+
DEFAULT_PATIENT_GENDER_NAME = "female"
|
|
52
|
+
DEFAULT_PATIENT_BIRTH_DATE = date(1970, 1, 1)
|
|
47
53
|
|
|
48
54
|
def get_information_source_prediction():
|
|
49
55
|
"""
|
|
@@ -170,33 +176,41 @@ def get_default_center() -> Center:
|
|
|
170
176
|
return center
|
|
171
177
|
|
|
172
178
|
def generate_patient(**kwargs) -> Patient:
|
|
173
|
-
"""
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
# Set default values
|
|
185
|
-
gender = kwargs.get("gender", get_random_gender())
|
|
186
|
-
if not isinstance(gender, Gender):
|
|
179
|
+
"""Create a Patient with deterministic defaults unless ``randomize=True`` is supplied."""
|
|
180
|
+
|
|
181
|
+
randomize = kwargs.pop("randomize", False)
|
|
182
|
+
|
|
183
|
+
gender = kwargs.get("gender")
|
|
184
|
+
if gender is None:
|
|
185
|
+
if randomize:
|
|
186
|
+
gender = get_random_gender()
|
|
187
|
+
else:
|
|
188
|
+
gender = Gender.objects.get(name=DEFAULT_PATIENT_GENDER_NAME)
|
|
189
|
+
elif not isinstance(gender, Gender):
|
|
187
190
|
gender = Gender.objects.get(name=gender)
|
|
188
191
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
first_name
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
192
|
+
first_name = kwargs.get("first_name")
|
|
193
|
+
last_name = kwargs.get("last_name")
|
|
194
|
+
if first_name is None or last_name is None:
|
|
195
|
+
if randomize:
|
|
196
|
+
generated_first, generated_last = create_mock_patient_name(gender=gender.name)
|
|
197
|
+
else:
|
|
198
|
+
generated_first, generated_last = DEFAULT_PATIENT_FIRST_NAME, DEFAULT_PATIENT_LAST_NAME
|
|
199
|
+
first_name = first_name or generated_first
|
|
200
|
+
last_name = last_name or generated_last
|
|
201
|
+
|
|
202
|
+
dob = kwargs.get("dob")
|
|
203
|
+
if dob is None:
|
|
204
|
+
birth_date = kwargs.get("birth_date", DEFAULT_PATIENT_BIRTH_DATE)
|
|
205
|
+
if isinstance(birth_date, date):
|
|
206
|
+
dob = birth_date
|
|
207
|
+
else:
|
|
208
|
+
dob = date.fromisoformat(str(birth_date))
|
|
209
|
+
|
|
210
|
+
center = kwargs.get("center")
|
|
197
211
|
if center is None:
|
|
198
212
|
center = get_default_center()
|
|
199
|
-
|
|
213
|
+
elif not isinstance(center, Center):
|
|
200
214
|
center = Center.objects.get(name=center)
|
|
201
215
|
|
|
202
216
|
patient = Patient(
|
|
@@ -261,6 +275,7 @@ def get_default_egd_pdf():
|
|
|
261
275
|
shutil.copy(egd_path, temp_file_path)
|
|
262
276
|
|
|
263
277
|
pdf_file = None
|
|
278
|
+
file_field: Optional[FieldFile] = None
|
|
264
279
|
try:
|
|
265
280
|
# Create the PDF record using the temporary file.
|
|
266
281
|
# delete_source=True will ensure temp_file_path is deleted by create_from_file
|
|
@@ -275,8 +290,11 @@ def get_default_egd_pdf():
|
|
|
275
290
|
raise RuntimeError("Failed to create PDF file object")
|
|
276
291
|
|
|
277
292
|
# Use storage API to check existence
|
|
278
|
-
|
|
279
|
-
|
|
293
|
+
file_field = pdf_file.file
|
|
294
|
+
if not isinstance(file_field, FieldFile):
|
|
295
|
+
raise RuntimeError("RawPdfFile.file did not return a FieldFile instance")
|
|
296
|
+
if not default_storage.exists(file_field.path):
|
|
297
|
+
raise RuntimeError(f"PDF file does not exist in storage at {file_field.path}")
|
|
280
298
|
|
|
281
299
|
# Check that the source temp file was deleted
|
|
282
300
|
if temp_file_path.exists():
|
|
@@ -308,10 +326,11 @@ def get_default_egd_pdf():
|
|
|
308
326
|
|
|
309
327
|
# pdf_file.file.path might fail if storage doesn't support direct paths (like S3)
|
|
310
328
|
# Prefer using storage API for checks. Logging path if available.
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
329
|
+
if file_field is not None:
|
|
330
|
+
try:
|
|
331
|
+
logger.info(f"PDF file created: {file_field.name}, Path: {file_field.path}")
|
|
332
|
+
except NotImplementedError:
|
|
333
|
+
logger.info(f"PDF file created: {file_field.name}, Path: (Not available from storage)")
|
|
315
334
|
|
|
316
335
|
|
|
317
336
|
return pdf_file
|
|
@@ -294,10 +294,12 @@ class Command(BaseCommand):
|
|
|
294
294
|
|
|
295
295
|
# Updated to handle new return signature (path, metadata)
|
|
296
296
|
cleaned_video_path, extracted_metadata = frame_cleaner.clean_video(
|
|
297
|
-
Path(video_file_obj.raw_file.path),
|
|
297
|
+
video_path=Path(video_file_obj.raw_file.path),
|
|
298
298
|
video_file_obj=video_file_obj, # Pass VideoFile object to store metadata
|
|
299
|
-
|
|
300
|
-
|
|
299
|
+
device_name=processor_name,
|
|
300
|
+
endoscope_roi=video_file_obj.processor.get_roi_endoscope_image if video_file_obj.processor else None,
|
|
301
|
+
output_path=video_file_obj.get_processed_file_path(),
|
|
302
|
+
technique="mask_overlay" # Use mask overlay technique as default, if not set this will be inferred.
|
|
301
303
|
)
|
|
302
304
|
|
|
303
305
|
# Save the cleaned video using Django's FileField
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from django.db import migrations, models
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def populate_display_name(apps, schema_editor):
|
|
5
|
+
Center = apps.get_model('endoreg_db', 'Center')
|
|
6
|
+
for center in Center.objects.all():
|
|
7
|
+
if not center.display_name:
|
|
8
|
+
center.display_name = center.name
|
|
9
|
+
center.save(update_fields=['display_name'])
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def reset_display_name(apps, schema_editor):
|
|
13
|
+
Center = apps.get_model('endoreg_db', 'Center')
|
|
14
|
+
Center.objects.update(display_name='')
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Migration(migrations.Migration):
|
|
18
|
+
|
|
19
|
+
dependencies = [
|
|
20
|
+
('endoreg_db', '0002_add_video_correction_models'),
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
operations = [
|
|
24
|
+
migrations.AddField(
|
|
25
|
+
model_name='center',
|
|
26
|
+
name='display_name',
|
|
27
|
+
field=models.CharField(blank=True, default='', max_length=255),
|
|
28
|
+
),
|
|
29
|
+
migrations.RunPython(populate_display_name, reset_display_name),
|
|
30
|
+
]
|
|
@@ -19,6 +19,7 @@ class Center(models.Model):
|
|
|
19
19
|
|
|
20
20
|
# import_id = models.IntegerField(primary_key=True)
|
|
21
21
|
name = models.CharField(max_length=255)
|
|
22
|
+
display_name = models.CharField(max_length=255, blank=True, default="")
|
|
22
23
|
|
|
23
24
|
first_names = models.ManyToManyField(
|
|
24
25
|
to="FirstName",
|
|
@@ -45,8 +46,13 @@ class Center(models.Model):
|
|
|
45
46
|
def natural_key(self) -> tuple[str]:
|
|
46
47
|
return (self.name,)
|
|
47
48
|
|
|
49
|
+
def save(self, *args, **kwargs):
|
|
50
|
+
if not self.display_name:
|
|
51
|
+
self.display_name = self.name
|
|
52
|
+
super().save(*args, **kwargs)
|
|
53
|
+
|
|
48
54
|
def __str__(self) -> str:
|
|
49
|
-
return str(object=self.name)
|
|
55
|
+
return str(object=self.display_name or self.name)
|
|
50
56
|
|
|
51
57
|
def get_first_names(self):
|
|
52
58
|
return self.first_names.all()
|
|
@@ -383,37 +383,42 @@ class RawPdfFile(models.Model):
|
|
|
383
383
|
new_file_name, _uuid = get_uuid_filename(file_path)
|
|
384
384
|
logger.info(f"Generated new filename: {new_file_name}")
|
|
385
385
|
|
|
386
|
-
# Create model instance
|
|
387
|
-
raw_pdf = cls(
|
|
388
|
-
pdf_hash=pdf_hash,
|
|
389
|
-
center=center,
|
|
390
|
-
)
|
|
391
|
-
|
|
392
|
-
# Assign file using Django's File wrapper and save
|
|
386
|
+
# Create model instance via manager so creation can be intercepted/mocked during tests
|
|
393
387
|
try:
|
|
394
388
|
with file_path.open("rb") as f:
|
|
395
389
|
django_file = File(f, name=new_file_name)
|
|
396
|
-
raw_pdf
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
390
|
+
raw_pdf = cls.objects.create(
|
|
391
|
+
pdf_hash=pdf_hash,
|
|
392
|
+
center=center,
|
|
393
|
+
file=django_file,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
_file = raw_pdf.file
|
|
397
|
+
assert _file is not None
|
|
398
|
+
logger.info(
|
|
399
|
+
"Created and saved new RawPdfFile %s with file %s", raw_pdf.pk, _file.name
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
if not _file.storage.exists(_file.name):
|
|
403
|
+
logger.error(
|
|
404
|
+
"File was not saved correctly to storage path %s after model save.",
|
|
405
|
+
_file.name,
|
|
406
|
+
)
|
|
407
|
+
raise IOError(
|
|
408
|
+
f"File not found at expected storage path after save: {_file.name}"
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
try:
|
|
412
|
+
logger.info("File saved to absolute path: %s", _file.path)
|
|
413
|
+
except NotImplementedError:
|
|
414
|
+
logger.info(
|
|
415
|
+
"File saved to storage path: %s (Absolute path not available from storage)",
|
|
416
|
+
_file.name,
|
|
417
|
+
)
|
|
412
418
|
|
|
413
419
|
except Exception as e:
|
|
414
|
-
logger.error(
|
|
415
|
-
|
|
416
|
-
raise # Re-raise the exception
|
|
420
|
+
logger.error("Error processing or saving file %s for new record: %s", file_path, e)
|
|
421
|
+
raise
|
|
417
422
|
|
|
418
423
|
# Delete source file *after* successful save and verification
|
|
419
424
|
if delete_source:
|
|
@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Optional, Type
|
|
|
6
6
|
|
|
7
7
|
# Import the new exceptions from the correct path
|
|
8
8
|
from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
|
|
9
|
-
from ...utils import VIDEO_DIR, TMP_VIDEO_DIR
|
|
9
|
+
from ...utils import VIDEO_DIR, TMP_VIDEO_DIR
|
|
10
|
+
from importlib import import_module
|
|
10
11
|
|
|
11
12
|
if TYPE_CHECKING:
|
|
12
13
|
from endoreg_db.models import VideoFile
|
|
@@ -170,6 +171,22 @@ def atomic_move_with_fallback(src_path: Path, dst_path: Path) -> bool:
|
|
|
170
171
|
raise
|
|
171
172
|
|
|
172
173
|
|
|
174
|
+
def _get_data_paths():
|
|
175
|
+
"""Return the current data_paths mapping (supports patched instances in tests)."""
|
|
176
|
+
utils_module = import_module("endoreg_db.utils")
|
|
177
|
+
return getattr(utils_module, "data_paths")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _get_path(mapping, key, default):
|
|
181
|
+
"""Access mapping by key using __getitem__ so MagicMocks with side effects work."""
|
|
182
|
+
if mapping is None:
|
|
183
|
+
return default
|
|
184
|
+
try:
|
|
185
|
+
return mapping[key]
|
|
186
|
+
except (KeyError, TypeError):
|
|
187
|
+
return default
|
|
188
|
+
|
|
189
|
+
|
|
173
190
|
def _create_from_file(
|
|
174
191
|
cls_model: Type["VideoFile"],
|
|
175
192
|
file_path: Path,
|
|
@@ -199,8 +216,12 @@ def _create_from_file(
|
|
|
199
216
|
|
|
200
217
|
try:
|
|
201
218
|
# Ensure we operate under the canonical video path root
|
|
202
|
-
|
|
203
|
-
|
|
219
|
+
data_paths = _get_data_paths()
|
|
220
|
+
resolved_video_dir = _get_path(data_paths, "video", video_dir)
|
|
221
|
+
video_dir = Path(resolved_video_dir)
|
|
222
|
+
storage_root_default = Path(video_dir).parent
|
|
223
|
+
resolved_storage_root = _get_path(data_paths, "storage", storage_root_default)
|
|
224
|
+
storage_root = Path(resolved_storage_root)
|
|
204
225
|
storage_root.mkdir(parents=True, exist_ok=True)
|
|
205
226
|
|
|
206
227
|
# Check storage capacity before starting any work
|
|
@@ -300,7 +321,8 @@ def _create_from_file(
|
|
|
300
321
|
# 8. Create the VideoFile instance
|
|
301
322
|
logger.info("Creating new VideoFile instance with UUID: %s", uuid_val)
|
|
302
323
|
# Store FileField path relative to storage root including the videos prefix
|
|
303
|
-
|
|
324
|
+
storage_base = Path(_get_path(data_paths, "storage", final_storage_path.parent))
|
|
325
|
+
relative_name = (final_storage_path.relative_to(storage_base)).as_posix()
|
|
304
326
|
video = cls_model(
|
|
305
327
|
uuid=uuid_val,
|
|
306
328
|
raw_file=relative_name,
|
|
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Optional, Union, cast
|
|
|
7
7
|
|
|
8
8
|
from django.db import models
|
|
9
9
|
from django.core.files import File
|
|
10
|
+
from django.db.models.fields.files import FieldFile
|
|
10
11
|
from django.core.validators import FileExtensionValidator
|
|
11
12
|
from django.db.models import F
|
|
12
13
|
from endoreg_db.utils.calc_duration_seconds import _calc_duration_vf
|
|
@@ -126,37 +127,37 @@ class VideoFile(models.Model):
|
|
|
126
127
|
|
|
127
128
|
sensitive_meta = models.OneToOneField(
|
|
128
129
|
"SensitiveMeta", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
|
|
129
|
-
)
|
|
130
|
-
center = models.ForeignKey("Center", on_delete=models.PROTECT)
|
|
130
|
+
) # type: ignore
|
|
131
|
+
center = models.ForeignKey("Center", on_delete=models.PROTECT) # type: ignore
|
|
131
132
|
processor = models.ForeignKey(
|
|
132
133
|
"EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True
|
|
133
|
-
)
|
|
134
|
+
) # type: ignore
|
|
134
135
|
video_meta = models.OneToOneField(
|
|
135
136
|
"VideoMeta", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
|
|
136
|
-
)
|
|
137
|
+
) # type: ignore
|
|
137
138
|
examination = models.ForeignKey(
|
|
138
139
|
"PatientExamination",
|
|
139
140
|
on_delete=models.SET_NULL,
|
|
140
141
|
blank=True,
|
|
141
142
|
null=True,
|
|
142
143
|
related_name="video_files",
|
|
143
|
-
)
|
|
144
|
+
) # type: ignore
|
|
144
145
|
patient = models.ForeignKey(
|
|
145
146
|
"Patient",
|
|
146
147
|
on_delete=models.SET_NULL,
|
|
147
148
|
blank=True,
|
|
148
149
|
null=True,
|
|
149
150
|
related_name="video_files",
|
|
150
|
-
)
|
|
151
|
+
) # type: ignore
|
|
151
152
|
ai_model_meta = models.ForeignKey(
|
|
152
153
|
"ModelMeta", on_delete=models.SET_NULL, blank=True, null=True
|
|
153
|
-
)
|
|
154
|
+
) # type: ignore
|
|
154
155
|
state = models.OneToOneField(
|
|
155
156
|
"VideoState", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
|
|
156
|
-
)
|
|
157
|
+
) # type: ignore
|
|
157
158
|
import_meta = models.OneToOneField(
|
|
158
159
|
"VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
|
|
159
|
-
)
|
|
160
|
+
) # type: ignore
|
|
160
161
|
|
|
161
162
|
original_file_name = models.CharField(max_length=255, blank=True, null=True)
|
|
162
163
|
uploaded_at = models.DateTimeField(auto_now_add=True)
|
|
@@ -197,7 +198,9 @@ class VideoFile(models.Model):
|
|
|
197
198
|
"""
|
|
198
199
|
from endoreg_db.models import FFMpegMeta
|
|
199
200
|
if self.video_meta is not None:
|
|
200
|
-
|
|
201
|
+
if self.video_meta.ffmpeg_meta is not None:
|
|
202
|
+
return self.video_meta.ffmpeg_meta
|
|
203
|
+
raise AssertionError("Expected FFMpegMeta instance.")
|
|
201
204
|
else:
|
|
202
205
|
self.initialize_video_specs()
|
|
203
206
|
ffmpeg_meta = self.video_meta.ffmpeg_meta if self.video_meta else None
|
|
@@ -216,20 +219,19 @@ class VideoFile(models.Model):
|
|
|
216
219
|
Raises:
|
|
217
220
|
Value Error if no active VideoFile is available.
|
|
218
221
|
"""
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
return url
|
|
222
|
+
active = self.active_file
|
|
223
|
+
if not isinstance(active, FieldFile):
|
|
224
|
+
raise ValueError("Active file is not a stored FieldFile instance.")
|
|
225
|
+
if not active.name:
|
|
226
|
+
raise ValueError("Active file has no associated name.")
|
|
227
|
+
return active.url
|
|
226
228
|
|
|
227
229
|
@property
|
|
228
|
-
def active_raw_file(self) ->
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
230
|
+
def active_raw_file(self) -> FieldFile:
|
|
231
|
+
raw = self.raw_file
|
|
232
|
+
if isinstance(raw, FieldFile) and raw.name:
|
|
233
|
+
return raw
|
|
234
|
+
raise ValueError("No raw file available for this video")
|
|
233
235
|
|
|
234
236
|
@property
|
|
235
237
|
def active_raw_file_url(self)-> str:
|
|
@@ -241,12 +243,10 @@ class VideoFile(models.Model):
|
|
|
241
243
|
|
|
242
244
|
Returns:
|
|
243
245
|
"""
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
url = _file.url
|
|
249
|
-
return url
|
|
246
|
+
raw = self.active_raw_file
|
|
247
|
+
if not raw.name:
|
|
248
|
+
raise ValueError("Active raw file has no associated name.")
|
|
249
|
+
return raw.url
|
|
250
250
|
|
|
251
251
|
|
|
252
252
|
# Pipeline Functions
|
|
@@ -363,7 +363,7 @@ class VideoFile(models.Model):
|
|
|
363
363
|
|
|
364
364
|
|
|
365
365
|
@property
|
|
366
|
-
def active_file(self) ->
|
|
366
|
+
def active_file(self) -> FieldFile:
|
|
367
367
|
"""
|
|
368
368
|
Return the active video file, preferring the processed file if available.
|
|
369
369
|
|
|
@@ -373,12 +373,15 @@ class VideoFile(models.Model):
|
|
|
373
373
|
Raises:
|
|
374
374
|
ValueError: If neither a processed nor a raw file is available.
|
|
375
375
|
"""
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
376
|
+
processed = self.processed_file
|
|
377
|
+
if isinstance(processed, FieldFile) and processed.name:
|
|
378
|
+
return processed
|
|
379
|
+
|
|
380
|
+
raw = self.raw_file
|
|
381
|
+
if isinstance(raw, FieldFile) and raw.name:
|
|
382
|
+
return raw
|
|
383
|
+
|
|
384
|
+
raise ValueError("No active file available. VideoFile has neither raw nor processed file.")
|
|
382
385
|
|
|
383
386
|
|
|
384
387
|
@property
|
|
@@ -393,13 +396,17 @@ class VideoFile(models.Model):
|
|
|
393
396
|
ValueError: If neither a processed nor raw file is present.
|
|
394
397
|
"""
|
|
395
398
|
active = self.active_file
|
|
396
|
-
if active
|
|
397
|
-
|
|
398
|
-
elif active
|
|
399
|
-
|
|
399
|
+
if active is self.processed_file:
|
|
400
|
+
path = _get_processed_file_path(self)
|
|
401
|
+
elif active is self.raw_file:
|
|
402
|
+
path = _get_raw_file_path(self)
|
|
400
403
|
else:
|
|
401
404
|
raise ValueError("No active file path available. VideoFile has neither raw nor processed file.")
|
|
402
405
|
|
|
406
|
+
if path is None:
|
|
407
|
+
raise ValueError("Active file path could not be resolved.")
|
|
408
|
+
return path
|
|
409
|
+
|
|
403
410
|
|
|
404
411
|
@classmethod
|
|
405
412
|
def create_from_file(cls, file_path: Union[str, Path], center_name: str, **kwargs) -> Optional["VideoFile"]:
|
|
@@ -448,13 +455,14 @@ class VideoFile(models.Model):
|
|
|
448
455
|
"""
|
|
449
456
|
# Ensure frames are deleted before the main instance
|
|
450
457
|
_delete_frames(self)
|
|
451
|
-
|
|
458
|
+
|
|
452
459
|
# Call the original delete method to remove the instance from the database
|
|
453
|
-
|
|
454
|
-
|
|
460
|
+
active_path = self.active_file_path
|
|
461
|
+
logger.info(f"Deleting VideoFile: {self.uuid} - {active_path}")
|
|
462
|
+
|
|
455
463
|
# Delete associated files if they exist
|
|
456
|
-
if
|
|
457
|
-
|
|
464
|
+
if active_path.exists():
|
|
465
|
+
active_path.unlink(missing_ok=True)
|
|
458
466
|
|
|
459
467
|
# Delete file storage
|
|
460
468
|
if self.raw_file and self.raw_file.storage.exists(self.raw_file.name):
|
|
@@ -465,11 +473,23 @@ class VideoFile(models.Model):
|
|
|
465
473
|
# Use proper database connection
|
|
466
474
|
if using is None:
|
|
467
475
|
using = 'default'
|
|
468
|
-
|
|
476
|
+
|
|
477
|
+
raw_file_path = self.get_raw_file_path()
|
|
478
|
+
if raw_file_path:
|
|
479
|
+
raw_file_path = Path(raw_file_path)
|
|
480
|
+
lock_path = raw_file_path.with_suffix(raw_file_path.suffix + ".lock")
|
|
481
|
+
if lock_path.exists():
|
|
482
|
+
try:
|
|
483
|
+
lock_path.unlink()
|
|
484
|
+
logger.info(f"Removed processing lock: {lock_path}")
|
|
485
|
+
except Exception as e:
|
|
486
|
+
logger.warning(f"Could not remove processing lock {lock_path}: {e}")
|
|
487
|
+
|
|
469
488
|
try:
|
|
470
489
|
# Call parent delete with proper parameters
|
|
471
|
-
super().delete(using=using, keep_parents=keep_parents)
|
|
490
|
+
result = super().delete(using=using, keep_parents=keep_parents)
|
|
472
491
|
logger.info(f"VideoFile {self.uuid} deleted successfully.")
|
|
492
|
+
return result
|
|
473
493
|
except Exception as e:
|
|
474
494
|
logger.error(f"Error deleting VideoFile {self.uuid}: {e}")
|
|
475
495
|
raise
|
|
@@ -572,15 +592,28 @@ class VideoFile(models.Model):
|
|
|
572
592
|
super().save(*args, **kwargs)
|
|
573
593
|
|
|
574
594
|
def get_or_create_state(self) -> "VideoState":
|
|
575
|
-
"""
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
595
|
+
"""Ensure this video has a persisted ``VideoState`` and return it."""
|
|
596
|
+
|
|
597
|
+
state = self.state
|
|
598
|
+
|
|
599
|
+
# When tests reuse cached instances across database flushes, ``state`` may reference
|
|
600
|
+
# a row that no longer exists. Guard against that by validating persistence.
|
|
601
|
+
state_pk = getattr(state, "pk", None)
|
|
602
|
+
if state is not None and state_pk is not None:
|
|
603
|
+
if not VideoState.objects.filter(pk=state_pk).exists():
|
|
604
|
+
state = None
|
|
605
|
+
|
|
606
|
+
if state is None:
|
|
607
|
+
# Create a fresh state to avoid refresh_from_db() failures on unsaved instances.
|
|
608
|
+
state = VideoState.objects.create()
|
|
609
|
+
self.state = state
|
|
610
|
+
|
|
611
|
+
# Persist the relation immediately if the VideoFile already exists in the DB so
|
|
612
|
+
# later refreshes see the association without requiring additional saves.
|
|
613
|
+
if self.pk:
|
|
614
|
+
self.save(update_fields=["state"])
|
|
615
|
+
|
|
616
|
+
return state
|
|
584
617
|
|
|
585
618
|
def get_or_create_sensitive_meta(self) -> "SensitiveMeta":
|
|
586
619
|
"""
|
|
@@ -592,8 +625,7 @@ class VideoFile(models.Model):
|
|
|
592
625
|
from endoreg_db.models import SensitiveMeta
|
|
593
626
|
if self.sensitive_meta is None:
|
|
594
627
|
self.sensitive_meta = SensitiveMeta.objects.create(center = self.center)
|
|
595
|
-
#
|
|
596
|
-
self.get_or_create_state().mark_sensitive_meta_processed(save=True)
|
|
628
|
+
# Do not mark processed here; it will be set after extraction/validation steps
|
|
597
629
|
return self.sensitive_meta
|
|
598
630
|
|
|
599
631
|
def get_outside_segments(self, only_validated: bool = False) -> models.QuerySet["LabelVideoSegment"]:
|
|
@@ -12,6 +12,7 @@ from django.conf import settings
|
|
|
12
12
|
|
|
13
13
|
from endoreg_db.utils.hashs import get_video_hash
|
|
14
14
|
from endoreg_db.utils.validate_endo_roi import validate_endo_roi
|
|
15
|
+
from endoreg_db.utils.paths import STORAGE_DIR
|
|
15
16
|
from ....utils.video.ffmpeg_wrapper import assemble_video_from_frames
|
|
16
17
|
from ...utils import anonymize_frame # Import from models.utils
|
|
17
18
|
from .video_file_segments import _get_outside_frames, _get_outside_frame_numbers
|
|
@@ -268,7 +269,7 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
|
|
|
268
269
|
raise ValueError(f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid}).")
|
|
269
270
|
|
|
270
271
|
video.processed_video_hash = new_processed_hash
|
|
271
|
-
video.processed_file.name = video.get_target_anonymized_video_path().relative_to(
|
|
272
|
+
video.processed_file.name = video.get_target_anonymized_video_path().relative_to(STORAGE_DIR).as_posix()
|
|
272
273
|
|
|
273
274
|
update_fields = [
|
|
274
275
|
"processed_video_hash",
|
|
@@ -97,6 +97,7 @@ def _extract_frame_range(
|
|
|
97
97
|
return True # Indicate success as frames are considered present
|
|
98
98
|
|
|
99
99
|
frame_dir.mkdir(parents=True, exist_ok=True)
|
|
100
|
+
extracted_paths = []
|
|
100
101
|
|
|
101
102
|
try:
|
|
102
103
|
logger.info("Starting frame range extraction [%d, %d) for video %s to %s", start_frame, end_frame, video.uuid, frame_dir)
|
|
@@ -111,6 +112,17 @@ def _extract_frame_range(
|
|
|
111
112
|
|
|
112
113
|
return True
|
|
113
114
|
|
|
115
|
+
except FileNotFoundError as err:
|
|
116
|
+
logger.error(
|
|
117
|
+
"Frame range extraction [%d, %d) failed for video %s: %s",
|
|
118
|
+
start_frame,
|
|
119
|
+
end_frame,
|
|
120
|
+
video.uuid,
|
|
121
|
+
err,
|
|
122
|
+
exc_info=True,
|
|
123
|
+
)
|
|
124
|
+
raise
|
|
125
|
+
|
|
114
126
|
except Exception as e:
|
|
115
127
|
logger.error("Frame range extraction [%d, %d) or DB update failed for video %s: %s", start_frame, end_frame, video.uuid, e, exc_info=True)
|
|
116
128
|
|
|
@@ -32,13 +32,15 @@ def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
|
|
|
32
32
|
if sensitive_path.exists():
|
|
33
33
|
return sensitive_path.resolve()
|
|
34
34
|
|
|
35
|
+
# Check direct raw_file.path if available
|
|
35
36
|
# Check direct raw_file.path if available
|
|
36
37
|
try:
|
|
37
38
|
direct_path = Path(video.raw_file.path)
|
|
38
39
|
if direct_path.exists():
|
|
39
40
|
return direct_path.resolve()
|
|
40
|
-
except Exception:
|
|
41
|
-
|
|
41
|
+
except Exception as e:
|
|
42
|
+
logger.debug("Could not access direct raw_file.path for video %s: %s", video.uuid, e)
|
|
43
|
+
# Fallback to checking alternative paths
|
|
42
44
|
|
|
43
45
|
# Check common alternative paths
|
|
44
46
|
alternative_paths = [
|