endoreg-db 0.8.4.3__py3-none-any.whl → 0.8.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/models/metadata/model_meta_logic.py +1 -96
- endoreg_db/models/metadata/sensitive_meta_logic.py +48 -143
- endoreg_db/services/video_import.py +6 -1
- {endoreg_db-0.8.4.3.dist-info → endoreg_db-0.8.4.4.dist-info}/METADATA +1 -2
- {endoreg_db-0.8.4.3.dist-info → endoreg_db-0.8.4.4.dist-info}/RECORD +7 -8
- endoreg_db/management/commands/validate_ai_models.py +0 -124
- {endoreg_db-0.8.4.3.dist-info → endoreg_db-0.8.4.4.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.4.3.dist-info → endoreg_db-0.8.4.4.dist-info}/licenses/LICENSE +0 -0
|
@@ -3,6 +3,7 @@ from logging import getLogger
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import TYPE_CHECKING, Any, Optional, Type
|
|
5
5
|
|
|
6
|
+
from django.core.files import File
|
|
6
7
|
from django.db import transaction
|
|
7
8
|
from huggingface_hub import hf_hub_download
|
|
8
9
|
|
|
@@ -326,99 +327,3 @@ def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str
|
|
|
326
327
|
size_y=meta["size_y"],
|
|
327
328
|
description=meta["description"],
|
|
328
329
|
)
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
def validate_and_fix_ai_model_metadata_logic():
|
|
332
|
-
"""
|
|
333
|
-
Validates that all AI models have proper active metadata and fixes any issues.
|
|
334
|
-
This prevents the "No model metadata found for this model" error.
|
|
335
|
-
|
|
336
|
-
Returns:
|
|
337
|
-
dict: Summary of fixes applied
|
|
338
|
-
"""
|
|
339
|
-
from ..administration.ai.ai_model import AiModel
|
|
340
|
-
from ..label.label_set import LabelSet
|
|
341
|
-
|
|
342
|
-
summary = {"models_checked": 0, "models_fixed": 0, "metadata_created": 0, "active_meta_set": 0, "errors": []}
|
|
343
|
-
|
|
344
|
-
try:
|
|
345
|
-
all_models = AiModel.objects.all()
|
|
346
|
-
summary["models_checked"] = all_models.count()
|
|
347
|
-
|
|
348
|
-
for model in all_models:
|
|
349
|
-
logger.info(f"Validating model: {model.name}")
|
|
350
|
-
|
|
351
|
-
# Check if model has metadata versions
|
|
352
|
-
metadata_count = model.metadata_versions.count()
|
|
353
|
-
|
|
354
|
-
if metadata_count == 0:
|
|
355
|
-
# Create metadata for models that don't have any
|
|
356
|
-
logger.info(f"Creating metadata for {model.name}")
|
|
357
|
-
|
|
358
|
-
# Use existing labelset or create default
|
|
359
|
-
labelset = LabelSet.objects.first()
|
|
360
|
-
if not labelset:
|
|
361
|
-
labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
|
|
362
|
-
|
|
363
|
-
# Import here to avoid circular imports
|
|
364
|
-
from .model_meta import ModelMeta
|
|
365
|
-
|
|
366
|
-
# Create basic metadata
|
|
367
|
-
meta = ModelMeta.objects.create(
|
|
368
|
-
name=model.name,
|
|
369
|
-
version="1.0",
|
|
370
|
-
model=model,
|
|
371
|
-
labelset=labelset,
|
|
372
|
-
activation="sigmoid" if "classification" in model.name else "sigmoid",
|
|
373
|
-
mean="0.485,0.456,0.406", # ImageNet defaults
|
|
374
|
-
std="0.229,0.224,0.225", # ImageNet defaults
|
|
375
|
-
size_x=224,
|
|
376
|
-
size_y=224,
|
|
377
|
-
axes="CHW",
|
|
378
|
-
batchsize=32,
|
|
379
|
-
num_workers=4,
|
|
380
|
-
description=f"Auto-generated metadata for {model.name}",
|
|
381
|
-
)
|
|
382
|
-
|
|
383
|
-
model.active_meta = meta
|
|
384
|
-
model.save()
|
|
385
|
-
summary["models_fixed"] += 1
|
|
386
|
-
summary["metadata_created"] += 1
|
|
387
|
-
logger.info(f"Created and set metadata for {model.name}")
|
|
388
|
-
|
|
389
|
-
elif not model.active_meta:
|
|
390
|
-
# Model has metadata but no active meta set
|
|
391
|
-
first_meta = model.metadata_versions.first()
|
|
392
|
-
if first_meta:
|
|
393
|
-
logger.info(f"Setting active metadata for {model.name}")
|
|
394
|
-
model.active_meta = first_meta
|
|
395
|
-
model.save()
|
|
396
|
-
summary["models_fixed"] += 1
|
|
397
|
-
summary["active_meta_set"] += 1
|
|
398
|
-
logger.info(f"Set active metadata: {first_meta.name} v{first_meta.version}")
|
|
399
|
-
else:
|
|
400
|
-
error_msg = f"No metadata versions available for {model.name}"
|
|
401
|
-
logger.warning(error_msg)
|
|
402
|
-
summary["errors"].append(error_msg)
|
|
403
|
-
|
|
404
|
-
else:
|
|
405
|
-
logger.info(f"Model {model.name} has valid active metadata: {model.active_meta}")
|
|
406
|
-
|
|
407
|
-
# Verify all models can get latest version
|
|
408
|
-
logger.info("Testing model metadata access...")
|
|
409
|
-
for model in all_models:
|
|
410
|
-
try:
|
|
411
|
-
latest = model.get_latest_version()
|
|
412
|
-
logger.info(f"✅ {model.name}: {latest}")
|
|
413
|
-
except Exception as e:
|
|
414
|
-
error_msg = f"Model {model.name} metadata test failed: {e}"
|
|
415
|
-
logger.error(error_msg)
|
|
416
|
-
summary["errors"].append(error_msg)
|
|
417
|
-
|
|
418
|
-
return summary
|
|
419
|
-
|
|
420
|
-
except Exception as e:
|
|
421
|
-
error_msg = f"Validation failed: {e}"
|
|
422
|
-
logger.error(error_msg)
|
|
423
|
-
summary["errors"].append(error_msg)
|
|
424
|
-
return summary
|
|
@@ -80,9 +80,7 @@ def parse_any_date(s: str) -> Optional[date]:
|
|
|
80
80
|
# Try dateparser with German locale preference
|
|
81
81
|
import dateparser
|
|
82
82
|
|
|
83
|
-
dt = dateparser.parse(
|
|
84
|
-
s, settings={"DATE_ORDER": "DMY", "PREFER_DAY_OF_MONTH": "first"}
|
|
85
|
-
)
|
|
83
|
+
dt = dateparser.parse(s, settings={"DATE_ORDER": "DMY", "PREFER_DAY_OF_MONTH": "first"})
|
|
86
84
|
return dt.date() if dt else None
|
|
87
85
|
except Exception as e:
|
|
88
86
|
logger.debug(f"Dateparser fallback failed for '{s}': {e}")
|
|
@@ -172,9 +170,7 @@ def calculate_patient_hash(instance: "SensitiveMeta", salt: str = SECRET_SALT) -
|
|
|
172
170
|
return sha256(hash_str.encode()).hexdigest()
|
|
173
171
|
|
|
174
172
|
|
|
175
|
-
def calculate_examination_hash(
|
|
176
|
-
instance: "SensitiveMeta", salt: str = SECRET_SALT
|
|
177
|
-
) -> str:
|
|
173
|
+
def calculate_examination_hash(instance: "SensitiveMeta", salt: str = SECRET_SALT) -> str:
|
|
178
174
|
"""Calculates the examination hash for the instance."""
|
|
179
175
|
dob = instance.patient_dob
|
|
180
176
|
first_name = instance.patient_first_name
|
|
@@ -207,25 +203,17 @@ def create_pseudo_examiner_logic(instance: "SensitiveMeta") -> "Examiner":
|
|
|
207
203
|
center = instance.center # Should be set before calling save
|
|
208
204
|
|
|
209
205
|
if not first_name or not last_name or not center:
|
|
210
|
-
logger.warning(
|
|
211
|
-
f"Incomplete examiner info for SensitiveMeta (pk={instance.pk}). Using default examiner."
|
|
212
|
-
)
|
|
206
|
+
logger.warning(f"Incomplete examiner info for SensitiveMeta (pk={instance.pk or 'new'}). Using default examiner.")
|
|
213
207
|
# Ensure default center exists or handle appropriately
|
|
214
208
|
try:
|
|
215
209
|
default_center = Center.objects.get_by_natural_key("endoreg_db_demo")
|
|
216
210
|
except Center.DoesNotExist:
|
|
217
|
-
logger.error(
|
|
218
|
-
"Default center 'endoreg_db_demo' not found. Cannot create default examiner."
|
|
219
|
-
)
|
|
211
|
+
logger.error("Default center 'endoreg_db_demo' not found. Cannot create default examiner.")
|
|
220
212
|
raise ValueError("Default center 'endoreg_db_demo' not found.")
|
|
221
213
|
|
|
222
|
-
examiner, _created = Examiner.custom_get_or_create(
|
|
223
|
-
first_name="Unknown", last_name="Unknown", center=default_center
|
|
224
|
-
)
|
|
214
|
+
examiner, _created = Examiner.custom_get_or_create(first_name="Unknown", last_name="Unknown", center=default_center)
|
|
225
215
|
else:
|
|
226
|
-
examiner, _created = Examiner.custom_get_or_create(
|
|
227
|
-
first_name=first_name, last_name=last_name, center=center
|
|
228
|
-
)
|
|
216
|
+
examiner, _created = Examiner.custom_get_or_create(first_name=first_name, last_name=last_name, center=center)
|
|
229
217
|
|
|
230
218
|
return examiner
|
|
231
219
|
|
|
@@ -271,13 +259,11 @@ def get_or_create_pseudo_patient_examination_logic(
|
|
|
271
259
|
pseudo_patient = get_or_create_pseudo_patient_logic(instance)
|
|
272
260
|
instance.pseudo_patient_id = pseudo_patient.pk # Assign FK directly
|
|
273
261
|
|
|
274
|
-
patient_examination, _created = (
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
# pseudo_patient=instance.pseudo_patient
|
|
280
|
-
)
|
|
262
|
+
patient_examination, _created = PatientExamination.get_or_create_pseudo_patient_examination_by_hash(
|
|
263
|
+
patient_hash=instance.patient_hash,
|
|
264
|
+
examination_hash=instance.examination_hash,
|
|
265
|
+
# Optionally pass pseudo_patient if the method requires it
|
|
266
|
+
# pseudo_patient=instance.pseudo_patient
|
|
281
267
|
)
|
|
282
268
|
return patient_examination
|
|
283
269
|
|
|
@@ -294,14 +280,10 @@ def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
|
|
|
294
280
|
|
|
295
281
|
# 1. Ensure DOB and Examination Date exist
|
|
296
282
|
if not instance.patient_dob:
|
|
297
|
-
logger.debug(
|
|
298
|
-
f"SensitiveMeta (pk={instance.pk}): Patient DOB missing, generating random."
|
|
299
|
-
)
|
|
283
|
+
logger.debug(f"SensitiveMeta (pk={instance.pk or 'new'}): Patient DOB missing, generating random.")
|
|
300
284
|
instance.patient_dob = generate_random_dob()
|
|
301
285
|
if not instance.examination_date:
|
|
302
|
-
logger.debug(
|
|
303
|
-
f"SensitiveMeta (pk={instance.pk}): Examination date missing, generating random."
|
|
304
|
-
)
|
|
286
|
+
logger.debug(f"SensitiveMeta (pk={instance.pk or 'new'}): Examination date missing, generating random.")
|
|
305
287
|
instance.examination_date = generate_random_examination_date()
|
|
306
288
|
|
|
307
289
|
# 2. Ensure Center exists (should be set before calling save)
|
|
@@ -314,9 +296,7 @@ def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
|
|
|
314
296
|
first_name = instance.patient_first_name or DEFAULT_UNKNOWN_NAME
|
|
315
297
|
gender = guess_name_gender(first_name)
|
|
316
298
|
if not gender:
|
|
317
|
-
raise ValueError(
|
|
318
|
-
"Patient gender could not be determined and must be set before saving."
|
|
319
|
-
)
|
|
299
|
+
raise ValueError("Patient gender could not be determined and must be set before saving.")
|
|
320
300
|
instance.patient_gender = gender
|
|
321
301
|
|
|
322
302
|
# 4. Calculate Hashes (depends on DOB, Exam Date, Center, Names)
|
|
@@ -344,16 +324,10 @@ def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
|
|
|
344
324
|
return examiner_instance
|
|
345
325
|
|
|
346
326
|
|
|
347
|
-
def create_sensitive_meta_from_dict(
|
|
348
|
-
cls: Type["SensitiveMeta"], data: Dict[str, Any]
|
|
349
|
-
) -> "SensitiveMeta":
|
|
327
|
+
def create_sensitive_meta_from_dict(cls: Type["SensitiveMeta"], data: Dict[str, Any]) -> "SensitiveMeta":
|
|
350
328
|
"""Logic to create a SensitiveMeta instance from a dictionary."""
|
|
351
329
|
|
|
352
|
-
field_names = {
|
|
353
|
-
f.name
|
|
354
|
-
for f in cls._meta.get_fields()
|
|
355
|
-
if not f.is_relation or f.one_to_one or (f.many_to_one and f.related_model)
|
|
356
|
-
}
|
|
330
|
+
field_names = {f.name for f in cls._meta.get_fields() if not f.is_relation or f.one_to_one or (f.many_to_one and f.related_model)}
|
|
357
331
|
selected_data = {k: v for k, v in data.items() if k in field_names}
|
|
358
332
|
|
|
359
333
|
# --- Convert patient_dob if it's a date object ---
|
|
@@ -380,13 +354,9 @@ def create_sensitive_meta_from_dict(
|
|
|
380
354
|
try:
|
|
381
355
|
import dateparser
|
|
382
356
|
|
|
383
|
-
parsed_dob = dateparser.parse(
|
|
384
|
-
dob, languages=["de"], settings={"DATE_ORDER": "DMY"}
|
|
385
|
-
)
|
|
357
|
+
parsed_dob = dateparser.parse(dob, languages=["de"], settings={"DATE_ORDER": "DMY"})
|
|
386
358
|
if parsed_dob:
|
|
387
|
-
aware_dob = timezone.make_aware(
|
|
388
|
-
parsed_dob.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
389
|
-
)
|
|
359
|
+
aware_dob = timezone.make_aware(parsed_dob.replace(hour=0, minute=0, second=0, microsecond=0))
|
|
390
360
|
selected_data["patient_dob"] = aware_dob
|
|
391
361
|
logger.debug(
|
|
392
362
|
"Parsed string patient_dob '%s' to aware datetime: %s",
|
|
@@ -440,9 +410,7 @@ def create_sensitive_meta_from_dict(
|
|
|
440
410
|
# Fall back to dateparser for complex formats
|
|
441
411
|
import dateparser
|
|
442
412
|
|
|
443
|
-
parsed_date = dateparser.parse(
|
|
444
|
-
exam_date, languages=["de"], settings={"DATE_ORDER": "DMY"}
|
|
445
|
-
)
|
|
413
|
+
parsed_date = dateparser.parse(exam_date, languages=["de"], settings={"DATE_ORDER": "DMY"})
|
|
446
414
|
if parsed_date:
|
|
447
415
|
selected_data["examination_date"] = parsed_date.date()
|
|
448
416
|
logger.debug(
|
|
@@ -460,9 +428,7 @@ def create_sensitive_meta_from_dict(
|
|
|
460
428
|
# Use dateparser for non-ISO formats
|
|
461
429
|
import dateparser
|
|
462
430
|
|
|
463
|
-
parsed_date = dateparser.parse(
|
|
464
|
-
exam_date, languages=["de"], settings={"DATE_ORDER": "DMY"}
|
|
465
|
-
)
|
|
431
|
+
parsed_date = dateparser.parse(exam_date, languages=["de"], settings={"DATE_ORDER": "DMY"})
|
|
466
432
|
if parsed_date:
|
|
467
433
|
selected_data["examination_date"] = parsed_date.date()
|
|
468
434
|
logger.debug(
|
|
@@ -508,34 +474,22 @@ def create_sensitive_meta_from_dict(
|
|
|
508
474
|
elif isinstance(patient_gender_input, str):
|
|
509
475
|
# Input is a string (gender name)
|
|
510
476
|
try:
|
|
511
|
-
selected_data["patient_gender"] = Gender.objects.get(
|
|
512
|
-
name=patient_gender_input
|
|
513
|
-
)
|
|
477
|
+
selected_data["patient_gender"] = Gender.objects.get(name=patient_gender_input)
|
|
514
478
|
except Gender.DoesNotExist:
|
|
515
|
-
logger.warning(
|
|
516
|
-
f"Gender with name '{patient_gender_input}' provided but not found. Attempting to guess or use default."
|
|
517
|
-
)
|
|
479
|
+
logger.warning(f"Gender with name '{patient_gender_input}' provided but not found. Attempting to guess or use default.")
|
|
518
480
|
# Fall through to guessing logic if provided string name is invalid
|
|
519
481
|
patient_gender_input = None # Reset to trigger guessing
|
|
520
482
|
|
|
521
|
-
if not isinstance(
|
|
522
|
-
selected_data.get("patient_gender"), Gender
|
|
523
|
-
): # If not already a Gender object (e.g. was None, or string lookup failed)
|
|
483
|
+
if not isinstance(selected_data.get("patient_gender"), Gender): # If not already a Gender object (e.g. was None, or string lookup failed)
|
|
524
484
|
gender_name_to_use = guess_name_gender(first_name)
|
|
525
485
|
if not gender_name_to_use:
|
|
526
|
-
logger.warning(
|
|
527
|
-
f"Could not guess gender for name '{first_name}'. Setting Gender to unknown."
|
|
528
|
-
)
|
|
486
|
+
logger.warning(f"Could not guess gender for name '{first_name}'. Setting Gender to unknown.")
|
|
529
487
|
gender_name_to_use = "unknown"
|
|
530
488
|
try:
|
|
531
|
-
selected_data["patient_gender"] = Gender.objects.get(
|
|
532
|
-
name=gender_name_to_use
|
|
533
|
-
)
|
|
489
|
+
selected_data["patient_gender"] = Gender.objects.get(name=gender_name_to_use)
|
|
534
490
|
except Gender.DoesNotExist:
|
|
535
491
|
# This should ideally not happen if "unknown" gender is guaranteed to exist
|
|
536
|
-
raise ValueError(
|
|
537
|
-
f"Default or guessed gender '{gender_name_to_use}' does not exist in Gender table."
|
|
538
|
-
)
|
|
492
|
+
raise ValueError(f"Default or guessed gender '{gender_name_to_use}' does not exist in Gender table.")
|
|
539
493
|
|
|
540
494
|
# Update name DB
|
|
541
495
|
update_name_db(first_name, last_name)
|
|
@@ -549,20 +503,12 @@ def create_sensitive_meta_from_dict(
|
|
|
549
503
|
return sensitive_meta
|
|
550
504
|
|
|
551
505
|
|
|
552
|
-
def update_sensitive_meta_from_dict(
|
|
553
|
-
instance: "SensitiveMeta", data: Dict[str, Any]
|
|
554
|
-
) -> "SensitiveMeta":
|
|
506
|
+
def update_sensitive_meta_from_dict(instance: "SensitiveMeta", data: Dict[str, Any]) -> "SensitiveMeta":
|
|
555
507
|
"""Logic to update a SensitiveMeta instance from a dictionary."""
|
|
556
|
-
field_names = {
|
|
557
|
-
f.name
|
|
558
|
-
for f in instance._meta.get_fields()
|
|
559
|
-
if not f.is_relation or f.one_to_one or (f.many_to_one and f.related_model)
|
|
560
|
-
}
|
|
508
|
+
field_names = {f.name for f in instance._meta.get_fields() if not f.is_relation or f.one_to_one or (f.many_to_one and f.related_model)}
|
|
561
509
|
# Exclude FKs that should not be updated directly from dict keys (handled separately or via save logic)
|
|
562
510
|
excluded_fields = {"pseudo_patient", "pseudo_examination"}
|
|
563
|
-
selected_data = {
|
|
564
|
-
k: v for k, v in data.items() if k in field_names and k not in excluded_fields
|
|
565
|
-
}
|
|
511
|
+
selected_data = {k: v for k, v in data.items() if k in field_names and k not in excluded_fields}
|
|
566
512
|
|
|
567
513
|
# Handle potential Center update
|
|
568
514
|
center_name = data.get("center_name")
|
|
@@ -571,9 +517,7 @@ def update_sensitive_meta_from_dict(
|
|
|
571
517
|
center = Center.objects.get_by_natural_key(center_name)
|
|
572
518
|
instance.center = center # Update center directly
|
|
573
519
|
except Center.DoesNotExist as exc:
|
|
574
|
-
logger.warning(
|
|
575
|
-
f"Center '{center_name}' not found during update. Keeping existing center."
|
|
576
|
-
)
|
|
520
|
+
logger.warning(f"Center '{center_name}' not found during update. Keeping existing center.")
|
|
577
521
|
selected_data.pop("center", None) # Remove from dict if not found
|
|
578
522
|
|
|
579
523
|
# Set examiner names if provided, before calling save
|
|
@@ -593,14 +537,10 @@ def update_sensitive_meta_from_dict(
|
|
|
593
537
|
elif isinstance(patient_gender_input, str):
|
|
594
538
|
gender_input_clean = patient_gender_input.strip()
|
|
595
539
|
# Try direct case-insensitive DB lookup first
|
|
596
|
-
gender_obj = Gender.objects.filter(
|
|
597
|
-
name__iexact=gender_input_clean
|
|
598
|
-
).first()
|
|
540
|
+
gender_obj = Gender.objects.filter(name__iexact=gender_input_clean).first()
|
|
599
541
|
if gender_obj:
|
|
600
542
|
selected_data["patient_gender"] = gender_obj
|
|
601
|
-
logger.debug(
|
|
602
|
-
f"Successfully matched gender string '{patient_gender_input}' to Gender object via iexact lookup"
|
|
603
|
-
)
|
|
543
|
+
logger.debug(f"Successfully matched gender string '{patient_gender_input}' to Gender object via iexact lookup")
|
|
604
544
|
else:
|
|
605
545
|
# Use mapping helper for fallback
|
|
606
546
|
mapped = _map_gender_string_to_standard(gender_input_clean)
|
|
@@ -608,50 +548,30 @@ def update_sensitive_meta_from_dict(
|
|
|
608
548
|
gender_obj = Gender.objects.filter(name__iexact=mapped).first()
|
|
609
549
|
if gender_obj:
|
|
610
550
|
selected_data["patient_gender"] = gender_obj
|
|
611
|
-
logger.info(
|
|
612
|
-
f"Mapped gender '{patient_gender_input}' to '{mapped}' via fallback mapping"
|
|
613
|
-
)
|
|
551
|
+
logger.info(f"Mapped gender '{patient_gender_input}' to '{mapped}' via fallback mapping")
|
|
614
552
|
else:
|
|
615
|
-
logger.warning(
|
|
616
|
-
|
|
617
|
-
)
|
|
618
|
-
unknown_gender = Gender.objects.filter(
|
|
619
|
-
name__iexact="unknown"
|
|
620
|
-
).first()
|
|
553
|
+
logger.warning(f"Mapped gender '{patient_gender_input}' to '{mapped}', but no such Gender in DB. Trying 'unknown'.")
|
|
554
|
+
unknown_gender = Gender.objects.filter(name__iexact="unknown").first()
|
|
621
555
|
if unknown_gender:
|
|
622
556
|
selected_data["patient_gender"] = unknown_gender
|
|
623
|
-
logger.warning(
|
|
624
|
-
f"Using 'unknown' gender as fallback for '{patient_gender_input}'"
|
|
625
|
-
)
|
|
557
|
+
logger.warning(f"Using 'unknown' gender as fallback for '{patient_gender_input}'")
|
|
626
558
|
else:
|
|
627
|
-
logger.error(
|
|
628
|
-
f"No 'unknown' gender found in database. Cannot handle gender '{patient_gender_input}'. Skipping gender update."
|
|
629
|
-
)
|
|
559
|
+
logger.error(f"No 'unknown' gender found in database. Cannot handle gender '{patient_gender_input}'. Skipping gender update.")
|
|
630
560
|
selected_data.pop("patient_gender", None)
|
|
631
561
|
else:
|
|
632
562
|
# Last resort: try to get 'unknown' gender
|
|
633
|
-
unknown_gender = Gender.objects.filter(
|
|
634
|
-
name__iexact="unknown"
|
|
635
|
-
).first()
|
|
563
|
+
unknown_gender = Gender.objects.filter(name__iexact="unknown").first()
|
|
636
564
|
if unknown_gender:
|
|
637
565
|
selected_data["patient_gender"] = unknown_gender
|
|
638
|
-
logger.warning(
|
|
639
|
-
f"Using 'unknown' gender as fallback for '{patient_gender_input}' (no mapping)"
|
|
640
|
-
)
|
|
566
|
+
logger.warning(f"Using 'unknown' gender as fallback for '{patient_gender_input}' (no mapping)")
|
|
641
567
|
else:
|
|
642
|
-
logger.error(
|
|
643
|
-
f"No 'unknown' gender found in database. Cannot handle gender '{patient_gender_input}'. Skipping gender update."
|
|
644
|
-
)
|
|
568
|
+
logger.error(f"No 'unknown' gender found in database. Cannot handle gender '{patient_gender_input}'. Skipping gender update.")
|
|
645
569
|
selected_data.pop("patient_gender", None)
|
|
646
570
|
else:
|
|
647
|
-
logger.warning(
|
|
648
|
-
f"Unexpected patient_gender type {type(patient_gender_input)}: {patient_gender_input}. Skipping gender update."
|
|
649
|
-
)
|
|
571
|
+
logger.warning(f"Unexpected patient_gender type {type(patient_gender_input)}: {patient_gender_input}. Skipping gender update.")
|
|
650
572
|
selected_data.pop("patient_gender", None)
|
|
651
573
|
except Exception as e:
|
|
652
|
-
logger.exception(
|
|
653
|
-
f"Error handling patient_gender '{patient_gender_input}': {e}. Skipping gender update."
|
|
654
|
-
)
|
|
574
|
+
logger.exception(f"Error handling patient_gender '{patient_gender_input}': {e}. Skipping gender update.")
|
|
655
575
|
selected_data.pop("patient_gender", None)
|
|
656
576
|
|
|
657
577
|
# Update other attributes from selected_data
|
|
@@ -668,9 +588,7 @@ def update_sensitive_meta_from_dict(
|
|
|
668
588
|
value_to_set = v
|
|
669
589
|
if k == "patient_dob":
|
|
670
590
|
if isinstance(v, date) and not isinstance(v, datetime):
|
|
671
|
-
aware_dob = timezone.make_aware(
|
|
672
|
-
datetime.combine(v, datetime.min.time())
|
|
673
|
-
)
|
|
591
|
+
aware_dob = timezone.make_aware(datetime.combine(v, datetime.min.time()))
|
|
674
592
|
value_to_set = aware_dob
|
|
675
593
|
logger.debug(
|
|
676
594
|
"Converted patient_dob from date to aware datetime during update: %s",
|
|
@@ -693,15 +611,9 @@ def update_sensitive_meta_from_dict(
|
|
|
693
611
|
try:
|
|
694
612
|
import dateparser
|
|
695
613
|
|
|
696
|
-
parsed_dob = dateparser.parse(
|
|
697
|
-
v, languages=["de"], settings={"DATE_ORDER": "DMY"}
|
|
698
|
-
)
|
|
614
|
+
parsed_dob = dateparser.parse(v, languages=["de"], settings={"DATE_ORDER": "DMY"})
|
|
699
615
|
if parsed_dob:
|
|
700
|
-
value_to_set = timezone.make_aware(
|
|
701
|
-
parsed_dob.replace(
|
|
702
|
-
hour=0, minute=0, second=0, microsecond=0
|
|
703
|
-
)
|
|
704
|
-
)
|
|
616
|
+
value_to_set = timezone.make_aware(parsed_dob.replace(hour=0, minute=0, second=0, microsecond=0))
|
|
705
617
|
logger.debug(
|
|
706
618
|
"Parsed string patient_dob '%s' during update to aware datetime: %s",
|
|
707
619
|
v,
|
|
@@ -736,9 +648,7 @@ def update_sensitive_meta_from_dict(
|
|
|
736
648
|
try:
|
|
737
649
|
import dateparser
|
|
738
650
|
|
|
739
|
-
parsed_date = dateparser.parse(
|
|
740
|
-
v, languages=["de"], settings={"DATE_ORDER": "DMY"}
|
|
741
|
-
)
|
|
651
|
+
parsed_date = dateparser.parse(v, languages=["de"], settings={"DATE_ORDER": "DMY"})
|
|
742
652
|
if parsed_date:
|
|
743
653
|
value_to_set = parsed_date.date()
|
|
744
654
|
logger.debug(
|
|
@@ -762,18 +672,13 @@ def update_sensitive_meta_from_dict(
|
|
|
762
672
|
# --- End Conversion ---
|
|
763
673
|
|
|
764
674
|
# Check if patient name is changing
|
|
765
|
-
if (
|
|
766
|
-
k in ["patient_first_name", "patient_last_name"]
|
|
767
|
-
and getattr(instance, k) != value_to_set
|
|
768
|
-
):
|
|
675
|
+
if k in ["patient_first_name", "patient_last_name"] and getattr(instance, k) != value_to_set:
|
|
769
676
|
patient_name_changed = True
|
|
770
677
|
|
|
771
678
|
setattr(instance, k, value_to_set) # Use value_to_set
|
|
772
679
|
|
|
773
680
|
except Exception as e:
|
|
774
|
-
logger.error(
|
|
775
|
-
f"Error setting attribute '{k}' to '{v}': {e}. Skipping this field."
|
|
776
|
-
)
|
|
681
|
+
logger.error(f"Error setting attribute '{k}' to '{v}': {e}. Skipping this field.")
|
|
777
682
|
continue
|
|
778
683
|
|
|
779
684
|
# Update name DB if patient names changed
|
|
@@ -19,6 +19,7 @@ from contextlib import contextmanager
|
|
|
19
19
|
from pathlib import Path
|
|
20
20
|
from typing import Union, Dict, Any, Optional, List, Tuple
|
|
21
21
|
from django.db import transaction
|
|
22
|
+
from lx_anonymizer import FrameCleaner
|
|
22
23
|
from moviepy import video
|
|
23
24
|
from endoreg_db.models import VideoFile, SensitiveMeta
|
|
24
25
|
from endoreg_db.utils.paths import STORAGE_DIR, VIDEO_DIR, ANONYM_VIDEO_DIR
|
|
@@ -76,6 +77,8 @@ class VideoImportService:
|
|
|
76
77
|
|
|
77
78
|
self.logger = logging.getLogger(__name__)
|
|
78
79
|
|
|
80
|
+
self.cleaner = None # This gets instantiated in the perform_frame_cleaning method
|
|
81
|
+
|
|
79
82
|
def _require_current_video(self) -> VideoFile:
|
|
80
83
|
"""Return the current VideoFile or raise if it has not been initialized."""
|
|
81
84
|
if self.current_video is None:
|
|
@@ -826,7 +829,7 @@ class VideoImportService:
|
|
|
826
829
|
from lx_anonymizer import FrameCleaner # type: ignore[import]
|
|
827
830
|
|
|
828
831
|
if FrameCleaner:
|
|
829
|
-
return True, FrameCleaner
|
|
832
|
+
return True, FrameCleaner()
|
|
830
833
|
|
|
831
834
|
except Exception as e:
|
|
832
835
|
self.logger.warning(f"Frame cleaning not available: {e} Please install or update lx_anonymizer.")
|
|
@@ -854,6 +857,8 @@ class VideoImportService:
|
|
|
854
857
|
# Create temporary output path for cleaned video
|
|
855
858
|
video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name)
|
|
856
859
|
cleaned_filename = f"cleaned_{video_filename}"
|
|
860
|
+
if not raw_video_path:
|
|
861
|
+
raise RuntimeError("raw_video_path is None after fallback, cannot construct cleaned_video_path")
|
|
857
862
|
cleaned_video_path = Path(raw_video_path).parent / cleaned_filename
|
|
858
863
|
|
|
859
864
|
# Clean video with ROI masking (heavy I/O operation)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: endoreg-db
|
|
3
|
-
Version: 0.8.4.
|
|
3
|
+
Version: 0.8.4.4
|
|
4
4
|
Summary: EndoReg Db Django App
|
|
5
5
|
Project-URL: Homepage, https://info.coloreg.de
|
|
6
6
|
Project-URL: Repository, https://github.com/wg-lux/endoreg-db
|
|
@@ -29,7 +29,6 @@ Requires-Dist: dotenv>=0.9.9
|
|
|
29
29
|
Requires-Dist: faker>=37.6.0
|
|
30
30
|
Requires-Dist: flake8>=7.3.0
|
|
31
31
|
Requires-Dist: gunicorn>=23.0.0
|
|
32
|
-
Requires-Dist: huggingface-hub>=0.35.3
|
|
33
32
|
Requires-Dist: icecream>=2.1.4
|
|
34
33
|
Requires-Dist: librosa==0.11.0
|
|
35
34
|
Requires-Dist: llvmlite>=0.44.0
|
|
@@ -295,7 +295,6 @@ endoreg_db/management/commands/setup_endoreg_db.py,sha256=C0ooVVmrzX-_ksoma3s3rZ
|
|
|
295
295
|
endoreg_db/management/commands/start_filewatcher.py,sha256=3jESBqRiYPa9f35--zd70qQaYnyT0tzRO_b_HJuyteQ,4093
|
|
296
296
|
endoreg_db/management/commands/storage_management.py,sha256=NpToX59ndwTFNmnSoeppmiPdMvpjSHH7mAdIe4SvUoI,22396
|
|
297
297
|
endoreg_db/management/commands/summarize_db_content.py,sha256=pOIz3qbY4Ktmh0zV_DKFx971VD0pPx027gCD7a47EL0,10766
|
|
298
|
-
endoreg_db/management/commands/validate_ai_models.py,sha256=Z7Ga-PndTFVG8GnkYbS58h8ofiyhnxZDcyP5Qqpl1c8,4684
|
|
299
298
|
endoreg_db/management/commands/validate_video.py,sha256=cns_kNgztyp6XTeXuDeLEet8vAATkpxZwJuSWuQ5Olk,11302
|
|
300
299
|
endoreg_db/management/commands/validate_video_files.py,sha256=0lvA0Z8BKiibjyqc4ueI646IIc5bKI3sIOxiiF5_bTk,6509
|
|
301
300
|
endoreg_db/management/commands/video_validation.py,sha256=xnAoCPB44dmnRbn6FqUjqRXQ-ZhDPNX1T5kCpAU8sgc,771
|
|
@@ -466,10 +465,10 @@ endoreg_db/models/medical/risk/risk_type.py,sha256=kEugcaWSTEWH_Vxq4dcF80Iv1L4_K
|
|
|
466
465
|
endoreg_db/models/metadata/__init__.py,sha256=8I6oLj3YTmeaPGJpL0AWG5gLwp38QzrEggxSkTisv7c,474
|
|
467
466
|
endoreg_db/models/metadata/frame_ocr_result.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
468
467
|
endoreg_db/models/metadata/model_meta.py,sha256=F_r-PTLeNi4J-4EaGCQkGIguhdl7Bwba7_i56ZAjc-4,7589
|
|
469
|
-
endoreg_db/models/metadata/model_meta_logic.py,sha256=
|
|
468
|
+
endoreg_db/models/metadata/model_meta_logic.py,sha256=OhPImTQaB_a3p-h7iaxNNQkip5aaDDIOMXCV2TEyNmk,12215
|
|
470
469
|
endoreg_db/models/metadata/pdf_meta.py,sha256=BTmpSgqxmPKi0apcNjyrZAS4AFKCPXVdBd6VBeyyv6E,3174
|
|
471
470
|
endoreg_db/models/metadata/sensitive_meta.py,sha256=ekLHrW-b5uYcjfkRd0EW5ncx5ef8Bu-K6msDkpWCAbk,13034
|
|
472
|
-
endoreg_db/models/metadata/sensitive_meta_logic.py,sha256=
|
|
471
|
+
endoreg_db/models/metadata/sensitive_meta_logic.py,sha256=XN3x3p0cqLlzPSZl7e35JBUXr_QKYSq48vwF1N60N4U,32134
|
|
473
472
|
endoreg_db/models/metadata/video_meta.py,sha256=c6xWdLW3uNqJ5VPJXHCxXA3mbXw-b0uR54-TOS3qL2Q,14966
|
|
474
473
|
endoreg_db/models/metadata/video_prediction_logic.py,sha256=j5N82mHtiomeeIaf1HA65kT5d0htQfJmbI2bJb8mpxQ,7677
|
|
475
474
|
endoreg_db/models/metadata/video_prediction_meta.py,sha256=EyfctAAAVcW9L0gf76ZBc9-G8MLMcD-tc2kkjaaLH4w,10592
|
|
@@ -604,7 +603,7 @@ endoreg_db/services/pseudonym_service.py,sha256=CJhbtRa6K6SPbphgCZgEMi8AFQtB18CU
|
|
|
604
603
|
endoreg_db/services/requirements_object.py,sha256=290zf8AEbVtCoHhW4Jr7_ud-RvrqYmb1Nz9UBHtTnc0,6164
|
|
605
604
|
endoreg_db/services/segment_sync.py,sha256=YgHvIHkbW4mqCu0ACf3zjRSZnNfxWwt4gh5syUVXuE0,6400
|
|
606
605
|
endoreg_db/services/storage_aware_video_processor.py,sha256=kKFK64vXLeBSVkp1YJonU3gFDTeXZ8C4qb9QZZB99SE,13420
|
|
607
|
-
endoreg_db/services/video_import.py,sha256=
|
|
606
|
+
endoreg_db/services/video_import.py,sha256=quQ1z_hqJFcrq5pOeq74pqqmnS2I9ybJIh5D0v8qtOk,46514
|
|
608
607
|
endoreg_db/tasks/upload_tasks.py,sha256=OJq7DhNwcbWdXzHY8jz5c51BCVkPN5gSWOz-6Fx6W5M,7799
|
|
609
608
|
endoreg_db/tasks/video_ingest.py,sha256=kxFuYkHijINV0VabQKCFVpJRv6eCAw07tviONurDgg8,5265
|
|
610
609
|
endoreg_db/tasks/video_processing_tasks.py,sha256=rZ7Kr49bAR4Q-vALO2SURebrhcJ5hSFGwjF4aULrOao,14089
|
|
@@ -789,7 +788,7 @@ endoreg_db/views/video/video_meta.py,sha256=C1wBMTtQb_yzEUrhFGAy2UHEWMk_CbU75WXX
|
|
|
789
788
|
endoreg_db/views/video/video_processing_history.py,sha256=mhFuS8RG5GV8E-lTtuD0qrq-bIpnUFp8vy9aERfC-J8,770
|
|
790
789
|
endoreg_db/views/video/video_remove_frames.py,sha256=2FmvNrSPM0fUXiBxINN6vBUUDCqDlBkNcGR3WsLDgKo,1696
|
|
791
790
|
endoreg_db/views/video/video_stream.py,sha256=kLyuf0ORTmsLeYUQkTQ6iRYqlIQozWhMMR3Lhfe_trk,12148
|
|
792
|
-
endoreg_db-0.8.4.
|
|
793
|
-
endoreg_db-0.8.4.
|
|
794
|
-
endoreg_db-0.8.4.
|
|
795
|
-
endoreg_db-0.8.4.
|
|
791
|
+
endoreg_db-0.8.4.4.dist-info/METADATA,sha256=GGQH5jWgXwF4K0rTdGb5jxxeVfRbbMdtmLf9b7MO_M0,14719
|
|
792
|
+
endoreg_db-0.8.4.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
793
|
+
endoreg_db-0.8.4.4.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
794
|
+
endoreg_db-0.8.4.4.dist-info/RECORD,,
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Django management command to validate and fix AI model metadata issues.
|
|
3
|
-
This command addresses the "No model metadata found for this model" error.
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
from django.core.management.base import BaseCommand
|
|
7
|
-
|
|
8
|
-
from endoreg_db.models.metadata.model_meta_logic import validate_and_fix_ai_model_metadata_logic
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class Command(BaseCommand):
|
|
12
|
-
help = """
|
|
13
|
-
Validate and fix AI model metadata to prevent processing errors.
|
|
14
|
-
|
|
15
|
-
This command:
|
|
16
|
-
- Checks all AI models for proper metadata configuration
|
|
17
|
-
- Creates missing metadata with sensible defaults
|
|
18
|
-
- Sets active metadata for models that don't have it
|
|
19
|
-
- Validates that all models can access their latest versions
|
|
20
|
-
|
|
21
|
-
Use this command to fix "No model metadata found for this model" errors.
|
|
22
|
-
"""
|
|
23
|
-
|
|
24
|
-
def add_arguments(self, parser):
|
|
25
|
-
parser.add_argument(
|
|
26
|
-
"--dry-run",
|
|
27
|
-
action="store_true",
|
|
28
|
-
help="Show what would be fixed without making changes",
|
|
29
|
-
)
|
|
30
|
-
parser.add_argument(
|
|
31
|
-
"--force",
|
|
32
|
-
action="store_true",
|
|
33
|
-
help="Force recreation of existing metadata",
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
def handle(self, *args, **options):
|
|
37
|
-
dry_run = options.get("dry_run", False)
|
|
38
|
-
force = options.get("force", False)
|
|
39
|
-
|
|
40
|
-
self.stdout.write(self.style.SUCCESS("🔍 Validating AI model metadata..."))
|
|
41
|
-
|
|
42
|
-
if dry_run:
|
|
43
|
-
self.stdout.write(self.style.WARNING("🧪 DRY RUN MODE - No changes will be made"))
|
|
44
|
-
|
|
45
|
-
try:
|
|
46
|
-
if dry_run:
|
|
47
|
-
# In dry run, we just check and report issues
|
|
48
|
-
summary = self._check_ai_models_dry_run()
|
|
49
|
-
else:
|
|
50
|
-
# Actually fix the issues
|
|
51
|
-
summary = validate_and_fix_ai_model_metadata_logic()
|
|
52
|
-
|
|
53
|
-
# Report results
|
|
54
|
-
self._report_summary(summary)
|
|
55
|
-
|
|
56
|
-
if summary["errors"]:
|
|
57
|
-
self.stdout.write(self.style.ERROR("❌ Validation completed with errors"))
|
|
58
|
-
for error in summary["errors"]:
|
|
59
|
-
self.stdout.write(self.style.ERROR(f" - {error}"))
|
|
60
|
-
return
|
|
61
|
-
|
|
62
|
-
self.stdout.write(self.style.SUCCESS("✅ AI model metadata validation completed successfully"))
|
|
63
|
-
|
|
64
|
-
except Exception as e:
|
|
65
|
-
self.stdout.write(self.style.ERROR(f"❌ Validation failed: {e}"))
|
|
66
|
-
raise
|
|
67
|
-
|
|
68
|
-
def _check_ai_models_dry_run(self):
|
|
69
|
-
"""Check AI models without making changes."""
|
|
70
|
-
from endoreg_db.models import AiModel
|
|
71
|
-
|
|
72
|
-
summary = {"models_checked": 0, "models_fixed": 0, "metadata_created": 0, "active_meta_set": 0, "errors": []}
|
|
73
|
-
|
|
74
|
-
all_models = AiModel.objects.all()
|
|
75
|
-
summary["models_checked"] = all_models.count()
|
|
76
|
-
|
|
77
|
-
for model in all_models:
|
|
78
|
-
self.stdout.write(f"Checking model: {model.name}")
|
|
79
|
-
|
|
80
|
-
metadata_count = model.metadata_versions.count()
|
|
81
|
-
self.stdout.write(f" Metadata versions: {metadata_count}")
|
|
82
|
-
|
|
83
|
-
if metadata_count == 0:
|
|
84
|
-
self.stdout.write(f" 🔧 Would create metadata for {model.name}")
|
|
85
|
-
summary["models_fixed"] += 1
|
|
86
|
-
summary["metadata_created"] += 1
|
|
87
|
-
|
|
88
|
-
elif not model.active_meta:
|
|
89
|
-
self.stdout.write(f" 🔧 Would set active metadata for {model.name}")
|
|
90
|
-
summary["models_fixed"] += 1
|
|
91
|
-
summary["active_meta_set"] += 1
|
|
92
|
-
|
|
93
|
-
else:
|
|
94
|
-
self.stdout.write(f" ✅ Model {model.name} has valid active metadata")
|
|
95
|
-
|
|
96
|
-
# Test metadata access
|
|
97
|
-
try:
|
|
98
|
-
latest = model.get_latest_version()
|
|
99
|
-
self.stdout.write(f" ✅ Can access latest version: {latest}")
|
|
100
|
-
except Exception as e:
|
|
101
|
-
error_msg = f"Model {model.name} metadata test failed: {e}"
|
|
102
|
-
self.stdout.write(self.style.ERROR(f" ❌ {error_msg}"))
|
|
103
|
-
summary["errors"].append(error_msg)
|
|
104
|
-
|
|
105
|
-
return summary
|
|
106
|
-
|
|
107
|
-
def _report_summary(self, summary):
|
|
108
|
-
"""Report the validation summary."""
|
|
109
|
-
self.stdout.write("\n📊 Validation Summary:")
|
|
110
|
-
self.stdout.write(f" Models checked: {summary['models_checked']}")
|
|
111
|
-
|
|
112
|
-
if summary["models_fixed"] > 0:
|
|
113
|
-
self.stdout.write(f" Models fixed: {summary['models_fixed']}")
|
|
114
|
-
|
|
115
|
-
if summary["metadata_created"] > 0:
|
|
116
|
-
self.stdout.write(f" Metadata created: {summary['metadata_created']}")
|
|
117
|
-
|
|
118
|
-
if summary["active_meta_set"] > 0:
|
|
119
|
-
self.stdout.write(f" Active metadata set: {summary['active_meta_set']}")
|
|
120
|
-
|
|
121
|
-
if summary["errors"]:
|
|
122
|
-
self.stdout.write(f" Errors encountered: {len(summary['errors'])}")
|
|
123
|
-
else:
|
|
124
|
-
self.stdout.write(" No errors found")
|
|
File without changes
|
|
File without changes
|