endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +146 -83
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/migrations/__init__.py +0 -0
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -3
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/ffmpeg_wrapper.py +217 -52
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +436 -433
- endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +0 -119
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import shutil
|
|
3
|
-
import uuid
|
|
4
3
|
from pathlib import Path
|
|
5
4
|
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
|
|
6
5
|
|
|
@@ -53,48 +52,92 @@ def _create_anonymized_frame_files(
|
|
|
53
52
|
generated_paths = []
|
|
54
53
|
frame_iterator = frames.filter(is_extracted=True).iterator()
|
|
55
54
|
total_frames = frames.filter(is_extracted=True).count()
|
|
56
|
-
progress_bar = tqdm(
|
|
55
|
+
progress_bar = tqdm(
|
|
56
|
+
frame_iterator,
|
|
57
|
+
total=total_frames,
|
|
58
|
+
desc=f"Anonymizing frames for {video.video_hash}",
|
|
59
|
+
)
|
|
57
60
|
|
|
58
61
|
for frame_obj in progress_bar:
|
|
59
62
|
try:
|
|
60
63
|
_frame_number = frame_obj.frame_number
|
|
61
|
-
target_path =
|
|
64
|
+
target_path = (
|
|
65
|
+
anonymized_frame_dir / f"frame_{frame_obj.frame_number:07d}.jpg"
|
|
66
|
+
)
|
|
62
67
|
make_all_black = frame_obj.frame_number in outside_frame_numbers
|
|
63
68
|
|
|
64
69
|
try:
|
|
65
70
|
source_path = frame_obj.file_path
|
|
66
71
|
if not isinstance(source_path, Path):
|
|
67
|
-
raise TypeError(
|
|
72
|
+
raise TypeError(
|
|
73
|
+
f"Frame.file_path did not return a Path object for frame {frame_obj.frame_number}"
|
|
74
|
+
)
|
|
68
75
|
except (AttributeError, TypeError, Exception) as path_err:
|
|
69
76
|
logger.error(
|
|
70
|
-
"Could not determine source path for Frame %d (PK: %s) using frame_obj.file_path: %s",
|
|
77
|
+
"Could not determine source path for Frame %d (PK: %s) using frame_obj.file_path: %s",
|
|
78
|
+
frame_obj.frame_number,
|
|
79
|
+
frame_obj.pk,
|
|
80
|
+
path_err,
|
|
71
81
|
)
|
|
72
|
-
raise RuntimeError(
|
|
82
|
+
raise RuntimeError(
|
|
83
|
+
f"Failed to get source path for frame {frame_obj.frame_number}"
|
|
84
|
+
) from path_err
|
|
73
85
|
|
|
74
86
|
if not source_path.exists():
|
|
75
87
|
error_msg = (
|
|
76
88
|
f"CRITICAL INCONSISTENCY: Source frame file missing for frame {frame_obj.frame_number} "
|
|
77
|
-
f"(PK: {frame_obj.pk}, Path: {source_path}) despite is_extracted=True for video {video.
|
|
89
|
+
f"(PK: {frame_obj.pk}, Path: {source_path}) despite is_extracted=True for video {video.video_hash}. "
|
|
78
90
|
f"Halting anonymization."
|
|
79
91
|
)
|
|
80
92
|
logger.error(error_msg)
|
|
81
93
|
raise FileNotFoundError(error_msg)
|
|
82
94
|
|
|
83
|
-
anonymize_frame(
|
|
95
|
+
anonymize_frame(
|
|
96
|
+
raw_frame_path=source_path,
|
|
97
|
+
target_frame_path=target_path,
|
|
98
|
+
endo_roi=endo_roi,
|
|
99
|
+
all_black=make_all_black,
|
|
100
|
+
censor_color=censor_color,
|
|
101
|
+
)
|
|
84
102
|
|
|
85
103
|
generated_paths.append(target_path)
|
|
86
|
-
except (
|
|
87
|
-
|
|
88
|
-
|
|
104
|
+
except (
|
|
105
|
+
FileNotFoundError,
|
|
106
|
+
IOError,
|
|
107
|
+
ValueError,
|
|
108
|
+
AttributeError,
|
|
109
|
+
TypeError,
|
|
110
|
+
Exception,
|
|
111
|
+
) as e:
|
|
112
|
+
logger.error(
|
|
113
|
+
"Error anonymizing frame %d (PK: %s): %s",
|
|
114
|
+
frame_obj.frame_number,
|
|
115
|
+
frame_obj.pk,
|
|
116
|
+
e,
|
|
117
|
+
exc_info=True,
|
|
118
|
+
)
|
|
119
|
+
raise RuntimeError(
|
|
120
|
+
f"Failed to anonymize frame {frame_obj.frame_number}"
|
|
121
|
+
) from e
|
|
89
122
|
|
|
90
123
|
if len(generated_paths) != total_frames:
|
|
91
|
-
logger.error(
|
|
92
|
-
|
|
124
|
+
logger.error(
|
|
125
|
+
"Mismatch in generated frames count. Expected %d, got %d.",
|
|
126
|
+
total_frames,
|
|
127
|
+
len(generated_paths),
|
|
128
|
+
)
|
|
129
|
+
raise RuntimeError(
|
|
130
|
+
"Anonymized frame generation resulted in incorrect number of files."
|
|
131
|
+
)
|
|
93
132
|
|
|
94
133
|
return generated_paths
|
|
95
134
|
|
|
96
135
|
|
|
97
|
-
def _censor_outside_frames(
|
|
136
|
+
def _censor_outside_frames(
|
|
137
|
+
video: "VideoFile",
|
|
138
|
+
outside_label_name: str = "outside",
|
|
139
|
+
censor_color: Tuple[int, int, int] = (0, 0, 0),
|
|
140
|
+
) -> bool:
|
|
98
141
|
"""
|
|
99
142
|
Overwrites frame files marked as 'outside' with a censored version (e.g., black).
|
|
100
143
|
This modifies the original raw frames directly. Use with caution. Requires frames to be extracted.
|
|
@@ -104,14 +147,21 @@ def _censor_outside_frames(video: "VideoFile", outside_label_name: str = "outsid
|
|
|
104
147
|
- Pre-condition: Requires state.frames_extracted=True.
|
|
105
148
|
- Post-condition: No state changes.
|
|
106
149
|
"""
|
|
107
|
-
logger.warning(
|
|
150
|
+
logger.warning(
|
|
151
|
+
"Starting direct censoring of 'outside' frames for video %s. This modifies raw frame files.",
|
|
152
|
+
video.video_hash,
|
|
153
|
+
)
|
|
108
154
|
state = video.get_or_create_state()
|
|
109
155
|
if not state.frames_extracted:
|
|
110
|
-
raise ValueError(
|
|
156
|
+
raise ValueError(
|
|
157
|
+
f"Frames not extracted for video {video.video_hash}. Cannot censor."
|
|
158
|
+
)
|
|
111
159
|
|
|
112
160
|
outside_frames = _get_outside_frames(video, outside_label_name)
|
|
113
161
|
if not outside_frames:
|
|
114
|
-
logger.info(
|
|
162
|
+
logger.info(
|
|
163
|
+
"No 'outside' frames found to censor for video %s.", video.video_hash
|
|
164
|
+
)
|
|
115
165
|
return True
|
|
116
166
|
|
|
117
167
|
censored_count = 0
|
|
@@ -120,17 +170,24 @@ def _censor_outside_frames(video: "VideoFile", outside_label_name: str = "outsid
|
|
|
120
170
|
try:
|
|
121
171
|
frame_path = frame_obj.file_path
|
|
122
172
|
if not frame_path:
|
|
123
|
-
logger.warning(
|
|
173
|
+
logger.warning(
|
|
174
|
+
"Could not get file path for frame %d. Skipping censoring.",
|
|
175
|
+
frame_obj.frame_number,
|
|
176
|
+
)
|
|
124
177
|
error_count += 1
|
|
125
178
|
continue
|
|
126
179
|
|
|
127
180
|
if not frame_path.exists():
|
|
128
|
-
logger.warning(
|
|
181
|
+
logger.warning(
|
|
182
|
+
"Frame file %s not found for censoring. Skipping.", frame_path
|
|
183
|
+
)
|
|
129
184
|
continue
|
|
130
185
|
|
|
131
186
|
img = cv2.imread(str(frame_path))
|
|
132
187
|
if img is None:
|
|
133
|
-
logger.warning(
|
|
188
|
+
logger.warning(
|
|
189
|
+
"Could not read frame %s for censoring. Skipping.", frame_path
|
|
190
|
+
)
|
|
134
191
|
continue
|
|
135
192
|
|
|
136
193
|
img[:] = censor_color
|
|
@@ -142,14 +199,27 @@ def _censor_outside_frames(video: "VideoFile", outside_label_name: str = "outsid
|
|
|
142
199
|
error_count += 1
|
|
143
200
|
|
|
144
201
|
except Exception as e:
|
|
145
|
-
logger.error(
|
|
202
|
+
logger.error(
|
|
203
|
+
"Error censoring frame %d (%s): %s",
|
|
204
|
+
frame_obj.frame_number,
|
|
205
|
+
getattr(frame_obj, "relative_path", "N/A"),
|
|
206
|
+
e,
|
|
207
|
+
exc_info=True,
|
|
208
|
+
)
|
|
146
209
|
error_count += 1
|
|
147
210
|
|
|
148
|
-
logger.info(
|
|
211
|
+
logger.info(
|
|
212
|
+
"Finished censoring for video %s. Censored: %d, Errors: %d",
|
|
213
|
+
video.video_hash,
|
|
214
|
+
censored_count,
|
|
215
|
+
error_count,
|
|
216
|
+
)
|
|
149
217
|
return error_count == 0
|
|
150
218
|
|
|
151
219
|
|
|
152
|
-
def _make_temporary_anonymized_frames(
|
|
220
|
+
def _make_temporary_anonymized_frames(
|
|
221
|
+
video: "VideoFile", roi_processing=True
|
|
222
|
+
) -> Tuple[Path, List[Path]]:
|
|
153
223
|
"""
|
|
154
224
|
Creates temporary anonymized frames in a separate directory.
|
|
155
225
|
Requires raw file and extracted frames. Raises ValueError or RuntimeError on failure.
|
|
@@ -159,43 +229,73 @@ def _make_temporary_anonymized_frames(video: "VideoFile", roi_processing = True)
|
|
|
159
229
|
- Post-condition: No state changes directly by this function.
|
|
160
230
|
"""
|
|
161
231
|
if video.is_processed:
|
|
162
|
-
raise ValueError(
|
|
232
|
+
raise ValueError(
|
|
233
|
+
f"Cannot create temporary anonymized frames for video {video.video_hash}: already processed."
|
|
234
|
+
)
|
|
163
235
|
if not video.has_raw:
|
|
164
|
-
raise ValueError(
|
|
236
|
+
raise ValueError(
|
|
237
|
+
f"Cannot create temporary anonymized frames for video {video.video_hash}: Raw file is missing."
|
|
238
|
+
)
|
|
165
239
|
|
|
166
240
|
temp_anonym_frame_dir = video.get_temp_anonymized_frame_dir()
|
|
167
241
|
temp_anonym_frame_dir.mkdir(parents=True, exist_ok=True)
|
|
168
|
-
logger.info(
|
|
242
|
+
logger.info(
|
|
243
|
+
"Creating temporary anonymized frames for video %s in %s",
|
|
244
|
+
video.video_hash,
|
|
245
|
+
temp_anonym_frame_dir,
|
|
246
|
+
)
|
|
169
247
|
if roi_processing:
|
|
170
248
|
endo_roi = video.get_endo_roi()
|
|
171
249
|
if not validate_endo_roi(endo_roi_dict=endo_roi):
|
|
172
|
-
raise ValueError(f"Endoscope ROI is not valid for video {video.
|
|
250
|
+
raise ValueError(f"Endoscope ROI is not valid for video {video.video_hash}")
|
|
173
251
|
else:
|
|
174
|
-
endo_roi = {
|
|
252
|
+
endo_roi = {
|
|
253
|
+
"x": 0,
|
|
254
|
+
"y": 0,
|
|
255
|
+
"width": 0,
|
|
256
|
+
"height": 0,
|
|
257
|
+
} # Dummy ROI to skip processing
|
|
175
258
|
assert endo_roi is not None # For type checker
|
|
176
|
-
|
|
177
|
-
|
|
178
259
|
|
|
179
260
|
state = video.get_or_create_state()
|
|
180
261
|
if not state.frames_extracted:
|
|
181
|
-
logger.info(
|
|
262
|
+
logger.info(
|
|
263
|
+
"Raw frames not extracted for %s, extracting now.", video.video_hash
|
|
264
|
+
)
|
|
182
265
|
try:
|
|
183
266
|
if not video.extract_frames(overwrite=False):
|
|
184
|
-
raise RuntimeError(
|
|
267
|
+
raise RuntimeError(
|
|
268
|
+
f"Frame extraction method returned False unexpectedly for video {video.video_hash}."
|
|
269
|
+
)
|
|
185
270
|
state.refresh_from_db()
|
|
186
271
|
if not state.frames_extracted:
|
|
187
|
-
raise RuntimeError(
|
|
272
|
+
raise RuntimeError(
|
|
273
|
+
f"Frame extraction did not update state for video {video.video_hash}, cannot create anonymized frames."
|
|
274
|
+
)
|
|
188
275
|
except Exception as extract_e:
|
|
189
|
-
logger.error(
|
|
190
|
-
|
|
276
|
+
logger.error(
|
|
277
|
+
"Frame extraction failed during anonymization prep for video %s: %s",
|
|
278
|
+
video.video_hash,
|
|
279
|
+
extract_e,
|
|
280
|
+
exc_info=True,
|
|
281
|
+
)
|
|
282
|
+
raise RuntimeError(
|
|
283
|
+
f"Frame extraction failed for video {video.video_hash}, cannot create anonymized frames."
|
|
284
|
+
) from extract_e
|
|
191
285
|
|
|
192
286
|
all_frames = video.get_frames()
|
|
193
287
|
if not all_frames.exists():
|
|
194
|
-
raise FileNotFoundError(
|
|
288
|
+
raise FileNotFoundError(
|
|
289
|
+
f"No frame objects found for video {video.video_hash} after extraction attempt."
|
|
290
|
+
)
|
|
195
291
|
|
|
196
292
|
outside_frame_numbers = _get_outside_frame_numbers(video)
|
|
197
293
|
|
|
198
|
-
logger.info(
|
|
294
|
+
logger.info(
|
|
295
|
+
"Generating %d temporary anonymized frame files for video %s...",
|
|
296
|
+
all_frames.filter(is_extracted=True).count(),
|
|
297
|
+
video.video_hash,
|
|
298
|
+
)
|
|
199
299
|
generated_frame_paths = _create_anonymized_frame_files(
|
|
200
300
|
video=video,
|
|
201
301
|
anonymized_frame_dir=temp_anonym_frame_dir,
|
|
@@ -203,7 +303,11 @@ def _make_temporary_anonymized_frames(video: "VideoFile", roi_processing = True)
|
|
|
203
303
|
frames=all_frames,
|
|
204
304
|
outside_frame_numbers=outside_frame_numbers,
|
|
205
305
|
)
|
|
206
|
-
logger.info(
|
|
306
|
+
logger.info(
|
|
307
|
+
"Generated %d temporary anonymized frame files for video %s.",
|
|
308
|
+
len(generated_frame_paths),
|
|
309
|
+
video.video_hash,
|
|
310
|
+
)
|
|
207
311
|
return temp_anonym_frame_dir, generated_frame_paths
|
|
208
312
|
|
|
209
313
|
|
|
@@ -223,25 +327,38 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
|
|
|
223
327
|
state = video.get_or_create_state()
|
|
224
328
|
|
|
225
329
|
if state.anonymized:
|
|
226
|
-
logger.info(
|
|
330
|
+
logger.info(
|
|
331
|
+
"Video %s is already marked as anonymized in state. Skipping.",
|
|
332
|
+
video.video_hash,
|
|
333
|
+
)
|
|
227
334
|
return True
|
|
228
335
|
if not video.has_raw:
|
|
229
|
-
raise FileNotFoundError(
|
|
336
|
+
raise FileNotFoundError(
|
|
337
|
+
f"Raw file is missing for video {video.video_hash}, cannot anonymize."
|
|
338
|
+
)
|
|
230
339
|
if not state.frames_extracted:
|
|
231
|
-
raise ValueError(
|
|
340
|
+
raise ValueError(
|
|
341
|
+
f"Frames not extracted for video {video.video_hash}, cannot anonymize."
|
|
342
|
+
)
|
|
232
343
|
if not video.sensitive_meta or not video.sensitive_meta.is_verified:
|
|
233
|
-
raise ValueError(
|
|
344
|
+
raise ValueError(
|
|
345
|
+
f"Sensitive metadata for video {video.video_hash} is not validated. Cannot anonymize."
|
|
346
|
+
)
|
|
234
347
|
# outside_segments = video.get_outside_segments(only_validated=False)
|
|
235
348
|
# unvalidated_outside = outside_segments.filter(state__is_validated=False)
|
|
236
349
|
|
|
237
|
-
logger.info("Starting anonymization process for video %s", video.
|
|
350
|
+
logger.info("Starting anonymization process for video %s", video.video_hash)
|
|
238
351
|
|
|
239
352
|
temp_anonym_frame_dir = None
|
|
240
353
|
anonymized_video_path = None
|
|
241
354
|
try:
|
|
242
|
-
temp_anonym_frame_dir, generated_frame_paths =
|
|
355
|
+
temp_anonym_frame_dir, generated_frame_paths = (
|
|
356
|
+
_make_temporary_anonymized_frames(video)
|
|
357
|
+
)
|
|
243
358
|
if not generated_frame_paths:
|
|
244
|
-
raise RuntimeError(
|
|
359
|
+
raise RuntimeError(
|
|
360
|
+
f"Failed to generate temporary anonymized frames for video {video.video_hash}."
|
|
361
|
+
)
|
|
245
362
|
|
|
246
363
|
anonymized_video_path = video.get_target_anonymized_video_path()
|
|
247
364
|
anonymized_video_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -250,9 +367,15 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
|
|
|
250
367
|
|
|
251
368
|
fps = video.get_fps()
|
|
252
369
|
if fps is None:
|
|
253
|
-
raise ValueError(
|
|
370
|
+
raise ValueError(
|
|
371
|
+
f"FPS could not be determined for {video.video_hash}, cannot assemble video."
|
|
372
|
+
)
|
|
254
373
|
|
|
255
|
-
logger.info(
|
|
374
|
+
logger.info(
|
|
375
|
+
"Assembling anonymized video for %s at %s",
|
|
376
|
+
video.video_hash,
|
|
377
|
+
anonymized_video_path,
|
|
378
|
+
)
|
|
256
379
|
assemble_video_from_frames(
|
|
257
380
|
frame_paths=generated_frame_paths,
|
|
258
381
|
output_path=anonymized_video_path,
|
|
@@ -260,14 +383,25 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
|
|
|
260
383
|
)
|
|
261
384
|
|
|
262
385
|
if not anonymized_video_path.exists():
|
|
263
|
-
raise RuntimeError(
|
|
386
|
+
raise RuntimeError(
|
|
387
|
+
f"Processed video file not found after assembly for {video.video_hash}: {anonymized_video_path}"
|
|
388
|
+
)
|
|
264
389
|
|
|
265
390
|
new_processed_hash = get_video_hash(anonymized_video_path)
|
|
266
|
-
if
|
|
267
|
-
|
|
391
|
+
if (
|
|
392
|
+
type(video)
|
|
393
|
+
.objects.filter(processed_video_hash=new_processed_hash)
|
|
394
|
+
.exclude(pk=video.pk)
|
|
395
|
+
.exists()
|
|
396
|
+
):
|
|
397
|
+
raise ValueError(
|
|
398
|
+
f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.video_hash})."
|
|
399
|
+
)
|
|
268
400
|
|
|
269
401
|
video.processed_video_hash = new_processed_hash
|
|
270
|
-
video.processed_file.name =
|
|
402
|
+
video.processed_file.name = (
|
|
403
|
+
video.get_target_anonymized_video_path().relative_to(STORAGE_DIR).as_posix()
|
|
404
|
+
)
|
|
271
405
|
|
|
272
406
|
update_fields = [
|
|
273
407
|
"processed_video_hash",
|
|
@@ -285,7 +419,9 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
|
|
|
285
419
|
|
|
286
420
|
transaction.on_commit(
|
|
287
421
|
lambda: _cleanup_raw_assets(
|
|
288
|
-
|
|
422
|
+
video_hash=video.video_hash,
|
|
423
|
+
raw_file_path=original_raw_file_path_to_delete,
|
|
424
|
+
raw_frame_dir=original_raw_frame_dir_to_delete,
|
|
289
425
|
)
|
|
290
426
|
)
|
|
291
427
|
|
|
@@ -296,19 +432,33 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
|
|
|
296
432
|
return True
|
|
297
433
|
|
|
298
434
|
except Exception as e:
|
|
299
|
-
logger.error(
|
|
435
|
+
logger.error(
|
|
436
|
+
"Anonymization failed for video %s: %s", video.video_hash, e, exc_info=True
|
|
437
|
+
)
|
|
300
438
|
if anonymized_video_path and anonymized_video_path.exists():
|
|
301
|
-
logger.warning(
|
|
439
|
+
logger.warning(
|
|
440
|
+
"Cleaning up potentially orphaned processed file for video %s due to error: %s",
|
|
441
|
+
video.video_hash,
|
|
442
|
+
anonymized_video_path,
|
|
443
|
+
)
|
|
302
444
|
anonymized_video_path.unlink(missing_ok=True)
|
|
303
|
-
raise RuntimeError(f"Anonymization failed for video {video.
|
|
445
|
+
raise RuntimeError(f"Anonymization failed for video {video.video_hash}") from e
|
|
304
446
|
|
|
305
447
|
finally:
|
|
306
448
|
if temp_anonym_frame_dir and temp_anonym_frame_dir.exists():
|
|
307
|
-
logger.info(
|
|
449
|
+
logger.info(
|
|
450
|
+
"Cleaning up temporary anonymized frame directory for video %s: %s",
|
|
451
|
+
video.video_hash,
|
|
452
|
+
temp_anonym_frame_dir,
|
|
453
|
+
)
|
|
308
454
|
shutil.rmtree(temp_anonym_frame_dir)
|
|
309
455
|
|
|
310
456
|
|
|
311
|
-
def _cleanup_raw_assets(
|
|
457
|
+
def _cleanup_raw_assets(
|
|
458
|
+
video_hash: "str",
|
|
459
|
+
raw_file_path: Optional[Path] = None,
|
|
460
|
+
raw_frame_dir: Optional[Path] = None,
|
|
461
|
+
):
|
|
312
462
|
"""
|
|
313
463
|
Deletes the original raw video file and its extracted frames directory.
|
|
314
464
|
Called via transaction.on_commit after successful anonymization.
|
|
@@ -318,32 +468,57 @@ def _cleanup_raw_assets(video_uuid: uuid.UUID, raw_file_path: Optional[Path] = N
|
|
|
318
468
|
"""
|
|
319
469
|
from endoreg_db.models import VideoFile, VideoState
|
|
320
470
|
|
|
321
|
-
logger.info(
|
|
471
|
+
logger.info(
|
|
472
|
+
"Performing post-commit cleanup of raw assets for video %s.", video_hash
|
|
473
|
+
)
|
|
322
474
|
try:
|
|
323
|
-
video_file =
|
|
475
|
+
video_file = (
|
|
476
|
+
VideoFile.objects.select_related("state")
|
|
477
|
+
.filter(content_hash=video_hash)
|
|
478
|
+
.first()
|
|
479
|
+
)
|
|
324
480
|
if not video_file:
|
|
325
|
-
logger.error(
|
|
481
|
+
logger.error(
|
|
482
|
+
"VideoFile %s not found during post-commit cleanup.", video_hash
|
|
483
|
+
)
|
|
326
484
|
return
|
|
327
485
|
if not video_file.state:
|
|
328
|
-
logger.error(
|
|
486
|
+
logger.error(
|
|
487
|
+
"VideoState not found for VideoFile %s during post-commit cleanup.",
|
|
488
|
+
video_hash,
|
|
489
|
+
)
|
|
329
490
|
video_file.state = VideoState.objects.create(video_file=video_file)
|
|
330
491
|
|
|
331
492
|
if raw_file_path and raw_file_path.exists():
|
|
332
493
|
logger.info("Deleting original raw video file: %s", raw_file_path)
|
|
333
494
|
raw_file_path.unlink()
|
|
334
495
|
elif raw_file_path:
|
|
335
|
-
logger.warning(
|
|
496
|
+
logger.warning(
|
|
497
|
+
"Original raw video file %s not found for post-commit deletion.",
|
|
498
|
+
raw_file_path,
|
|
499
|
+
)
|
|
336
500
|
|
|
337
501
|
if raw_frame_dir and raw_frame_dir.exists():
|
|
338
502
|
logger.info("Deleting original raw frame directory: %s", raw_frame_dir)
|
|
339
503
|
shutil.rmtree(raw_frame_dir, ignore_errors=True)
|
|
340
504
|
elif raw_frame_dir:
|
|
341
|
-
logger.warning(
|
|
505
|
+
logger.warning(
|
|
506
|
+
"Original raw frame directory %s not found for post-commit deletion.",
|
|
507
|
+
raw_frame_dir,
|
|
508
|
+
)
|
|
342
509
|
|
|
343
510
|
if video_file.state.frames_extracted:
|
|
344
511
|
video_file.state.frames_extracted = False
|
|
345
512
|
video_file.state.save(update_fields=["frames_extracted"])
|
|
346
|
-
logger.info(
|
|
513
|
+
logger.info(
|
|
514
|
+
"Set state.frames_extracted=False for video %s after raw asset cleanup.",
|
|
515
|
+
video_hash,
|
|
516
|
+
)
|
|
347
517
|
|
|
348
518
|
except Exception as e:
|
|
349
|
-
logger.error(
|
|
519
|
+
logger.error(
|
|
520
|
+
"Error during post-commit cleanup of raw assets for video %s: %s",
|
|
521
|
+
video_hash,
|
|
522
|
+
e,
|
|
523
|
+
exc_info=True,
|
|
524
|
+
)
|
|
@@ -16,5 +16,10 @@ def _bulk_create_frames(video: "VideoFile", frames_to_create: List["Frame"]):
|
|
|
16
16
|
try:
|
|
17
17
|
Frame.objects.bulk_create(frames_to_create, ignore_conflicts=True)
|
|
18
18
|
except Exception as e:
|
|
19
|
-
logger.error(
|
|
19
|
+
logger.error(
|
|
20
|
+
"Error during bulk creation of frames for video %s: %s",
|
|
21
|
+
video.video_hash,
|
|
22
|
+
e,
|
|
23
|
+
exc_info=True,
|
|
24
|
+
)
|
|
20
25
|
raise
|
|
@@ -9,7 +9,9 @@ logger = logging.getLogger(__name__)
|
|
|
9
9
|
__all__ = ["_create_frame_object"]
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
def _create_frame_object(
|
|
12
|
+
def _create_frame_object(
|
|
13
|
+
video: "VideoFile", frame_number: int, relative_path: str, extracted: bool = False
|
|
14
|
+
) -> "Frame":
|
|
13
15
|
"""Instantiates a Frame object (does not save it)."""
|
|
14
16
|
from endoreg_db.models import Frame
|
|
15
17
|
|
|
@@ -4,7 +4,10 @@ from typing import TYPE_CHECKING
|
|
|
4
4
|
|
|
5
5
|
from django.db import transaction
|
|
6
6
|
|
|
7
|
-
from endoreg_db.models.media.video.video_file_io import
|
|
7
|
+
from endoreg_db.models.media.video.video_file_io import (
|
|
8
|
+
_get_frame_dir_path,
|
|
9
|
+
_get_temp_anonymized_frame_dir,
|
|
10
|
+
)
|
|
8
11
|
|
|
9
12
|
if TYPE_CHECKING:
|
|
10
13
|
from endoreg_db.models import VideoFile, VideoState
|
|
@@ -44,7 +47,7 @@ def _delete_frames(video: "VideoFile") -> str:
|
|
|
44
47
|
msg = f"Frame directory not found, skipping deletion: {frame_dir}"
|
|
45
48
|
logger.debug(msg)
|
|
46
49
|
else:
|
|
47
|
-
msg = f"Frame directory path not set for video {video.
|
|
50
|
+
msg = f"Frame directory path not set for video {video.video_hash}, cannot delete standard frames."
|
|
48
51
|
logger.warning(msg)
|
|
49
52
|
|
|
50
53
|
temp_anonym_frame_dir = None
|
|
@@ -52,7 +55,9 @@ def _delete_frames(video: "VideoFile") -> str:
|
|
|
52
55
|
temp_anonym_frame_dir = _get_temp_anonymized_frame_dir(video)
|
|
53
56
|
if temp_anonym_frame_dir and temp_anonym_frame_dir.exists():
|
|
54
57
|
shutil.rmtree(temp_anonym_frame_dir)
|
|
55
|
-
msg =
|
|
58
|
+
msg = (
|
|
59
|
+
f"Deleted temporary anonymized frame directory: {temp_anonym_frame_dir}"
|
|
60
|
+
)
|
|
56
61
|
logger.info(msg)
|
|
57
62
|
deleted_messages.append(msg)
|
|
58
63
|
except Exception as e:
|
|
@@ -69,16 +74,28 @@ def _delete_frames(video: "VideoFile") -> str:
|
|
|
69
74
|
|
|
70
75
|
if update_fields_state:
|
|
71
76
|
state.save(update_fields=update_fields_state)
|
|
72
|
-
logger.info(
|
|
77
|
+
logger.info(
|
|
78
|
+
"Reset frame state flags (%s) for video %s.",
|
|
79
|
+
", ".join(update_fields_state),
|
|
80
|
+
video.video_hash,
|
|
81
|
+
)
|
|
73
82
|
state_updated = True
|
|
74
83
|
else:
|
|
75
|
-
logger.info(
|
|
84
|
+
logger.info(
|
|
85
|
+
"Frame state flags already False for video %s.", video.video_hash
|
|
86
|
+
)
|
|
76
87
|
state_updated = True
|
|
77
88
|
|
|
78
89
|
try:
|
|
79
|
-
update_count = Frame.objects.filter(video=video, is_extracted=True).update(
|
|
90
|
+
update_count = Frame.objects.filter(video=video, is_extracted=True).update(
|
|
91
|
+
is_extracted=False
|
|
92
|
+
)
|
|
80
93
|
if update_count > 0:
|
|
81
|
-
logger.info(
|
|
94
|
+
logger.info(
|
|
95
|
+
"Marked %d Frame objects as is_extracted=False for video %s.",
|
|
96
|
+
update_count,
|
|
97
|
+
video.video_hash,
|
|
98
|
+
)
|
|
82
99
|
db_updated = True
|
|
83
100
|
except Exception as db_err:
|
|
84
101
|
msg = f"Failed to update is_extracted flag for Frame objects for video %s: {db_err}"
|
|
@@ -87,10 +104,14 @@ def _delete_frames(video: "VideoFile") -> str:
|
|
|
87
104
|
db_updated = False
|
|
88
105
|
|
|
89
106
|
except Exception as state_e:
|
|
90
|
-
msg =
|
|
107
|
+
msg = (
|
|
108
|
+
f"Failed to update state after deleting frame files for video %s: {state_e}"
|
|
109
|
+
)
|
|
91
110
|
logger.error(msg, exc_info=True)
|
|
92
111
|
error_messages.append(msg)
|
|
93
|
-
raise RuntimeError(
|
|
112
|
+
raise RuntimeError(
|
|
113
|
+
f"Failed to update state during frame file deletion for video {video.video_hash}"
|
|
114
|
+
) from state_e
|
|
94
115
|
|
|
95
116
|
final_message = "; ".join(deleted_messages)
|
|
96
117
|
if error_messages:
|