endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +89 -122
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -4
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -7,14 +7,17 @@ from typing import TYPE_CHECKING, Optional, Type
|
|
|
7
7
|
|
|
8
8
|
# Import the new exceptions from the correct path
|
|
9
9
|
from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
|
|
10
|
-
from endoreg_db.utils.paths import
|
|
10
|
+
from endoreg_db.utils.paths import (
|
|
11
|
+
IMPORT_VIDEO_DIR,
|
|
12
|
+
SENSITIVE_VIDEO_DIR,
|
|
13
|
+
TRANSCODING_DIR,
|
|
14
|
+
)
|
|
11
15
|
|
|
12
16
|
if TYPE_CHECKING:
|
|
13
17
|
from endoreg_db.models import VideoFile
|
|
14
18
|
|
|
15
19
|
import endoreg_db.utils.paths as path_utils
|
|
16
|
-
|
|
17
|
-
from ....utils.hashs import get_video_hash
|
|
20
|
+
|
|
18
21
|
from ....utils.video.ffmpeg_wrapper import transcode_videofile_if_required
|
|
19
22
|
|
|
20
23
|
logger = logging.getLogger(__name__)
|
|
@@ -57,7 +60,9 @@ def check_storage_capacity(
|
|
|
57
60
|
# Don't fail the operation, just log the warning
|
|
58
61
|
|
|
59
62
|
|
|
60
|
-
def atomic_copy_with_fallback(
|
|
63
|
+
def atomic_copy_with_fallback(
|
|
64
|
+
src_path: Path = IMPORT_VIDEO_DIR, dst_path: Path = SENSITIVE_VIDEO_DIR
|
|
65
|
+
) -> bool:
|
|
61
66
|
"""
|
|
62
67
|
Atomically copy file from src to dst, preserving the source file.
|
|
63
68
|
|
|
@@ -192,7 +197,8 @@ def _create_from_file(
|
|
|
192
197
|
cls_model: Type["VideoFile"],
|
|
193
198
|
file_path: Path,
|
|
194
199
|
center_name: str,
|
|
195
|
-
processor_name: Optional[str]
|
|
200
|
+
processor_name: Optional[str],
|
|
201
|
+
video_hash: str,
|
|
196
202
|
video_dir: Path = IMPORT_VIDEO_DIR,
|
|
197
203
|
save: bool = True,
|
|
198
204
|
delete_source: bool = False,
|
|
@@ -218,7 +224,7 @@ def _create_from_file(
|
|
|
218
224
|
try:
|
|
219
225
|
# Ensure we operate under the canonical video path root
|
|
220
226
|
data_paths = _get_data_paths()
|
|
221
|
-
resolved_video_dir = _get_path(data_paths, "
|
|
227
|
+
resolved_video_dir = _get_path(data_paths, "sensitive_video", video_dir)
|
|
222
228
|
video_dir = Path(resolved_video_dir)
|
|
223
229
|
storage_root_default = Path(video_dir).parent
|
|
224
230
|
resolved_storage_root = _get_path(data_paths, "storage", storage_root_default)
|
|
@@ -251,23 +257,13 @@ def _create_from_file(
|
|
|
251
257
|
|
|
252
258
|
logger.debug("Using file for hashing: %s", transcoded_file_path)
|
|
253
259
|
|
|
254
|
-
# 2. Calculate hash (this will be the raw_video_hash)
|
|
255
|
-
video_hash = get_video_hash(transcoded_file_path)
|
|
256
|
-
if not video_hash:
|
|
257
|
-
raise ValueError(
|
|
258
|
-
f"Could not calculate video hash for {transcoded_file_path}"
|
|
259
|
-
)
|
|
260
|
-
logger.info(
|
|
261
|
-
"Calculated raw video hash: %s for %s", video_hash, original_file_name
|
|
262
|
-
)
|
|
263
|
-
|
|
264
260
|
# 3. Check if hash already exists
|
|
265
261
|
if cls_model.check_hash_exists(video_hash=video_hash):
|
|
266
262
|
existing_video = cls_model.objects.get(video_hash=video_hash)
|
|
267
263
|
logger.warning(
|
|
268
264
|
"Video with hash %s already exists (UUID: %s)",
|
|
269
265
|
video_hash,
|
|
270
|
-
existing_video.
|
|
266
|
+
existing_video.video_hash,
|
|
271
267
|
)
|
|
272
268
|
|
|
273
269
|
# Check if the existing video has a valid file
|
|
@@ -292,9 +288,7 @@ def _create_from_file(
|
|
|
292
288
|
)
|
|
293
289
|
existing_video.delete()
|
|
294
290
|
|
|
295
|
-
|
|
296
|
-
new_file_name, uuid_val = get_uuid_filename(transcoded_file_path)
|
|
297
|
-
final_storage_path = video_dir / new_file_name
|
|
291
|
+
final_storage_path = video_dir / video_hash / transcoded_file_path.suffix
|
|
298
292
|
final_storage_path.parent.mkdir(parents=True, exist_ok=True)
|
|
299
293
|
|
|
300
294
|
# 5. Move or Copy the file to final storage using improved method
|
|
@@ -327,19 +321,6 @@ def _create_from_file(
|
|
|
327
321
|
except Exception as e:
|
|
328
322
|
raise RuntimeError(f"Failed to move file to final storage: {e}") from e
|
|
329
323
|
|
|
330
|
-
# 6. Verify hash after move/copy
|
|
331
|
-
final_hash = get_video_hash(final_storage_path)
|
|
332
|
-
if final_hash != video_hash:
|
|
333
|
-
logger.error(
|
|
334
|
-
"Hash mismatch after file operation! Expected %s, got %s",
|
|
335
|
-
video_hash,
|
|
336
|
-
final_hash,
|
|
337
|
-
)
|
|
338
|
-
final_storage_path.unlink(missing_ok=True)
|
|
339
|
-
raise RuntimeError(
|
|
340
|
-
f"Hash mismatch after file operation for {final_storage_path}"
|
|
341
|
-
)
|
|
342
|
-
|
|
343
324
|
# 7. Get related objects
|
|
344
325
|
try:
|
|
345
326
|
center = Center.objects.get(name=center_name)
|
|
@@ -365,14 +346,11 @@ def _create_from_file(
|
|
|
365
346
|
raise ValueError(f"Processor '{processor_name}' not found.") from e
|
|
366
347
|
|
|
367
348
|
# 8. Create the VideoFile instance
|
|
368
|
-
logger.info("Creating new VideoFile instance with
|
|
349
|
+
logger.info("Creating new VideoFile instance with hash: %s", video_hash)
|
|
369
350
|
# Store FileField path relative to storage root including the videos prefix
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
relative_name = path_utils.to_storage_relative(final_storage_path)
|
|
351
|
+
|
|
352
|
+
relative_name = path_utils.to_storage_relative(final_storage_path)
|
|
374
353
|
video = cls_model(
|
|
375
|
-
uuid=uuid_val,
|
|
376
354
|
raw_file=relative_name,
|
|
377
355
|
processed_file=None,
|
|
378
356
|
center=center,
|
|
@@ -385,10 +363,11 @@ def _create_from_file(
|
|
|
385
363
|
|
|
386
364
|
# 9. Save the instance if requested
|
|
387
365
|
if save:
|
|
388
|
-
logger.info("Saving new VideoFile instance (
|
|
366
|
+
logger.info("Saving new VideoFile instance (Hash:%s", video_hash)
|
|
389
367
|
video.save()
|
|
390
368
|
logger.info(
|
|
391
|
-
"Successfully created VideoFile PK %s
|
|
369
|
+
"Successfully created VideoFile PK %s ",
|
|
370
|
+
video.pk,
|
|
392
371
|
)
|
|
393
372
|
|
|
394
373
|
return video
|
|
@@ -40,18 +40,24 @@ def _pipe_1(
|
|
|
40
40
|
video_file.refresh_from_db()
|
|
41
41
|
video_file.update_video_meta()
|
|
42
42
|
|
|
43
|
-
logger.info(f"Starting Pipe 1 for video {video_file.
|
|
43
|
+
logger.info(f"Starting Pipe 1 for video {video_file.video_hash}")
|
|
44
44
|
try:
|
|
45
45
|
# 1. Heavy I/O operations outside the transaction block
|
|
46
46
|
logger.info("Pipe 1: Extracting frames...")
|
|
47
|
-
video_file.extract_frames(
|
|
47
|
+
video_file.extract_frames(
|
|
48
|
+
overwrite=False
|
|
49
|
+
) # Avoid overwriting if already extracted
|
|
48
50
|
|
|
49
51
|
logger.info("Pipe 1: Extracting text metadata...")
|
|
50
|
-
video_file.update_text_metadata(
|
|
52
|
+
video_file.update_text_metadata(
|
|
53
|
+
ocr_frame_fraction=ocr_frame_fraction, cap=ocr_cap, overwrite=False
|
|
54
|
+
)
|
|
51
55
|
with transaction.atomic():
|
|
52
56
|
state = video_file.get_or_create_state()
|
|
53
57
|
if not state.frames_extracted:
|
|
54
|
-
logger.error(
|
|
58
|
+
logger.error(
|
|
59
|
+
"Pipe 1 failed: Frame extraction did not complete successfully."
|
|
60
|
+
)
|
|
55
61
|
return False
|
|
56
62
|
|
|
57
63
|
# 3. Perform Initial Prediction
|
|
@@ -59,7 +65,9 @@ def _pipe_1(
|
|
|
59
65
|
try:
|
|
60
66
|
ai_model_obj = AiModel.objects.get(name=model_name)
|
|
61
67
|
if model_meta_version is not None:
|
|
62
|
-
model_meta = ai_model_obj.metadata_versions.get(
|
|
68
|
+
model_meta = ai_model_obj.metadata_versions.get(
|
|
69
|
+
version=model_meta_version
|
|
70
|
+
)
|
|
63
71
|
else:
|
|
64
72
|
model_meta = ai_model_obj.get_latest_version()
|
|
65
73
|
except AiModel.DoesNotExist:
|
|
@@ -68,7 +76,9 @@ def _pipe_1(
|
|
|
68
76
|
model_name = download_segmentation_model()
|
|
69
77
|
ai_model_obj = AiModel.objects.get(name=model_name)
|
|
70
78
|
if model_meta_version is not None:
|
|
71
|
-
model_meta = ai_model_obj.metadata_versions.get(
|
|
79
|
+
model_meta = ai_model_obj.metadata_versions.get(
|
|
80
|
+
version=model_meta_version
|
|
81
|
+
)
|
|
72
82
|
else:
|
|
73
83
|
model_meta = ai_model_obj.get_latest_version()
|
|
74
84
|
except AiModel.DoesNotExist:
|
|
@@ -80,19 +90,25 @@ def _pipe_1(
|
|
|
80
90
|
model_name = download_segmentation_model()
|
|
81
91
|
ai_model_obj = AiModel.objects.get(name=model_name)
|
|
82
92
|
if model_meta_version is not None:
|
|
83
|
-
model_meta = ai_model_obj.metadata_versions.get(
|
|
93
|
+
model_meta = ai_model_obj.metadata_versions.get(
|
|
94
|
+
version=model_meta_version
|
|
95
|
+
)
|
|
84
96
|
else:
|
|
85
97
|
model_meta = ai_model_obj.get_latest_version()
|
|
86
98
|
except ModelMeta.DoesNotExist:
|
|
87
|
-
logger.error(
|
|
99
|
+
logger.error(
|
|
100
|
+
f"Pipe 1 failed: ModelMeta version {model_meta_version} for model '{model_name}' not found."
|
|
101
|
+
)
|
|
88
102
|
return False
|
|
89
103
|
try:
|
|
90
|
-
sequences: Optional[Dict[str, List[Tuple[int, int]]]] =
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
104
|
+
sequences: Optional[Dict[str, List[Tuple[int, int]]]] = (
|
|
105
|
+
video_file.predict_video(
|
|
106
|
+
model_meta=model_meta,
|
|
107
|
+
smooth_window_size_s=smooth_window_size_s,
|
|
108
|
+
binarize_threshold=binarize_threshold,
|
|
109
|
+
test_run=test_run,
|
|
110
|
+
n_test_frames=n_test_frames,
|
|
111
|
+
)
|
|
96
112
|
)
|
|
97
113
|
except Exception as e:
|
|
98
114
|
logger.error(f"Pipe 1 failed during prediction: {e}", exc_info=True)
|
|
@@ -110,13 +126,19 @@ def _pipe_1(
|
|
|
110
126
|
|
|
111
127
|
logger.info(f"Pipe 1: Sequences returned from prediction: {sequences}")
|
|
112
128
|
if not sequences:
|
|
113
|
-
logger.warning(
|
|
129
|
+
logger.warning(
|
|
130
|
+
"Pipe 1: Prediction returned empty sequences dictionary. No LabelVideoSegments will be created."
|
|
131
|
+
)
|
|
114
132
|
|
|
115
133
|
# 4. Create LabelVideoSegments
|
|
116
134
|
logger.info("Pipe 1: Creating LabelVideoSegments from predictions...")
|
|
117
135
|
try:
|
|
118
|
-
video_prediction_meta = VideoPredictionMeta.objects.get(
|
|
119
|
-
|
|
136
|
+
video_prediction_meta = VideoPredictionMeta.objects.get(
|
|
137
|
+
video_file=video_file, model_meta=model_meta
|
|
138
|
+
)
|
|
139
|
+
logger.info(
|
|
140
|
+
f"Pipe 1: Calling _convert_sequences_to_db_segments for video {video_file.video_hash} with prediction meta {video_prediction_meta.pk}"
|
|
141
|
+
)
|
|
120
142
|
_convert_sequences_to_db_segments(
|
|
121
143
|
video=video_file,
|
|
122
144
|
sequences=sequences,
|
|
@@ -128,18 +150,26 @@ def _pipe_1(
|
|
|
128
150
|
state.save(update_fields=["lvs_created"])
|
|
129
151
|
logger.info("Pipe 1: Set lvs_created state to True.")
|
|
130
152
|
logger.info("Pipe 1: LabelVideoSegment creation complete.")
|
|
131
|
-
lvs_count_after = LabelVideoSegment.objects.filter(
|
|
132
|
-
|
|
153
|
+
lvs_count_after = LabelVideoSegment.objects.filter(
|
|
154
|
+
video_file=video_file
|
|
155
|
+
).count()
|
|
156
|
+
logger.info(
|
|
157
|
+
f"Pipe 1: Found {lvs_count_after} LabelVideoSegments after conversion attempt."
|
|
158
|
+
)
|
|
133
159
|
except VideoPredictionMeta.DoesNotExist:
|
|
134
|
-
logger.error(
|
|
160
|
+
logger.error(
|
|
161
|
+
"Pipe 1 failed: Could not find VideoPredictionMeta after prediction."
|
|
162
|
+
)
|
|
135
163
|
raise
|
|
136
164
|
|
|
137
|
-
logger.info(f"Pipe 1 completed successfully for video {video_file.
|
|
165
|
+
logger.info(f"Pipe 1 completed successfully for video {video_file.video_hash}")
|
|
138
166
|
success = True # Set success flag
|
|
139
167
|
return True
|
|
140
168
|
|
|
141
169
|
except Exception as e:
|
|
142
|
-
logger.error(
|
|
170
|
+
logger.error(
|
|
171
|
+
f"Pipe 1 failed for video {video_file.video_hash}: {e}", exc_info=True
|
|
172
|
+
)
|
|
143
173
|
return False
|
|
144
174
|
finally:
|
|
145
175
|
# 5. Optionally delete frames
|
|
@@ -155,19 +185,23 @@ def _pipe_1(
|
|
|
155
185
|
|
|
156
186
|
|
|
157
187
|
# --- Test after Pipe 1 ---
|
|
158
|
-
def _test_after_pipe_1(
|
|
188
|
+
def _test_after_pipe_1(
|
|
189
|
+
video_file: "VideoFile", start_frame: int = 0, end_frame: int = 100
|
|
190
|
+
) -> bool:
|
|
159
191
|
"""
|
|
160
192
|
Simulates human annotation validation after Pipe 1.
|
|
161
193
|
Creates 'outside' segments and marks sensitive meta as verified.
|
|
162
194
|
"""
|
|
163
195
|
from ...label import Label, LabelVideoSegment
|
|
164
196
|
|
|
165
|
-
logger.info(f"Starting _test_after_pipe_1 for video {video_file.
|
|
197
|
+
logger.info(f"Starting _test_after_pipe_1 for video {video_file.video_hash}")
|
|
166
198
|
try:
|
|
167
199
|
# 1. Create 'outside' LabelVideoSegments
|
|
168
200
|
try:
|
|
169
201
|
outside_label = Label.objects.get(name__iexact="outside")
|
|
170
|
-
logger.info(
|
|
202
|
+
logger.info(
|
|
203
|
+
f"Creating 'outside' annotation segment [{start_frame}-{end_frame}]"
|
|
204
|
+
)
|
|
171
205
|
# Create a segment - assuming custom_create handles saving
|
|
172
206
|
outside_segment = LabelVideoSegment.objects.create( # Assign to variable
|
|
173
207
|
video_file=video_file,
|
|
@@ -177,16 +211,22 @@ def _test_after_pipe_1(video_file: "VideoFile", start_frame: int = 0, end_frame:
|
|
|
177
211
|
prediction_meta=None,
|
|
178
212
|
)
|
|
179
213
|
# Ensure the segment has a state and mark it as validated
|
|
180
|
-
segment_state, created =
|
|
214
|
+
segment_state, created = (
|
|
215
|
+
outside_segment.get_or_create_state()
|
|
216
|
+
) # Unpack the tuple
|
|
181
217
|
segment_state.is_validated = True
|
|
182
218
|
segment_state.save()
|
|
183
|
-
logger.info(
|
|
219
|
+
logger.info(
|
|
220
|
+
f"Marked 'outside' segment {outside_segment.pk} as validated. Created: {created}"
|
|
221
|
+
)
|
|
184
222
|
|
|
185
223
|
except Label.DoesNotExist:
|
|
186
224
|
logger.error("_test_after_pipe_1 failed: 'outside' Label not found.")
|
|
187
225
|
return False
|
|
188
226
|
except Exception as e:
|
|
189
|
-
logger.error(
|
|
227
|
+
logger.error(
|
|
228
|
+
f"_test_after_pipe_1 failed during segment creation: {e}", exc_info=True
|
|
229
|
+
)
|
|
190
230
|
return False
|
|
191
231
|
|
|
192
232
|
# 2. Set Sensitive Metadata state to verified
|
|
@@ -205,9 +245,14 @@ def _test_after_pipe_1(video_file: "VideoFile", start_frame: int = 0, end_frame:
|
|
|
205
245
|
else:
|
|
206
246
|
logger.warning("_test_after_pipe_1: No sensitive meta found to verify.")
|
|
207
247
|
|
|
208
|
-
logger.info(
|
|
248
|
+
logger.info(
|
|
249
|
+
f"_test_after_pipe_1 completed successfully for video {video_file.video_hash}"
|
|
250
|
+
)
|
|
209
251
|
return True
|
|
210
252
|
|
|
211
253
|
except Exception as e:
|
|
212
|
-
logger.error(
|
|
254
|
+
logger.error(
|
|
255
|
+
f"_test_after_pipe_1 failed for video {video_file.video_hash}: {e}",
|
|
256
|
+
exc_info=True,
|
|
257
|
+
)
|
|
213
258
|
return False
|
|
@@ -22,7 +22,7 @@ def _pipe_2(video_file: "VideoFile") -> bool:
|
|
|
22
22
|
Returns:
|
|
23
23
|
bool: True if all operations complete successfully; otherwise, False.
|
|
24
24
|
"""
|
|
25
|
-
logger.info("Starting Pipe 2 for video %s", video_file.
|
|
25
|
+
logger.info("Starting Pipe 2 for video %s", video_file.video_hash)
|
|
26
26
|
try:
|
|
27
27
|
# --- Part 1: Frame Extraction ---
|
|
28
28
|
# Determine if frames are needed (short transaction for state read)
|
|
@@ -31,7 +31,9 @@ def _pipe_2(video_file: "VideoFile") -> bool:
|
|
|
31
31
|
frames_needed = not state.frames_extracted
|
|
32
32
|
|
|
33
33
|
if frames_needed:
|
|
34
|
-
logger.info(
|
|
34
|
+
logger.info(
|
|
35
|
+
"Pipe 2: Frames not extracted. Extracting outside main DB transaction..."
|
|
36
|
+
)
|
|
35
37
|
if not video_file.extract_frames(overwrite=False): # Heavy I/O work
|
|
36
38
|
logger.error("Pipe 2 failed: Frame extraction method returned False.")
|
|
37
39
|
return False
|
|
@@ -40,7 +42,9 @@ def _pipe_2(video_file: "VideoFile") -> bool:
|
|
|
40
42
|
with transaction.atomic():
|
|
41
43
|
video_file.refresh_from_db()
|
|
42
44
|
if not video_file.state or not video_file.state.frames_extracted:
|
|
43
|
-
logger.error(
|
|
45
|
+
logger.error(
|
|
46
|
+
"Pipe 2 failed: Frame extraction did not update state successfully."
|
|
47
|
+
)
|
|
44
48
|
return False
|
|
45
49
|
logger.info("Pipe 2: Frame extraction complete.")
|
|
46
50
|
else:
|
|
@@ -55,17 +59,25 @@ def _pipe_2(video_file: "VideoFile") -> bool:
|
|
|
55
59
|
state.sensitive_meta_processed = False
|
|
56
60
|
|
|
57
61
|
if anonymization_needed:
|
|
58
|
-
logger.info(
|
|
59
|
-
|
|
62
|
+
logger.info(
|
|
63
|
+
"Pipe 2: Video not anonymized. Anonymizing outside main DB transaction..."
|
|
64
|
+
)
|
|
65
|
+
anonymize_success = video_file.anonymize(
|
|
66
|
+
delete_original_raw=True
|
|
67
|
+
) # Heavy I/O work
|
|
60
68
|
if not anonymize_success:
|
|
61
|
-
logger.error(
|
|
69
|
+
logger.error(
|
|
70
|
+
"Pipe 2 failed: Anonymization process failed (returned False)."
|
|
71
|
+
)
|
|
62
72
|
return False
|
|
63
73
|
|
|
64
74
|
# Verify anonymization and update state (short transaction)
|
|
65
75
|
with transaction.atomic():
|
|
66
76
|
video_file.refresh_from_db()
|
|
67
77
|
if not video_file.state or not video_file.state.anonymized:
|
|
68
|
-
logger.error(
|
|
78
|
+
logger.error(
|
|
79
|
+
"Pipe 2 Error: State.anonymized is False even after anonymize() call."
|
|
80
|
+
)
|
|
69
81
|
return False
|
|
70
82
|
logger.info("Pipe 2: Anonymization complete.")
|
|
71
83
|
else:
|
|
@@ -86,19 +98,32 @@ def _pipe_2(video_file: "VideoFile") -> bool:
|
|
|
86
98
|
sm_pk = video_file.sensitive_meta.pk
|
|
87
99
|
video_file.sensitive_meta.delete()
|
|
88
100
|
video_file.sensitive_meta = None # Important after SET_NULL
|
|
89
|
-
video_file.save(
|
|
90
|
-
|
|
101
|
+
video_file.save(
|
|
102
|
+
update_fields=["sensitive_meta"]
|
|
103
|
+
) # Persist the null relation
|
|
104
|
+
logger.info(
|
|
105
|
+
"Pipe 2: Deleted sensitive meta object (PK: %s).", sm_pk
|
|
106
|
+
)
|
|
91
107
|
except Exception as e:
|
|
92
|
-
logger.error(
|
|
108
|
+
logger.error(
|
|
109
|
+
"Pipe 2: Failed to delete sensitive meta object: %s",
|
|
110
|
+
e,
|
|
111
|
+
exc_info=True,
|
|
112
|
+
)
|
|
93
113
|
raise # Reraise to ensure this transaction rolls back
|
|
94
114
|
else:
|
|
95
115
|
logger.info("Pipe 2: No sensitive meta object found to delete.")
|
|
96
116
|
|
|
97
|
-
logger.info(
|
|
117
|
+
logger.info(
|
|
118
|
+
f"Pipe 2 completed successfully for video {video_file.video_hash}"
|
|
119
|
+
)
|
|
98
120
|
return True
|
|
99
121
|
|
|
100
122
|
except Exception as e:
|
|
101
123
|
# This will catch exceptions from I/O operations if they raise,
|
|
102
124
|
# or from the final transaction block, or any other unhandled error.
|
|
103
|
-
logger.error(
|
|
125
|
+
logger.error(
|
|
126
|
+
f"Pipe 2 failed for video {video_file.video_hash} with unhandled exception: {e}",
|
|
127
|
+
exc_info=True,
|
|
128
|
+
)
|
|
104
129
|
return False
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
import os
|
|
5
|
-
import uuid
|
|
6
5
|
from pathlib import Path
|
|
7
|
-
from typing import TYPE_CHECKING, Optional,
|
|
6
|
+
from typing import TYPE_CHECKING, Optional, Union, cast
|
|
8
7
|
|
|
9
8
|
from django.core.files import File
|
|
10
9
|
from django.core.validators import FileExtensionValidator
|
|
@@ -13,11 +12,11 @@ from django.db.models import F
|
|
|
13
12
|
from django.db.models.fields.files import FieldFile
|
|
14
13
|
|
|
15
14
|
from endoreg_db.utils.calc_duration_seconds import _calc_duration_vf
|
|
15
|
+
from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, SENSITIVE_VIDEO_DIR
|
|
16
16
|
from endoreg_db.utils.video.ffmpeg_wrapper import assemble_video_from_frames
|
|
17
17
|
|
|
18
18
|
from ...label import Label, LabelVideoSegment
|
|
19
19
|
from ...state import VideoState
|
|
20
|
-
from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, SENSITIVE_VIDEO_DIR
|
|
21
20
|
|
|
22
21
|
# --- Import model-specific function modules ---
|
|
23
22
|
from .create_from_file import _create_from_file
|
|
@@ -112,8 +111,6 @@ class VideoQuerySet(models.QuerySet):
|
|
|
112
111
|
|
|
113
112
|
|
|
114
113
|
class VideoFile(models.Model):
|
|
115
|
-
uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
|
|
116
|
-
|
|
117
114
|
objects = VideoQuerySet.as_manager()
|
|
118
115
|
|
|
119
116
|
raw_file = models.FileField(
|
|
@@ -447,7 +444,7 @@ class VideoFile(models.Model):
|
|
|
447
444
|
except Exception as exc: # storage backends may raise when missing
|
|
448
445
|
logger.warning(
|
|
449
446
|
"Active file URL unavailable for video %s: %s",
|
|
450
|
-
self.
|
|
447
|
+
self.video_hash,
|
|
451
448
|
exc,
|
|
452
449
|
)
|
|
453
450
|
raise ValueError(
|
|
@@ -482,9 +479,10 @@ class VideoFile(models.Model):
|
|
|
482
479
|
cls,
|
|
483
480
|
file_path: Union[str, Path],
|
|
484
481
|
center_name: str,
|
|
485
|
-
processor_name: Optional[str]
|
|
482
|
+
processor_name: Optional[str],
|
|
483
|
+
video_hash: str,
|
|
486
484
|
delete_source: bool = False,
|
|
487
|
-
save_video_file: bool = True,
|
|
485
|
+
save_video_file: bool = True,
|
|
488
486
|
):
|
|
489
487
|
"""
|
|
490
488
|
Creates a VideoFile instance from a given video file path.
|
|
@@ -501,6 +499,7 @@ class VideoFile(models.Model):
|
|
|
501
499
|
file_path=file_path,
|
|
502
500
|
center_name=center_name,
|
|
503
501
|
processor_name=processor_name,
|
|
502
|
+
video_hash=video_hash,
|
|
504
503
|
delete_source=delete_source,
|
|
505
504
|
save=save_video_file, # Add this line
|
|
506
505
|
)
|
|
@@ -520,10 +519,12 @@ class VideoFile(models.Model):
|
|
|
520
519
|
# Call the original delete method to remove the instance from the database
|
|
521
520
|
try:
|
|
522
521
|
active_path = self.active_file_path
|
|
523
|
-
logger.info(f"Deleting VideoFile: {self.
|
|
522
|
+
logger.info(f"Deleting VideoFile: {self.video_hash} - {active_path}")
|
|
524
523
|
|
|
525
524
|
except ValueError:
|
|
526
|
-
logger.info(
|
|
525
|
+
logger.info(
|
|
526
|
+
f"Deleting VideoFile: {self.video_hash} - No active file path found."
|
|
527
|
+
)
|
|
527
528
|
active_path = None
|
|
528
529
|
|
|
529
530
|
# Delete associated files if they exist
|
|
@@ -556,10 +557,10 @@ class VideoFile(models.Model):
|
|
|
556
557
|
try:
|
|
557
558
|
# Call parent delete with proper parameters
|
|
558
559
|
result = super().delete(using=using, keep_parents=keep_parents)
|
|
559
|
-
logger.info(f"VideoFile {self.
|
|
560
|
+
logger.info(f"VideoFile {self.video_hash} deleted successfully.")
|
|
560
561
|
return result
|
|
561
562
|
except Exception as e:
|
|
562
|
-
logger.error(f"Error deleting VideoFile {self.
|
|
563
|
+
logger.error(f"Error deleting VideoFile {self.video_hash}: {e}")
|
|
563
564
|
raise
|
|
564
565
|
|
|
565
566
|
def validate_metadata_annotation(
|
|
@@ -596,12 +597,12 @@ class VideoFile(models.Model):
|
|
|
596
597
|
if self.raw_file:
|
|
597
598
|
self.raw_file.delete(save=False)
|
|
598
599
|
logger.info(
|
|
599
|
-
f"Raw video deleted for {self.
|
|
600
|
+
f"Raw video deleted for {self.video_hash}. Anonymized video preserved."
|
|
600
601
|
)
|
|
601
602
|
else:
|
|
602
603
|
logger.warning(
|
|
603
604
|
"Raw video file not found for deletion during validation %s.",
|
|
604
|
-
self.
|
|
605
|
+
self.video_hash,
|
|
605
606
|
)
|
|
606
607
|
|
|
607
608
|
if self.sensitive_meta:
|
|
@@ -610,12 +611,12 @@ class VideoFile(models.Model):
|
|
|
610
611
|
# Save the VideoFile instance to persist changes
|
|
611
612
|
self.save()
|
|
612
613
|
logger.info(
|
|
613
|
-
f"Metadata annotation validated and saved for video {self.
|
|
614
|
+
f"Metadata annotation validated and saved for video {self.video_hash}."
|
|
614
615
|
)
|
|
615
616
|
return True
|
|
616
617
|
else:
|
|
617
618
|
logger.error(
|
|
618
|
-
f"Failed to validate metadata annotation for video {self.
|
|
619
|
+
f"Failed to validate metadata annotation for video {self.video_hash}."
|
|
619
620
|
)
|
|
620
621
|
return False
|
|
621
622
|
|
|
@@ -652,7 +653,7 @@ class VideoFile(models.Model):
|
|
|
652
653
|
state = (
|
|
653
654
|
"Processed" if self.is_processed else ("Raw" if self.has_raw else "No File")
|
|
654
655
|
)
|
|
655
|
-
return f"VideoFile ({state}): {file_name} (UUID: {self.
|
|
656
|
+
return f"VideoFile ({state}): {file_name} (UUID: {self.video_hash})"
|
|
656
657
|
|
|
657
658
|
# --- Convenience state/meta helpers used in tests and admin workflows ---
|
|
658
659
|
def mark_sensitive_meta_processed(self, *, save: bool = True) -> "VideoFile":
|
|
@@ -746,7 +747,7 @@ class VideoFile(models.Model):
|
|
|
746
747
|
except Exception as e:
|
|
747
748
|
logger.error(
|
|
748
749
|
"Error getting outside segments for video %s: %s",
|
|
749
|
-
self.
|
|
750
|
+
self.video_hash,
|
|
750
751
|
e,
|
|
751
752
|
exc_info=True,
|
|
752
753
|
)
|
|
@@ -769,7 +770,8 @@ class VideoFile(models.Model):
|
|
|
769
770
|
|
|
770
771
|
if not video:
|
|
771
772
|
logger.warning(
|
|
772
|
-
"No processed video file available for VideoFile %s.",
|
|
773
|
+
"No processed video file available for VideoFile %s.",
|
|
774
|
+
instance.video_hash,
|
|
773
775
|
)
|
|
774
776
|
return False
|
|
775
777
|
try:
|
|
@@ -805,8 +807,10 @@ class VideoFile(models.Model):
|
|
|
805
807
|
# assert isinstance(frames, list[Path]) #TODO improve TypeCheck
|
|
806
808
|
|
|
807
809
|
# Step 2: Reassemble the video with frames excluding the 'outside' labeled frames
|
|
808
|
-
output_video_path = Path(f"/path/to/output/{
|
|
809
|
-
fps =
|
|
810
|
+
output_video_path = Path(f"/path/to/output/{video.video_hash}_filtered.mp4")
|
|
811
|
+
fps = (
|
|
812
|
+
video.fps if video.fps else 30.0
|
|
813
|
+
) # Default to 30 FPS if fps is not set
|
|
810
814
|
new_video_file = assemble_video_from_frames(
|
|
811
815
|
frames, output_video_path, fps, width=video.width, height=video.height
|
|
812
816
|
)
|
|
@@ -814,7 +818,7 @@ class VideoFile(models.Model):
|
|
|
814
818
|
return True
|
|
815
819
|
except AssertionError as ae:
|
|
816
820
|
logger.error(
|
|
817
|
-
f"Assertion error while creating video without 'outside' frames for VideoFile {
|
|
821
|
+
f"Assertion error while creating video without 'outside' frames for VideoFile {video.video_hash}: {ae}",
|
|
818
822
|
exc_info=True,
|
|
819
823
|
)
|
|
820
824
|
return False
|
|
@@ -823,7 +827,7 @@ class VideoFile(models.Model):
|
|
|
823
827
|
return False
|
|
824
828
|
except Exception as e:
|
|
825
829
|
logger.error(
|
|
826
|
-
f"Error creating video without 'outside' frames for VideoFile {
|
|
830
|
+
f"Error creating video without 'outside' frames for VideoFile {video.video_hash}: {e}",
|
|
827
831
|
exc_info=True,
|
|
828
832
|
)
|
|
829
833
|
return False
|
|
@@ -884,4 +888,12 @@ class VideoFile(models.Model):
|
|
|
884
888
|
Raises:
|
|
885
889
|
VideoFile.DoesNotExist: If no VideoFile with the given ID exists.
|
|
886
890
|
"""
|
|
887
|
-
return VideoFile.objects.get(pk=
|
|
891
|
+
return VideoFile.objects.get(pk=pk)
|
|
892
|
+
|
|
893
|
+
@staticmethod
|
|
894
|
+
def get_video_by_content_hash(hash: str) -> "VideoFile":
|
|
895
|
+
try:
|
|
896
|
+
return VideoFile.objects.get(video_hash=hash)
|
|
897
|
+
except Exception as e:
|
|
898
|
+
logger.error(f"Video cant be returned for known hash. {e}")
|
|
899
|
+
raise
|