endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +89 -122
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -4
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -15,7 +15,9 @@ def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
|
|
|
15
15
|
Returns None if VideoMeta is not linked or ROI is not properly defined.
|
|
16
16
|
"""
|
|
17
17
|
if not video.video_meta:
|
|
18
|
-
logger.warning(
|
|
18
|
+
logger.warning(
|
|
19
|
+
"VideoMeta not linked for video %s. Cannot get endo ROI.", video.video_hash
|
|
20
|
+
)
|
|
19
21
|
return None
|
|
20
22
|
|
|
21
23
|
try:
|
|
@@ -25,17 +27,36 @@ def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
|
|
|
25
27
|
if (
|
|
26
28
|
isinstance(endo_roi, dict)
|
|
27
29
|
and all(k in endo_roi for k in ("x", "y", "width", "height"))
|
|
28
|
-
and all(
|
|
30
|
+
and all(
|
|
31
|
+
isinstance(v, int) and not isinstance(v, bool)
|
|
32
|
+
for v in endo_roi.values()
|
|
33
|
+
)
|
|
29
34
|
):
|
|
30
|
-
cleaned_roi = {
|
|
31
|
-
|
|
35
|
+
cleaned_roi = {
|
|
36
|
+
k: int(endo_roi[k] or 0) for k in ("x", "y", "width", "height")
|
|
37
|
+
}
|
|
38
|
+
logger.debug(
|
|
39
|
+
"Retrieved endo ROI for video %s: %s", video.video_hash, cleaned_roi
|
|
40
|
+
)
|
|
32
41
|
return cleaned_roi
|
|
33
42
|
else:
|
|
34
|
-
logger.warning(
|
|
43
|
+
logger.warning(
|
|
44
|
+
"Endo ROI not fully defined or invalid in VideoMeta for video %s. ROI: %s",
|
|
45
|
+
video.video_hash,
|
|
46
|
+
endo_roi,
|
|
47
|
+
)
|
|
35
48
|
return None
|
|
36
49
|
except AttributeError:
|
|
37
|
-
logger.error(
|
|
50
|
+
logger.error(
|
|
51
|
+
"VideoMeta object for video %s does not have a 'get_endo_roi' method.",
|
|
52
|
+
video.video_hash,
|
|
53
|
+
)
|
|
38
54
|
return None
|
|
39
55
|
except Exception as e:
|
|
40
|
-
logger.error(
|
|
56
|
+
logger.error(
|
|
57
|
+
"Error getting endo ROI from VideoMeta for video %s: %s",
|
|
58
|
+
video.video_hash,
|
|
59
|
+
e,
|
|
60
|
+
exc_info=True,
|
|
61
|
+
)
|
|
41
62
|
return None
|
|
@@ -45,10 +45,10 @@ def _get_fps(video: "VideoFile") -> float:
|
|
|
45
45
|
if video.fps is not None:
|
|
46
46
|
return video.fps
|
|
47
47
|
|
|
48
|
-
logger.debug("FPS not set on instance %s, checking VideoMeta.", video.
|
|
48
|
+
logger.debug("FPS not set on instance %s, checking VideoMeta.", video.video_hash)
|
|
49
49
|
|
|
50
50
|
if not video.video_meta:
|
|
51
|
-
logger.info("VideoMeta not linked for %s, attempting update.", video.
|
|
51
|
+
logger.info("VideoMeta not linked for %s, attempting update.", video.video_hash)
|
|
52
52
|
|
|
53
53
|
_update_video_meta(video, save_instance=True) # Call the helper function
|
|
54
54
|
|
|
@@ -56,20 +56,33 @@ def _get_fps(video: "VideoFile") -> float:
|
|
|
56
56
|
if video.fps is not None:
|
|
57
57
|
return video.fps
|
|
58
58
|
elif video.video_meta and video.video_meta.fps is not None:
|
|
59
|
-
logger.info(
|
|
59
|
+
logger.info(
|
|
60
|
+
"Retrieved FPS %.2f from VideoMeta for %s.",
|
|
61
|
+
video.video_meta.fps,
|
|
62
|
+
video.video_hash,
|
|
63
|
+
)
|
|
60
64
|
_fps = video.video_meta.fps
|
|
61
65
|
try:
|
|
62
66
|
_fps = float(_fps)
|
|
63
67
|
except (TypeError, ValueError):
|
|
64
|
-
logger.warning(
|
|
65
|
-
|
|
68
|
+
logger.warning(
|
|
69
|
+
"Invalid FPS value %.2f in VideoMeta for video %s.",
|
|
70
|
+
video.video_meta.fps,
|
|
71
|
+
video.video_hash,
|
|
72
|
+
)
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f"Could not determine FPS for video {video.video_hash} due to invalid VideoMeta FPS value."
|
|
75
|
+
)
|
|
66
76
|
video.fps = _fps
|
|
67
77
|
# Avoid saving if called from within the save method itself
|
|
68
78
|
if not getattr(video, "_saving", False):
|
|
69
79
|
video.save(update_fields=["fps"])
|
|
70
80
|
return _fps
|
|
71
81
|
else:
|
|
72
|
-
logger.warning(
|
|
82
|
+
logger.warning(
|
|
83
|
+
"Could not determine FPS from VideoMeta for video %s. Trying direct raw file access.",
|
|
84
|
+
video.video_hash,
|
|
85
|
+
)
|
|
73
86
|
try:
|
|
74
87
|
if video.has_raw:
|
|
75
88
|
video_path = video.get_raw_file_path() # Use helper
|
|
@@ -88,23 +101,39 @@ def _get_fps(video: "VideoFile") -> float:
|
|
|
88
101
|
cap.release()
|
|
89
102
|
if fps and fps > 0:
|
|
90
103
|
video.fps = fps
|
|
91
|
-
logger.info(
|
|
104
|
+
logger.info(
|
|
105
|
+
"Determined FPS %.2f directly from file for %s.",
|
|
106
|
+
video.fps,
|
|
107
|
+
video.video_hash,
|
|
108
|
+
)
|
|
92
109
|
if not getattr(video, "_saving", False):
|
|
93
110
|
video.save(update_fields=["fps"])
|
|
94
111
|
return fps
|
|
95
112
|
else:
|
|
96
|
-
logger.warning(
|
|
113
|
+
logger.warning(
|
|
114
|
+
"Could not determine a valid FPS for video file %s.",
|
|
115
|
+
video_path,
|
|
116
|
+
)
|
|
97
117
|
elif video_path:
|
|
98
|
-
logger.warning(
|
|
118
|
+
logger.warning(
|
|
119
|
+
"Raw file path %s does not exist for direct FPS check.",
|
|
120
|
+
video_path,
|
|
121
|
+
)
|
|
99
122
|
else:
|
|
100
123
|
logger.warning("Raw file path is None for direct FPS check.")
|
|
101
124
|
else:
|
|
102
125
|
logger.warning("Raw file not available for direct FPS check.")
|
|
103
126
|
|
|
104
127
|
except Exception as e:
|
|
105
|
-
logger.error(
|
|
128
|
+
logger.error(
|
|
129
|
+
"Error getting FPS directly from file %s: %s",
|
|
130
|
+
video.raw_file.name if video.has_raw else "N/A",
|
|
131
|
+
e,
|
|
132
|
+
)
|
|
106
133
|
|
|
107
|
-
raise ValueError(
|
|
134
|
+
raise ValueError(
|
|
135
|
+
f"Could not determine FPS for video {video.video_hash}. Ensure the video file is valid and accessible."
|
|
136
|
+
)
|
|
108
137
|
|
|
109
138
|
|
|
110
139
|
# TODO Refactor to utils / check if similar function exists in utils
|
|
@@ -132,7 +161,9 @@ def _calculate_fps_manually(cap, video_path: Path) -> float:
|
|
|
132
161
|
Returns:
|
|
133
162
|
float: The estimated FPS, or 0.0 if the duration is zero or calculation fails.
|
|
134
163
|
"""
|
|
135
|
-
logger.warning(
|
|
164
|
+
logger.warning(
|
|
165
|
+
f"Could not get a valid FPS for {video_path}. Trying to calculate manually."
|
|
166
|
+
)
|
|
136
167
|
# This is less accurate and slower
|
|
137
168
|
num_frames = 0
|
|
138
169
|
start_time = cv2.getTickCount()
|
|
@@ -146,5 +177,7 @@ def _calculate_fps_manually(cap, video_path: Path) -> float:
|
|
|
146
177
|
if seconds > 0:
|
|
147
178
|
return num_frames / seconds
|
|
148
179
|
|
|
149
|
-
logger.error(
|
|
180
|
+
logger.error(
|
|
181
|
+
f"Manual FPS calculation failed for {video_path} due to zero duration."
|
|
182
|
+
)
|
|
150
183
|
return 0.0
|
|
@@ -31,24 +31,38 @@ def _initialize_video_specs(video: "VideoFile", use_raw: bool = True) -> bool:
|
|
|
31
31
|
video_path = video.active_file_path # Use property relying on IO helpers
|
|
32
32
|
target_file_name = video.active_file.name
|
|
33
33
|
else:
|
|
34
|
-
logger.error(
|
|
34
|
+
logger.error(
|
|
35
|
+
"No suitable video file found for spec initialization for %s.",
|
|
36
|
+
video.video_hash,
|
|
37
|
+
)
|
|
35
38
|
return False
|
|
36
39
|
|
|
37
40
|
if not video_path:
|
|
38
41
|
# Raise exception
|
|
39
|
-
raise FileNotFoundError(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
+
raise FileNotFoundError(
|
|
43
|
+
f"Could not determine video file path for spec initialization for {video.video_hash}."
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
logger.info(
|
|
47
|
+
"Initializing video specs directly from file %s (%s) for %s",
|
|
48
|
+
target_file_name,
|
|
49
|
+
video_path,
|
|
50
|
+
video.video_hash,
|
|
51
|
+
)
|
|
42
52
|
try:
|
|
43
53
|
if not video_path.exists():
|
|
44
54
|
# Raise exception
|
|
45
|
-
raise FileNotFoundError(
|
|
55
|
+
raise FileNotFoundError(
|
|
56
|
+
f"Video file not found at {video_path} for spec initialization (Video: {video.video_hash})."
|
|
57
|
+
)
|
|
46
58
|
|
|
47
59
|
video_cap = cv2.VideoCapture(video_path.as_posix())
|
|
48
60
|
if not video_cap.isOpened():
|
|
49
61
|
# Raise exception
|
|
50
62
|
video_cap.release() # Ensure release
|
|
51
|
-
raise RuntimeError(
|
|
63
|
+
raise RuntimeError(
|
|
64
|
+
f"Could not open video file {video_path} with OpenCV for spec initialization (Video: {video.video_hash})."
|
|
65
|
+
)
|
|
52
66
|
|
|
53
67
|
updated = False
|
|
54
68
|
fields_to_update = []
|
|
@@ -67,8 +81,16 @@ def _initialize_video_specs(video: "VideoFile", use_raw: bool = True) -> bool:
|
|
|
67
81
|
file_height = int(video_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
|
68
82
|
file_frame_count = int(video_cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
|
69
83
|
except Exception as cv_err:
|
|
70
|
-
logger.error(
|
|
71
|
-
|
|
84
|
+
logger.error(
|
|
85
|
+
"Error getting properties from OpenCV for %s (Video: %s): %s",
|
|
86
|
+
video_path,
|
|
87
|
+
video.video_hash,
|
|
88
|
+
cv_err,
|
|
89
|
+
exc_info=True,
|
|
90
|
+
)
|
|
91
|
+
raise RuntimeError(
|
|
92
|
+
f"OpenCV failed to get properties for {video_path}"
|
|
93
|
+
) from cv_err
|
|
72
94
|
finally:
|
|
73
95
|
video_cap.release() # Ensure release after getting props
|
|
74
96
|
|
|
@@ -96,42 +118,81 @@ def _initialize_video_specs(video: "VideoFile", use_raw: bool = True) -> bool:
|
|
|
96
118
|
video.frame_count = file_frame_count
|
|
97
119
|
fields_to_update.append("frame_count")
|
|
98
120
|
updated = True
|
|
99
|
-
elif
|
|
100
|
-
|
|
121
|
+
elif (
|
|
122
|
+
file_frame_count is None or file_frame_count <= 0
|
|
123
|
+
): # Log if not updated due to invalid file_frame_count
|
|
124
|
+
logger.warning(
|
|
125
|
+
"Invalid frame count (value: %s) obtained from OpenCV for %s. Video frame_count not updated.",
|
|
126
|
+
file_frame_count,
|
|
127
|
+
video_path,
|
|
128
|
+
)
|
|
101
129
|
|
|
102
130
|
# --- Update Duration ---
|
|
103
131
|
if current_duration is None: # Only if duration isn't already set
|
|
104
132
|
# Use the potentially updated video.frame_count and current_fps (which reflects video.fps or file_fps)
|
|
105
133
|
final_frame_count_for_duration = video.frame_count
|
|
106
|
-
final_fps_for_duration =
|
|
107
|
-
|
|
108
|
-
|
|
134
|
+
final_fps_for_duration = (
|
|
135
|
+
current_fps # This is video.fps after potential update from file_fps
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
if (
|
|
139
|
+
final_frame_count_for_duration
|
|
140
|
+
and final_frame_count_for_duration > 0
|
|
141
|
+
and final_fps_for_duration
|
|
142
|
+
and final_fps_for_duration > 0
|
|
143
|
+
):
|
|
109
144
|
video.duration = final_frame_count_for_duration / final_fps_for_duration
|
|
110
145
|
fields_to_update.append("duration")
|
|
111
146
|
updated = True
|
|
112
147
|
else:
|
|
113
148
|
# Log if duration could not be calculated, indicating which component was missing/invalid
|
|
114
|
-
if not (
|
|
149
|
+
if not (
|
|
150
|
+
final_frame_count_for_duration
|
|
151
|
+
and final_frame_count_for_duration > 0
|
|
152
|
+
):
|
|
115
153
|
logger.warning(
|
|
116
|
-
"Duration not calculated for %s: frame count is unavailable or invalid (value: %s).",
|
|
154
|
+
"Duration not calculated for %s: frame count is unavailable or invalid (value: %s).",
|
|
155
|
+
video_path,
|
|
156
|
+
final_frame_count_for_duration,
|
|
117
157
|
)
|
|
118
158
|
if not (final_fps_for_duration and final_fps_for_duration > 0):
|
|
119
|
-
logger.warning(
|
|
159
|
+
logger.warning(
|
|
160
|
+
"Duration not calculated for %s: FPS is unavailable or invalid (value: %s).",
|
|
161
|
+
video_path,
|
|
162
|
+
final_fps_for_duration,
|
|
163
|
+
)
|
|
120
164
|
|
|
121
165
|
# --- Save if updated ---
|
|
122
166
|
if updated:
|
|
123
|
-
logger.info(
|
|
167
|
+
logger.info(
|
|
168
|
+
"Updated video specs for %s from file %s: %s",
|
|
169
|
+
video.video_hash,
|
|
170
|
+
target_file_name,
|
|
171
|
+
", ".join(fields_to_update),
|
|
172
|
+
)
|
|
124
173
|
video.save(update_fields=fields_to_update)
|
|
125
174
|
return True
|
|
126
175
|
else:
|
|
127
|
-
logger.info(
|
|
176
|
+
logger.info(
|
|
177
|
+
"No video specs needed updating for %s from file %s.",
|
|
178
|
+
video.video_hash,
|
|
179
|
+
target_file_name,
|
|
180
|
+
)
|
|
128
181
|
return True
|
|
129
182
|
|
|
130
183
|
except Exception as e:
|
|
131
184
|
# Log and re-raise exception
|
|
132
|
-
logger.error(
|
|
185
|
+
logger.error(
|
|
186
|
+
"Error initializing video specs for %s from file %s: %s",
|
|
187
|
+
video.video_hash,
|
|
188
|
+
video_path,
|
|
189
|
+
e,
|
|
190
|
+
exc_info=True,
|
|
191
|
+
)
|
|
133
192
|
# Ensure capture is released in case of unexpected error
|
|
134
193
|
if "video_cap" in locals() and video_cap.isOpened():
|
|
135
194
|
video_cap.release()
|
|
136
195
|
# Re-raise as RuntimeError
|
|
137
|
-
raise RuntimeError(
|
|
196
|
+
raise RuntimeError(
|
|
197
|
+
f"Failed to initialize video specs for {video.video_hash} from {video_path}"
|
|
198
|
+
) from e
|
|
@@ -10,6 +10,7 @@ from ....metadata.sensitive_meta_logic import update_or_create_sensitive_meta_fr
|
|
|
10
10
|
|
|
11
11
|
if TYPE_CHECKING:
|
|
12
12
|
from ..video_file import VideoFile
|
|
13
|
+
|
|
13
14
|
# SensitiveMeta is already imported above
|
|
14
15
|
|
|
15
16
|
logger = logging.getLogger(__name__)
|
|
@@ -31,35 +32,46 @@ def _update_text_metadata(
|
|
|
31
32
|
- Pre-condition: Requires state.frames_extracted=True.
|
|
32
33
|
- Post-condition: Sets state.text_meta_extracted=True (even if no text found).
|
|
33
34
|
"""
|
|
34
|
-
logger.debug(f"Updating text metadata for video {video.
|
|
35
|
+
logger.debug(f"Updating text metadata for video {video.video_hash}")
|
|
35
36
|
state = video.get_or_create_state()
|
|
36
37
|
|
|
37
38
|
# --- Pre-condition Checks ---
|
|
38
39
|
if not state.frames_extracted:
|
|
39
40
|
# Attempt to extract frames automatically if they're not available
|
|
40
|
-
logger.warning(
|
|
41
|
+
logger.warning(
|
|
42
|
+
f"Frames not extracted for video {video.video_hash}. Attempting automatic frame extraction..."
|
|
43
|
+
)
|
|
41
44
|
try:
|
|
42
45
|
success = video.extract_frames(overwrite=False)
|
|
43
46
|
except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc:
|
|
44
47
|
logger.error(
|
|
45
48
|
"Failed to extract frames for video %s: %s",
|
|
46
|
-
video.
|
|
49
|
+
video.video_hash,
|
|
47
50
|
exc,
|
|
48
51
|
exc_info=True,
|
|
49
52
|
)
|
|
50
|
-
raise ValueError(
|
|
53
|
+
raise ValueError(
|
|
54
|
+
f"Cannot update text metadata for video {video.video_hash}: Frames not extracted and automatic extraction failed"
|
|
55
|
+
) from exc
|
|
51
56
|
|
|
52
57
|
if not success:
|
|
53
|
-
raise ValueError(
|
|
58
|
+
raise ValueError(
|
|
59
|
+
f"Cannot update text metadata for video {video.video_hash}: Frame extraction returned False"
|
|
60
|
+
)
|
|
54
61
|
|
|
55
62
|
state.refresh_from_db()
|
|
56
63
|
if not state.frames_extracted:
|
|
57
|
-
raise ValueError(
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"Cannot update text metadata for video {video.video_hash}: Frame extraction completed but state was not updated"
|
|
66
|
+
)
|
|
58
67
|
|
|
59
|
-
logger.info(f"Successfully extracted frames for video {video.
|
|
68
|
+
logger.info(f"Successfully extracted frames for video {video.video_hash}")
|
|
60
69
|
|
|
61
70
|
if state.text_meta_extracted and not overwrite:
|
|
62
|
-
logger.info(
|
|
71
|
+
logger.info(
|
|
72
|
+
"Text already extracted for video %s and overwrite=False. Skipping.",
|
|
73
|
+
video.video_hash,
|
|
74
|
+
) # Changed to info
|
|
63
75
|
return video.sensitive_meta # Return existing meta if available
|
|
64
76
|
# --- End Pre-condition Checks ---
|
|
65
77
|
|
|
@@ -67,10 +79,19 @@ def _update_text_metadata(
|
|
|
67
79
|
# _extract_text_from_video_frames raises ValueError on pre-condition failure
|
|
68
80
|
try:
|
|
69
81
|
if not extracted_data_dict:
|
|
70
|
-
extracted_data_dict = video.extract_text_from_frames(
|
|
82
|
+
extracted_data_dict = video.extract_text_from_frames(
|
|
83
|
+
frame_fraction=ocr_frame_fraction, cap=cap
|
|
84
|
+
)
|
|
71
85
|
except Exception as text_extract_e:
|
|
72
|
-
logger.error(
|
|
73
|
-
|
|
86
|
+
logger.error(
|
|
87
|
+
"Failed during text extraction step for video %s: %s",
|
|
88
|
+
video.video_hash,
|
|
89
|
+
text_extract_e,
|
|
90
|
+
exc_info=True,
|
|
91
|
+
)
|
|
92
|
+
raise RuntimeError(
|
|
93
|
+
f"Text extraction failed for video {video.video_hash}"
|
|
94
|
+
) from text_extract_e
|
|
74
95
|
|
|
75
96
|
# --- Atomic Update Block ---
|
|
76
97
|
try:
|
|
@@ -80,7 +101,10 @@ def _update_text_metadata(
|
|
|
80
101
|
sensitive_meta_instance = video.sensitive_meta # Get current instance
|
|
81
102
|
|
|
82
103
|
if not extracted_data_dict:
|
|
83
|
-
logger.warning(
|
|
104
|
+
logger.warning(
|
|
105
|
+
"No text extracted for video %s; skipping SensitiveMeta update.",
|
|
106
|
+
video.video_hash,
|
|
107
|
+
)
|
|
84
108
|
# Mark state as retrieved even if no data found, to avoid re-running unless overwrite=True
|
|
85
109
|
if not state.text_meta_extracted:
|
|
86
110
|
state.text_meta_extracted = True
|
|
@@ -90,7 +114,11 @@ def _update_text_metadata(
|
|
|
90
114
|
# Add center info if not already present in extracted data
|
|
91
115
|
if "center_name" not in extracted_data_dict and video.center:
|
|
92
116
|
extracted_data_dict["center_name"] = video.center.name
|
|
93
|
-
logger.debug(
|
|
117
|
+
logger.debug(
|
|
118
|
+
"Data for SensitiveMeta update for video %s: %s",
|
|
119
|
+
video.video_hash,
|
|
120
|
+
extracted_data_dict,
|
|
121
|
+
)
|
|
94
122
|
|
|
95
123
|
# Pass the Class, the data dict, and the current instance (or None)
|
|
96
124
|
# This function might raise exceptions if data is invalid
|
|
@@ -102,7 +130,9 @@ def _update_text_metadata(
|
|
|
102
130
|
|
|
103
131
|
# Update VideoFile fields if necessary
|
|
104
132
|
update_fields_video = []
|
|
105
|
-
if
|
|
133
|
+
if (
|
|
134
|
+
created or sensitive_meta != sensitive_meta_instance
|
|
135
|
+
): # Check if relation needs update
|
|
106
136
|
video.sensitive_meta = sensitive_meta
|
|
107
137
|
update_fields_video.append("sensitive_meta")
|
|
108
138
|
|
|
@@ -111,7 +141,6 @@ def _update_text_metadata(
|
|
|
111
141
|
if extracted_date: # Ensure date is not None or empty
|
|
112
142
|
video.date = extracted_date
|
|
113
143
|
update_fields_video.append("date")
|
|
114
|
-
|
|
115
144
|
|
|
116
145
|
# Save VideoFile if fields changed
|
|
117
146
|
if update_fields_video:
|
|
@@ -125,13 +154,25 @@ def _update_text_metadata(
|
|
|
125
154
|
# Mark sensitive meta as processed when updated via text metadata
|
|
126
155
|
if sensitive_meta:
|
|
127
156
|
state.mark_sensitive_meta_processed(save=True)
|
|
128
|
-
logger.info(
|
|
129
|
-
|
|
130
|
-
|
|
157
|
+
logger.info(
|
|
158
|
+
f"Marked sensitive_meta_processed=True for video {video.video_hash} after text metadata update"
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
logger.info(
|
|
162
|
+
"Successfully updated/created SensitiveMeta and state for video %s.",
|
|
163
|
+
video.video_hash,
|
|
164
|
+
) # Changed to info
|
|
131
165
|
return sensitive_meta
|
|
132
166
|
|
|
133
167
|
except Exception as e:
|
|
134
|
-
logger.error(
|
|
168
|
+
logger.error(
|
|
169
|
+
"Failed to update/create SensitiveMeta or state for video %s: %s",
|
|
170
|
+
video.video_hash,
|
|
171
|
+
e,
|
|
172
|
+
exc_info=True,
|
|
173
|
+
)
|
|
135
174
|
# Re-raise exception for the pipeline to catch
|
|
136
|
-
raise RuntimeError(
|
|
175
|
+
raise RuntimeError(
|
|
176
|
+
f"Failed to update/create SensitiveMeta or state for video {video.video_hash}"
|
|
177
|
+
) from e
|
|
137
178
|
# --- End Atomic Update Block ---
|
|
@@ -14,12 +14,14 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
|
|
|
14
14
|
"""
|
|
15
15
|
from ....metadata import VideoMeta # Local import
|
|
16
16
|
|
|
17
|
-
logger.debug(
|
|
17
|
+
logger.debug(
|
|
18
|
+
"Updating technical VideoMeta for video %s (from raw file).", video.video_hash
|
|
19
|
+
)
|
|
18
20
|
|
|
19
21
|
if not video.has_raw:
|
|
20
22
|
# DEFENSIVE: Log warning and skip instead of crashing
|
|
21
23
|
logger.warning(
|
|
22
|
-
f"Raw video file path not available for {video.
|
|
24
|
+
f"Raw video file path not available for {video.video_hash}. Skipping VideoMeta update - this may indicate the video was processed and raw file moved."
|
|
23
25
|
)
|
|
24
26
|
return # Graceful skip instead of FileNotFoundError
|
|
25
27
|
|
|
@@ -27,22 +29,30 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
|
|
|
27
29
|
if not raw_video_path or not raw_video_path.exists():
|
|
28
30
|
# DEFENSIVE: Log warning and skip instead of crashing production pipeline
|
|
29
31
|
logger.warning(
|
|
30
|
-
f"Raw video file path {raw_video_path} does not exist for video {video.
|
|
32
|
+
f"Raw video file path {raw_video_path} does not exist for video {video.video_hash}. Skipping VideoMeta update - this typically happens after video processing when raw files are moved to processed location."
|
|
31
33
|
)
|
|
32
34
|
return # Graceful skip instead of FileNotFoundError that crashes production
|
|
33
35
|
|
|
34
36
|
try:
|
|
35
37
|
vm = video.video_meta
|
|
36
38
|
if vm:
|
|
37
|
-
logger.info(
|
|
38
|
-
|
|
39
|
+
logger.info(
|
|
40
|
+
"Updating existing VideoMeta (PK: %s) for video %s.",
|
|
41
|
+
vm.pk,
|
|
42
|
+
video.video_hash,
|
|
43
|
+
)
|
|
44
|
+
vm.update_meta(
|
|
45
|
+
raw_video_path
|
|
46
|
+
) # Assuming this method exists and raises on error
|
|
39
47
|
vm.save()
|
|
40
48
|
else:
|
|
41
49
|
if not video.center or not video.processor:
|
|
42
50
|
# Raise exception
|
|
43
|
-
raise ValueError(
|
|
51
|
+
raise ValueError(
|
|
52
|
+
f"Cannot create VideoMeta for {video.video_hash}: Center or Processor is missing."
|
|
53
|
+
)
|
|
44
54
|
|
|
45
|
-
logger.info("Creating new VideoMeta for video %s.", video.
|
|
55
|
+
logger.info("Creating new VideoMeta for video %s.", video.video_hash)
|
|
46
56
|
# Assuming create_from_file exists and raises on error
|
|
47
57
|
video.video_meta = VideoMeta.create_from_file(
|
|
48
58
|
video_path=raw_video_path,
|
|
@@ -52,7 +62,11 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
|
|
|
52
62
|
)
|
|
53
63
|
vm = video.video_meta
|
|
54
64
|
assert vm is not None # For type checker
|
|
55
|
-
logger.info(
|
|
65
|
+
logger.info(
|
|
66
|
+
"Created and linked VideoMeta (PK: %s) for video %s.",
|
|
67
|
+
vm.pk,
|
|
68
|
+
video.video_hash,
|
|
69
|
+
)
|
|
56
70
|
|
|
57
71
|
# Save the VideoFile instance itself if requested and if video_meta was linked/updated
|
|
58
72
|
if save_instance:
|
|
@@ -62,16 +76,30 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
|
|
|
62
76
|
meta_fields = ["fps", "duration", "frame_count", "width", "height"]
|
|
63
77
|
for field in meta_fields:
|
|
64
78
|
# Check if field is None on video but has value on meta
|
|
65
|
-
if
|
|
79
|
+
if (
|
|
80
|
+
getattr(video, field) is None
|
|
81
|
+
and getattr(video.video_meta, field, None) is not None
|
|
82
|
+
):
|
|
66
83
|
# No need to set attribute here, save method handles it
|
|
67
84
|
update_fields.append(field)
|
|
68
85
|
# Ensure update_fields has unique values before saving
|
|
69
86
|
unique_update_fields = list(set(update_fields))
|
|
70
87
|
if unique_update_fields:
|
|
71
88
|
video.save(update_fields=unique_update_fields)
|
|
72
|
-
logger.info(
|
|
89
|
+
logger.info(
|
|
90
|
+
"Saved video %s after VideoMeta update (Fields: %s).",
|
|
91
|
+
video.video_hash,
|
|
92
|
+
unique_update_fields,
|
|
93
|
+
)
|
|
73
94
|
|
|
74
95
|
except Exception as e:
|
|
75
|
-
logger.error(
|
|
96
|
+
logger.error(
|
|
97
|
+
"Failed to update/create VideoMeta for video %s: %s",
|
|
98
|
+
video.video_hash,
|
|
99
|
+
e,
|
|
100
|
+
exc_info=True,
|
|
101
|
+
)
|
|
76
102
|
# Re-raise exception
|
|
77
|
-
raise RuntimeError(
|
|
103
|
+
raise RuntimeError(
|
|
104
|
+
f"Failed to update/create VideoMeta for video {video.video_hash}"
|
|
105
|
+
) from e
|