endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +89 -122
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -4
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -7,7 +7,9 @@ from django.db import transaction
|
|
|
7
7
|
from endoreg_db.models.media.video.video_file_io import _get_frame_dir_path
|
|
8
8
|
|
|
9
9
|
# Assuming ffmpeg_wrapper has or will have this function
|
|
10
|
-
from endoreg_db.utils.video.ffmpeg_wrapper import
|
|
10
|
+
from endoreg_db.utils.video.ffmpeg_wrapper import (
|
|
11
|
+
extract_frame_range as ffmpeg_extract_frame_range,
|
|
12
|
+
)
|
|
11
13
|
|
|
12
14
|
if TYPE_CHECKING:
|
|
13
15
|
from endoreg_db.models import VideoFile
|
|
@@ -21,15 +23,30 @@ def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
|
|
|
21
23
|
and updates their is_extracted status to False. Runs within the caller's transaction.
|
|
22
24
|
"""
|
|
23
25
|
|
|
24
|
-
logger.info(
|
|
25
|
-
|
|
26
|
+
logger.info(
|
|
27
|
+
"Deleting frame files for video %s in range [%d, %d)",
|
|
28
|
+
video.video_hash,
|
|
29
|
+
start_frame,
|
|
30
|
+
end_frame,
|
|
31
|
+
)
|
|
32
|
+
frames_to_delete = video.frames.filter(
|
|
33
|
+
frame_number__gte=start_frame, frame_number__lt=end_frame, is_extracted=True
|
|
34
|
+
)
|
|
26
35
|
|
|
27
36
|
deleted_count = 0
|
|
28
|
-
paths_to_delete = [
|
|
37
|
+
paths_to_delete = [
|
|
38
|
+
frame.file_path for frame in frames_to_delete
|
|
39
|
+
] # Get paths before potential DB changes
|
|
29
40
|
|
|
30
41
|
# Update DB first
|
|
31
42
|
update_count = frames_to_delete.update(is_extracted=False)
|
|
32
|
-
logger.info(
|
|
43
|
+
logger.info(
|
|
44
|
+
"Marked %d Frame objects as is_extracted=False for video %s range [%d, %d).",
|
|
45
|
+
update_count,
|
|
46
|
+
video.video_hash,
|
|
47
|
+
start_frame,
|
|
48
|
+
end_frame,
|
|
49
|
+
)
|
|
33
50
|
|
|
34
51
|
# Then delete files
|
|
35
52
|
for frame_path in paths_to_delete:
|
|
@@ -39,12 +56,17 @@ def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
|
|
|
39
56
|
deleted_count += 1
|
|
40
57
|
except Exception as e:
|
|
41
58
|
# Log warning but continue; DB state is already updated.
|
|
42
|
-
logger.warning(
|
|
59
|
+
logger.warning(
|
|
60
|
+
"Could not delete frame file %s for video %s: %s",
|
|
61
|
+
frame_path,
|
|
62
|
+
video.video_hash,
|
|
63
|
+
e,
|
|
64
|
+
)
|
|
43
65
|
|
|
44
66
|
logger.info(
|
|
45
67
|
"Attempted deletion of %d files for video %s range [%d, %d). Actual deleted: %d.",
|
|
46
68
|
len(paths_to_delete),
|
|
47
|
-
video.
|
|
69
|
+
video.video_hash,
|
|
48
70
|
start_frame,
|
|
49
71
|
end_frame,
|
|
50
72
|
deleted_count,
|
|
@@ -69,49 +91,88 @@ def _extract_frame_range(
|
|
|
69
91
|
"""
|
|
70
92
|
|
|
71
93
|
if not video.has_raw:
|
|
72
|
-
raise FileNotFoundError(
|
|
94
|
+
raise FileNotFoundError(
|
|
95
|
+
f"Raw video file not available for {video.video_hash}. Cannot extract frame range."
|
|
96
|
+
)
|
|
73
97
|
|
|
74
98
|
raw_file_path = video.get_raw_file_path()
|
|
75
99
|
if not raw_file_path or not raw_file_path.exists():
|
|
76
|
-
raise FileNotFoundError(
|
|
100
|
+
raise FileNotFoundError(
|
|
101
|
+
f"Raw video file not found at {raw_file_path} for video {video.video_hash}. Cannot extract frame range."
|
|
102
|
+
)
|
|
77
103
|
|
|
78
104
|
frame_dir = _get_frame_dir_path(video)
|
|
79
105
|
if not frame_dir:
|
|
80
|
-
raise ValueError(
|
|
106
|
+
raise ValueError(
|
|
107
|
+
f"Cannot determine frame directory path for video {video.video_hash}."
|
|
108
|
+
)
|
|
81
109
|
|
|
82
110
|
# Check frames within the range that might already exist
|
|
83
|
-
frames_in_range = video.frames.filter(
|
|
111
|
+
frames_in_range = video.frames.filter(
|
|
112
|
+
frame_number__gte=start_frame, frame_number__lt=end_frame
|
|
113
|
+
)
|
|
84
114
|
existing_extracted_in_range = frames_in_range.filter(is_extracted=True)
|
|
85
115
|
|
|
86
116
|
if existing_extracted_in_range.exists():
|
|
87
117
|
if overwrite:
|
|
88
|
-
logger.info(
|
|
118
|
+
logger.info(
|
|
119
|
+
"Overwrite=True, deleting existing frame files in range [%d, %d) for video %s before extraction.",
|
|
120
|
+
start_frame,
|
|
121
|
+
end_frame,
|
|
122
|
+
video.video_hash,
|
|
123
|
+
)
|
|
89
124
|
_delete_frame_range(video, start_frame, end_frame)
|
|
90
125
|
else:
|
|
91
126
|
logger.info(
|
|
92
127
|
"Frames already exist in range [%d, %d) for video %s and overwrite=False. Skipping extraction for this range.",
|
|
93
128
|
start_frame,
|
|
94
129
|
end_frame,
|
|
95
|
-
video.
|
|
130
|
+
video.video_hash,
|
|
131
|
+
)
|
|
132
|
+
updated_count = frames_in_range.filter(is_extracted=False).update(
|
|
133
|
+
is_extracted=True
|
|
96
134
|
)
|
|
97
|
-
updated_count = frames_in_range.filter(is_extracted=False).update(is_extracted=True)
|
|
98
135
|
if updated_count > 0:
|
|
99
|
-
logger.info(
|
|
136
|
+
logger.info(
|
|
137
|
+
"Marked %d existing Frame objects in range [%d, %d) as extracted for video %s.",
|
|
138
|
+
updated_count,
|
|
139
|
+
start_frame,
|
|
140
|
+
end_frame,
|
|
141
|
+
video.video_hash,
|
|
142
|
+
)
|
|
100
143
|
return True # Indicate success as frames are considered present
|
|
101
144
|
|
|
102
145
|
frame_dir.mkdir(parents=True, exist_ok=True)
|
|
103
146
|
extracted_paths = []
|
|
104
147
|
|
|
105
148
|
try:
|
|
106
|
-
logger.info(
|
|
107
|
-
|
|
149
|
+
logger.info(
|
|
150
|
+
"Starting frame range extraction [%d, %d) for video %s to %s",
|
|
151
|
+
start_frame,
|
|
152
|
+
end_frame,
|
|
153
|
+
video.video_hash,
|
|
154
|
+
frame_dir,
|
|
155
|
+
)
|
|
156
|
+
extracted_paths = ffmpeg_extract_frame_range(
|
|
157
|
+
raw_file_path, frame_dir, start_frame, end_frame, quality=quality, ext=ext
|
|
158
|
+
)
|
|
108
159
|
|
|
109
160
|
logger.info(
|
|
110
|
-
"ffmpeg extraction process completed for video %s range [%d, %d). Found %d files.",
|
|
161
|
+
"ffmpeg extraction process completed for video %s range [%d, %d). Found %d files.",
|
|
162
|
+
video.video_hash,
|
|
163
|
+
start_frame,
|
|
164
|
+
end_frame,
|
|
165
|
+
len(extracted_paths),
|
|
111
166
|
)
|
|
112
167
|
|
|
113
168
|
update_count = frames_in_range.update(is_extracted=True)
|
|
114
|
-
logger.info(
|
|
169
|
+
logger.info(
|
|
170
|
+
"Marked %d Frame objects in range [%d, %d) as is_extracted=True for video %s.",
|
|
171
|
+
update_count,
|
|
172
|
+
start_frame,
|
|
173
|
+
end_frame,
|
|
174
|
+
video.video_hash,
|
|
175
|
+
)
|
|
115
176
|
|
|
116
177
|
return True
|
|
117
178
|
|
|
@@ -120,25 +181,48 @@ def _extract_frame_range(
|
|
|
120
181
|
"Frame range extraction [%d, %d) failed for video %s: %s",
|
|
121
182
|
start_frame,
|
|
122
183
|
end_frame,
|
|
123
|
-
video.
|
|
184
|
+
video.video_hash,
|
|
124
185
|
err,
|
|
125
186
|
exc_info=True,
|
|
126
187
|
)
|
|
127
188
|
raise
|
|
128
189
|
|
|
129
190
|
except Exception as e:
|
|
130
|
-
logger.error(
|
|
191
|
+
logger.error(
|
|
192
|
+
"Frame range extraction [%d, %d) or DB update failed for video %s: %s",
|
|
193
|
+
start_frame,
|
|
194
|
+
end_frame,
|
|
195
|
+
video.video_hash,
|
|
196
|
+
e,
|
|
197
|
+
exc_info=True,
|
|
198
|
+
)
|
|
131
199
|
|
|
132
|
-
logger.warning(
|
|
133
|
-
|
|
200
|
+
logger.warning(
|
|
201
|
+
"Attempting file cleanup in range [%d, %d) for video %s due to extraction error.",
|
|
202
|
+
start_frame,
|
|
203
|
+
end_frame,
|
|
204
|
+
video.video_hash,
|
|
205
|
+
)
|
|
206
|
+
files_to_check = (
|
|
207
|
+
extracted_paths if "extracted_paths" in locals() and extracted_paths else []
|
|
208
|
+
)
|
|
134
209
|
if not files_to_check:
|
|
135
|
-
files_to_check = [
|
|
210
|
+
files_to_check = [
|
|
211
|
+
frame_dir / f"frame_{i:07d}.{ext}"
|
|
212
|
+
for i in range(start_frame, end_frame)
|
|
213
|
+
]
|
|
136
214
|
|
|
137
215
|
for potential_file in files_to_check:
|
|
138
216
|
if potential_file.exists():
|
|
139
217
|
try:
|
|
140
218
|
os.remove(potential_file)
|
|
141
219
|
except OSError as unlink_err:
|
|
142
|
-
logger.error(
|
|
143
|
-
|
|
144
|
-
|
|
220
|
+
logger.error(
|
|
221
|
+
"Failed to delete potential frame %s during cleanup: %s",
|
|
222
|
+
potential_file,
|
|
223
|
+
unlink_err,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
raise RuntimeError(
|
|
227
|
+
f"Frame range extraction or update failed for video {video.video_hash} range [{start_frame}, {end_frame})."
|
|
228
|
+
) from e
|
|
@@ -10,14 +10,18 @@ logger = logging.getLogger(__name__)
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@transaction.atomic
|
|
13
|
-
def _mark_frames_extracted_status(
|
|
13
|
+
def _mark_frames_extracted_status(
|
|
14
|
+
video: "VideoFile", extracted_frame_numbers: Set[int], status: bool
|
|
15
|
+
):
|
|
14
16
|
"""
|
|
15
17
|
Bulk updates the is_extracted status for a set of frame numbers.
|
|
16
18
|
"""
|
|
17
19
|
from endoreg_db.models.media.frame import Frame
|
|
18
20
|
|
|
19
21
|
if not extracted_frame_numbers:
|
|
20
|
-
logger.warning(
|
|
22
|
+
logger.warning(
|
|
23
|
+
"No frame numbers provided to update status for video %s.", video.video_hash
|
|
24
|
+
)
|
|
21
25
|
return 0
|
|
22
26
|
|
|
23
27
|
# --- Enhanced Logging ---
|
|
@@ -28,7 +32,7 @@ def _mark_frames_extracted_status(video: "VideoFile", extracted_frame_numbers: S
|
|
|
28
32
|
"Attempting to mark %d Frame objects as is_extracted=%s for video %s. Frame numbers range: [%s-%s]. Contains frame 0: %s",
|
|
29
33
|
len(extracted_frame_numbers),
|
|
30
34
|
status,
|
|
31
|
-
video.
|
|
35
|
+
video.video_hash,
|
|
32
36
|
min_frame,
|
|
33
37
|
max_frame,
|
|
34
38
|
contains_zero,
|
|
@@ -38,37 +42,66 @@ def _mark_frames_extracted_status(video: "VideoFile", extracted_frame_numbers: S
|
|
|
38
42
|
try:
|
|
39
43
|
# Update Frame objects based on frame_number
|
|
40
44
|
# Convert set to list for potentially better compatibility with some DB backends
|
|
41
|
-
updated_count = Frame.objects.filter(
|
|
45
|
+
updated_count = Frame.objects.filter(
|
|
46
|
+
video=video, frame_number__in=list(extracted_frame_numbers)
|
|
47
|
+
).update(is_extracted=status)
|
|
42
48
|
|
|
43
|
-
logger.info(
|
|
49
|
+
logger.info(
|
|
50
|
+
"Database reported updating %d Frame objects to is_extracted=%s for video %s.",
|
|
51
|
+
updated_count,
|
|
52
|
+
status,
|
|
53
|
+
video.video_hash,
|
|
54
|
+
)
|
|
44
55
|
|
|
45
56
|
# Verification step
|
|
46
57
|
if updated_count != len(extracted_frame_numbers):
|
|
47
58
|
logger.warning(
|
|
48
59
|
"Mismatch during status update for video %s. Expected to update %d frames, but DB reported updating %d.",
|
|
49
|
-
video.
|
|
60
|
+
video.video_hash,
|
|
50
61
|
len(extracted_frame_numbers),
|
|
51
62
|
updated_count,
|
|
52
63
|
)
|
|
53
64
|
# --- Add detailed check for frame 0 if status is True and it should have been updated ---
|
|
54
|
-
if
|
|
65
|
+
if (
|
|
66
|
+
status is True
|
|
67
|
+
and contains_zero
|
|
68
|
+
and updated_count < len(extracted_frame_numbers)
|
|
69
|
+
):
|
|
55
70
|
try:
|
|
56
71
|
# Check the status of frame 0 directly after the update attempt
|
|
57
|
-
frame_zero = Frame.objects.get(
|
|
72
|
+
frame_zero = Frame.objects.get(video=video, frame_number=0)
|
|
58
73
|
if not frame_zero.is_extracted:
|
|
59
|
-
logger.error(
|
|
74
|
+
logger.error(
|
|
75
|
+
"Verification check: Frame 0 (PK: %s) was NOT updated to is_extracted=True for video %s.",
|
|
76
|
+
frame_zero.pk,
|
|
77
|
+
video.video_hash,
|
|
78
|
+
)
|
|
60
79
|
else:
|
|
61
80
|
# This case should ideally not happen if updated_count < expected count, but log just in case
|
|
62
81
|
logger.info(
|
|
63
|
-
"Verification check: Frame 0 (PK: %s) IS is_extracted=True for video %s, despite count mismatch.",
|
|
82
|
+
"Verification check: Frame 0 (PK: %s) IS is_extracted=True for video %s, despite count mismatch.",
|
|
83
|
+
frame_zero.pk,
|
|
84
|
+
video.video_hash,
|
|
64
85
|
)
|
|
65
86
|
except Frame.DoesNotExist:
|
|
66
|
-
logger.error(
|
|
87
|
+
logger.error(
|
|
88
|
+
"Verification check: Frame 0 does not exist for video %s during status check.",
|
|
89
|
+
video.video_hash,
|
|
90
|
+
)
|
|
67
91
|
except Exception as verify_e:
|
|
68
|
-
logger.error(
|
|
92
|
+
logger.error(
|
|
93
|
+
"Verification check: Error checking frame 0 status for video %s: %s",
|
|
94
|
+
video.video_hash,
|
|
95
|
+
verify_e,
|
|
96
|
+
)
|
|
69
97
|
# --- End detailed check ---
|
|
70
98
|
|
|
71
99
|
return updated_count
|
|
72
100
|
except Exception as e:
|
|
73
|
-
logger.error(
|
|
101
|
+
logger.error(
|
|
102
|
+
"Failed to bulk update is_extracted status for video %s: %s",
|
|
103
|
+
video.video_hash,
|
|
104
|
+
e,
|
|
105
|
+
exc_info=True,
|
|
106
|
+
)
|
|
74
107
|
raise # Re-raise to ensure transaction rollback if needed
|
|
@@ -5,12 +5,9 @@ from typing import TYPE_CHECKING, Iterator, Optional
|
|
|
5
5
|
|
|
6
6
|
from django.db import transaction
|
|
7
7
|
|
|
8
|
+
from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, data_paths
|
|
9
|
+
|
|
8
10
|
from ....utils import delete_field_file, ensure_local_file, storage_file_exists
|
|
9
|
-
from endoreg_db.utils.paths import (
|
|
10
|
-
ANONYM_VIDEO_DIR,
|
|
11
|
-
SENSITIVE_VIDEO_DIR,
|
|
12
|
-
data_paths,
|
|
13
|
-
)
|
|
14
11
|
|
|
15
12
|
if TYPE_CHECKING:
|
|
16
13
|
from .video_file import VideoFile
|
|
@@ -31,13 +28,13 @@ def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
|
|
|
31
28
|
else:
|
|
32
29
|
logger.debug(
|
|
33
30
|
"raw_file.path for video %s is not a regular file: %s",
|
|
34
|
-
video.
|
|
31
|
+
video.video_hash,
|
|
35
32
|
direct_path,
|
|
36
33
|
)
|
|
37
34
|
except Exception as exc:
|
|
38
35
|
logger.debug(
|
|
39
36
|
"Could not access raw_file.path for video %s: %s",
|
|
40
|
-
video.
|
|
37
|
+
video.video_hash,
|
|
41
38
|
exc,
|
|
42
39
|
)
|
|
43
40
|
|
|
@@ -50,15 +47,14 @@ def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
|
|
|
50
47
|
data_paths["sensitive_video"] / filename,
|
|
51
48
|
]
|
|
52
49
|
|
|
53
|
-
|
|
54
50
|
for candidate in candidates:
|
|
55
51
|
if candidate.is_file():
|
|
56
|
-
return candidate
|
|
52
|
+
return candidate
|
|
57
53
|
|
|
58
54
|
logger.warning(
|
|
59
55
|
"Raw video file '%s' not found in import/sensitive paths or via stored FileField path for video %s.",
|
|
60
56
|
video.raw_file.name,
|
|
61
|
-
video.
|
|
57
|
+
video.video_hash,
|
|
62
58
|
)
|
|
63
59
|
return None
|
|
64
60
|
|
|
@@ -67,7 +63,7 @@ def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
|
|
|
67
63
|
def _ensure_local_raw_file(video: "VideoFile") -> Iterator[Path]:
|
|
68
64
|
"""Yield a local filesystem path for the raw file, downloading if required."""
|
|
69
65
|
if not video.has_raw:
|
|
70
|
-
raise ValueError(f"Video {video.
|
|
66
|
+
raise ValueError(f"Video {video.video_hash} has no raw file")
|
|
71
67
|
|
|
72
68
|
with ensure_local_file(video.raw_file) as local_path:
|
|
73
69
|
yield local_path
|
|
@@ -85,17 +81,19 @@ def _get_processed_file_path(video: "VideoFile") -> Optional[Path]:
|
|
|
85
81
|
except Exception as exc:
|
|
86
82
|
logger.debug(
|
|
87
83
|
"Could not access direct processed_file.path for video %s: %s",
|
|
88
|
-
video.
|
|
84
|
+
video.video_hash,
|
|
89
85
|
exc,
|
|
90
86
|
)
|
|
91
87
|
direct_path = None
|
|
92
88
|
|
|
93
89
|
if processed_field and storage_file_exists(processed_field):
|
|
94
|
-
logger.debug(
|
|
90
|
+
logger.debug(
|
|
91
|
+
"Processed file for %s available only via storage backend", video.video_hash
|
|
92
|
+
)
|
|
95
93
|
else:
|
|
96
94
|
logger.warning(
|
|
97
95
|
"Could not get path for processed file of VideoFile %s: %s",
|
|
98
|
-
video.
|
|
96
|
+
video.video_hash,
|
|
99
97
|
"path unavailable",
|
|
100
98
|
)
|
|
101
99
|
return None
|
|
@@ -105,7 +103,7 @@ def _get_processed_file_path(video: "VideoFile") -> Optional[Path]:
|
|
|
105
103
|
def _ensure_local_processed_file(video: "VideoFile") -> Iterator[Path]:
|
|
106
104
|
"""Yield a local path to the processed file, downloading if necessary."""
|
|
107
105
|
if not video.is_processed:
|
|
108
|
-
raise ValueError(f"Video {video.
|
|
106
|
+
raise ValueError(f"Video {video.video_hash} has no processed file")
|
|
109
107
|
|
|
110
108
|
with ensure_local_file(video.processed_file) as local_path:
|
|
111
109
|
yield local_path
|
|
@@ -118,10 +116,17 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
|
|
|
118
116
|
try:
|
|
119
117
|
# delete_frames raises RuntimeError on state update failure
|
|
120
118
|
frame_delete_msg = video.delete_frames()
|
|
121
|
-
logger.info(
|
|
119
|
+
logger.info(
|
|
120
|
+
"Frame deletion result for video %s: %s", video.video_hash, frame_delete_msg
|
|
121
|
+
)
|
|
122
122
|
except Exception as frame_del_e:
|
|
123
123
|
# Log error but continue, as file deletion might still be possible
|
|
124
|
-
logger.error(
|
|
124
|
+
logger.error(
|
|
125
|
+
"Error during frame file/state deletion for video %s: %s",
|
|
126
|
+
video.video_hash,
|
|
127
|
+
frame_del_e,
|
|
128
|
+
exc_info=True,
|
|
129
|
+
)
|
|
125
130
|
|
|
126
131
|
# 2. Delete Raw File
|
|
127
132
|
raw_file_path = _get_raw_file_path(video)
|
|
@@ -129,18 +134,33 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
|
|
|
129
134
|
try:
|
|
130
135
|
if raw_file_path.exists():
|
|
131
136
|
raw_file_path.unlink()
|
|
132
|
-
logger.info(
|
|
137
|
+
logger.info(
|
|
138
|
+
"Deleted raw video file for %s: %s", video.video_hash, raw_file_path
|
|
139
|
+
)
|
|
133
140
|
else:
|
|
134
|
-
logger.warning(
|
|
141
|
+
logger.warning(
|
|
142
|
+
"Raw video file not found at %s for video %s, skipping deletion.",
|
|
143
|
+
raw_file_path,
|
|
144
|
+
video.video_hash,
|
|
145
|
+
)
|
|
135
146
|
|
|
136
147
|
except Exception as e:
|
|
137
148
|
# Log error but continue
|
|
138
|
-
logger.error(
|
|
149
|
+
logger.error(
|
|
150
|
+
"Error deleting raw video file %s for video %s: %s",
|
|
151
|
+
raw_file_path,
|
|
152
|
+
video.video_hash,
|
|
153
|
+
e,
|
|
154
|
+
exc_info=True,
|
|
155
|
+
)
|
|
139
156
|
else:
|
|
140
157
|
if delete_field_file(getattr(video, "raw_file", None), save=False):
|
|
141
|
-
logger.info("Deleted raw file from storage for video %s", video.
|
|
158
|
+
logger.info("Deleted raw file from storage for video %s", video.video_hash)
|
|
142
159
|
else:
|
|
143
|
-
logger.warning(
|
|
160
|
+
logger.warning(
|
|
161
|
+
"Raw video file not found during deletion for video %s.",
|
|
162
|
+
video.video_hash,
|
|
163
|
+
)
|
|
144
164
|
|
|
145
165
|
# 3. Delete Processed File
|
|
146
166
|
processed_file_path = _get_processed_file_path(video)
|
|
@@ -148,34 +168,62 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
|
|
|
148
168
|
try:
|
|
149
169
|
if processed_file_path.exists():
|
|
150
170
|
processed_file_path.unlink()
|
|
151
|
-
logger.info(
|
|
171
|
+
logger.info(
|
|
172
|
+
"Deleted processed video file for %s: %s",
|
|
173
|
+
video.video_hash,
|
|
174
|
+
processed_file_path,
|
|
175
|
+
)
|
|
152
176
|
else:
|
|
153
|
-
logger.warning(
|
|
177
|
+
logger.warning(
|
|
178
|
+
"Processed video file not found at %s for video %s, skipping deletion.",
|
|
179
|
+
processed_file_path,
|
|
180
|
+
video.video_hash,
|
|
181
|
+
)
|
|
154
182
|
except Exception as e:
|
|
155
183
|
# Log error but continue
|
|
156
|
-
logger.error(
|
|
184
|
+
logger.error(
|
|
185
|
+
"Error deleting processed video file %s for video %s: %s",
|
|
186
|
+
processed_file_path,
|
|
187
|
+
video.video_hash,
|
|
188
|
+
e,
|
|
189
|
+
exc_info=True,
|
|
190
|
+
)
|
|
157
191
|
else:
|
|
158
192
|
if delete_field_file(getattr(video, "processed_file", None), save=False):
|
|
159
|
-
logger.info(
|
|
193
|
+
logger.info(
|
|
194
|
+
"Deleted processed file from storage for video %s", video.video_hash
|
|
195
|
+
)
|
|
160
196
|
else:
|
|
161
|
-
logger.warning(
|
|
197
|
+
logger.warning(
|
|
198
|
+
"Processed file missing in storage for video %s", video.video_hash
|
|
199
|
+
)
|
|
162
200
|
|
|
163
201
|
# 4. Delete Database Record
|
|
164
202
|
try:
|
|
165
203
|
# Use 'super(type(video), video)' to call the parent's delete method
|
|
166
204
|
super(type(video), video).delete(*args, **kwargs)
|
|
167
|
-
logger.info(
|
|
205
|
+
logger.info(
|
|
206
|
+
"Deleted VideoFile database record PK %s (UUID: %s).",
|
|
207
|
+
video.pk,
|
|
208
|
+
video.video_hash,
|
|
209
|
+
)
|
|
168
210
|
|
|
169
|
-
return f"Successfully deleted VideoFile {video.
|
|
211
|
+
return f"Successfully deleted VideoFile {video.video_hash} and attempted file cleanup."
|
|
170
212
|
except Exception as e:
|
|
171
|
-
logger.error(
|
|
213
|
+
logger.error(
|
|
214
|
+
"Error deleting VideoFile database record PK %s (UUID: %s): %s",
|
|
215
|
+
video.pk,
|
|
216
|
+
video.video_hash,
|
|
217
|
+
e,
|
|
218
|
+
exc_info=True,
|
|
219
|
+
)
|
|
172
220
|
raise # Re-raise the exception for DB deletion failure
|
|
173
221
|
|
|
174
222
|
|
|
175
223
|
def _get_base_frame_dir(video: "VideoFile") -> Path:
|
|
176
224
|
"""Gets the base directory path for storing extracted frames."""
|
|
177
225
|
# Assuming data_paths['frame'] is the root directory for all frame storage
|
|
178
|
-
return data_paths["frame"] / str(video.
|
|
226
|
+
return data_paths["frame"] / str(video.video_hash)
|
|
179
227
|
|
|
180
228
|
|
|
181
229
|
def _set_frame_dir(video: "VideoFile", force_update: bool = False):
|
|
@@ -185,7 +233,9 @@ def _set_frame_dir(video: "VideoFile", force_update: bool = False):
|
|
|
185
233
|
|
|
186
234
|
if not video.frame_dir or video.frame_dir != target_path_str or force_update:
|
|
187
235
|
video.frame_dir = target_path_str
|
|
188
|
-
logger.info(
|
|
236
|
+
logger.info(
|
|
237
|
+
"Set frame_dir for video %s to %s", video.video_hash, video.frame_dir
|
|
238
|
+
)
|
|
189
239
|
# Avoid saving if called from within the save method itself
|
|
190
240
|
if not getattr(video, "_saving", False):
|
|
191
241
|
video.save(update_fields=["frame_dir"])
|
|
@@ -212,7 +262,9 @@ def _get_target_anonymized_video_path(video: "VideoFile") -> Path:
|
|
|
212
262
|
if not video.has_raw or not video.raw_file.name:
|
|
213
263
|
# If raw is gone, maybe base it on UUID? Requires careful thought.
|
|
214
264
|
# For now, assume raw is needed to determine the original filename base.
|
|
215
|
-
raise ValueError(
|
|
265
|
+
raise ValueError(
|
|
266
|
+
"Cannot determine target anonymized path without a raw file reference."
|
|
267
|
+
)
|
|
216
268
|
|
|
217
269
|
# Use the filename part of the raw file's relative path
|
|
218
270
|
raw_path_relative = Path(video.raw_file.name)
|
|
@@ -13,10 +13,10 @@ logger = logging.getLogger(__name__)
|
|
|
13
13
|
|
|
14
14
|
# Define __all__ if you want to control what `from .video_file_meta import *` imports
|
|
15
15
|
__all__ = [
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
16
|
+
"_update_text_metadata",
|
|
17
|
+
"_update_video_meta",
|
|
18
|
+
"_initialize_video_specs",
|
|
19
|
+
"_get_fps",
|
|
20
|
+
"_get_endo_roi",
|
|
21
|
+
"_get_crop_template",
|
|
22
22
|
]
|
|
@@ -13,7 +13,10 @@ def _get_crop_template(video: "VideoFile") -> Optional[List[int]]:
|
|
|
13
13
|
"""Generates a crop template [y1, y2, x1, x2] from the endo ROI."""
|
|
14
14
|
endo_roi = _get_endo_roi(video) # Use the helper function
|
|
15
15
|
if not endo_roi:
|
|
16
|
-
logger.warning(
|
|
16
|
+
logger.warning(
|
|
17
|
+
"Cannot generate crop template for video %s: Endo ROI not available.",
|
|
18
|
+
video.video_hash,
|
|
19
|
+
)
|
|
17
20
|
return None
|
|
18
21
|
|
|
19
22
|
x = endo_roi["x"]
|
|
@@ -23,7 +26,9 @@ def _get_crop_template(video: "VideoFile") -> Optional[List[int]]:
|
|
|
23
26
|
|
|
24
27
|
# Validate dimensions
|
|
25
28
|
if None in [x, y, width, height] or width <= 0 or height <= 0:
|
|
26
|
-
logger.warning(
|
|
29
|
+
logger.warning(
|
|
30
|
+
"Invalid ROI dimensions for video %s: %s", video.video_hash, endo_roi
|
|
31
|
+
)
|
|
27
32
|
return None
|
|
28
33
|
|
|
29
34
|
# Ensure crop boundaries are within image dimensions if available
|
|
@@ -34,12 +39,20 @@ def _get_crop_template(video: "VideoFile") -> Optional[List[int]]:
|
|
|
34
39
|
x1 = max(0, x)
|
|
35
40
|
x2 = min(img_w, x + width)
|
|
36
41
|
if y1 >= y2 or x1 >= x2:
|
|
37
|
-
logger.warning(
|
|
42
|
+
logger.warning(
|
|
43
|
+
"Calculated crop template has zero or negative size for video %s. ROI: %s, Img: %dx%d",
|
|
44
|
+
video.video_hash,
|
|
45
|
+
endo_roi,
|
|
46
|
+
img_w,
|
|
47
|
+
img_h,
|
|
48
|
+
)
|
|
38
49
|
return None
|
|
39
50
|
crop_template = [y1, y2, x1, x2]
|
|
40
51
|
else:
|
|
41
52
|
# Proceed without boundary check if image dimensions unknown
|
|
42
53
|
crop_template = [y, y + height, x, x + width]
|
|
43
54
|
|
|
44
|
-
logger.debug(
|
|
55
|
+
logger.debug(
|
|
56
|
+
"Generated crop template for video %s: %s", video.video_hash, crop_template
|
|
57
|
+
)
|
|
45
58
|
return crop_template
|