endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/authz/auth.py +74 -0
- endoreg_db/authz/backends.py +168 -0
- endoreg_db/authz/management/commands/list_routes.py +18 -0
- endoreg_db/authz/middleware.py +83 -0
- endoreg_db/authz/permissions.py +127 -0
- endoreg_db/authz/policy.py +218 -0
- endoreg_db/authz/views_auth.py +66 -0
- endoreg_db/config/env.py +13 -8
- endoreg_db/data/__init__.py +8 -31
- endoreg_db/data/_examples/disease.yaml +55 -0
- endoreg_db/data/_examples/disease_classification.yaml +13 -0
- endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
- endoreg_db/data/_examples/event.yaml +64 -0
- endoreg_db/data/_examples/examination.yaml +72 -0
- endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
- endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
- endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
- endoreg_db/data/_examples/finding/complication.yaml +16 -0
- endoreg_db/data/_examples/finding/data.yaml +105 -0
- endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
- endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
- endoreg_db/data/_examples/finding/outcome.yaml +12 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
- endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
- endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
- endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
- endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
- endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
- endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
- endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
- endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
- endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
- endoreg_db/data/_examples/finding_type/data.yaml +43 -0
- endoreg_db/data/_examples/requirement/age.yaml +26 -0
- endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
- endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
- endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
- endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
- endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
- endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
- endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
- endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
- endoreg_db/data/_examples/requirement/gender.yaml +25 -0
- endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
- endoreg_db/data/_examples/requirement/medication.yaml +93 -0
- endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
- endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
- endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
- endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
- endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
- endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
- endoreg_db/data/event_classification/data.yaml +4 -0
- endoreg_db/data/event_classification_choice/data.yaml +9 -0
- endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
- endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
- endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
- endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
- endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
- endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
- endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
- endoreg_db/data/requirement_set/_old_ +109 -0
- endoreg_db/data/requirement_set_type/data.yaml +21 -0
- endoreg_db/data/setup_config.yaml +4 -4
- endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
- endoreg_db/exceptions.py +5 -2
- endoreg_db/helpers/data_loader.py +1 -1
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
- endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
- endoreg_db/management/commands/import_video.py +9 -10
- endoreg_db/management/commands/import_video_with_classification.py +1 -1
- endoreg_db/management/commands/init_default_ai_model.py +1 -1
- endoreg_db/management/commands/list_routes.py +18 -0
- endoreg_db/management/commands/load_ai_model_data.py +2 -1
- endoreg_db/management/commands/load_center_data.py +12 -12
- endoreg_db/management/commands/load_requirement_data.py +60 -31
- endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
- endoreg_db/management/commands/setup_endoreg_db.py +14 -10
- endoreg_db/management/commands/storage_management.py +271 -203
- endoreg_db/migrations/0001_initial.py +1799 -1300
- endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
- endoreg_db/migrations/_old/0001_initial.py +1857 -0
- endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
- endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
- endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
- endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
- endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
- endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
- endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
- endoreg_db/models/__init__.py +78 -123
- endoreg_db/models/administration/__init__.py +21 -42
- endoreg_db/models/administration/ai/active_model.py +2 -2
- endoreg_db/models/administration/ai/ai_model.py +7 -6
- endoreg_db/models/administration/case/__init__.py +1 -15
- endoreg_db/models/administration/case/case.py +3 -3
- endoreg_db/models/administration/case/case_template/__init__.py +2 -14
- endoreg_db/models/administration/case/case_template/case_template.py +2 -124
- endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
- endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
- endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
- endoreg_db/models/administration/center/center.py +33 -19
- endoreg_db/models/administration/center/center_product.py +12 -9
- endoreg_db/models/administration/center/center_resource.py +25 -19
- endoreg_db/models/administration/center/center_shift.py +21 -17
- endoreg_db/models/administration/center/center_waste.py +16 -8
- endoreg_db/models/administration/person/__init__.py +2 -0
- endoreg_db/models/administration/person/employee/employee.py +10 -5
- endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
- endoreg_db/models/administration/person/employee/employee_type.py +12 -6
- endoreg_db/models/administration/person/examiner/examiner.py +13 -11
- endoreg_db/models/administration/person/patient/__init__.py +2 -0
- endoreg_db/models/administration/person/patient/patient.py +103 -100
- endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
- endoreg_db/models/administration/person/person.py +4 -0
- endoreg_db/models/administration/person/profession/__init__.py +8 -4
- endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
- endoreg_db/models/administration/product/product.py +20 -15
- endoreg_db/models/administration/product/product_material.py +17 -18
- endoreg_db/models/administration/product/product_weight.py +12 -8
- endoreg_db/models/administration/product/reference_product.py +23 -55
- endoreg_db/models/administration/qualification/qualification.py +7 -3
- endoreg_db/models/administration/qualification/qualification_type.py +7 -3
- endoreg_db/models/administration/shift/scheduled_days.py +8 -5
- endoreg_db/models/administration/shift/shift.py +16 -12
- endoreg_db/models/administration/shift/shift_type.py +23 -31
- endoreg_db/models/label/__init__.py +7 -8
- endoreg_db/models/label/annotation/image_classification.py +10 -9
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
- endoreg_db/models/label/label.py +15 -15
- endoreg_db/models/label/label_set.py +19 -6
- endoreg_db/models/label/label_type.py +1 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
- endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
- endoreg_db/models/label/video_segmentation_label.py +4 -0
- endoreg_db/models/label/video_segmentation_labelset.py +4 -3
- endoreg_db/models/media/frame/frame.py +22 -22
- endoreg_db/models/media/pdf/raw_pdf.py +249 -177
- endoreg_db/models/media/pdf/report_file.py +25 -29
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
- endoreg_db/models/media/video/__init__.py +1 -0
- endoreg_db/models/media/video/create_from_file.py +48 -56
- endoreg_db/models/media/video/pipe_1.py +30 -33
- endoreg_db/models/media/video/pipe_2.py +8 -9
- endoreg_db/models/media/video/video_file.py +359 -204
- endoreg_db/models/media/video/video_file_ai.py +288 -74
- endoreg_db/models/media/video/video_file_anonymize.py +38 -38
- endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
- endoreg_db/models/media/video/video_file_io.py +109 -62
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
- endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
- endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
- endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
- endoreg_db/models/media/video/video_file_segments.py +24 -17
- endoreg_db/models/media/video/video_metadata.py +19 -35
- endoreg_db/models/media/video/video_processing.py +96 -95
- endoreg_db/models/medical/contraindication/__init__.py +13 -3
- endoreg_db/models/medical/disease.py +22 -16
- endoreg_db/models/medical/event.py +31 -18
- endoreg_db/models/medical/examination/__init__.py +13 -6
- endoreg_db/models/medical/examination/examination.py +17 -18
- endoreg_db/models/medical/examination/examination_indication.py +26 -25
- endoreg_db/models/medical/examination/examination_time.py +16 -6
- endoreg_db/models/medical/examination/examination_time_type.py +9 -6
- endoreg_db/models/medical/examination/examination_type.py +3 -4
- endoreg_db/models/medical/finding/finding.py +38 -39
- endoreg_db/models/medical/finding/finding_classification.py +37 -48
- endoreg_db/models/medical/finding/finding_intervention.py +27 -22
- endoreg_db/models/medical/finding/finding_type.py +13 -12
- endoreg_db/models/medical/hardware/endoscope.py +20 -26
- endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
- endoreg_db/models/medical/laboratory/lab_value.py +62 -91
- endoreg_db/models/medical/medication/medication.py +22 -10
- endoreg_db/models/medical/medication/medication_indication.py +29 -3
- endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
- endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
- endoreg_db/models/medical/medication/medication_schedule.py +27 -16
- endoreg_db/models/medical/organ/__init__.py +15 -12
- endoreg_db/models/medical/patient/medication_examples.py +1 -5
- endoreg_db/models/medical/patient/patient_disease.py +20 -23
- endoreg_db/models/medical/patient/patient_event.py +19 -22
- endoreg_db/models/medical/patient/patient_examination.py +48 -54
- endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
- endoreg_db/models/medical/patient/patient_finding.py +122 -139
- endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
- endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
- endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
- endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
- endoreg_db/models/medical/patient/patient_medication.py +27 -38
- endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
- endoreg_db/models/medical/risk/risk.py +7 -6
- endoreg_db/models/medical/risk/risk_type.py +8 -5
- endoreg_db/models/metadata/model_meta.py +60 -29
- endoreg_db/models/metadata/model_meta_logic.py +139 -18
- endoreg_db/models/metadata/pdf_meta.py +19 -24
- endoreg_db/models/metadata/sensitive_meta.py +102 -85
- endoreg_db/models/metadata/sensitive_meta_logic.py +383 -43
- endoreg_db/models/metadata/video_meta.py +51 -31
- endoreg_db/models/metadata/video_prediction_logic.py +16 -23
- endoreg_db/models/metadata/video_prediction_meta.py +29 -33
- endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
- endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
- endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
- endoreg_db/models/other/emission/emission_factor.py +18 -8
- endoreg_db/models/other/gender.py +10 -5
- endoreg_db/models/other/information_source.py +25 -25
- endoreg_db/models/other/material.py +9 -5
- endoreg_db/models/other/resource.py +6 -4
- endoreg_db/models/other/tag.py +10 -5
- endoreg_db/models/other/transport_route.py +13 -8
- endoreg_db/models/other/unit.py +10 -6
- endoreg_db/models/other/waste.py +6 -5
- endoreg_db/models/requirement/requirement.py +580 -272
- endoreg_db/models/requirement/requirement_error.py +85 -0
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
- endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
- endoreg_db/models/requirement/requirement_operator.py +36 -33
- endoreg_db/models/requirement/requirement_set.py +74 -57
- endoreg_db/models/state/__init__.py +4 -4
- endoreg_db/models/state/abstract.py +2 -2
- endoreg_db/models/state/anonymization.py +12 -0
- endoreg_db/models/state/audit_ledger.py +46 -47
- endoreg_db/models/state/label_video_segment.py +9 -0
- endoreg_db/models/state/raw_pdf.py +40 -46
- endoreg_db/models/state/sensitive_meta.py +6 -2
- endoreg_db/models/state/video.py +58 -53
- endoreg_db/models/upload_job.py +32 -55
- endoreg_db/models/utils.py +1 -2
- endoreg_db/root_urls.py +21 -2
- endoreg_db/serializers/__init__.py +26 -57
- endoreg_db/serializers/anonymization.py +18 -10
- endoreg_db/serializers/meta/report_meta.py +1 -1
- endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
- endoreg_db/serializers/misc/__init__.py +1 -1
- endoreg_db/serializers/misc/file_overview.py +33 -91
- endoreg_db/serializers/misc/{vop_patient_data.py → sensitive_patient_data.py} +1 -1
- endoreg_db/serializers/requirements/requirement_sets.py +92 -22
- endoreg_db/serializers/video/segmentation.py +2 -1
- endoreg_db/serializers/video/video_processing_history.py +20 -5
- endoreg_db/serializers/video_examination.py +198 -0
- endoreg_db/services/anonymization.py +75 -73
- endoreg_db/services/lookup_service.py +256 -73
- endoreg_db/services/lookup_store.py +174 -30
- endoreg_db/services/pdf_import.py +711 -310
- endoreg_db/services/storage_aware_video_processor.py +140 -114
- endoreg_db/services/video_import.py +266 -117
- endoreg_db/urls/__init__.py +27 -27
- endoreg_db/urls/label_video_segments.py +2 -0
- endoreg_db/urls/media.py +108 -66
- endoreg_db/urls/root_urls.py +29 -0
- endoreg_db/utils/__init__.py +15 -5
- endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
- endoreg_db/utils/case_generator/__init__.py +3 -0
- endoreg_db/utils/dataloader.py +88 -16
- endoreg_db/utils/defaults/set_default_center.py +32 -0
- endoreg_db/utils/names.py +22 -16
- endoreg_db/utils/permissions.py +2 -1
- endoreg_db/utils/pipelines/process_video_dir.py +1 -1
- endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
- endoreg_db/utils/setup_config.py +8 -5
- endoreg_db/utils/storage.py +115 -0
- endoreg_db/utils/validate_endo_roi.py +8 -2
- endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
- endoreg_db/views/__init__.py +5 -12
- endoreg_db/views/anonymization/media_management.py +198 -163
- endoreg_db/views/anonymization/overview.py +4 -1
- endoreg_db/views/anonymization/validate.py +174 -40
- endoreg_db/views/media/__init__.py +2 -0
- endoreg_db/views/media/pdf_media.py +131 -150
- endoreg_db/views/media/sensitive_metadata.py +46 -6
- endoreg_db/views/media/video_media.py +89 -82
- endoreg_db/views/media/video_segments.py +187 -260
- endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
- endoreg_db/views/patient/patient.py +5 -4
- endoreg_db/views/pdf/__init__.py +5 -8
- endoreg_db/views/pdf/pdf_stream.py +186 -0
- endoreg_db/views/pdf/pdf_stream_views.py +0 -127
- endoreg_db/views/pdf/reimport.py +86 -91
- endoreg_db/views/requirement/evaluate.py +188 -187
- endoreg_db/views/requirement/lookup.py +186 -288
- endoreg_db/views/requirement/requirement_utils.py +89 -0
- endoreg_db/views/video/__init__.py +0 -4
- endoreg_db/views/video/correction.py +2 -2
- endoreg_db/views/video/video_examination_viewset.py +202 -289
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +350 -255
- endoreg_db/models/administration/permissions/__init__.py +0 -44
- endoreg_db/models/media/video/refactor_plan.md +0 -0
- endoreg_db/models/media/video/video_file_frames.py +0 -0
- endoreg_db/models/metadata/frame_ocr_result.py +0 -0
- endoreg_db/models/rule/__init__.py +0 -13
- endoreg_db/models/rule/rule.py +0 -27
- endoreg_db/models/rule/rule_applicator.py +0 -224
- endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
- endoreg_db/models/rule/rule_type.py +0 -20
- endoreg_db/models/rule/ruleset.py +0 -17
- endoreg_db/serializers/video/video_metadata.py +0 -105
- endoreg_db/urls/report.py +0 -48
- endoreg_db/urls/video.py +0 -61
- endoreg_db/utils/case_generator/case_generator.py +0 -159
- endoreg_db/utils/case_generator/utils.py +0 -30
- endoreg_db/views/pdf/pdf_media.py +0 -239
- endoreg_db/views/report/__init__.py +0 -9
- endoreg_db/views/report/report_list.py +0 -112
- endoreg_db/views/report/report_with_secure_url.py +0 -28
- endoreg_db/views/report/start_examination.py +0 -7
- endoreg_db/views/video/video_media.py +0 -158
- endoreg_db/views.py +0 -0
- /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
- /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
- /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
from tqdm import tqdm
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
from typing import List
|
|
4
|
-
from typing import TYPE_CHECKING, Optional
|
|
5
1
|
import logging
|
|
6
|
-
from django.db import OperationalError
|
|
7
2
|
import time
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
5
|
+
|
|
6
|
+
from django.db import OperationalError
|
|
7
|
+
from tqdm import tqdm
|
|
8
8
|
|
|
9
9
|
if TYPE_CHECKING:
|
|
10
10
|
from endoreg_db.models import VideoFile
|
|
11
|
-
|
|
11
|
+
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
14
|
|
|
@@ -35,7 +35,6 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
35
35
|
from endoreg_db.models.media.video.video_file_frames._bulk_create_frames import _bulk_create_frames
|
|
36
36
|
from endoreg_db.models.media.video.video_file_frames._create_frame_object import _create_frame_object
|
|
37
37
|
|
|
38
|
-
|
|
39
38
|
frames_to_create = []
|
|
40
39
|
num_expected_or_provided = 0
|
|
41
40
|
mark_as_extracted = False
|
|
@@ -46,11 +45,9 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
46
45
|
num_expected_or_provided = len(frame_paths)
|
|
47
46
|
for frame_path in tqdm(frame_paths, desc=f"Initializing Frames from Paths {video.uuid}", unit="frame"):
|
|
48
47
|
try:
|
|
49
|
-
frame_number = int(frame_path.stem.split(
|
|
48
|
+
frame_number = int(frame_path.stem.split("_")[-1])
|
|
50
49
|
relative_path_str = frame_path.name
|
|
51
|
-
frames_to_create.append(
|
|
52
|
-
_create_frame_object(video, frame_number, relative_path_str, extracted=mark_as_extracted)
|
|
53
|
-
)
|
|
50
|
+
frames_to_create.append(_create_frame_object(video, frame_number, relative_path_str, extracted=mark_as_extracted))
|
|
54
51
|
except (ValueError, IndexError) as e:
|
|
55
52
|
logger.warning("Could not parse frame number from %s: %s", frame_path.name, e)
|
|
56
53
|
continue
|
|
@@ -63,7 +60,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
63
60
|
if state.frames_initialized or state.frame_count is not None:
|
|
64
61
|
state.frames_initialized = False
|
|
65
62
|
state.frame_count = None
|
|
66
|
-
state.save(update_fields=[
|
|
63
|
+
state.save(update_fields=["frames_initialized", "frame_count"])
|
|
67
64
|
except Exception as state_e:
|
|
68
65
|
logger.error("Failed to reset state during empty initialization for video %s: %s", video.uuid, state_e, exc_info=True)
|
|
69
66
|
return
|
|
@@ -73,13 +70,10 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
73
70
|
num_expected_or_provided = expected_frame_count
|
|
74
71
|
for frame_number in tqdm(range(expected_frame_count), desc=f"Initializing Expected Frames {video.uuid}", unit="frame"):
|
|
75
72
|
relative_path_str = f"frame_{frame_number:07d}.jpg"
|
|
76
|
-
frames_to_create.append(
|
|
77
|
-
|
|
78
|
-
)
|
|
79
|
-
|
|
73
|
+
frames_to_create.append(_create_frame_object(video, frame_number, relative_path_str, extracted=mark_as_extracted))
|
|
74
|
+
|
|
80
75
|
if frames_to_create:
|
|
81
76
|
for attempt in range(5):
|
|
82
|
-
|
|
83
77
|
try:
|
|
84
78
|
_bulk_create_frames(video, frames_to_create)
|
|
85
79
|
num_created_or_ignored = len(frames_to_create)
|
|
@@ -88,11 +82,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
88
82
|
if mark_as_extracted:
|
|
89
83
|
frame_numbers_to_update = [f.frame_number for f in frames_to_create]
|
|
90
84
|
if frame_numbers_to_update:
|
|
91
|
-
update_count = Frame.objects.filter(
|
|
92
|
-
video=video,
|
|
93
|
-
frame_number__in=frame_numbers_to_update,
|
|
94
|
-
is_extracted=False
|
|
95
|
-
).update(is_extracted=True)
|
|
85
|
+
update_count = Frame.objects.filter(video=video, frame_number__in=frame_numbers_to_update, is_extracted=False).update(is_extracted=True)
|
|
96
86
|
if update_count > 0:
|
|
97
87
|
logger.info("Marked %d existing Frame objects as is_extracted=True for video %s.", update_count, video.uuid)
|
|
98
88
|
|
|
@@ -100,7 +90,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
100
90
|
state = video.get_or_create_state()
|
|
101
91
|
state.frames_initialized = True
|
|
102
92
|
state.frame_count = num_expected_or_provided
|
|
103
|
-
state.save(update_fields=[
|
|
93
|
+
state.save(update_fields=["frames_initialized", "frame_count"])
|
|
104
94
|
logger.info("Set frames_initialized=True and frame_count=%d for video %s.", num_expected_or_provided, video.uuid)
|
|
105
95
|
except Exception as state_e:
|
|
106
96
|
logger.error("Failed to update state after frame initialization for video %s: %s", video.uuid, state_e, exc_info=True)
|
|
@@ -109,7 +99,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
109
99
|
except OperationalError as e:
|
|
110
100
|
if "database is locked" in str(e):
|
|
111
101
|
logger.warning("Database is locked, retrying frame initialization for video %s (attempt %d/5).", video.uuid, attempt + 1)
|
|
112
|
-
time.sleep(2
|
|
102
|
+
time.sleep(2**attempt)
|
|
113
103
|
if attempt < 4:
|
|
114
104
|
continue
|
|
115
105
|
else:
|
|
@@ -124,6 +114,6 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
|
|
|
124
114
|
if state.frames_initialized or state.frame_count is not None:
|
|
125
115
|
state.frames_initialized = False
|
|
126
116
|
state.frame_count = None
|
|
127
|
-
state.save(update_fields=[
|
|
117
|
+
state.save(update_fields=["frames_initialized", "frame_count"])
|
|
128
118
|
except Exception as state_e:
|
|
129
119
|
logger.error("Failed to reset state during empty initialization for video %s: %s", video.uuid, state_e, exc_info=True)
|
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
4
5
|
from django.db import transaction
|
|
5
6
|
|
|
7
|
+
from endoreg_db.models.media.video.video_file_io import _get_frame_dir_path
|
|
8
|
+
|
|
6
9
|
# Assuming ffmpeg_wrapper has or will have this function
|
|
7
10
|
from endoreg_db.utils.video.ffmpeg_wrapper import extract_frame_range as ffmpeg_extract_frame_range
|
|
8
11
|
|
|
9
|
-
from endoreg_db.models.media.video.video_file_io import _get_frame_dir_path
|
|
10
|
-
|
|
11
12
|
if TYPE_CHECKING:
|
|
12
13
|
from endoreg_db.models import VideoFile
|
|
13
14
|
|
|
14
15
|
logger = logging.getLogger(__name__)
|
|
15
16
|
|
|
17
|
+
|
|
16
18
|
def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
|
|
17
19
|
"""
|
|
18
20
|
Deletes frame image files within the specified range [start_frame, end_frame)
|
|
@@ -20,19 +22,14 @@ def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
|
|
|
20
22
|
"""
|
|
21
23
|
|
|
22
24
|
logger.info("Deleting frame files for video %s in range [%d, %d)", video.uuid, start_frame, end_frame)
|
|
23
|
-
frames_to_delete = video.frames.filter(
|
|
24
|
-
frame_number__gte=start_frame,
|
|
25
|
-
frame_number__lt=end_frame,
|
|
26
|
-
is_extracted=True
|
|
27
|
-
)
|
|
25
|
+
frames_to_delete = video.frames.filter(frame_number__gte=start_frame, frame_number__lt=end_frame, is_extracted=True)
|
|
28
26
|
|
|
29
27
|
deleted_count = 0
|
|
30
28
|
paths_to_delete = [frame.file_path for frame in frames_to_delete] # Get paths before potential DB changes
|
|
31
29
|
|
|
32
30
|
# Update DB first
|
|
33
31
|
update_count = frames_to_delete.update(is_extracted=False)
|
|
34
|
-
logger.info("Marked %d Frame objects as is_extracted=False for video %s range [%d, %d).",
|
|
35
|
-
update_count, video.uuid, start_frame, end_frame)
|
|
32
|
+
logger.info("Marked %d Frame objects as is_extracted=False for video %s range [%d, %d).", update_count, video.uuid, start_frame, end_frame)
|
|
36
33
|
|
|
37
34
|
# Then delete files
|
|
38
35
|
for frame_path in paths_to_delete:
|
|
@@ -44,10 +41,14 @@ def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
|
|
|
44
41
|
# Log warning but continue; DB state is already updated.
|
|
45
42
|
logger.warning("Could not delete frame file %s for video %s: %s", frame_path, video.uuid, e)
|
|
46
43
|
|
|
47
|
-
logger.info(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
44
|
+
logger.info(
|
|
45
|
+
"Attempted deletion of %d files for video %s range [%d, %d). Actual deleted: %d.",
|
|
46
|
+
len(paths_to_delete),
|
|
47
|
+
video.uuid,
|
|
48
|
+
start_frame,
|
|
49
|
+
end_frame,
|
|
50
|
+
deleted_count,
|
|
51
|
+
)
|
|
51
52
|
|
|
52
53
|
|
|
53
54
|
@transaction.atomic
|
|
@@ -79,10 +80,7 @@ def _extract_frame_range(
|
|
|
79
80
|
raise ValueError(f"Cannot determine frame directory path for video {video.uuid}.")
|
|
80
81
|
|
|
81
82
|
# Check frames within the range that might already exist
|
|
82
|
-
frames_in_range = video.frames.filter(
|
|
83
|
-
frame_number__gte=start_frame,
|
|
84
|
-
frame_number__lt=end_frame
|
|
85
|
-
)
|
|
83
|
+
frames_in_range = video.frames.filter(frame_number__gte=start_frame, frame_number__lt=end_frame)
|
|
86
84
|
existing_extracted_in_range = frames_in_range.filter(is_extracted=True)
|
|
87
85
|
|
|
88
86
|
if existing_extracted_in_range.exists():
|
|
@@ -90,7 +88,12 @@ def _extract_frame_range(
|
|
|
90
88
|
logger.info("Overwrite=True, deleting existing frame files in range [%d, %d) for video %s before extraction.", start_frame, end_frame, video.uuid)
|
|
91
89
|
_delete_frame_range(video, start_frame, end_frame)
|
|
92
90
|
else:
|
|
93
|
-
logger.info(
|
|
91
|
+
logger.info(
|
|
92
|
+
"Frames already exist in range [%d, %d) for video %s and overwrite=False. Skipping extraction for this range.",
|
|
93
|
+
start_frame,
|
|
94
|
+
end_frame,
|
|
95
|
+
video.uuid,
|
|
96
|
+
)
|
|
94
97
|
updated_count = frames_in_range.filter(is_extracted=False).update(is_extracted=True)
|
|
95
98
|
if updated_count > 0:
|
|
96
99
|
logger.info("Marked %d existing Frame objects in range [%d, %d) as extracted for video %s.", updated_count, start_frame, end_frame, video.uuid)
|
|
@@ -101,11 +104,11 @@ def _extract_frame_range(
|
|
|
101
104
|
|
|
102
105
|
try:
|
|
103
106
|
logger.info("Starting frame range extraction [%d, %d) for video %s to %s", start_frame, end_frame, video.uuid, frame_dir)
|
|
104
|
-
extracted_paths = ffmpeg_extract_frame_range(
|
|
105
|
-
raw_file_path, frame_dir, start_frame, end_frame, quality=quality, ext=ext
|
|
106
|
-
)
|
|
107
|
+
extracted_paths = ffmpeg_extract_frame_range(raw_file_path, frame_dir, start_frame, end_frame, quality=quality, ext=ext)
|
|
107
108
|
|
|
108
|
-
logger.info(
|
|
109
|
+
logger.info(
|
|
110
|
+
"ffmpeg extraction process completed for video %s range [%d, %d). Found %d files.", video.uuid, start_frame, end_frame, len(extracted_paths)
|
|
111
|
+
)
|
|
109
112
|
|
|
110
113
|
update_count = frames_in_range.update(is_extracted=True)
|
|
111
114
|
logger.info("Marked %d Frame objects in range [%d, %d) as is_extracted=True for video %s.", update_count, start_frame, end_frame, video.uuid)
|
|
@@ -127,7 +130,7 @@ def _extract_frame_range(
|
|
|
127
130
|
logger.error("Frame range extraction [%d, %d) or DB update failed for video %s: %s", start_frame, end_frame, video.uuid, e, exc_info=True)
|
|
128
131
|
|
|
129
132
|
logger.warning("Attempting file cleanup in range [%d, %d) for video %s due to extraction error.", start_frame, end_frame, video.uuid)
|
|
130
|
-
files_to_check = extracted_paths if
|
|
133
|
+
files_to_check = extracted_paths if "extracted_paths" in locals() and extracted_paths else []
|
|
131
134
|
if not files_to_check:
|
|
132
135
|
files_to_check = [frame_dir / f"frame_{i:07d}.{ext}" for i in range(start_frame, end_frame)]
|
|
133
136
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from typing import TYPE_CHECKING, Set
|
|
3
|
+
|
|
3
4
|
from django.db import transaction
|
|
4
5
|
|
|
5
6
|
if TYPE_CHECKING:
|
|
@@ -7,43 +8,49 @@ if TYPE_CHECKING:
|
|
|
7
8
|
|
|
8
9
|
logger = logging.getLogger(__name__)
|
|
9
10
|
|
|
11
|
+
|
|
10
12
|
@transaction.atomic
|
|
11
13
|
def _mark_frames_extracted_status(video: "VideoFile", extracted_frame_numbers: Set[int], status: bool):
|
|
12
14
|
"""
|
|
13
15
|
Bulk updates the is_extracted status for a set of frame numbers.
|
|
14
16
|
"""
|
|
15
17
|
from endoreg_db.models.media.frame import Frame
|
|
18
|
+
|
|
16
19
|
if not extracted_frame_numbers:
|
|
17
20
|
logger.warning("No frame numbers provided to update status for video %s.", video.uuid)
|
|
18
21
|
return 0
|
|
19
22
|
|
|
20
23
|
# --- Enhanced Logging ---
|
|
21
|
-
min_frame = min(extracted_frame_numbers) if extracted_frame_numbers else
|
|
22
|
-
max_frame = max(extracted_frame_numbers) if extracted_frame_numbers else
|
|
24
|
+
min_frame = min(extracted_frame_numbers) if extracted_frame_numbers else "N/A"
|
|
25
|
+
max_frame = max(extracted_frame_numbers) if extracted_frame_numbers else "N/A"
|
|
23
26
|
contains_zero = 0 in extracted_frame_numbers
|
|
24
27
|
logger.info(
|
|
25
28
|
"Attempting to mark %d Frame objects as is_extracted=%s for video %s. Frame numbers range: [%s-%s]. Contains frame 0: %s",
|
|
26
|
-
len(extracted_frame_numbers),
|
|
29
|
+
len(extracted_frame_numbers),
|
|
30
|
+
status,
|
|
31
|
+
video.uuid,
|
|
32
|
+
min_frame,
|
|
33
|
+
max_frame,
|
|
34
|
+
contains_zero,
|
|
27
35
|
)
|
|
28
36
|
# --- End Enhanced Logging ---
|
|
29
37
|
|
|
30
38
|
try:
|
|
31
39
|
# Update Frame objects based on frame_number
|
|
32
40
|
# Convert set to list for potentially better compatibility with some DB backends
|
|
33
|
-
updated_count = Frame.objects.filter(
|
|
34
|
-
video=video,
|
|
35
|
-
frame_number__in=list(extracted_frame_numbers)
|
|
36
|
-
).update(is_extracted=status)
|
|
41
|
+
updated_count = Frame.objects.filter(video=video, frame_number__in=list(extracted_frame_numbers)).update(is_extracted=status)
|
|
37
42
|
|
|
38
43
|
logger.info("Database reported updating %d Frame objects to is_extracted=%s for video %s.", updated_count, status, video.uuid)
|
|
39
44
|
|
|
40
45
|
# Verification step
|
|
41
46
|
if updated_count != len(extracted_frame_numbers):
|
|
42
47
|
logger.warning(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
48
|
+
"Mismatch during status update for video %s. Expected to update %d frames, but DB reported updating %d.",
|
|
49
|
+
video.uuid,
|
|
50
|
+
len(extracted_frame_numbers),
|
|
51
|
+
updated_count,
|
|
52
|
+
)
|
|
53
|
+
# --- Add detailed check for frame 0 if status is True and it should have been updated ---
|
|
47
54
|
if status is True and contains_zero and updated_count < len(extracted_frame_numbers):
|
|
48
55
|
try:
|
|
49
56
|
# Check the status of frame 0 directly after the update attempt
|
|
@@ -52,14 +59,16 @@ def _mark_frames_extracted_status(video: "VideoFile", extracted_frame_numbers: S
|
|
|
52
59
|
logger.error("Verification check: Frame 0 (PK: %s) was NOT updated to is_extracted=True for video %s.", frame_zero.pk, video.uuid)
|
|
53
60
|
else:
|
|
54
61
|
# This case should ideally not happen if updated_count < expected count, but log just in case
|
|
55
|
-
logger.info(
|
|
62
|
+
logger.info(
|
|
63
|
+
"Verification check: Frame 0 (PK: %s) IS is_extracted=True for video %s, despite count mismatch.", frame_zero.pk, video.uuid
|
|
64
|
+
)
|
|
56
65
|
except Frame.DoesNotExist:
|
|
57
66
|
logger.error("Verification check: Frame 0 does not exist for video %s during status check.", video.uuid)
|
|
58
67
|
except Exception as verify_e:
|
|
59
68
|
logger.error("Verification check: Error checking frame 0 status for video %s: %s", video.uuid, verify_e)
|
|
60
|
-
|
|
69
|
+
# --- End detailed check ---
|
|
61
70
|
|
|
62
71
|
return updated_count
|
|
63
72
|
except Exception as e:
|
|
64
73
|
logger.error("Failed to bulk update is_extracted status for video %s: %s", video.uuid, e, exc_info=True)
|
|
65
|
-
raise
|
|
74
|
+
raise # Re-raise to ensure transaction rollback if needed
|
|
@@ -1,75 +1,110 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
from contextlib import contextmanager
|
|
2
3
|
from pathlib import Path
|
|
3
|
-
from typing import TYPE_CHECKING, Optional
|
|
4
|
+
from typing import TYPE_CHECKING, Iterator, Optional
|
|
5
|
+
|
|
4
6
|
from django.db import transaction
|
|
5
7
|
|
|
6
|
-
from
|
|
8
|
+
from ....utils import delete_field_file, ensure_local_file, storage_file_exists
|
|
9
|
+
from ...utils import (
|
|
10
|
+
ANONYM_VIDEO_DIR,
|
|
11
|
+
VIDEO_DIR,
|
|
12
|
+
data_paths,
|
|
13
|
+
)
|
|
7
14
|
|
|
8
15
|
if TYPE_CHECKING:
|
|
9
16
|
from .video_file import VideoFile
|
|
10
17
|
|
|
11
18
|
logger = logging.getLogger("video_file")
|
|
12
19
|
|
|
20
|
+
|
|
13
21
|
def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
|
|
14
|
-
"""
|
|
15
|
-
|
|
16
|
-
The FileField stores a path relative to the storage root. We need to join
|
|
17
|
-
that relative path onto the actual video directory under STORAGE_DIR.
|
|
18
|
-
"""
|
|
19
|
-
try:
|
|
20
|
-
if video.has_raw and video.raw_file.name:
|
|
21
|
-
# raw_file.name is a relative storage path like 'videos/<filename>'
|
|
22
|
-
raw_rel = Path(video.raw_file.name)
|
|
23
|
-
|
|
24
|
-
# If it already contains the video directory name, keep the tail
|
|
25
|
-
rel_name = raw_rel.name if raw_rel.parent.name == VIDEO_DIR.name else raw_rel
|
|
26
|
-
full_path = data_paths["video"] / rel_name
|
|
27
|
-
|
|
28
|
-
# If primary path doesn't exist, check alternative locations
|
|
29
|
-
if not full_path.exists():
|
|
30
|
-
# Check if file is in sensitive subdirectory
|
|
31
|
-
sensitive_path = data_paths["video"] / "sensitive" / rel_name
|
|
32
|
-
if sensitive_path.exists():
|
|
33
|
-
return sensitive_path.resolve()
|
|
34
|
-
|
|
35
|
-
# Check direct raw_file.path if available
|
|
36
|
-
# Check direct raw_file.path if available
|
|
37
|
-
try:
|
|
38
|
-
direct_path = Path(video.raw_file.path)
|
|
39
|
-
if direct_path.exists():
|
|
40
|
-
return direct_path.resolve()
|
|
41
|
-
except Exception as e:
|
|
42
|
-
logger.debug("Could not access direct raw_file.path for video %s: %s", video.uuid, e)
|
|
43
|
-
# Fallback to checking alternative paths
|
|
44
|
-
|
|
45
|
-
# Check common alternative paths
|
|
46
|
-
alternative_paths = [
|
|
47
|
-
Path("/home/admin/dev/lx-annotate/libs/data/videos") / rel_name,
|
|
48
|
-
Path("/home/admin/dev/lx-annotate/libs/data/videos/sensitive") / rel_name,
|
|
49
|
-
data_paths["video"].parent / "libs" / "data" / "videos" / rel_name,
|
|
50
|
-
]
|
|
51
|
-
|
|
52
|
-
for alt_path in alternative_paths:
|
|
53
|
-
if alt_path.exists():
|
|
54
|
-
return alt_path.resolve()
|
|
55
|
-
|
|
56
|
-
return full_path.resolve()
|
|
57
|
-
else:
|
|
58
|
-
return None
|
|
59
|
-
except Exception as e:
|
|
60
|
-
logger.warning("Could not get path for raw file of VideoFile %s: %s", video.uuid, e)
|
|
22
|
+
"""Return the best-effort absolute path to the raw video on disk."""
|
|
23
|
+
if not (video.has_raw and video.raw_file.name):
|
|
61
24
|
return None
|
|
62
25
|
|
|
26
|
+
# raw_file.name is a relative storage path like 'videos/<filename>'
|
|
27
|
+
raw_rel = Path(video.raw_file.name)
|
|
28
|
+
|
|
29
|
+
# If it already contains the video directory name, keep the tail
|
|
30
|
+
rel_name = raw_rel.name if raw_rel.parent.name == VIDEO_DIR.name else raw_rel
|
|
31
|
+
full_path = data_paths["video"] / rel_name
|
|
32
|
+
|
|
33
|
+
if full_path.exists():
|
|
34
|
+
return full_path.resolve()
|
|
35
|
+
|
|
36
|
+
sensitive_path = data_paths["video"] / "sensitive" / rel_name
|
|
37
|
+
if sensitive_path.exists():
|
|
38
|
+
return sensitive_path.resolve()
|
|
39
|
+
|
|
40
|
+
if storage_file_exists(video.raw_file):
|
|
41
|
+
try:
|
|
42
|
+
direct_path = Path(video.raw_file.path)
|
|
43
|
+
if direct_path.exists():
|
|
44
|
+
return direct_path.resolve()
|
|
45
|
+
except Exception as exc:
|
|
46
|
+
logger.debug(
|
|
47
|
+
"Could not access direct raw_file.path for video %s: %s",
|
|
48
|
+
video.uuid,
|
|
49
|
+
exc,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
logger.warning(
|
|
53
|
+
"Raw video file '%s' not found under %s or via stored FileField path for video %s.",
|
|
54
|
+
rel_name,
|
|
55
|
+
data_paths["video"],
|
|
56
|
+
video.uuid,
|
|
57
|
+
)
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@contextmanager
|
|
62
|
+
def _ensure_local_raw_file(video: "VideoFile") -> Iterator[Path]:
|
|
63
|
+
"""Yield a local filesystem path for the raw file, downloading if required."""
|
|
64
|
+
if not video.has_raw:
|
|
65
|
+
raise ValueError(f"Video {video.uuid} has no raw file")
|
|
66
|
+
|
|
67
|
+
with ensure_local_file(video.raw_file) as local_path:
|
|
68
|
+
yield local_path
|
|
69
|
+
|
|
70
|
+
|
|
63
71
|
def _get_processed_file_path(video: "VideoFile") -> Optional[Path]:
|
|
64
72
|
"""Returns the absolute Path object for the processed file, if it exists."""
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
return Path(video.processed_file.path)
|
|
68
|
-
else:
|
|
69
|
-
return None
|
|
70
|
-
except Exception as e:
|
|
71
|
-
logger.warning("Could not get path for processed file of VideoFile %s: %s", video.uuid, e)
|
|
73
|
+
processed_field = getattr(video, "processed_file", None)
|
|
74
|
+
if not (video.is_processed and processed_field and processed_field.name):
|
|
72
75
|
return None
|
|
76
|
+
try:
|
|
77
|
+
direct_path = Path(processed_field.path)
|
|
78
|
+
if direct_path.exists():
|
|
79
|
+
return direct_path.resolve()
|
|
80
|
+
except Exception as exc:
|
|
81
|
+
logger.debug(
|
|
82
|
+
"Could not access direct processed_file.path for video %s: %s",
|
|
83
|
+
video.uuid,
|
|
84
|
+
exc,
|
|
85
|
+
)
|
|
86
|
+
direct_path = None
|
|
87
|
+
|
|
88
|
+
if processed_field and storage_file_exists(processed_field):
|
|
89
|
+
logger.debug("Processed file for %s available only via storage backend", video.uuid)
|
|
90
|
+
else:
|
|
91
|
+
logger.warning(
|
|
92
|
+
"Could not get path for processed file of VideoFile %s: %s",
|
|
93
|
+
video.uuid,
|
|
94
|
+
"path unavailable",
|
|
95
|
+
)
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@contextmanager
|
|
100
|
+
def _ensure_local_processed_file(video: "VideoFile") -> Iterator[Path]:
|
|
101
|
+
"""Yield a local path to the processed file, downloading if necessary."""
|
|
102
|
+
if not video.is_processed:
|
|
103
|
+
raise ValueError(f"Video {video.uuid} has no processed file")
|
|
104
|
+
|
|
105
|
+
with ensure_local_file(video.processed_file) as local_path:
|
|
106
|
+
yield local_path
|
|
107
|
+
|
|
73
108
|
|
|
74
109
|
@transaction.atomic
|
|
75
110
|
def _delete_with_file(video: "VideoFile", *args, **kwargs):
|
|
@@ -96,6 +131,11 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
|
|
|
96
131
|
except Exception as e:
|
|
97
132
|
# Log error but continue
|
|
98
133
|
logger.error("Error deleting raw video file %s for video %s: %s", raw_file_path, video.uuid, e, exc_info=True)
|
|
134
|
+
else:
|
|
135
|
+
if delete_field_file(getattr(video, "raw_file", None), save=False):
|
|
136
|
+
logger.info("Deleted raw file from storage for video %s", video.uuid)
|
|
137
|
+
else:
|
|
138
|
+
logger.warning("Raw video file not found during deletion for video %s.", video.uuid)
|
|
99
139
|
|
|
100
140
|
# 3. Delete Processed File
|
|
101
141
|
processed_file_path = _get_processed_file_path(video)
|
|
@@ -109,6 +149,11 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
|
|
|
109
149
|
except Exception as e:
|
|
110
150
|
# Log error but continue
|
|
111
151
|
logger.error("Error deleting processed video file %s for video %s: %s", processed_file_path, video.uuid, e, exc_info=True)
|
|
152
|
+
else:
|
|
153
|
+
if delete_field_file(getattr(video, "processed_file", None), save=False):
|
|
154
|
+
logger.info("Deleted processed file from storage for video %s", video.uuid)
|
|
155
|
+
else:
|
|
156
|
+
logger.warning("Processed file missing in storage for video %s", video.uuid)
|
|
112
157
|
|
|
113
158
|
# 4. Delete Database Record
|
|
114
159
|
try:
|
|
@@ -119,7 +164,8 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
|
|
|
119
164
|
return f"Successfully deleted VideoFile {video.uuid} and attempted file cleanup."
|
|
120
165
|
except Exception as e:
|
|
121
166
|
logger.error("Error deleting VideoFile database record PK %s (UUID: %s): %s", video.pk, video.uuid, e, exc_info=True)
|
|
122
|
-
raise
|
|
167
|
+
raise # Re-raise the exception for DB deletion failure
|
|
168
|
+
|
|
123
169
|
|
|
124
170
|
def _get_base_frame_dir(video: "VideoFile") -> Path:
|
|
125
171
|
"""Gets the base directory path for storing extracted frames."""
|
|
@@ -130,22 +176,23 @@ def _get_base_frame_dir(video: "VideoFile") -> Path:
|
|
|
130
176
|
def _set_frame_dir(video: "VideoFile", force_update: bool = False):
|
|
131
177
|
"""Sets the frame_dir field based on the video's UUID."""
|
|
132
178
|
target_dir = _get_base_frame_dir(video)
|
|
133
|
-
target_path_str = target_dir.as_posix()
|
|
179
|
+
target_path_str = target_dir.as_posix() # Store as POSIX path string
|
|
134
180
|
|
|
135
181
|
if not video.frame_dir or video.frame_dir != target_path_str or force_update:
|
|
136
182
|
video.frame_dir = target_path_str
|
|
137
183
|
logger.info("Set frame_dir for video %s to %s", video.uuid, video.frame_dir)
|
|
138
184
|
# Avoid saving if called from within the save method itself
|
|
139
|
-
if not getattr(video,
|
|
140
|
-
video.save(update_fields=[
|
|
185
|
+
if not getattr(video, "_saving", False):
|
|
186
|
+
video.save(update_fields=["frame_dir"])
|
|
141
187
|
|
|
142
188
|
|
|
143
189
|
def _get_frame_dir_path(video: "VideoFile") -> Optional[Path]:
|
|
144
190
|
"""Returns the Path object for the frame directory, if set."""
|
|
145
191
|
if not video.frame_dir:
|
|
146
192
|
_set_frame_dir(video)
|
|
147
|
-
|
|
148
|
-
return Path(video.frame_dir)
|
|
193
|
+
|
|
194
|
+
return Path(video.frame_dir)
|
|
195
|
+
|
|
149
196
|
|
|
150
197
|
def _get_temp_anonymized_frame_dir(video: "VideoFile") -> Path:
|
|
151
198
|
"""Gets the path for the temporary directory used during anonymization frame creation."""
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import TYPE_CHECKING,
|
|
2
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
3
|
+
|
|
3
4
|
from .get_endo_roi import _get_endo_roi
|
|
4
5
|
|
|
5
6
|
if TYPE_CHECKING:
|
|
@@ -10,7 +11,7 @@ logger = logging.getLogger(__name__)
|
|
|
10
11
|
|
|
11
12
|
def _get_crop_template(video: "VideoFile") -> Optional[List[int]]:
|
|
12
13
|
"""Generates a crop template [y1, y2, x1, x2] from the endo ROI."""
|
|
13
|
-
endo_roi = _get_endo_roi(video)
|
|
14
|
+
endo_roi = _get_endo_roi(video) # Use the helper function
|
|
14
15
|
if not endo_roi:
|
|
15
16
|
logger.warning("Cannot generate crop template for video %s: Endo ROI not available.", video.uuid)
|
|
16
17
|
return None
|
|
@@ -40,6 +41,5 @@ def _get_crop_template(video: "VideoFile") -> Optional[List[int]]:
|
|
|
40
41
|
# Proceed without boundary check if image dimensions unknown
|
|
41
42
|
crop_template = [y, y + height, x, x + width]
|
|
42
43
|
|
|
43
|
-
|
|
44
44
|
logger.debug("Generated crop template for video %s: %s", video.uuid, crop_template)
|
|
45
45
|
return crop_template
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import TYPE_CHECKING,
|
|
2
|
+
from typing import TYPE_CHECKING, Dict, Optional
|
|
3
3
|
|
|
4
4
|
if TYPE_CHECKING:
|
|
5
5
|
from ..video_file import VideoFile
|
|
6
6
|
|
|
7
7
|
logger = logging.getLogger(__name__)
|
|
8
8
|
|
|
9
|
+
|
|
9
10
|
def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
|
|
10
11
|
"""
|
|
11
12
|
Gets the endoscope region of interest (ROI) dictionary from the linked VideoMeta.
|
|
@@ -26,8 +27,9 @@ def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
|
|
|
26
27
|
and all(k in endo_roi for k in ("x", "y", "width", "height"))
|
|
27
28
|
and all(isinstance(v, int) and not isinstance(v, bool) for v in endo_roi.values())
|
|
28
29
|
):
|
|
29
|
-
|
|
30
|
-
|
|
30
|
+
cleaned_roi = {k: int(endo_roi[k] or 0) for k in ("x", "y", "width", "height")}
|
|
31
|
+
logger.debug("Retrieved endo ROI for video %s: %s", video.uuid, cleaned_roi)
|
|
32
|
+
return cleaned_roi
|
|
31
33
|
else:
|
|
32
34
|
logger.warning("Endo ROI not fully defined or invalid in VideoMeta for video %s. ROI: %s", video.uuid, endo_roi)
|
|
33
35
|
return None
|