endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/authz/auth.py +74 -0
- endoreg_db/authz/backends.py +168 -0
- endoreg_db/authz/management/commands/list_routes.py +18 -0
- endoreg_db/authz/middleware.py +83 -0
- endoreg_db/authz/permissions.py +127 -0
- endoreg_db/authz/policy.py +218 -0
- endoreg_db/authz/views_auth.py +66 -0
- endoreg_db/config/env.py +13 -8
- endoreg_db/data/__init__.py +8 -31
- endoreg_db/data/_examples/disease.yaml +55 -0
- endoreg_db/data/_examples/disease_classification.yaml +13 -0
- endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
- endoreg_db/data/_examples/event.yaml +64 -0
- endoreg_db/data/_examples/examination.yaml +72 -0
- endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
- endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
- endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
- endoreg_db/data/_examples/finding/complication.yaml +16 -0
- endoreg_db/data/_examples/finding/data.yaml +105 -0
- endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
- endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
- endoreg_db/data/_examples/finding/outcome.yaml +12 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
- endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
- endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
- endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
- endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
- endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
- endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
- endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
- endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
- endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
- endoreg_db/data/_examples/finding_type/data.yaml +43 -0
- endoreg_db/data/_examples/requirement/age.yaml +26 -0
- endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
- endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
- endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
- endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
- endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
- endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
- endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
- endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
- endoreg_db/data/_examples/requirement/gender.yaml +25 -0
- endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
- endoreg_db/data/_examples/requirement/medication.yaml +93 -0
- endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
- endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
- endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
- endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
- endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
- endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
- endoreg_db/data/event_classification/data.yaml +4 -0
- endoreg_db/data/event_classification_choice/data.yaml +9 -0
- endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
- endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
- endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
- endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
- endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
- endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
- endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
- endoreg_db/data/requirement_set/_old_ +109 -0
- endoreg_db/data/requirement_set_type/data.yaml +21 -0
- endoreg_db/data/setup_config.yaml +4 -4
- endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
- endoreg_db/exceptions.py +5 -2
- endoreg_db/helpers/data_loader.py +1 -1
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
- endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
- endoreg_db/management/commands/import_video.py +9 -10
- endoreg_db/management/commands/import_video_with_classification.py +1 -1
- endoreg_db/management/commands/init_default_ai_model.py +1 -1
- endoreg_db/management/commands/list_routes.py +18 -0
- endoreg_db/management/commands/load_ai_model_data.py +2 -1
- endoreg_db/management/commands/load_center_data.py +12 -12
- endoreg_db/management/commands/load_requirement_data.py +60 -31
- endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
- endoreg_db/management/commands/setup_endoreg_db.py +14 -10
- endoreg_db/management/commands/storage_management.py +271 -203
- endoreg_db/migrations/0001_initial.py +1799 -1300
- endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
- endoreg_db/migrations/_old/0001_initial.py +1857 -0
- endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
- endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
- endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
- endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
- endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
- endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
- endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
- endoreg_db/models/__init__.py +78 -123
- endoreg_db/models/administration/__init__.py +21 -42
- endoreg_db/models/administration/ai/active_model.py +2 -2
- endoreg_db/models/administration/ai/ai_model.py +7 -6
- endoreg_db/models/administration/case/__init__.py +1 -15
- endoreg_db/models/administration/case/case.py +3 -3
- endoreg_db/models/administration/case/case_template/__init__.py +2 -14
- endoreg_db/models/administration/case/case_template/case_template.py +2 -124
- endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
- endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
- endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
- endoreg_db/models/administration/center/center.py +33 -19
- endoreg_db/models/administration/center/center_product.py +12 -9
- endoreg_db/models/administration/center/center_resource.py +25 -19
- endoreg_db/models/administration/center/center_shift.py +21 -17
- endoreg_db/models/administration/center/center_waste.py +16 -8
- endoreg_db/models/administration/person/__init__.py +2 -0
- endoreg_db/models/administration/person/employee/employee.py +10 -5
- endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
- endoreg_db/models/administration/person/employee/employee_type.py +12 -6
- endoreg_db/models/administration/person/examiner/examiner.py +13 -11
- endoreg_db/models/administration/person/patient/__init__.py +2 -0
- endoreg_db/models/administration/person/patient/patient.py +103 -100
- endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
- endoreg_db/models/administration/person/person.py +4 -0
- endoreg_db/models/administration/person/profession/__init__.py +8 -4
- endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
- endoreg_db/models/administration/product/product.py +20 -15
- endoreg_db/models/administration/product/product_material.py +17 -18
- endoreg_db/models/administration/product/product_weight.py +12 -8
- endoreg_db/models/administration/product/reference_product.py +23 -55
- endoreg_db/models/administration/qualification/qualification.py +7 -3
- endoreg_db/models/administration/qualification/qualification_type.py +7 -3
- endoreg_db/models/administration/shift/scheduled_days.py +8 -5
- endoreg_db/models/administration/shift/shift.py +16 -12
- endoreg_db/models/administration/shift/shift_type.py +23 -31
- endoreg_db/models/label/__init__.py +7 -8
- endoreg_db/models/label/annotation/image_classification.py +10 -9
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
- endoreg_db/models/label/label.py +15 -15
- endoreg_db/models/label/label_set.py +19 -6
- endoreg_db/models/label/label_type.py +1 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
- endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
- endoreg_db/models/label/video_segmentation_label.py +4 -0
- endoreg_db/models/label/video_segmentation_labelset.py +4 -3
- endoreg_db/models/media/frame/frame.py +22 -22
- endoreg_db/models/media/pdf/raw_pdf.py +249 -177
- endoreg_db/models/media/pdf/report_file.py +25 -29
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
- endoreg_db/models/media/video/__init__.py +1 -0
- endoreg_db/models/media/video/create_from_file.py +48 -56
- endoreg_db/models/media/video/pipe_1.py +30 -33
- endoreg_db/models/media/video/pipe_2.py +8 -9
- endoreg_db/models/media/video/video_file.py +359 -204
- endoreg_db/models/media/video/video_file_ai.py +288 -74
- endoreg_db/models/media/video/video_file_anonymize.py +38 -38
- endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
- endoreg_db/models/media/video/video_file_io.py +109 -62
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
- endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
- endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
- endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
- endoreg_db/models/media/video/video_file_segments.py +24 -17
- endoreg_db/models/media/video/video_metadata.py +19 -35
- endoreg_db/models/media/video/video_processing.py +96 -95
- endoreg_db/models/medical/contraindication/__init__.py +13 -3
- endoreg_db/models/medical/disease.py +22 -16
- endoreg_db/models/medical/event.py +31 -18
- endoreg_db/models/medical/examination/__init__.py +13 -6
- endoreg_db/models/medical/examination/examination.py +17 -18
- endoreg_db/models/medical/examination/examination_indication.py +26 -25
- endoreg_db/models/medical/examination/examination_time.py +16 -6
- endoreg_db/models/medical/examination/examination_time_type.py +9 -6
- endoreg_db/models/medical/examination/examination_type.py +3 -4
- endoreg_db/models/medical/finding/finding.py +38 -39
- endoreg_db/models/medical/finding/finding_classification.py +37 -48
- endoreg_db/models/medical/finding/finding_intervention.py +27 -22
- endoreg_db/models/medical/finding/finding_type.py +13 -12
- endoreg_db/models/medical/hardware/endoscope.py +20 -26
- endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
- endoreg_db/models/medical/laboratory/lab_value.py +62 -91
- endoreg_db/models/medical/medication/medication.py +22 -10
- endoreg_db/models/medical/medication/medication_indication.py +29 -3
- endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
- endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
- endoreg_db/models/medical/medication/medication_schedule.py +27 -16
- endoreg_db/models/medical/organ/__init__.py +15 -12
- endoreg_db/models/medical/patient/medication_examples.py +1 -5
- endoreg_db/models/medical/patient/patient_disease.py +20 -23
- endoreg_db/models/medical/patient/patient_event.py +19 -22
- endoreg_db/models/medical/patient/patient_examination.py +48 -54
- endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
- endoreg_db/models/medical/patient/patient_finding.py +122 -139
- endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
- endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
- endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
- endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
- endoreg_db/models/medical/patient/patient_medication.py +27 -38
- endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
- endoreg_db/models/medical/risk/risk.py +7 -6
- endoreg_db/models/medical/risk/risk_type.py +8 -5
- endoreg_db/models/metadata/model_meta.py +60 -29
- endoreg_db/models/metadata/model_meta_logic.py +139 -18
- endoreg_db/models/metadata/pdf_meta.py +19 -24
- endoreg_db/models/metadata/sensitive_meta.py +102 -85
- endoreg_db/models/metadata/sensitive_meta_logic.py +383 -43
- endoreg_db/models/metadata/video_meta.py +51 -31
- endoreg_db/models/metadata/video_prediction_logic.py +16 -23
- endoreg_db/models/metadata/video_prediction_meta.py +29 -33
- endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
- endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
- endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
- endoreg_db/models/other/emission/emission_factor.py +18 -8
- endoreg_db/models/other/gender.py +10 -5
- endoreg_db/models/other/information_source.py +25 -25
- endoreg_db/models/other/material.py +9 -5
- endoreg_db/models/other/resource.py +6 -4
- endoreg_db/models/other/tag.py +10 -5
- endoreg_db/models/other/transport_route.py +13 -8
- endoreg_db/models/other/unit.py +10 -6
- endoreg_db/models/other/waste.py +6 -5
- endoreg_db/models/requirement/requirement.py +580 -272
- endoreg_db/models/requirement/requirement_error.py +85 -0
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
- endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
- endoreg_db/models/requirement/requirement_operator.py +36 -33
- endoreg_db/models/requirement/requirement_set.py +74 -57
- endoreg_db/models/state/__init__.py +4 -4
- endoreg_db/models/state/abstract.py +2 -2
- endoreg_db/models/state/anonymization.py +12 -0
- endoreg_db/models/state/audit_ledger.py +46 -47
- endoreg_db/models/state/label_video_segment.py +9 -0
- endoreg_db/models/state/raw_pdf.py +40 -46
- endoreg_db/models/state/sensitive_meta.py +6 -2
- endoreg_db/models/state/video.py +58 -53
- endoreg_db/models/upload_job.py +32 -55
- endoreg_db/models/utils.py +1 -2
- endoreg_db/root_urls.py +21 -2
- endoreg_db/serializers/__init__.py +26 -57
- endoreg_db/serializers/anonymization.py +18 -10
- endoreg_db/serializers/meta/report_meta.py +1 -1
- endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
- endoreg_db/serializers/misc/__init__.py +1 -1
- endoreg_db/serializers/misc/file_overview.py +33 -91
- endoreg_db/serializers/misc/{vop_patient_data.py → sensitive_patient_data.py} +1 -1
- endoreg_db/serializers/requirements/requirement_sets.py +92 -22
- endoreg_db/serializers/video/segmentation.py +2 -1
- endoreg_db/serializers/video/video_processing_history.py +20 -5
- endoreg_db/serializers/video_examination.py +198 -0
- endoreg_db/services/anonymization.py +75 -73
- endoreg_db/services/lookup_service.py +256 -73
- endoreg_db/services/lookup_store.py +174 -30
- endoreg_db/services/pdf_import.py +711 -310
- endoreg_db/services/storage_aware_video_processor.py +140 -114
- endoreg_db/services/video_import.py +266 -117
- endoreg_db/urls/__init__.py +27 -27
- endoreg_db/urls/label_video_segments.py +2 -0
- endoreg_db/urls/media.py +108 -66
- endoreg_db/urls/root_urls.py +29 -0
- endoreg_db/utils/__init__.py +15 -5
- endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
- endoreg_db/utils/case_generator/__init__.py +3 -0
- endoreg_db/utils/dataloader.py +88 -16
- endoreg_db/utils/defaults/set_default_center.py +32 -0
- endoreg_db/utils/names.py +22 -16
- endoreg_db/utils/permissions.py +2 -1
- endoreg_db/utils/pipelines/process_video_dir.py +1 -1
- endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
- endoreg_db/utils/setup_config.py +8 -5
- endoreg_db/utils/storage.py +115 -0
- endoreg_db/utils/validate_endo_roi.py +8 -2
- endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
- endoreg_db/views/__init__.py +5 -12
- endoreg_db/views/anonymization/media_management.py +198 -163
- endoreg_db/views/anonymization/overview.py +4 -1
- endoreg_db/views/anonymization/validate.py +174 -40
- endoreg_db/views/media/__init__.py +2 -0
- endoreg_db/views/media/pdf_media.py +131 -150
- endoreg_db/views/media/sensitive_metadata.py +46 -6
- endoreg_db/views/media/video_media.py +89 -82
- endoreg_db/views/media/video_segments.py +187 -260
- endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
- endoreg_db/views/patient/patient.py +5 -4
- endoreg_db/views/pdf/__init__.py +5 -8
- endoreg_db/views/pdf/pdf_stream.py +186 -0
- endoreg_db/views/pdf/pdf_stream_views.py +0 -127
- endoreg_db/views/pdf/reimport.py +86 -91
- endoreg_db/views/requirement/evaluate.py +188 -187
- endoreg_db/views/requirement/lookup.py +186 -288
- endoreg_db/views/requirement/requirement_utils.py +89 -0
- endoreg_db/views/video/__init__.py +0 -4
- endoreg_db/views/video/correction.py +2 -2
- endoreg_db/views/video/video_examination_viewset.py +202 -289
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +350 -255
- endoreg_db/models/administration/permissions/__init__.py +0 -44
- endoreg_db/models/media/video/refactor_plan.md +0 -0
- endoreg_db/models/media/video/video_file_frames.py +0 -0
- endoreg_db/models/metadata/frame_ocr_result.py +0 -0
- endoreg_db/models/rule/__init__.py +0 -13
- endoreg_db/models/rule/rule.py +0 -27
- endoreg_db/models/rule/rule_applicator.py +0 -224
- endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
- endoreg_db/models/rule/rule_type.py +0 -20
- endoreg_db/models/rule/ruleset.py +0 -17
- endoreg_db/serializers/video/video_metadata.py +0 -105
- endoreg_db/urls/report.py +0 -48
- endoreg_db/urls/video.py +0 -61
- endoreg_db/utils/case_generator/case_generator.py +0 -159
- endoreg_db/utils/case_generator/utils.py +0 -30
- endoreg_db/views/pdf/pdf_media.py +0 -239
- endoreg_db/views/report/__init__.py +0 -9
- endoreg_db/views/report/report_list.py +0 -112
- endoreg_db/views/report/report_with_secure_url.py +0 -28
- endoreg_db/views/report/start_examination.py +0 -7
- endoreg_db/views/video/video_media.py +0 -158
- endoreg_db/views.py +0 -0
- /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
- /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
- /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -9,25 +9,24 @@ Changelog:
|
|
|
9
9
|
during concurrent video imports (matches PDF import pattern)
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
|
-
from datetime import date
|
|
13
12
|
import logging
|
|
14
|
-
import sys
|
|
15
13
|
import os
|
|
16
14
|
import shutil
|
|
17
15
|
import time
|
|
18
16
|
from contextlib import contextmanager
|
|
17
|
+
from datetime import date
|
|
19
18
|
from pathlib import Path
|
|
20
|
-
from typing import
|
|
19
|
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
20
|
+
import subprocess
|
|
21
21
|
from django.db import transaction
|
|
22
|
-
from lx_anonymizer import FrameCleaner
|
|
23
|
-
from moviepy import video
|
|
24
|
-
from endoreg_db.models import VideoFile, SensitiveMeta
|
|
25
|
-
from endoreg_db.utils.paths import STORAGE_DIR, VIDEO_DIR, ANONYM_VIDEO_DIR
|
|
26
|
-
import random
|
|
27
|
-
from endoreg_db.utils.hashs import get_video_hash
|
|
28
|
-
from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
|
|
29
22
|
from django.db.models.fields.files import FieldFile
|
|
30
|
-
|
|
23
|
+
|
|
24
|
+
from endoreg_db.models import EndoscopyProcessor, SensitiveMeta, VideoFile
|
|
25
|
+
from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
|
|
26
|
+
from endoreg_db.utils import ensure_local_file, storage_file_exists
|
|
27
|
+
from endoreg_db.utils.hashs import get_video_hash
|
|
28
|
+
from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, STORAGE_DIR, VIDEO_DIR
|
|
29
|
+
from endoreg_db.models.state import VideoState
|
|
31
30
|
|
|
32
31
|
# File lock configuration (matches PDF import)
|
|
33
32
|
STALE_LOCK_SECONDS = 6000 # 100 minutes - reclaim locks older than this
|
|
@@ -74,8 +73,11 @@ class VideoImportService:
|
|
|
74
73
|
self.processing_context: Dict[str, Any] = {}
|
|
75
74
|
|
|
76
75
|
self.delete_source = True
|
|
76
|
+
self.original_file_path = None
|
|
77
77
|
|
|
78
78
|
self.logger = logging.getLogger(__name__)
|
|
79
|
+
|
|
80
|
+
self.current_video_id = Optional[int]
|
|
79
81
|
|
|
80
82
|
self.cleaner = None # This gets instantiated in the perform_frame_cleaning method
|
|
81
83
|
|
|
@@ -116,7 +118,11 @@ class VideoImportService:
|
|
|
116
118
|
|
|
117
119
|
if age is not None and age > STALE_LOCK_SECONDS:
|
|
118
120
|
try:
|
|
119
|
-
logger.warning(
|
|
121
|
+
logger.warning(
|
|
122
|
+
"Stale lock detected for %s (age %.0fs). Reclaiming lock...",
|
|
123
|
+
path,
|
|
124
|
+
age,
|
|
125
|
+
)
|
|
120
126
|
lock_path.unlink()
|
|
121
127
|
except Exception as e:
|
|
122
128
|
logger.warning("Failed to remove stale lock %s: %s", lock_path, e)
|
|
@@ -206,7 +212,14 @@ class VideoImportService:
|
|
|
206
212
|
finally:
|
|
207
213
|
self._cleanup_processing_context()
|
|
208
214
|
|
|
209
|
-
def _initialize_processing_context(
|
|
215
|
+
def _initialize_processing_context(
|
|
216
|
+
self,
|
|
217
|
+
file_path: Union[Path, str],
|
|
218
|
+
center_name: str,
|
|
219
|
+
processor_name: str,
|
|
220
|
+
save_video: bool,
|
|
221
|
+
delete_source: bool,
|
|
222
|
+
):
|
|
210
223
|
"""Initialize the processing context for the current video import."""
|
|
211
224
|
self.processing_context = {
|
|
212
225
|
"file_path": Path(file_path),
|
|
@@ -219,6 +232,7 @@ class VideoImportService:
|
|
|
219
232
|
"anonymization_completed": False,
|
|
220
233
|
"error_reason": None,
|
|
221
234
|
}
|
|
235
|
+
self.original_file_path = str(file_path)
|
|
222
236
|
|
|
223
237
|
self.logger.info(f"Initialized processing context for: {file_path}")
|
|
224
238
|
|
|
@@ -270,6 +284,7 @@ class VideoImportService:
|
|
|
270
284
|
delete_source=self.processing_context["delete_source"],
|
|
271
285
|
save_video_file=self.processing_context["save_video"],
|
|
272
286
|
)
|
|
287
|
+
self.current_video_id = self.current_video.pk
|
|
273
288
|
|
|
274
289
|
if not self.current_video:
|
|
275
290
|
raise RuntimeError("Failed to create VideoFile instance")
|
|
@@ -319,12 +334,21 @@ class VideoImportService:
|
|
|
319
334
|
except Exception:
|
|
320
335
|
stored_raw_path = None
|
|
321
336
|
|
|
322
|
-
# Fallback: derive from UUID + suffix
|
|
337
|
+
# Fallback: derive from UUID + suffix - ALWAYS use UUID for consistency
|
|
323
338
|
if not stored_raw_path:
|
|
324
339
|
suffix = source_path.suffix or ".mp4"
|
|
325
340
|
uuid_str = getattr(_current_video, "uuid", None)
|
|
326
|
-
|
|
341
|
+
if uuid_str:
|
|
342
|
+
filename = f"{uuid_str}{suffix}"
|
|
343
|
+
else:
|
|
344
|
+
# Emergency fallback with timestamp to avoid conflicts
|
|
345
|
+
import time
|
|
346
|
+
|
|
347
|
+
timestamp = int(time.time())
|
|
348
|
+
filename = f"video_{timestamp}{suffix}"
|
|
349
|
+
self.logger.warning("No UUID available, using timestamp-based filename: %s", filename)
|
|
327
350
|
stored_raw_path = videos_dir / filename
|
|
351
|
+
self.logger.debug("Using UUID-based raw filename: %s", filename)
|
|
328
352
|
|
|
329
353
|
delete_source = bool(self.processing_context.get("delete_source", True))
|
|
330
354
|
stored_raw_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -369,9 +393,6 @@ class VideoImportService:
|
|
|
369
393
|
# Initialize video specifications
|
|
370
394
|
video.initialize_video_specs()
|
|
371
395
|
|
|
372
|
-
# Initialize frame objects in database
|
|
373
|
-
video.initialize_frames()
|
|
374
|
-
|
|
375
396
|
# Extract frames BEFORE processing to prevent pipeline 1 conflicts
|
|
376
397
|
self.logger.info("Pre-extracting frames to avoid pipeline conflicts...")
|
|
377
398
|
try:
|
|
@@ -379,6 +400,8 @@ class VideoImportService:
|
|
|
379
400
|
if frames_extracted:
|
|
380
401
|
self.processing_context["frames_extracted"] = True
|
|
381
402
|
self.logger.info("Frame extraction completed successfully")
|
|
403
|
+
# Initialize frame objects in database
|
|
404
|
+
video.initialize_frames(video.get_frame_paths())
|
|
382
405
|
|
|
383
406
|
# CRITICAL: Immediately save the frames_extracted state to database
|
|
384
407
|
# to prevent refresh_from_db() in pipeline 1 from overriding it
|
|
@@ -420,20 +443,23 @@ class VideoImportService:
|
|
|
420
443
|
endoscope_data_roi_nested, endoscope_image_roi = self._get_processor_roi_info()
|
|
421
444
|
|
|
422
445
|
# Perform frame cleaning with timeout to prevent blocking
|
|
423
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
446
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
447
|
+
from concurrent.futures import TimeoutError as FutureTimeoutError
|
|
424
448
|
|
|
425
449
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
426
|
-
future = executor.submit(
|
|
450
|
+
future = executor.submit(
|
|
451
|
+
self._perform_frame_cleaning,
|
|
452
|
+
endoscope_data_roi_nested,
|
|
453
|
+
endoscope_image_roi,
|
|
454
|
+
)
|
|
427
455
|
try:
|
|
428
456
|
# Increased timeout to better accommodate ffmpeg + OCR
|
|
429
|
-
future.result(timeout=
|
|
457
|
+
future.result(timeout=5000)
|
|
430
458
|
self.processing_context["anonymization_completed"] = True
|
|
431
459
|
self.logger.info("Frame cleaning completed successfully within timeout")
|
|
432
460
|
except FutureTimeoutError:
|
|
433
461
|
self.logger.warning("Frame cleaning timed out; entering grace period check for cleaned output")
|
|
434
462
|
# Grace period: detect if cleaned file appears shortly after timeout
|
|
435
|
-
raw_video_path = self.processing_context.get("raw_video_path")
|
|
436
|
-
video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name if raw_video_path else "video.mp4")
|
|
437
463
|
grace_seconds = 60
|
|
438
464
|
expected_cleaned_path: Optional[Path] = None
|
|
439
465
|
processed_field = video.processed_file
|
|
@@ -448,7 +474,10 @@ class VideoImportService:
|
|
|
448
474
|
if expected_cleaned_path.exists():
|
|
449
475
|
self.processing_context["cleaned_video_path"] = expected_cleaned_path
|
|
450
476
|
self.processing_context["anonymization_completed"] = True
|
|
451
|
-
self.logger.info(
|
|
477
|
+
self.logger.info(
|
|
478
|
+
"Detected cleaned video during grace period: %s",
|
|
479
|
+
expected_cleaned_path,
|
|
480
|
+
)
|
|
452
481
|
found = True
|
|
453
482
|
break
|
|
454
483
|
time.sleep(1)
|
|
@@ -494,21 +523,31 @@ class VideoImportService:
|
|
|
494
523
|
original_raw_file_path_to_delete = video.get_raw_file_path()
|
|
495
524
|
original_raw_frame_dir_to_delete = video.get_frame_dir_path()
|
|
496
525
|
|
|
497
|
-
video.raw_file.name =
|
|
526
|
+
video.raw_file.name = ""
|
|
498
527
|
|
|
499
528
|
update_fields.extend(["raw_file", "video_hash"])
|
|
500
529
|
|
|
501
530
|
transaction.on_commit(
|
|
502
531
|
lambda: _cleanup_raw_assets(
|
|
503
|
-
video_uuid=video.uuid,
|
|
532
|
+
video_uuid=video.uuid,
|
|
533
|
+
raw_file_path=original_raw_file_path_to_delete,
|
|
534
|
+
raw_frame_dir=original_raw_frame_dir_to_delete,
|
|
504
535
|
)
|
|
505
536
|
)
|
|
506
537
|
|
|
507
538
|
video.save(update_fields=update_fields)
|
|
508
|
-
video.state
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
539
|
+
if not isinstance(video.state, VideoState):
|
|
540
|
+
try:
|
|
541
|
+
video.get_or_create_state()
|
|
542
|
+
except ValueError as e:
|
|
543
|
+
raise RuntimeError(f"Video state not found for video {video.uuid}. Error {e}")
|
|
544
|
+
|
|
545
|
+
else:
|
|
546
|
+
video.state.mark_anonymized(save=True)
|
|
547
|
+
video.refresh_from_db()
|
|
548
|
+
self.current_video = video
|
|
549
|
+
|
|
550
|
+
return True
|
|
512
551
|
|
|
513
552
|
def _fallback_anonymize_video(self):
|
|
514
553
|
"""
|
|
@@ -539,7 +578,11 @@ class VideoImportService:
|
|
|
539
578
|
try:
|
|
540
579
|
video.refresh_from_db()
|
|
541
580
|
except Exception as refresh_error:
|
|
542
|
-
self.logger.warning(
|
|
581
|
+
self.logger.warning(
|
|
582
|
+
"Could not refresh VideoFile %s from DB: %s",
|
|
583
|
+
video.uuid,
|
|
584
|
+
refresh_error,
|
|
585
|
+
)
|
|
543
586
|
|
|
544
587
|
state = video.get_or_create_state()
|
|
545
588
|
|
|
@@ -587,8 +630,9 @@ class VideoImportService:
|
|
|
587
630
|
else:
|
|
588
631
|
raw_video_path = self.processing_context.get("raw_video_path")
|
|
589
632
|
if raw_video_path and Path(raw_video_path).exists():
|
|
590
|
-
|
|
591
|
-
|
|
633
|
+
# Use UUID-based naming to avoid conflicts
|
|
634
|
+
suffix = Path(raw_video_path).suffix or ".mp4"
|
|
635
|
+
processed_filename = f"processed_{video.uuid}{suffix}"
|
|
592
636
|
processed_video_path = Path(raw_video_path).parent / processed_filename
|
|
593
637
|
try:
|
|
594
638
|
shutil.copy2(str(raw_video_path), str(processed_video_path))
|
|
@@ -624,7 +668,10 @@ class VideoImportService:
|
|
|
624
668
|
|
|
625
669
|
self.processing_context["anonymization_completed"] = True
|
|
626
670
|
else:
|
|
627
|
-
self.logger.warning(
|
|
671
|
+
self.logger.warning(
|
|
672
|
+
"Processed video file not found after move: %s",
|
|
673
|
+
anonym_target_path,
|
|
674
|
+
)
|
|
628
675
|
except Exception as exc:
|
|
629
676
|
self.logger.error("Failed to move processed video to anonym_videos: %s", exc)
|
|
630
677
|
else:
|
|
@@ -646,7 +693,7 @@ class VideoImportService:
|
|
|
646
693
|
except Exception as exc:
|
|
647
694
|
self.logger.warning("Failed to remove source file %s: %s", source_path, exc)
|
|
648
695
|
|
|
649
|
-
if not video.processed_file or not
|
|
696
|
+
if not video.processed_file or not storage_file_exists(video.processed_file):
|
|
650
697
|
self.logger.warning("No processed_file found after cleanup - video will be unprocessed")
|
|
651
698
|
try:
|
|
652
699
|
video.anonymize(delete_original_raw=self.delete_source)
|
|
@@ -663,6 +710,12 @@ class VideoImportService:
|
|
|
663
710
|
with transaction.atomic():
|
|
664
711
|
video.refresh_from_db()
|
|
665
712
|
if hasattr(video, "state") and self.processing_context.get("anonymization_completed"):
|
|
713
|
+
if not isinstance(video.state, VideoState):
|
|
714
|
+
try:
|
|
715
|
+
video.get_or_create_state()
|
|
716
|
+
except:
|
|
717
|
+
raise RuntimeError(f"Video state not found for video {video.uuid}")
|
|
718
|
+
|
|
666
719
|
video.state.mark_sensitive_meta_processed(save=True)
|
|
667
720
|
|
|
668
721
|
self.logger.info("Import and anonymization completed for VideoFile UUID: %s", video.uuid)
|
|
@@ -677,48 +730,82 @@ class VideoImportService:
|
|
|
677
730
|
"""Create or move a sensitive copy of the raw video file inside storage."""
|
|
678
731
|
|
|
679
732
|
video = video_instance or self._require_current_video()
|
|
680
|
-
|
|
681
733
|
raw_field: FieldFile | None = getattr(video, "raw_file", None)
|
|
682
|
-
source_path: Path | None = None
|
|
683
|
-
try:
|
|
684
|
-
if raw_field and raw_field.path:
|
|
685
|
-
source_path = Path(raw_field.path)
|
|
686
|
-
except Exception:
|
|
687
|
-
source_path = None
|
|
688
734
|
|
|
689
|
-
|
|
690
|
-
|
|
735
|
+
def copy_into_sensitive(source: Path) -> Path:
|
|
736
|
+
target_dir = VIDEO_DIR / "sensitive"
|
|
737
|
+
if not target_dir.exists():
|
|
738
|
+
self.logger.info("Creating sensitive file directory: %s", target_dir)
|
|
739
|
+
os.makedirs(target_dir, exist_ok=True)
|
|
691
740
|
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
741
|
+
target_name = source.name or "raw_video"
|
|
742
|
+
target_file_path = target_dir / target_name
|
|
743
|
+
|
|
744
|
+
if source != target_file_path:
|
|
745
|
+
try:
|
|
746
|
+
shutil.copy2(source, target_file_path)
|
|
747
|
+
self.logger.info("Copied raw file to sensitive directory: %s", target_file_path)
|
|
748
|
+
except Exception as exc:
|
|
749
|
+
self.logger.warning("Failed to copy raw file to sensitive dir: %s", exc)
|
|
750
|
+
shutil.copy(source, target_file_path)
|
|
751
|
+
self.logger.info(
|
|
752
|
+
"Fallback copy succeeded for sensitive directory: %s",
|
|
753
|
+
target_file_path,
|
|
754
|
+
)
|
|
755
|
+
else:
|
|
756
|
+
self.logger.debug(
|
|
757
|
+
"Source path already in sensitive directory: %s",
|
|
758
|
+
target_file_path,
|
|
759
|
+
)
|
|
696
760
|
|
|
697
|
-
|
|
698
|
-
if not target_dir.exists():
|
|
699
|
-
self.logger.info("Creating sensitive file directory: %s", target_dir)
|
|
700
|
-
os.makedirs(target_dir, exist_ok=True)
|
|
761
|
+
return target_file_path
|
|
701
762
|
|
|
702
|
-
target_file_path
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
except Exception as exc:
|
|
707
|
-
self.logger.warning("Failed to move raw file to sensitive dir, copying instead: %s", exc)
|
|
708
|
-
shutil.copy(str(source_path), str(target_file_path))
|
|
763
|
+
target_file_path: Path | None = None
|
|
764
|
+
|
|
765
|
+
# Prefer an on-disk path from the FieldFile when available
|
|
766
|
+
if raw_field:
|
|
709
767
|
try:
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
768
|
+
local_candidate = Path(raw_field.path)
|
|
769
|
+
if local_candidate.exists():
|
|
770
|
+
target_file_path = copy_into_sensitive(local_candidate)
|
|
771
|
+
except Exception:
|
|
772
|
+
target_file_path = None
|
|
773
|
+
|
|
774
|
+
if target_file_path is None and storage_file_exists(raw_field):
|
|
775
|
+
try:
|
|
776
|
+
with ensure_local_file(raw_field) as temp_source:
|
|
777
|
+
target_file_path = copy_into_sensitive(Path(temp_source))
|
|
778
|
+
except Exception as exc:
|
|
779
|
+
self.logger.warning("Failed to download raw_field for sensitive copy: %s", exc)
|
|
780
|
+
|
|
781
|
+
if target_file_path is None and file_path is not None:
|
|
782
|
+
file_candidate = Path(file_path)
|
|
783
|
+
if file_candidate.exists():
|
|
784
|
+
target_file_path = copy_into_sensitive(file_candidate)
|
|
785
|
+
|
|
786
|
+
if target_file_path is None:
|
|
787
|
+
context_path = self.processing_context.get("raw_video_path")
|
|
788
|
+
if context_path:
|
|
789
|
+
context_candidate = Path(context_path)
|
|
790
|
+
if context_candidate.exists():
|
|
791
|
+
target_file_path = copy_into_sensitive(context_candidate)
|
|
792
|
+
|
|
793
|
+
if target_file_path is None:
|
|
794
|
+
raise ValueError("No file path available for creating sensitive file")
|
|
795
|
+
if not raw_field:
|
|
796
|
+
raise ValueError("VideoFile must have a raw_file to create a sensitive file")
|
|
713
797
|
|
|
714
798
|
try:
|
|
715
799
|
from endoreg_db.utils import data_paths
|
|
716
800
|
|
|
717
801
|
storage_root = data_paths["storage"]
|
|
718
802
|
relative_path = target_file_path.relative_to(storage_root)
|
|
719
|
-
video.raw_file.name =
|
|
803
|
+
video.raw_file.name = relative_path.as_posix()
|
|
720
804
|
video.save(update_fields=["raw_file"])
|
|
721
|
-
self.logger.info(
|
|
805
|
+
self.logger.info(
|
|
806
|
+
"Updated video.raw_file to point to sensitive location: %s",
|
|
807
|
+
relative_path,
|
|
808
|
+
)
|
|
722
809
|
except Exception as exc:
|
|
723
810
|
self.logger.warning("Failed to set relative path, using fallback: %s", exc)
|
|
724
811
|
video.raw_file.name = f"videos/sensitive/{target_file_path.name}"
|
|
@@ -734,7 +821,9 @@ class VideoImportService:
|
|
|
734
821
|
self.logger.info("Created sensitive file for %s at %s", video.uuid, target_file_path)
|
|
735
822
|
return target_file_path
|
|
736
823
|
|
|
737
|
-
def _get_processor_roi_info(
|
|
824
|
+
def _get_processor_roi_info(
|
|
825
|
+
self,
|
|
826
|
+
) -> Tuple[Optional[List[List[Dict[str, Any]]]], Optional[Dict[str, Any]]]:
|
|
738
827
|
"""Get processor ROI information for masking."""
|
|
739
828
|
endoscope_data_roi_nested = None
|
|
740
829
|
endoscope_image_roi = None
|
|
@@ -748,7 +837,10 @@ class VideoImportService:
|
|
|
748
837
|
assert isinstance(processor, EndoscopyProcessor), "Processor is not of type EndoscopyProcessor"
|
|
749
838
|
endoscope_image_roi = processor.get_roi_endoscope_image()
|
|
750
839
|
endoscope_data_roi_nested = processor.get_sensitive_rois()
|
|
751
|
-
self.logger.info(
|
|
840
|
+
self.logger.info(
|
|
841
|
+
"Retrieved processor ROI information: endoscope_image_roi=%s",
|
|
842
|
+
endoscope_image_roi,
|
|
843
|
+
)
|
|
752
844
|
else:
|
|
753
845
|
self.logger.warning(
|
|
754
846
|
"No processor found for video %s, proceeding without ROI masking",
|
|
@@ -791,31 +883,16 @@ class VideoImportService:
|
|
|
791
883
|
video.save(update_fields=["sensitive_meta"])
|
|
792
884
|
self.logger.info("Created default SensitiveMeta for video %s", video.uuid)
|
|
793
885
|
except Exception as exc:
|
|
794
|
-
self.logger.error(
|
|
886
|
+
self.logger.error(
|
|
887
|
+
"Failed to create default SensitiveMeta for video %s: %s",
|
|
888
|
+
video.uuid,
|
|
889
|
+
exc,
|
|
890
|
+
)
|
|
795
891
|
return
|
|
796
892
|
else:
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
if not sensitive_meta.patient_last_name:
|
|
801
|
-
update_data["patient_last_name"] = "Unknown"
|
|
802
|
-
if not sensitive_meta.patient_dob:
|
|
803
|
-
update_data["patient_dob"] = date(1990, 1, 1)
|
|
804
|
-
if not sensitive_meta.examination_date:
|
|
805
|
-
update_data["examination_date"] = date.today()
|
|
806
|
-
|
|
807
|
-
if update_data:
|
|
808
|
-
try:
|
|
809
|
-
sensitive_meta.update_from_dict(update_data)
|
|
810
|
-
state = video.get_or_create_state()
|
|
811
|
-
state.mark_sensitive_meta_processed(save=True)
|
|
812
|
-
self.logger.info(
|
|
813
|
-
"Updated missing SensitiveMeta fields for video %s: %s",
|
|
814
|
-
video.uuid,
|
|
815
|
-
list(update_data.keys()),
|
|
816
|
-
)
|
|
817
|
-
except Exception as exc:
|
|
818
|
-
self.logger.error("Failed to update SensitiveMeta for video %s: %s", video.uuid, exc)
|
|
893
|
+
state = video.get_or_create_state()
|
|
894
|
+
state.mark_sensitive_meta_processed(save=True)
|
|
895
|
+
|
|
819
896
|
|
|
820
897
|
def _ensure_frame_cleaning_available(self):
|
|
821
898
|
"""
|
|
@@ -825,23 +902,24 @@ class VideoImportService:
|
|
|
825
902
|
Tuple of (availability_flag, FrameCleaner_class, ReportReader_class)
|
|
826
903
|
"""
|
|
827
904
|
try:
|
|
828
|
-
|
|
829
|
-
from lx_anonymizer import FrameCleaner # type: ignore[import]
|
|
830
|
-
|
|
831
|
-
if FrameCleaner:
|
|
832
|
-
return True, FrameCleaner()
|
|
833
|
-
|
|
905
|
+
from lx_anonymizer import FrameCleaner
|
|
834
906
|
except Exception as e:
|
|
835
907
|
self.logger.warning(f"Frame cleaning not available: {e} Please install or update lx_anonymizer.")
|
|
908
|
+
_available = False
|
|
909
|
+
FrameCleaner = None
|
|
836
910
|
|
|
837
|
-
|
|
911
|
+
assert FrameCleaner is not None
|
|
912
|
+
frame_cleaner = FrameCleaner()
|
|
913
|
+
_available = True
|
|
914
|
+
|
|
915
|
+
return _available, frame_cleaner
|
|
838
916
|
|
|
839
917
|
def _perform_frame_cleaning(self, endoscope_data_roi_nested, endoscope_image_roi):
|
|
840
918
|
"""Perform frame cleaning and anonymization."""
|
|
841
919
|
# Instantiate frame cleaner
|
|
842
920
|
is_available, frame_cleaner = self._ensure_frame_cleaning_available()
|
|
843
921
|
|
|
844
|
-
if not is_available:
|
|
922
|
+
if not is_available or frame_cleaner is None:
|
|
845
923
|
raise RuntimeError("Frame cleaning not available")
|
|
846
924
|
|
|
847
925
|
# Prepare parameters for frame cleaning
|
|
@@ -854,12 +932,15 @@ class VideoImportService:
|
|
|
854
932
|
except Exception:
|
|
855
933
|
raise RuntimeError(f"Raw video path not found: {raw_video_path}")
|
|
856
934
|
|
|
857
|
-
# Create temporary output path for cleaned video
|
|
858
|
-
|
|
859
|
-
|
|
935
|
+
# Create temporary output path for cleaned video using UUID to avoid naming conflicts
|
|
936
|
+
video = self._require_current_video()
|
|
937
|
+
# Ensure raw_video_path is not None
|
|
860
938
|
if not raw_video_path:
|
|
861
|
-
raise RuntimeError("raw_video_path is None
|
|
939
|
+
raise RuntimeError("raw_video_path is None, cannot construct cleaned_video_path")
|
|
940
|
+
suffix = Path(raw_video_path).suffix or ".mp4"
|
|
941
|
+
cleaned_filename = f"cleaned_{video.uuid}{suffix}"
|
|
862
942
|
cleaned_video_path = Path(raw_video_path).parent / cleaned_filename
|
|
943
|
+
self.logger.debug("Using UUID-based cleaned filename: %s", cleaned_filename)
|
|
863
944
|
|
|
864
945
|
# Clean video with ROI masking (heavy I/O operation)
|
|
865
946
|
actual_cleaned_path, extracted_metadata = frame_cleaner.clean_video(
|
|
@@ -896,21 +977,58 @@ class VideoImportService:
|
|
|
896
977
|
sm = sensitive_meta
|
|
897
978
|
updated_fields = []
|
|
898
979
|
|
|
980
|
+
# Ensure center is set from video.center if not in extracted_metadata
|
|
981
|
+
metadata_to_update = extracted_metadata.copy()
|
|
982
|
+
|
|
983
|
+
# FIX: Set center object instead of center_name string
|
|
984
|
+
if not hasattr(sm, "center") or not sm.center:
|
|
985
|
+
if video.center:
|
|
986
|
+
metadata_to_update["center"] = video.center
|
|
987
|
+
self.logger.debug(
|
|
988
|
+
"Added center object '%s' to metadata for SensitiveMeta update",
|
|
989
|
+
video.center.name,
|
|
990
|
+
)
|
|
991
|
+
else:
|
|
992
|
+
center_name = metadata_to_update.get("center_name")
|
|
993
|
+
if center_name:
|
|
994
|
+
try:
|
|
995
|
+
from ..models.administration import Center
|
|
996
|
+
|
|
997
|
+
center_obj = Center.objects.get(name=center_name)
|
|
998
|
+
metadata_to_update["center"] = center_obj
|
|
999
|
+
self.logger.debug("Loaded center object '%s' from center_name", center_name)
|
|
1000
|
+
metadata_to_update.pop("center_name", None)
|
|
1001
|
+
except Center.DoesNotExist:
|
|
1002
|
+
self.logger.error("Center '%s' not found in database", center_name)
|
|
1003
|
+
return
|
|
1004
|
+
|
|
899
1005
|
try:
|
|
900
|
-
sm.update_from_dict(
|
|
901
|
-
updated_fields = list(extracted_metadata.keys())
|
|
1006
|
+
sm.update_from_dict(metadata_to_update)
|
|
1007
|
+
updated_fields = list(extracted_metadata.keys()) # Only log originally extracted fields
|
|
902
1008
|
except KeyError as e:
|
|
903
1009
|
self.logger.warning(f"Failed to update SensitiveMeta field {e}")
|
|
1010
|
+
return
|
|
904
1011
|
|
|
905
1012
|
if updated_fields:
|
|
906
|
-
|
|
907
|
-
|
|
1013
|
+
try:
|
|
1014
|
+
sm.save() # Remove update_fields to allow all necessary fields to be saved
|
|
1015
|
+
self.logger.info(
|
|
1016
|
+
"Updated SensitiveMeta fields for video %s: %s",
|
|
1017
|
+
video.uuid,
|
|
1018
|
+
updated_fields,
|
|
1019
|
+
)
|
|
908
1020
|
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
1021
|
+
state = video.get_or_create_state()
|
|
1022
|
+
state.mark_sensitive_meta_processed(save=True)
|
|
1023
|
+
self.logger.info("Marked sensitive metadata as processed for video %s", video.uuid)
|
|
1024
|
+
except Exception as e:
|
|
1025
|
+
self.logger.error(f"Failed to save SensitiveMeta: {e}")
|
|
1026
|
+
raise # Re-raise to trigger fallback in calling method
|
|
912
1027
|
else:
|
|
913
|
-
self.logger.info(
|
|
1028
|
+
self.logger.info(
|
|
1029
|
+
"No SensitiveMeta fields updated for video %s - all existing values preserved",
|
|
1030
|
+
video.uuid,
|
|
1031
|
+
)
|
|
914
1032
|
|
|
915
1033
|
def _signal_completion(self):
|
|
916
1034
|
"""Signal completion to the tracking system."""
|
|
@@ -918,21 +1036,23 @@ class VideoImportService:
|
|
|
918
1036
|
video = self._require_current_video()
|
|
919
1037
|
|
|
920
1038
|
raw_field: FieldFile | None = getattr(video, "raw_file", None)
|
|
921
|
-
raw_exists =
|
|
922
|
-
if raw_field and getattr(raw_field, "path", None):
|
|
923
|
-
try:
|
|
924
|
-
raw_exists = Path(raw_field.path).exists()
|
|
925
|
-
except (ValueError, OSError):
|
|
926
|
-
raw_exists = False
|
|
1039
|
+
raw_exists = storage_file_exists(raw_field)
|
|
927
1040
|
|
|
928
1041
|
video_processing_complete = video.sensitive_meta is not None and video.video_meta is not None and raw_exists
|
|
929
1042
|
|
|
930
1043
|
if video_processing_complete:
|
|
931
|
-
self.logger.info(
|
|
1044
|
+
self.logger.info(
|
|
1045
|
+
"Video %s processing completed successfully - ready for validation",
|
|
1046
|
+
video.uuid,
|
|
1047
|
+
)
|
|
932
1048
|
|
|
933
1049
|
# Update completion flags if they exist
|
|
934
1050
|
completion_fields = []
|
|
935
|
-
for field_name in [
|
|
1051
|
+
for field_name in [
|
|
1052
|
+
"import_completed",
|
|
1053
|
+
"processing_complete",
|
|
1054
|
+
"ready_for_validation",
|
|
1055
|
+
]:
|
|
936
1056
|
if hasattr(video, field_name):
|
|
937
1057
|
setattr(video, field_name, True)
|
|
938
1058
|
completion_fields.append(field_name)
|
|
@@ -952,15 +1072,44 @@ class VideoImportService:
|
|
|
952
1072
|
def _cleanup_on_error(self):
|
|
953
1073
|
"""Cleanup processing context on error."""
|
|
954
1074
|
if self.current_video and hasattr(self.current_video, "state"):
|
|
1075
|
+
if self.current_video.state is None:
|
|
1076
|
+
try:
|
|
1077
|
+
self.current_video.get_or_create_state()
|
|
1078
|
+
except Exception as e:
|
|
1079
|
+
self.logger.warning(f"Video state not found for video {self.current_video.uuid} during error cleanup {e}")
|
|
1080
|
+
return
|
|
1081
|
+
self.current_video.state = self.current_video.get_or_create_state()
|
|
1082
|
+
try:
|
|
1083
|
+
if self.original_file_path is not None:
|
|
1084
|
+
assert Path(self.original_file_path).exists()
|
|
1085
|
+
else:
|
|
1086
|
+
self.logger.warning("Original file path is None")
|
|
1087
|
+
self.logger.info("Marked video import as failed in state")
|
|
1088
|
+
raw_file_path = getattr(self.current_video.raw_file, "path", None)
|
|
1089
|
+
original_file_path = self.original_file_path
|
|
1090
|
+
if raw_file_path and original_file_path:
|
|
1091
|
+
shutil.copy2(str(raw_file_path), str(original_file_path))
|
|
1092
|
+
else:
|
|
1093
|
+
self.logger.warning("Cannot restore original raw file: path is None")
|
|
1094
|
+
except AssertionError:
|
|
1095
|
+
self.logger.warning("Original file path does not exist")
|
|
955
1096
|
try:
|
|
1097
|
+
|
|
1098
|
+
if not isinstance(self.current_video.state, VideoState):
|
|
1099
|
+
logger.error("Current video is none after Assertion for Video File")
|
|
1100
|
+
raise AssertionError
|
|
1101
|
+
|
|
1102
|
+
|
|
956
1103
|
if self.processing_context.get("processing_started"):
|
|
957
1104
|
self.current_video.state.frames_extracted = False
|
|
958
1105
|
self.current_video.state.frames_initialized = False
|
|
959
1106
|
self.current_video.state.video_meta_extracted = False
|
|
960
1107
|
self.current_video.state.text_meta_extracted = False
|
|
961
1108
|
self.current_video.state.save()
|
|
1109
|
+
|
|
962
1110
|
except Exception as e:
|
|
963
1111
|
self.logger.warning(f"Error during cleanup: {e}")
|
|
1112
|
+
|
|
964
1113
|
|
|
965
1114
|
def _cleanup_processing_context(self):
|
|
966
1115
|
"""
|