endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/authz/auth.py +74 -0
- endoreg_db/authz/backends.py +168 -0
- endoreg_db/authz/management/commands/list_routes.py +18 -0
- endoreg_db/authz/middleware.py +83 -0
- endoreg_db/authz/permissions.py +127 -0
- endoreg_db/authz/policy.py +218 -0
- endoreg_db/authz/views_auth.py +66 -0
- endoreg_db/config/env.py +13 -8
- endoreg_db/data/__init__.py +8 -31
- endoreg_db/data/_examples/disease.yaml +55 -0
- endoreg_db/data/_examples/disease_classification.yaml +13 -0
- endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
- endoreg_db/data/_examples/event.yaml +64 -0
- endoreg_db/data/_examples/examination.yaml +72 -0
- endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
- endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
- endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
- endoreg_db/data/_examples/finding/complication.yaml +16 -0
- endoreg_db/data/_examples/finding/data.yaml +105 -0
- endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
- endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
- endoreg_db/data/_examples/finding/outcome.yaml +12 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
- endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
- endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
- endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
- endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
- endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
- endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
- endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
- endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
- endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
- endoreg_db/data/_examples/finding_type/data.yaml +43 -0
- endoreg_db/data/_examples/requirement/age.yaml +26 -0
- endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
- endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
- endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
- endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
- endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
- endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
- endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
- endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
- endoreg_db/data/_examples/requirement/gender.yaml +25 -0
- endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
- endoreg_db/data/_examples/requirement/medication.yaml +93 -0
- endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
- endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
- endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
- endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
- endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
- endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
- endoreg_db/data/event_classification/data.yaml +4 -0
- endoreg_db/data/event_classification_choice/data.yaml +9 -0
- endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
- endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
- endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
- endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
- endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
- endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
- endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
- endoreg_db/data/requirement_set/_old_ +109 -0
- endoreg_db/data/requirement_set_type/data.yaml +21 -0
- endoreg_db/data/setup_config.yaml +4 -4
- endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
- endoreg_db/exceptions.py +5 -2
- endoreg_db/helpers/data_loader.py +1 -1
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
- endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
- endoreg_db/management/commands/import_video.py +9 -10
- endoreg_db/management/commands/import_video_with_classification.py +1 -1
- endoreg_db/management/commands/init_default_ai_model.py +1 -1
- endoreg_db/management/commands/list_routes.py +18 -0
- endoreg_db/management/commands/load_ai_model_data.py +2 -1
- endoreg_db/management/commands/load_center_data.py +12 -12
- endoreg_db/management/commands/load_requirement_data.py +60 -31
- endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
- endoreg_db/management/commands/setup_endoreg_db.py +14 -10
- endoreg_db/management/commands/storage_management.py +271 -203
- endoreg_db/migrations/0001_initial.py +1799 -1300
- endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
- endoreg_db/migrations/_old/0001_initial.py +1857 -0
- endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
- endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
- endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
- endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
- endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
- endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
- endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
- endoreg_db/models/__init__.py +78 -123
- endoreg_db/models/administration/__init__.py +21 -42
- endoreg_db/models/administration/ai/active_model.py +2 -2
- endoreg_db/models/administration/ai/ai_model.py +7 -6
- endoreg_db/models/administration/case/__init__.py +1 -15
- endoreg_db/models/administration/case/case.py +3 -3
- endoreg_db/models/administration/case/case_template/__init__.py +2 -14
- endoreg_db/models/administration/case/case_template/case_template.py +2 -124
- endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
- endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
- endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
- endoreg_db/models/administration/center/center.py +33 -19
- endoreg_db/models/administration/center/center_product.py +12 -9
- endoreg_db/models/administration/center/center_resource.py +25 -19
- endoreg_db/models/administration/center/center_shift.py +21 -17
- endoreg_db/models/administration/center/center_waste.py +16 -8
- endoreg_db/models/administration/person/__init__.py +2 -0
- endoreg_db/models/administration/person/employee/employee.py +10 -5
- endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
- endoreg_db/models/administration/person/employee/employee_type.py +12 -6
- endoreg_db/models/administration/person/examiner/examiner.py +13 -11
- endoreg_db/models/administration/person/patient/__init__.py +2 -0
- endoreg_db/models/administration/person/patient/patient.py +103 -100
- endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
- endoreg_db/models/administration/person/person.py +4 -0
- endoreg_db/models/administration/person/profession/__init__.py +8 -4
- endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
- endoreg_db/models/administration/product/product.py +20 -15
- endoreg_db/models/administration/product/product_material.py +17 -18
- endoreg_db/models/administration/product/product_weight.py +12 -8
- endoreg_db/models/administration/product/reference_product.py +23 -55
- endoreg_db/models/administration/qualification/qualification.py +7 -3
- endoreg_db/models/administration/qualification/qualification_type.py +7 -3
- endoreg_db/models/administration/shift/scheduled_days.py +8 -5
- endoreg_db/models/administration/shift/shift.py +16 -12
- endoreg_db/models/administration/shift/shift_type.py +23 -31
- endoreg_db/models/label/__init__.py +7 -8
- endoreg_db/models/label/annotation/image_classification.py +10 -9
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
- endoreg_db/models/label/label.py +15 -15
- endoreg_db/models/label/label_set.py +19 -6
- endoreg_db/models/label/label_type.py +1 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
- endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
- endoreg_db/models/label/video_segmentation_label.py +4 -0
- endoreg_db/models/label/video_segmentation_labelset.py +4 -3
- endoreg_db/models/media/frame/frame.py +22 -22
- endoreg_db/models/media/pdf/raw_pdf.py +249 -177
- endoreg_db/models/media/pdf/report_file.py +25 -29
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
- endoreg_db/models/media/video/__init__.py +1 -0
- endoreg_db/models/media/video/create_from_file.py +48 -56
- endoreg_db/models/media/video/pipe_1.py +30 -33
- endoreg_db/models/media/video/pipe_2.py +8 -9
- endoreg_db/models/media/video/video_file.py +359 -204
- endoreg_db/models/media/video/video_file_ai.py +288 -74
- endoreg_db/models/media/video/video_file_anonymize.py +38 -38
- endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
- endoreg_db/models/media/video/video_file_io.py +109 -62
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
- endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
- endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
- endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
- endoreg_db/models/media/video/video_file_segments.py +24 -17
- endoreg_db/models/media/video/video_metadata.py +19 -35
- endoreg_db/models/media/video/video_processing.py +96 -95
- endoreg_db/models/medical/contraindication/__init__.py +13 -3
- endoreg_db/models/medical/disease.py +22 -16
- endoreg_db/models/medical/event.py +31 -18
- endoreg_db/models/medical/examination/__init__.py +13 -6
- endoreg_db/models/medical/examination/examination.py +17 -18
- endoreg_db/models/medical/examination/examination_indication.py +26 -25
- endoreg_db/models/medical/examination/examination_time.py +16 -6
- endoreg_db/models/medical/examination/examination_time_type.py +9 -6
- endoreg_db/models/medical/examination/examination_type.py +3 -4
- endoreg_db/models/medical/finding/finding.py +38 -39
- endoreg_db/models/medical/finding/finding_classification.py +37 -48
- endoreg_db/models/medical/finding/finding_intervention.py +27 -22
- endoreg_db/models/medical/finding/finding_type.py +13 -12
- endoreg_db/models/medical/hardware/endoscope.py +20 -26
- endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
- endoreg_db/models/medical/laboratory/lab_value.py +62 -91
- endoreg_db/models/medical/medication/medication.py +22 -10
- endoreg_db/models/medical/medication/medication_indication.py +29 -3
- endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
- endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
- endoreg_db/models/medical/medication/medication_schedule.py +27 -16
- endoreg_db/models/medical/organ/__init__.py +15 -12
- endoreg_db/models/medical/patient/medication_examples.py +1 -5
- endoreg_db/models/medical/patient/patient_disease.py +20 -23
- endoreg_db/models/medical/patient/patient_event.py +19 -22
- endoreg_db/models/medical/patient/patient_examination.py +48 -54
- endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
- endoreg_db/models/medical/patient/patient_finding.py +122 -139
- endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
- endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
- endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
- endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
- endoreg_db/models/medical/patient/patient_medication.py +27 -38
- endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
- endoreg_db/models/medical/risk/risk.py +7 -6
- endoreg_db/models/medical/risk/risk_type.py +8 -5
- endoreg_db/models/metadata/model_meta.py +60 -29
- endoreg_db/models/metadata/model_meta_logic.py +139 -18
- endoreg_db/models/metadata/pdf_meta.py +19 -24
- endoreg_db/models/metadata/sensitive_meta.py +102 -85
- endoreg_db/models/metadata/sensitive_meta_logic.py +383 -43
- endoreg_db/models/metadata/video_meta.py +51 -31
- endoreg_db/models/metadata/video_prediction_logic.py +16 -23
- endoreg_db/models/metadata/video_prediction_meta.py +29 -33
- endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
- endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
- endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
- endoreg_db/models/other/emission/emission_factor.py +18 -8
- endoreg_db/models/other/gender.py +10 -5
- endoreg_db/models/other/information_source.py +25 -25
- endoreg_db/models/other/material.py +9 -5
- endoreg_db/models/other/resource.py +6 -4
- endoreg_db/models/other/tag.py +10 -5
- endoreg_db/models/other/transport_route.py +13 -8
- endoreg_db/models/other/unit.py +10 -6
- endoreg_db/models/other/waste.py +6 -5
- endoreg_db/models/requirement/requirement.py +580 -272
- endoreg_db/models/requirement/requirement_error.py +85 -0
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
- endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
- endoreg_db/models/requirement/requirement_operator.py +36 -33
- endoreg_db/models/requirement/requirement_set.py +74 -57
- endoreg_db/models/state/__init__.py +4 -4
- endoreg_db/models/state/abstract.py +2 -2
- endoreg_db/models/state/anonymization.py +12 -0
- endoreg_db/models/state/audit_ledger.py +46 -47
- endoreg_db/models/state/label_video_segment.py +9 -0
- endoreg_db/models/state/raw_pdf.py +40 -46
- endoreg_db/models/state/sensitive_meta.py +6 -2
- endoreg_db/models/state/video.py +58 -53
- endoreg_db/models/upload_job.py +32 -55
- endoreg_db/models/utils.py +1 -2
- endoreg_db/root_urls.py +21 -2
- endoreg_db/serializers/__init__.py +26 -57
- endoreg_db/serializers/anonymization.py +18 -10
- endoreg_db/serializers/meta/report_meta.py +1 -1
- endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
- endoreg_db/serializers/misc/__init__.py +1 -1
- endoreg_db/serializers/misc/file_overview.py +33 -91
- endoreg_db/serializers/misc/{vop_patient_data.py → sensitive_patient_data.py} +1 -1
- endoreg_db/serializers/requirements/requirement_sets.py +92 -22
- endoreg_db/serializers/video/segmentation.py +2 -1
- endoreg_db/serializers/video/video_processing_history.py +20 -5
- endoreg_db/serializers/video_examination.py +198 -0
- endoreg_db/services/anonymization.py +75 -73
- endoreg_db/services/lookup_service.py +256 -73
- endoreg_db/services/lookup_store.py +174 -30
- endoreg_db/services/pdf_import.py +711 -310
- endoreg_db/services/storage_aware_video_processor.py +140 -114
- endoreg_db/services/video_import.py +266 -117
- endoreg_db/urls/__init__.py +27 -27
- endoreg_db/urls/label_video_segments.py +2 -0
- endoreg_db/urls/media.py +108 -66
- endoreg_db/urls/root_urls.py +29 -0
- endoreg_db/utils/__init__.py +15 -5
- endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
- endoreg_db/utils/case_generator/__init__.py +3 -0
- endoreg_db/utils/dataloader.py +88 -16
- endoreg_db/utils/defaults/set_default_center.py +32 -0
- endoreg_db/utils/names.py +22 -16
- endoreg_db/utils/permissions.py +2 -1
- endoreg_db/utils/pipelines/process_video_dir.py +1 -1
- endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
- endoreg_db/utils/setup_config.py +8 -5
- endoreg_db/utils/storage.py +115 -0
- endoreg_db/utils/validate_endo_roi.py +8 -2
- endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
- endoreg_db/views/__init__.py +5 -12
- endoreg_db/views/anonymization/media_management.py +198 -163
- endoreg_db/views/anonymization/overview.py +4 -1
- endoreg_db/views/anonymization/validate.py +174 -40
- endoreg_db/views/media/__init__.py +2 -0
- endoreg_db/views/media/pdf_media.py +131 -150
- endoreg_db/views/media/sensitive_metadata.py +46 -6
- endoreg_db/views/media/video_media.py +89 -82
- endoreg_db/views/media/video_segments.py +187 -260
- endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
- endoreg_db/views/patient/patient.py +5 -4
- endoreg_db/views/pdf/__init__.py +5 -8
- endoreg_db/views/pdf/pdf_stream.py +186 -0
- endoreg_db/views/pdf/pdf_stream_views.py +0 -127
- endoreg_db/views/pdf/reimport.py +86 -91
- endoreg_db/views/requirement/evaluate.py +188 -187
- endoreg_db/views/requirement/lookup.py +186 -288
- endoreg_db/views/requirement/requirement_utils.py +89 -0
- endoreg_db/views/video/__init__.py +0 -4
- endoreg_db/views/video/correction.py +2 -2
- endoreg_db/views/video/video_examination_viewset.py +202 -289
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +350 -255
- endoreg_db/models/administration/permissions/__init__.py +0 -44
- endoreg_db/models/media/video/refactor_plan.md +0 -0
- endoreg_db/models/media/video/video_file_frames.py +0 -0
- endoreg_db/models/metadata/frame_ocr_result.py +0 -0
- endoreg_db/models/rule/__init__.py +0 -13
- endoreg_db/models/rule/rule.py +0 -27
- endoreg_db/models/rule/rule_applicator.py +0 -224
- endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
- endoreg_db/models/rule/rule_type.py +0 -20
- endoreg_db/models/rule/ruleset.py +0 -17
- endoreg_db/serializers/video/video_metadata.py +0 -105
- endoreg_db/urls/report.py +0 -48
- endoreg_db/urls/video.py +0 -61
- endoreg_db/utils/case_generator/case_generator.py +0 -159
- endoreg_db/utils/case_generator/utils.py +0 -30
- endoreg_db/views/pdf/pdf_media.py +0 -239
- endoreg_db/views/report/__init__.py +0 -9
- endoreg_db/views/report/report_list.py +0 -112
- endoreg_db/views/report/report_with_secure_url.py +0 -28
- endoreg_db/views/report/start_examination.py +0 -7
- endoreg_db/views/video/video_media.py +0 -158
- endoreg_db/views.py +0 -0
- /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
- /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
- /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
from django.db import models
|
|
2
1
|
import logging
|
|
3
2
|
from pathlib import Path
|
|
4
|
-
from typing import
|
|
3
|
+
from typing import TYPE_CHECKING, Optional
|
|
5
4
|
|
|
6
5
|
# import endoreg_center_id from django settings
|
|
7
6
|
from django.conf import settings
|
|
7
|
+
from django.db import models
|
|
8
8
|
|
|
9
9
|
# check if endoreg_center_id is set
|
|
10
10
|
if not hasattr(settings, "ENDOREG_CENTER_ID"):
|
|
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
|
|
|
19
19
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from ..administration import Center
|
|
22
|
-
from ..medical.hardware import
|
|
22
|
+
from ..medical.hardware import Endoscope, EndoscopyProcessor
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
# VideoMeta
|
|
@@ -29,19 +29,40 @@ class VideoMeta(models.Model):
|
|
|
29
29
|
|
|
30
30
|
Links to hardware (processor, endoscope), center, import details, and FFmpeg technical specs.
|
|
31
31
|
"""
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
)
|
|
35
|
-
endoscope = models.ForeignKey(
|
|
36
|
-
"Endoscope", on_delete=models.CASCADE, blank=True, null=True
|
|
37
|
-
)
|
|
32
|
+
|
|
33
|
+
processor = models.ForeignKey("EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True)
|
|
34
|
+
endoscope = models.ForeignKey("Endoscope", on_delete=models.CASCADE, blank=True, null=True)
|
|
38
35
|
center = models.ForeignKey("Center", on_delete=models.CASCADE)
|
|
39
|
-
import_meta = models.OneToOneField(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
36
|
+
import_meta = models.OneToOneField("VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True)
|
|
37
|
+
ffmpeg_meta = models.OneToOneField("FFMpegMeta", on_delete=models.CASCADE, blank=True, null=True)
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
processor: models.ForeignKey["EndoscopyProcessor|None"]
|
|
41
|
+
endoscope: models.ForeignKey["Endoscope|None"]
|
|
42
|
+
center: models.ForeignKey["Center|None"]
|
|
43
|
+
import_meta: models.OneToOneField["VideoImportMeta|None"]
|
|
44
|
+
ffmpeg_meta: models.OneToOneField["FFMpegMeta|None"]
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def center_safe(self) -> "Center":
|
|
48
|
+
center = self.center
|
|
49
|
+
if not center:
|
|
50
|
+
raise Center.DoesNotExist("Center does not exist for this VideoMeta instance.")
|
|
51
|
+
return center
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def processor_safe(self) -> "EndoscopyProcessor":
|
|
55
|
+
processor = self.processor
|
|
56
|
+
if not processor:
|
|
57
|
+
raise EndoscopyProcessor.DoesNotExist("EndoscopyProcessor does not exist for this VideoMeta instance.")
|
|
58
|
+
return processor
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def ffmpeg_meta_safe(self) -> "FFMpegMeta":
|
|
62
|
+
ffmpeg_meta = self.ffmpeg_meta
|
|
63
|
+
if not ffmpeg_meta:
|
|
64
|
+
raise FFMpegMeta.DoesNotExist("FFMpegMeta does not exist for this VideoMeta instance.")
|
|
65
|
+
return ffmpeg_meta
|
|
45
66
|
|
|
46
67
|
@classmethod
|
|
47
68
|
def create_from_file(
|
|
@@ -71,7 +92,7 @@ class VideoMeta(models.Model):
|
|
|
71
92
|
raise RuntimeError(f"Failed to initialize FFMpeg metadata for {video_path.name}") from e
|
|
72
93
|
|
|
73
94
|
if save_instance:
|
|
74
|
-
meta.save()
|
|
95
|
+
meta.save() # This ensures VideoImportMeta is created too
|
|
75
96
|
logger.info("Created and saved VideoMeta instance PK %s from %s", meta.pk, video_path.name)
|
|
76
97
|
else:
|
|
77
98
|
logger.info("Instantiated VideoMeta from %s (not saved yet)", video_path.name)
|
|
@@ -120,8 +141,8 @@ class VideoMeta(models.Model):
|
|
|
120
141
|
# If the VideoMeta instance is already saved, save the link immediately.
|
|
121
142
|
# Otherwise, the link will be saved when VideoMeta itself is saved.
|
|
122
143
|
if self.pk:
|
|
123
|
-
self.save(update_fields=[
|
|
124
|
-
logger.info("Successfully created and linked FFMpegMeta PK %s", self.
|
|
144
|
+
self.save(update_fields=["ffmpeg_meta"])
|
|
145
|
+
logger.info("Successfully created and linked FFMpegMeta PK %s", self.ffmpeg_meta_safe.pk)
|
|
125
146
|
|
|
126
147
|
except Exception as e:
|
|
127
148
|
# Log the error and re-raise it
|
|
@@ -140,8 +161,8 @@ class VideoMeta(models.Model):
|
|
|
140
161
|
logger.debug("Deleting existing FFMpegMeta PK %s before update.", existing_ffmpeg_pk)
|
|
141
162
|
# Nullify the relation first before deleting the related object
|
|
142
163
|
self.ffmpeg_meta = None
|
|
143
|
-
self.save(update_fields=[
|
|
144
|
-
FFMpegMeta.objects.filter(pk=existing_ffmpeg_pk).delete()
|
|
164
|
+
self.save(update_fields=["ffmpeg_meta"]) # Save the null relation
|
|
165
|
+
FFMpegMeta.objects.filter(pk=existing_ffmpeg_pk).delete() # Delete the old object
|
|
145
166
|
|
|
146
167
|
# initialize_ffmpeg_meta handles creation, linking, saving the link, and raises exceptions
|
|
147
168
|
self.initialize_ffmpeg_meta(video_path)
|
|
@@ -150,7 +171,7 @@ class VideoMeta(models.Model):
|
|
|
150
171
|
"""Retrieves the endoscope region of interest (ROI) from the associated processor."""
|
|
151
172
|
from ..medical.hardware import EndoscopyProcessor
|
|
152
173
|
|
|
153
|
-
processor: EndoscopyProcessor = self.
|
|
174
|
+
processor: EndoscopyProcessor = self.processor_safe
|
|
154
175
|
endo_roi = processor.get_roi_endoscope_image()
|
|
155
176
|
return endo_roi
|
|
156
177
|
|
|
@@ -189,6 +210,7 @@ class FFMpegMeta(models.Model):
|
|
|
189
210
|
"""
|
|
190
211
|
Stores technical video stream information extracted using FFmpeg (ffprobe).
|
|
191
212
|
"""
|
|
213
|
+
|
|
192
214
|
width = models.IntegerField(null=True, blank=True)
|
|
193
215
|
height = models.IntegerField(null=True, blank=True)
|
|
194
216
|
duration = models.FloatField(null=True, blank=True) # Duration in seconds
|
|
@@ -219,7 +241,6 @@ class FFMpegMeta(models.Model):
|
|
|
219
241
|
logger.error("ffprobe execution failed for %s: %s", file_path, probe_err, exc_info=True)
|
|
220
242
|
raise RuntimeError(f"ffprobe execution failed for {file_path}") from probe_err
|
|
221
243
|
|
|
222
|
-
|
|
223
244
|
if not probe_data or "streams" not in probe_data:
|
|
224
245
|
logger.error("Failed to get valid stream info from ffprobe for %s", file_path)
|
|
225
246
|
# Raise exception instead of returning None
|
|
@@ -237,8 +258,8 @@ class FFMpegMeta(models.Model):
|
|
|
237
258
|
height = video_stream.get("height")
|
|
238
259
|
duration_str = video_stream.get("duration")
|
|
239
260
|
# --- FIX: Handle potential format key ---
|
|
240
|
-
if duration_str is None and
|
|
241
|
-
duration_str = probe_data[
|
|
261
|
+
if duration_str is None and "format" in probe_data and "duration" in probe_data["format"]:
|
|
262
|
+
duration_str = probe_data["format"]["duration"]
|
|
242
263
|
logger.debug("Using duration from format block: %s", duration_str)
|
|
243
264
|
# --- End Fix ---
|
|
244
265
|
duration = float(duration_str) if duration_str else None
|
|
@@ -253,10 +274,10 @@ class FFMpegMeta(models.Model):
|
|
|
253
274
|
frame_rate_num, frame_rate_den = None, None
|
|
254
275
|
if frame_rate_str and "/" in frame_rate_str:
|
|
255
276
|
try:
|
|
256
|
-
num_str, den_str = frame_rate_str.split(
|
|
277
|
+
num_str, den_str = frame_rate_str.split("/")
|
|
257
278
|
frame_rate_num = int(num_str)
|
|
258
279
|
frame_rate_den = int(den_str)
|
|
259
|
-
if frame_rate_den == 0:
|
|
280
|
+
if frame_rate_den == 0: # Avoid division by zero
|
|
260
281
|
logger.warning("Invalid frame rate denominator (0) for %s", file_path)
|
|
261
282
|
frame_rate_num, frame_rate_den = None, None
|
|
262
283
|
except ValueError:
|
|
@@ -267,8 +288,8 @@ class FFMpegMeta(models.Model):
|
|
|
267
288
|
pixel_format = video_stream.get("pix_fmt")
|
|
268
289
|
bit_rate_str = video_stream.get("bit_rate")
|
|
269
290
|
# --- FIX: Handle potential format key for bit_rate ---
|
|
270
|
-
if bit_rate_str is None and
|
|
271
|
-
bit_rate_str = probe_data[
|
|
291
|
+
if bit_rate_str is None and "format" in probe_data and "bit_rate" in probe_data["format"]:
|
|
292
|
+
bit_rate_str = probe_data["format"]["bit_rate"]
|
|
272
293
|
logger.debug("Using bit_rate from format block: %s", bit_rate_str)
|
|
273
294
|
# --- End Fix ---
|
|
274
295
|
bit_rate = int(bit_rate_str) if bit_rate_str else None
|
|
@@ -311,6 +332,7 @@ class VideoImportMeta(models.Model):
|
|
|
311
332
|
"""
|
|
312
333
|
Stores metadata related to the import and processing status of a video.
|
|
313
334
|
"""
|
|
335
|
+
|
|
314
336
|
file_name = models.CharField(max_length=255, blank=True, null=True)
|
|
315
337
|
video_anonymized = models.BooleanField(default=False)
|
|
316
338
|
video_patient_data_detected = models.BooleanField(default=False)
|
|
@@ -323,9 +345,7 @@ class VideoImportMeta(models.Model):
|
|
|
323
345
|
result_html = ""
|
|
324
346
|
|
|
325
347
|
result_html += f"Video anonymized: {self.video_anonymized}\n"
|
|
326
|
-
result_html +=
|
|
327
|
-
f"Video patient data detected: {self.video_patient_data_detected}\n"
|
|
328
|
-
)
|
|
348
|
+
result_html += f"Video patient data detected: {self.video_patient_data_detected}\n"
|
|
329
349
|
result_html += f"Outside detected: {self.outside_detected}\n"
|
|
330
350
|
result_html += f"Patient data removed: {self.patient_data_removed}\n"
|
|
331
351
|
result_html += f"Outside removed: {self.outside_removed}\n"
|
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from typing import TYPE_CHECKING, List,
|
|
2
|
+
from typing import TYPE_CHECKING, List, Optional, Tuple
|
|
3
|
+
|
|
3
4
|
import numpy as np
|
|
4
5
|
|
|
5
|
-
# Import necessary models and utils used by the logic
|
|
6
|
-
from ..utils import find_segments_in_prediction_array
|
|
7
6
|
from ..label.label_video_segment import LabelVideoSegment
|
|
8
7
|
|
|
8
|
+
# Import necessary models and utils used by the logic
|
|
9
|
+
from ..utils import find_segments_in_prediction_array
|
|
9
10
|
|
|
10
11
|
logger = logging.getLogger(__name__)
|
|
11
12
|
|
|
13
|
+
# TODO configure via settings
|
|
12
14
|
DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
|
|
13
15
|
DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
|
|
14
16
|
|
|
15
17
|
if TYPE_CHECKING:
|
|
16
|
-
from .video_prediction_meta import VideoPredictionMeta
|
|
17
18
|
from ..label import Label
|
|
19
|
+
from .video_prediction_meta import VideoPredictionMeta
|
|
18
20
|
|
|
19
21
|
|
|
20
22
|
def apply_running_mean_logic(instance: "VideoPredictionMeta", confidence_array: np.ndarray, window_size_in_seconds: Optional[float] = None) -> np.ndarray:
|
|
@@ -63,6 +65,7 @@ def calculate_prediction_array_logic(instance: "VideoPredictionMeta", window_siz
|
|
|
63
65
|
Does not save the array itself.
|
|
64
66
|
"""
|
|
65
67
|
from ..label import ImageClassificationAnnotation
|
|
68
|
+
|
|
66
69
|
video_obj = instance.get_video()
|
|
67
70
|
model_meta = instance.model_meta
|
|
68
71
|
label_list = instance.get_label_list()
|
|
@@ -78,15 +81,10 @@ def calculate_prediction_array_logic(instance: "VideoPredictionMeta", window_siz
|
|
|
78
81
|
|
|
79
82
|
prediction_array = np.zeros((num_frames, len(label_list)))
|
|
80
83
|
|
|
81
|
-
base_pred_qs = ImageClassificationAnnotation.objects.filter(
|
|
82
|
-
model_meta=model_meta,
|
|
83
|
-
frame__video_file=video_obj
|
|
84
|
-
)
|
|
84
|
+
base_pred_qs = ImageClassificationAnnotation.objects.filter(model_meta=model_meta, frame__video_file=video_obj)
|
|
85
85
|
|
|
86
86
|
for i, label in enumerate(label_list):
|
|
87
|
-
predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list(
|
|
88
|
-
"frame__frame_number", "confidence"
|
|
89
|
-
)
|
|
87
|
+
predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list("frame__frame_number", "confidence")
|
|
90
88
|
|
|
91
89
|
# Initialize with 0.5 (neutral confidence)
|
|
92
90
|
confidences = np.full(num_frames, 0.5)
|
|
@@ -96,14 +94,12 @@ def calculate_prediction_array_logic(instance: "VideoPredictionMeta", window_siz
|
|
|
96
94
|
confidences[frame_num] = confidence
|
|
97
95
|
found_predictions = True
|
|
98
96
|
else:
|
|
99
|
-
logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames-1}). Skipping.")
|
|
97
|
+
logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames - 1}). Skipping.")
|
|
100
98
|
|
|
101
99
|
if not found_predictions:
|
|
102
100
|
logger.warning(f"No predictions found for label '{label.name}' in {video_obj}. Using default confidence.")
|
|
103
101
|
|
|
104
|
-
smooth_confidences = apply_running_mean_logic(
|
|
105
|
-
instance, confidences, window_size_in_seconds
|
|
106
|
-
)
|
|
102
|
+
smooth_confidences = apply_running_mean_logic(instance, confidences, window_size_in_seconds)
|
|
107
103
|
# Threshold smoothed confidences
|
|
108
104
|
binary_predictions = smooth_confidences > 0.5
|
|
109
105
|
prediction_array[:, i] = binary_predictions
|
|
@@ -116,6 +112,7 @@ def create_video_segments_for_label_logic(instance: "VideoPredictionMeta", segme
|
|
|
116
112
|
Creates LabelVideoSegment instances for the given label and segments.
|
|
117
113
|
"""
|
|
118
114
|
from ..other import InformationSource
|
|
115
|
+
|
|
119
116
|
video_obj = instance.get_video()
|
|
120
117
|
information_source, _ = InformationSource.objects.get_or_create(name="prediction")
|
|
121
118
|
|
|
@@ -131,11 +128,7 @@ def create_video_segments_for_label_logic(instance: "VideoPredictionMeta", segme
|
|
|
131
128
|
}
|
|
132
129
|
# Check for existence before creating the object instance
|
|
133
130
|
if not LabelVideoSegment.objects.filter(
|
|
134
|
-
video_file=video_obj,
|
|
135
|
-
prediction_meta=instance,
|
|
136
|
-
label=label,
|
|
137
|
-
start_frame_number=start_frame,
|
|
138
|
-
end_frame_number=end_frame
|
|
131
|
+
video_file=video_obj, prediction_meta=instance, label=label, start_frame_number=start_frame, end_frame_number=end_frame
|
|
139
132
|
).exists():
|
|
140
133
|
segments_to_create.append(LabelVideoSegment(**segment_data))
|
|
141
134
|
|
|
@@ -161,7 +154,7 @@ def create_video_segments_logic(instance: "VideoPredictionMeta", segment_length_
|
|
|
161
154
|
return
|
|
162
155
|
|
|
163
156
|
min_frame_length = int(segment_length_threshold_in_s * fps)
|
|
164
|
-
min_frame_length = max(min_frame_length, 1)
|
|
157
|
+
min_frame_length = max(min_frame_length, 1) # Ensure minimum length is at least 1
|
|
165
158
|
|
|
166
159
|
label_list = instance.get_label_list()
|
|
167
160
|
if not label_list:
|
|
@@ -171,8 +164,8 @@ def create_video_segments_logic(instance: "VideoPredictionMeta", segment_length_
|
|
|
171
164
|
prediction_array = instance.get_prediction_array()
|
|
172
165
|
if prediction_array is None:
|
|
173
166
|
logger.info(f"Prediction array not found for {instance}. Calculating...")
|
|
174
|
-
instance.calculate_prediction_array()
|
|
175
|
-
prediction_array = instance.get_prediction_array()
|
|
167
|
+
instance.calculate_prediction_array() # This will save the array internally
|
|
168
|
+
prediction_array = instance.get_prediction_array() # Fetch again
|
|
176
169
|
if prediction_array is None:
|
|
177
170
|
logger.error(f"Failed to get or calculate prediction array for {instance}. Cannot create segments.")
|
|
178
171
|
return
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
from typing import TYPE_CHECKING, Optional, List, Tuple
|
|
2
|
-
from django.db import models
|
|
3
1
|
import logging
|
|
2
|
+
import pickle
|
|
3
|
+
from typing import TYPE_CHECKING, List, Optional, Tuple
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from django.db import models
|
|
4
7
|
|
|
5
8
|
from endoreg_db.models.label import LabelSet
|
|
6
9
|
|
|
7
10
|
from ..label.label_video_segment import (
|
|
8
11
|
LabelVideoSegment,
|
|
9
|
-
|
|
10
12
|
)
|
|
11
13
|
from ..utils import find_segments_in_prediction_array
|
|
12
14
|
|
|
13
|
-
import numpy as np
|
|
14
|
-
import pickle
|
|
15
|
-
|
|
16
15
|
logger = logging.getLogger(__name__)
|
|
17
16
|
|
|
18
17
|
DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
|
|
19
18
|
DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
|
|
20
19
|
|
|
21
20
|
if TYPE_CHECKING:
|
|
22
|
-
from endoreg_db.models import
|
|
21
|
+
from endoreg_db.models import Label, ModelMeta
|
|
22
|
+
|
|
23
23
|
from ..media.video.video_file import VideoFile
|
|
24
24
|
|
|
25
25
|
|
|
@@ -29,6 +29,7 @@ class VideoPredictionMeta(models.Model):
|
|
|
29
29
|
|
|
30
30
|
Must be associated with exactly one `VideoFile`.
|
|
31
31
|
"""
|
|
32
|
+
|
|
32
33
|
model_meta = models.ForeignKey("ModelMeta", on_delete=models.CASCADE)
|
|
33
34
|
date_created = models.DateTimeField(auto_now_add=True)
|
|
34
35
|
date_modified = models.DateTimeField(auto_now=True)
|
|
@@ -43,18 +44,24 @@ class VideoPredictionMeta(models.Model):
|
|
|
43
44
|
)
|
|
44
45
|
|
|
45
46
|
if TYPE_CHECKING:
|
|
46
|
-
model_meta: "ModelMeta"
|
|
47
|
-
video_file: "VideoFile"
|
|
47
|
+
model_meta: models.ForeignKey["ModelMeta"]
|
|
48
|
+
video_file: models.ForeignKey["VideoFile|None"]
|
|
48
49
|
label_video_segments: "models.Manager[LabelVideoSegment]"
|
|
49
50
|
|
|
50
51
|
class Meta:
|
|
51
|
-
constraints = [
|
|
52
|
-
models.UniqueConstraint(fields=['model_meta', 'video_file'], name='unique_prediction_per_video_model')
|
|
53
|
-
]
|
|
52
|
+
constraints = [models.UniqueConstraint(fields=["model_meta", "video_file"], name="unique_prediction_per_video_model")]
|
|
54
53
|
indexes = [
|
|
55
54
|
models.Index(fields=["model_meta", "video_file"]),
|
|
56
55
|
]
|
|
57
56
|
|
|
57
|
+
@property
|
|
58
|
+
def video_file_safe(self) -> "VideoFile":
|
|
59
|
+
"""Returns the associated VideoFile instance."""
|
|
60
|
+
video_file = self.video_file
|
|
61
|
+
if not video_file:
|
|
62
|
+
raise ValueError("VideoPredictionMeta is not associated with a VideoFile.")
|
|
63
|
+
return video_file
|
|
64
|
+
|
|
58
65
|
def get_video(self) -> "VideoFile":
|
|
59
66
|
"""Returns the associated VideoFile instance."""
|
|
60
67
|
if self.video_file:
|
|
@@ -83,12 +90,12 @@ class VideoPredictionMeta(models.Model):
|
|
|
83
90
|
return labelset.get_labels_in_order()
|
|
84
91
|
return []
|
|
85
92
|
|
|
86
|
-
def save_prediction_array(self, prediction_array: np.
|
|
93
|
+
def save_prediction_array(self, prediction_array: np.typing.NDArray):
|
|
87
94
|
"""
|
|
88
95
|
Save the prediction array to the database.
|
|
89
96
|
"""
|
|
90
97
|
self.prediction_array = pickle.dumps(prediction_array)
|
|
91
|
-
self.save(update_fields=[
|
|
98
|
+
self.save(update_fields=["prediction_array", "date_modified"])
|
|
92
99
|
|
|
93
100
|
def get_prediction_array(self):
|
|
94
101
|
"""
|
|
@@ -103,7 +110,7 @@ class VideoPredictionMeta(models.Model):
|
|
|
103
110
|
logger.error(f"Error unpickling prediction array for {self}: {e}")
|
|
104
111
|
return None
|
|
105
112
|
|
|
106
|
-
def calculate_prediction_array(self, window_size_in_seconds: int = None):
|
|
113
|
+
def calculate_prediction_array(self, window_size_in_seconds: Optional[int] = None):
|
|
107
114
|
"""
|
|
108
115
|
Fetches all predictions for the associated video, labelset, and model meta,
|
|
109
116
|
applies smoothing, and saves the resulting binary prediction array.
|
|
@@ -125,15 +132,10 @@ class VideoPredictionMeta(models.Model):
|
|
|
125
132
|
|
|
126
133
|
prediction_array = np.zeros((num_frames, len(label_list)))
|
|
127
134
|
|
|
128
|
-
base_pred_qs = ImageClassificationAnnotation.objects.filter(
|
|
129
|
-
model_meta=model_meta,
|
|
130
|
-
frame__video_file=video_obj
|
|
131
|
-
)
|
|
135
|
+
base_pred_qs = ImageClassificationAnnotation.objects.filter(model_meta=model_meta, frame__video_file=video_obj)
|
|
132
136
|
|
|
133
137
|
for i, label in enumerate(label_list):
|
|
134
|
-
predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list(
|
|
135
|
-
"frame__frame_number", "float_value"
|
|
136
|
-
)
|
|
138
|
+
predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list("frame__frame_number", "float_value")
|
|
137
139
|
|
|
138
140
|
confidences = np.full(num_frames, 0.5)
|
|
139
141
|
found_predictions = False
|
|
@@ -142,21 +144,19 @@ class VideoPredictionMeta(models.Model):
|
|
|
142
144
|
confidences[frame_num] = confidence
|
|
143
145
|
found_predictions = True
|
|
144
146
|
else:
|
|
145
|
-
logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames-1}). Skipping.")
|
|
147
|
+
logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames - 1}). Skipping.")
|
|
146
148
|
|
|
147
149
|
if not found_predictions:
|
|
148
150
|
logger.warning(f"No predictions found for label '{label.name}' in {video_obj}. Using default confidence.")
|
|
149
151
|
|
|
150
|
-
smooth_confidences = self.apply_running_mean(
|
|
151
|
-
confidences, window_size_in_seconds
|
|
152
|
-
)
|
|
152
|
+
smooth_confidences = self.apply_running_mean(confidences, window_size_in_seconds)
|
|
153
153
|
binary_predictions = smooth_confidences > 0.5
|
|
154
154
|
prediction_array[:, i] = binary_predictions
|
|
155
155
|
|
|
156
156
|
self.save_prediction_array(prediction_array)
|
|
157
157
|
logger.info(f"Calculated and saved prediction array for {self}")
|
|
158
158
|
|
|
159
|
-
def apply_running_mean(self, confidence_array, window_size_in_seconds:
|
|
159
|
+
def apply_running_mean(self, confidence_array, window_size_in_seconds: Optional[float] = None):
|
|
160
160
|
"""
|
|
161
161
|
Apply a running mean filter to the confidence array for smoothing, assuming a padding
|
|
162
162
|
of 0.5 for the edges.
|
|
@@ -215,11 +215,7 @@ class VideoPredictionMeta(models.Model):
|
|
|
215
215
|
"video_file": video_obj,
|
|
216
216
|
}
|
|
217
217
|
if not LabelVideoSegment.objects.filter(
|
|
218
|
-
video_file=video_obj,
|
|
219
|
-
prediction_meta=self,
|
|
220
|
-
label=label,
|
|
221
|
-
start_frame_number=start_frame,
|
|
222
|
-
end_frame_number=end_frame
|
|
218
|
+
video_file=video_obj, prediction_meta=self, label=label, start_frame_number=start_frame, end_frame_number=end_frame
|
|
223
219
|
).exists():
|
|
224
220
|
segments_to_create.append(LabelVideoSegment(**segment_data))
|
|
225
221
|
|
|
@@ -229,7 +225,7 @@ class VideoPredictionMeta(models.Model):
|
|
|
229
225
|
else:
|
|
230
226
|
logger.info(f"No new video segments needed for label '{label.name}' in {video_obj}.")
|
|
231
227
|
|
|
232
|
-
def create_video_segments(self, segment_length_threshold_in_s: float = None):
|
|
228
|
+
def create_video_segments(self, segment_length_threshold_in_s: Optional[float] = None):
|
|
233
229
|
"""
|
|
234
230
|
Generates LabelVideoSegments based on the stored prediction array.
|
|
235
231
|
"""
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
|
|
1
|
+
"""Model for date value distribution"""
|
|
2
2
|
|
|
3
3
|
from datetime import date, timedelta
|
|
4
|
-
|
|
4
|
+
|
|
5
5
|
import numpy as np
|
|
6
|
+
from django.db import models
|
|
6
7
|
|
|
7
8
|
from .base_value_distribution import BaseValueDistribution
|
|
8
9
|
|
|
10
|
+
|
|
9
11
|
class DateValueDistributionManager(models.Manager):
|
|
10
|
-
|
|
12
|
+
"""Object manager for DateValueDistribution"""
|
|
13
|
+
|
|
11
14
|
def get_by_natural_key(self, name):
|
|
12
|
-
|
|
15
|
+
"""Retrieve a DateValueDistribution by its natural key."""
|
|
13
16
|
return self.get(name=name)
|
|
14
17
|
|
|
15
18
|
|
|
@@ -20,16 +23,17 @@ class DateValueDistribution(BaseValueDistribution):
|
|
|
20
23
|
date_min, date_max, date_mean, date_std_dev or
|
|
21
24
|
timedelta_days_min, timedelta_days_max, timedelta_days_mean, timedelta_days_std_dev
|
|
22
25
|
"""
|
|
26
|
+
|
|
23
27
|
objects = DateValueDistributionManager()
|
|
24
28
|
name = models.CharField(max_length=100)
|
|
25
29
|
description = models.TextField(blank=True, null=True)
|
|
26
30
|
DISTRIBUTION_CHOICES = [
|
|
27
|
-
(
|
|
28
|
-
(
|
|
31
|
+
("uniform", "Uniform"),
|
|
32
|
+
("normal", "Normal"),
|
|
29
33
|
]
|
|
30
34
|
MODE_CHOICES = [
|
|
31
|
-
(
|
|
32
|
-
(
|
|
35
|
+
("date", "Date"),
|
|
36
|
+
("timedelta", "Timedelta"),
|
|
33
37
|
]
|
|
34
38
|
|
|
35
39
|
distribution_type = models.CharField(max_length=20, choices=DISTRIBUTION_CHOICES)
|
|
@@ -47,43 +51,99 @@ class DateValueDistribution(BaseValueDistribution):
|
|
|
47
51
|
timedelta_days_mean = models.IntegerField(blank=True, null=True)
|
|
48
52
|
timedelta_days_std_dev = models.IntegerField(blank=True, null=True)
|
|
49
53
|
|
|
54
|
+
# create *_safe properties for dates and timedeltas
|
|
55
|
+
@property
|
|
56
|
+
def date_min_safe(self):
|
|
57
|
+
_date = self.date_min
|
|
58
|
+
if _date is None:
|
|
59
|
+
raise ValueError("date_min is not set")
|
|
60
|
+
return _date
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def date_max_safe(self):
|
|
64
|
+
_date = self.date_max
|
|
65
|
+
if _date is None:
|
|
66
|
+
raise ValueError("date_max is not set")
|
|
67
|
+
return _date
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def date_mean_safe(self):
|
|
71
|
+
_date = self.date_mean
|
|
72
|
+
if _date is None:
|
|
73
|
+
raise ValueError("date_mean is not set")
|
|
74
|
+
return _date
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def date_std_dev_safe(self):
|
|
78
|
+
_std_dev = self.date_std_dev
|
|
79
|
+
if _std_dev is None:
|
|
80
|
+
raise ValueError("date_std_dev is not set")
|
|
81
|
+
return _std_dev
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def timedelta_days_min_safe(self):
|
|
85
|
+
_min = self.timedelta_days_min
|
|
86
|
+
if _min is None:
|
|
87
|
+
raise ValueError("timedelta_days_min is not set")
|
|
88
|
+
return _min
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def timedelta_days_max_safe(self):
|
|
92
|
+
_max = self.timedelta_days_max
|
|
93
|
+
if _max is None:
|
|
94
|
+
raise ValueError("timedelta_days_max is not set")
|
|
95
|
+
return _max
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def timedelta_days_mean_safe(self):
|
|
99
|
+
_mean = self.timedelta_days_mean
|
|
100
|
+
if _mean is None:
|
|
101
|
+
raise ValueError("timedelta_days_mean is not set")
|
|
102
|
+
return _mean
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def timedelta_days_std_dev_safe(self):
|
|
106
|
+
_std_dev = self.timedelta_days_std_dev
|
|
107
|
+
if _std_dev is None:
|
|
108
|
+
raise ValueError("timedelta_days_std_dev is not set")
|
|
109
|
+
return _std_dev
|
|
110
|
+
|
|
50
111
|
def generate_value(self):
|
|
51
|
-
if self.mode ==
|
|
112
|
+
if self.mode == "date":
|
|
52
113
|
return self._generate_date_value()
|
|
53
|
-
elif self.mode ==
|
|
114
|
+
elif self.mode == "timedelta":
|
|
54
115
|
return self._generate_timedelta_value()
|
|
55
116
|
else:
|
|
56
117
|
raise ValueError("Unsupported mode")
|
|
57
118
|
|
|
58
119
|
def _generate_date_value(self):
|
|
59
|
-
#UNTESTED
|
|
60
|
-
if self.distribution_type ==
|
|
61
|
-
start_date = self.
|
|
62
|
-
end_date = self.
|
|
120
|
+
# UNTESTED
|
|
121
|
+
if self.distribution_type == "uniform":
|
|
122
|
+
start_date = self.date_min_safe.toordinal()
|
|
123
|
+
end_date = self.date_max_safe.toordinal()
|
|
63
124
|
random_ordinal = np.random.randint(start_date, end_date)
|
|
64
125
|
return date.fromordinal(random_ordinal)
|
|
65
|
-
elif self.distribution_type ==
|
|
66
|
-
mean_ordinal = self.
|
|
67
|
-
std_dev_days = self.
|
|
126
|
+
elif self.distribution_type == "normal":
|
|
127
|
+
mean_ordinal = self.date_mean_safe.toordinal()
|
|
128
|
+
std_dev_days = self.date_std_dev_safe
|
|
68
129
|
random_ordinal = int(np.random.normal(mean_ordinal, std_dev_days))
|
|
69
|
-
random_ordinal = np.clip(random_ordinal, self.
|
|
130
|
+
random_ordinal = np.clip(random_ordinal, self.date_min_safe.toordinal(), self.date_max_safe.toordinal())
|
|
70
131
|
return date.fromordinal(random_ordinal)
|
|
71
132
|
else:
|
|
72
133
|
raise ValueError("Unsupported distribution type")
|
|
73
134
|
|
|
74
135
|
def _generate_timedelta_value(self):
|
|
75
|
-
if self.distribution_type ==
|
|
76
|
-
random_days = np.random.randint(self.
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
random_days =
|
|
81
|
-
|
|
82
|
-
|
|
136
|
+
if self.distribution_type == "uniform":
|
|
137
|
+
random_days = np.random.randint(self.timedelta_days_min_safe, self.timedelta_days_max_safe + 1)
|
|
138
|
+
|
|
139
|
+
elif self.distribution_type == "normal":
|
|
140
|
+
random_days = int(np.random.normal(self.timedelta_days_mean_safe, self.timedelta_days_std_dev_safe))
|
|
141
|
+
random_days = np.clip(random_days, self.timedelta_days_min_safe, self.timedelta_days_max_safe)
|
|
142
|
+
|
|
83
143
|
else:
|
|
84
144
|
raise ValueError("Unsupported distribution type")
|
|
85
|
-
|
|
145
|
+
|
|
86
146
|
current_date = date.today()
|
|
87
147
|
generated_date = current_date - timedelta(days=random_days)
|
|
88
|
-
|
|
89
|
-
return
|
|
148
|
+
|
|
149
|
+
return generated_date
|
|
@@ -1,29 +1,45 @@
|
|
|
1
|
-
from django.db import models
|
|
2
1
|
import numpy as np
|
|
2
|
+
from django.db import models
|
|
3
|
+
|
|
3
4
|
from .base_value_distribution import BaseValueDistribution
|
|
4
5
|
|
|
6
|
+
|
|
5
7
|
class MultipleCategoricalValueDistributionManager(models.Manager):
|
|
6
8
|
def get_by_natural_key(self, name):
|
|
7
9
|
return self.get(name=name)
|
|
8
10
|
|
|
11
|
+
|
|
9
12
|
class MultipleCategoricalValueDistribution(BaseValueDistribution):
|
|
10
13
|
"""
|
|
11
14
|
Multiple categorical value distribution model.
|
|
12
15
|
Assigns a specific number or varying number of values based on probabilities.
|
|
13
16
|
"""
|
|
17
|
+
|
|
14
18
|
objects = MultipleCategoricalValueDistributionManager()
|
|
15
19
|
categories = models.JSONField() # { "category": "probability", ... }
|
|
16
20
|
min_count = models.IntegerField()
|
|
17
21
|
max_count = models.IntegerField()
|
|
18
|
-
count_distribution_type = models.CharField(max_length=20, choices=[(
|
|
22
|
+
count_distribution_type = models.CharField(max_length=20, choices=[("uniform", "Uniform"), ("normal", "Normal")])
|
|
19
23
|
count_mean = models.FloatField(null=True, blank=True)
|
|
20
24
|
count_std_dev = models.FloatField(null=True, blank=True)
|
|
21
25
|
|
|
26
|
+
@property
|
|
27
|
+
def count_mean_safe(self):
|
|
28
|
+
if self.count_mean is None:
|
|
29
|
+
raise ValueError("count_mean is not set")
|
|
30
|
+
return self.count_mean
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def count_std_dev_safe(self):
|
|
34
|
+
if self.count_std_dev is None:
|
|
35
|
+
raise ValueError("count_std_dev is not set")
|
|
36
|
+
return self.count_std_dev
|
|
37
|
+
|
|
22
38
|
def generate_value(self):
|
|
23
|
-
if self.count_distribution_type ==
|
|
39
|
+
if self.count_distribution_type == "uniform":
|
|
24
40
|
count = np.random.randint(self.min_count, self.max_count + 1)
|
|
25
|
-
elif self.count_distribution_type ==
|
|
26
|
-
count = int(np.random.normal(self.
|
|
41
|
+
elif self.count_distribution_type == "normal":
|
|
42
|
+
count = int(np.random.normal(self.count_mean_safe, self.count_std_dev_safe))
|
|
27
43
|
count = np.clip(count, self.min_count, self.max_count)
|
|
28
44
|
else:
|
|
29
45
|
raise ValueError("Unsupported count distribution type")
|