endoreg-db 0.4.5__py3-none-any.whl → 0.8.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- endoreg_db/admin.py +90 -1
- endoreg_db/api_urls.py +4 -0
- endoreg_db/apps.py +12 -0
- endoreg_db/assets/dummy_model.ckpt +1 -0
- endoreg_db/codemods/readme.md +88 -0
- endoreg_db/codemods/rename_datetime_fields.py +92 -0
- endoreg_db/config/env.py +101 -0
- endoreg_db/data/__init__.py +76 -4
- endoreg_db/data/ai_model/data.yaml +7 -0
- endoreg_db/data/{label → ai_model_label}/label/data.yaml +27 -1
- endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
- endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
- endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
- endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
- endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
- endoreg_db/data/center/data.yaml +40 -9
- endoreg_db/data/center_shift/ukw.yaml +9 -0
- endoreg_db/data/contraindication/bleeding.yaml +11 -0
- endoreg_db/data/db_summary.csv +58 -0
- endoreg_db/data/db_summary.xlsx +0 -0
- endoreg_db/data/disease/misc.yaml +1 -2
- endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +2 -2
- endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +6 -6
- endoreg_db/data/distribution/numeric/data.yaml +14 -0
- endoreg_db/data/endoscope/data.yaml +93 -0
- endoreg_db/data/endoscopy_processor/data.yaml +3 -0
- endoreg_db/data/event/cardiology.yaml +0 -13
- endoreg_db/data/examination/examinations/data.yaml +34 -28
- endoreg_db/data/examination/type/data.yaml +12 -0
- endoreg_db/data/examination_indication/endoscopy.yaml +424 -0
- endoreg_db/data/examination_indication_classification/endoscopy.yaml +160 -0
- endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +101 -0
- endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
- endoreg_db/data/finding/anatomy_colon.yaml +128 -0
- endoreg_db/data/finding/colonoscopy.yaml +40 -0
- endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +56 -0
- endoreg_db/data/finding/complication.yaml +16 -0
- endoreg_db/data/finding/data.yaml +105 -0
- endoreg_db/data/finding/examination_setting.yaml +16 -0
- endoreg_db/data/finding/medication_related.yaml +18 -0
- endoreg_db/data/finding/outcome.yaml +12 -0
- endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +95 -0
- endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +22 -0
- endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +25 -0
- endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
- endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +68 -0
- endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +80 -0
- endoreg_db/data/finding_classification/colonoscopy_lst.yaml +21 -0
- endoreg_db/data/finding_classification/colonoscopy_nice.yaml +20 -0
- endoreg_db/data/finding_classification/colonoscopy_paris.yaml +26 -0
- endoreg_db/data/finding_classification/colonoscopy_sano.yaml +22 -0
- endoreg_db/data/finding_classification/colonoscopy_summary.yaml +53 -0
- endoreg_db/data/finding_classification/complication_generic.yaml +25 -0
- endoreg_db/data/finding_classification/examination_setting_generic.yaml +40 -0
- endoreg_db/data/finding_classification/histology_colo.yaml +51 -0
- endoreg_db/data/finding_classification/intervention_required.yaml +26 -0
- endoreg_db/data/finding_classification/medication_related.yaml +23 -0
- endoreg_db/data/finding_classification/visualized.yaml +33 -0
- endoreg_db/data/finding_classification_choice/bowel_preparation.yaml +78 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +15 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +17 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +57 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +14 -0
- endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
- endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +229 -0
- endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
- endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +82 -0
- endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
- endoreg_db/data/finding_classification_choice/complication_generic_types.yaml +15 -0
- endoreg_db/data/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
- endoreg_db/data/finding_classification_choice/histology.yaml +24 -0
- endoreg_db/data/finding_classification_choice/histology_polyp.yaml +20 -0
- endoreg_db/data/finding_classification_choice/outcome.yaml +19 -0
- endoreg_db/data/finding_classification_choice/yes_no_na.yaml +11 -0
- endoreg_db/data/finding_classification_type/colonoscopy_basic.yaml +48 -0
- endoreg_db/data/finding_intervention/endoscopy.yaml +43 -0
- endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
- endoreg_db/data/finding_intervention/endoscopy_egd.yaml +128 -0
- endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +32 -0
- endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +9 -0
- endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +36 -0
- endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
- endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +79 -0
- endoreg_db/data/finding_type/data.yaml +43 -0
- endoreg_db/data/gender/data.yaml +24 -0
- endoreg_db/data/information_source/annotation.yaml +6 -0
- endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
- endoreg_db/data/information_source/prediction.yaml +7 -0
- endoreg_db/data/information_source_type/data.yaml +8 -0
- endoreg_db/data/lab_value/cardiac_enzymes.yaml +7 -1
- endoreg_db/data/lab_value/coagulation.yaml +6 -1
- endoreg_db/data/lab_value/electrolytes.yaml +39 -1
- endoreg_db/data/lab_value/gastrointestinal_function.yaml +12 -0
- endoreg_db/data/lab_value/hematology.yaml +17 -2
- endoreg_db/data/lab_value/hormones.yaml +6 -0
- endoreg_db/data/lab_value/lipids.yaml +12 -3
- endoreg_db/data/lab_value/misc.yaml +48 -2
- endoreg_db/data/lab_value/renal_function.yaml +2 -1
- endoreg_db/data/lx_client_tag/base.yaml +54 -0
- endoreg_db/data/lx_client_type/base.yaml +30 -0
- endoreg_db/data/lx_permission/base.yaml +24 -0
- endoreg_db/data/lx_permission/endoreg.yaml +52 -0
- endoreg_db/data/medication/anticoagulation.yaml +5 -5
- endoreg_db/data/medication/tah.yaml +5 -5
- endoreg_db/data/medication_indication/anticoagulation.yaml +48 -53
- endoreg_db/data/medication_intake_time/base.yaml +4 -4
- endoreg_db/data/names_first/first_names.yaml +54 -0
- endoreg_db/data/names_last/last_names.yaml +51 -0
- endoreg_db/data/network_device/data.yaml +30 -0
- endoreg_db/data/organ/data.yaml +29 -0
- endoreg_db/data/pdf_type/data.yaml +27 -9
- endoreg_db/data/qualification/endoscopy.yaml +36 -0
- endoreg_db/data/qualification/m2.yaml +39 -0
- endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
- endoreg_db/data/qualification/sonography.yaml +36 -0
- endoreg_db/data/qualification_type/base.yaml +29 -0
- endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
- endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +4 -0
- endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +5 -0
- endoreg_db/data/requirement/age.yaml +26 -0
- endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +45 -0
- endoreg_db/data/requirement/disease_cardiovascular.yaml +79 -0
- endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
- endoreg_db/data/requirement/disease_hepatology.yaml +12 -0
- endoreg_db/data/requirement/disease_misc.yaml +12 -0
- endoreg_db/data/requirement/disease_renal.yaml +96 -0
- endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +59 -0
- endoreg_db/data/requirement/event_cardiology.yaml +251 -0
- endoreg_db/data/requirement/event_requirements.yaml +145 -0
- endoreg_db/data/requirement/finding_colon_polyp.yaml +50 -0
- endoreg_db/data/requirement/gender.yaml +25 -0
- endoreg_db/data/requirement/lab_value.yaml +441 -0
- endoreg_db/data/requirement/medication.yaml +93 -0
- endoreg_db/data/requirement_operator/age.yaml +13 -0
- endoreg_db/data/requirement_operator/lab_operators.yaml +129 -0
- endoreg_db/data/requirement_operator/model_operators.yaml +96 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +48 -0
- endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
- endoreg_db/data/requirement_set/endoscopy_bleeding_risk.yaml +52 -0
- endoreg_db/data/requirement_set_type/data.yaml +20 -0
- endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
- endoreg_db/data/risk/bleeding.yaml +26 -0
- endoreg_db/data/risk/thrombosis.yaml +37 -0
- endoreg_db/data/risk_type/data.yaml +27 -0
- endoreg_db/data/setup_config.yaml +38 -0
- endoreg_db/data/shift/endoscopy.yaml +21 -0
- endoreg_db/data/shift_type/base.yaml +35 -0
- endoreg_db/data/tag/requirement_set_tags.yaml +11 -0
- endoreg_db/data/unit/concentration.yaml +23 -0
- endoreg_db/data/unit/time.yaml +36 -1
- endoreg_db/exceptions.py +19 -0
- endoreg_db/forms/__init__.py +3 -1
- endoreg_db/forms/examination_form.py +11 -0
- endoreg_db/forms/patient_finding_intervention_form.py +18 -0
- endoreg_db/forms/patient_form.py +27 -0
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/questionnaires/tto_questionnaire.py +19 -19
- endoreg_db/helpers/count_db.py +45 -0
- endoreg_db/helpers/data_loader.py +208 -0
- endoreg_db/helpers/default_objects.py +378 -0
- endoreg_db/helpers/download_segmentation_model.py +31 -0
- endoreg_db/helpers/interact.py +6 -0
- endoreg_db/helpers/test_video_helper.py +119 -0
- endoreg_db/logger_conf.py +140 -0
- endoreg_db/management/__init__.py +1 -0
- endoreg_db/management/commands/__init__.py +1 -0
- endoreg_db/management/commands/anonymize_video.py +0 -0
- endoreg_db/management/commands/check_auth.py +125 -0
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +115 -0
- endoreg_db/management/commands/create_multilabel_model_meta.py +214 -0
- endoreg_db/management/commands/fix_missing_patient_data.py +172 -0
- endoreg_db/management/commands/fix_video_paths.py +165 -0
- endoreg_db/management/commands/import_fallback_video.py +203 -0
- endoreg_db/management/commands/import_report.py +298 -0
- endoreg_db/management/commands/import_video.py +423 -0
- endoreg_db/management/commands/import_video_with_classification.py +367 -0
- endoreg_db/management/commands/init_default_ai_model.py +112 -0
- endoreg_db/management/commands/load_ai_model_data.py +58 -26
- endoreg_db/management/commands/load_ai_model_label_data.py +59 -0
- endoreg_db/management/commands/load_base_db_data.py +174 -118
- endoreg_db/management/commands/load_center_data.py +46 -21
- endoreg_db/management/commands/{load_logging_data.py → load_contraindication_data.py} +4 -2
- endoreg_db/management/commands/load_disease_data.py +29 -7
- endoreg_db/management/commands/{load_endoscope_type_data.py → load_endoscope_data.py} +30 -7
- endoreg_db/management/commands/load_examination_indication_data.py +86 -0
- endoreg_db/management/commands/load_finding_data.py +128 -0
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +0 -1
- endoreg_db/management/commands/load_information_source.py +13 -7
- endoreg_db/management/commands/load_lab_value_data.py +3 -3
- endoreg_db/management/commands/load_medication_data.py +83 -21
- endoreg_db/management/commands/load_name_data.py +37 -0
- endoreg_db/management/commands/{load_medication_intake_time_data.py → load_organ_data.py} +7 -5
- endoreg_db/management/commands/load_qualification_data.py +59 -0
- endoreg_db/management/commands/load_requirement_data.py +180 -0
- endoreg_db/management/commands/load_risk_data.py +56 -0
- endoreg_db/management/commands/load_shift_data.py +60 -0
- endoreg_db/management/commands/load_tag_data.py +57 -0
- endoreg_db/management/commands/register_ai_model.py +1 -1
- endoreg_db/management/commands/setup_endoreg_db.py +381 -0
- endoreg_db/management/commands/start_filewatcher.py +106 -0
- endoreg_db/management/commands/storage_management.py +548 -0
- endoreg_db/management/commands/summarize_db_content.py +189 -0
- endoreg_db/management/commands/validate_video.py +204 -0
- endoreg_db/management/commands/validate_video_files.py +161 -0
- endoreg_db/management/commands/video_validation.py +22 -0
- endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
- endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
- endoreg_db/mermaid/binary_classification_annotation.md +50 -0
- endoreg_db/mermaid/classification.md +8 -0
- endoreg_db/mermaid/examination.md +8 -0
- endoreg_db/mermaid/findings.md +7 -0
- endoreg_db/mermaid/image_classification.md +28 -0
- endoreg_db/mermaid/interventions.md +8 -0
- endoreg_db/mermaid/morphology.md +8 -0
- endoreg_db/mermaid/patient_creation.md +14 -0
- endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
- endoreg_db/migrations/0001_initial.py +1234 -944
- endoreg_db/migrations/0002_add_video_correction_models.py +52 -0
- endoreg_db/migrations/0003_add_center_display_name.py +30 -0
- endoreg_db/models/__init__.py +339 -53
- endoreg_db/models/administration/__init__.py +116 -0
- endoreg_db/models/administration/ai/__init__.py +9 -0
- endoreg_db/models/administration/ai/active_model.py +35 -0
- endoreg_db/models/administration/ai/ai_model.py +156 -0
- endoreg_db/models/{ai_model → administration/ai}/model_type.py +19 -4
- endoreg_db/models/administration/case/__init__.py +19 -0
- endoreg_db/models/administration/case/case.py +114 -0
- endoreg_db/models/{case_template → administration/case/case_template}/__init__.py +10 -1
- endoreg_db/models/{case_template → administration/case/case_template}/case_template.py +60 -16
- endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule.py +6 -13
- endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule_value.py +21 -8
- endoreg_db/models/{case_template → administration/case/case_template}/case_template_type.py +1 -3
- endoreg_db/models/{center → administration/center}/__init__.py +9 -0
- endoreg_db/models/administration/center/center.py +67 -0
- endoreg_db/models/administration/center/center_product.py +64 -0
- endoreg_db/models/administration/center/center_resource.py +49 -0
- endoreg_db/models/administration/center/center_shift.py +88 -0
- endoreg_db/models/administration/center/center_waste.py +30 -0
- endoreg_db/models/administration/permissions/__init__.py +44 -0
- endoreg_db/models/administration/person/__init__.py +24 -0
- endoreg_db/models/administration/person/employee/__init__.py +3 -0
- endoreg_db/models/administration/person/employee/employee.py +35 -0
- endoreg_db/models/administration/person/employee/employee_qualification.py +39 -0
- endoreg_db/models/administration/person/employee/employee_type.py +42 -0
- endoreg_db/models/administration/person/examiner/__init__.py +4 -0
- endoreg_db/models/administration/person/examiner/examiner.py +54 -0
- endoreg_db/models/administration/person/names/__init__.py +0 -0
- endoreg_db/models/{persons → administration/person/names}/first_name.py +1 -1
- endoreg_db/models/{persons → administration/person/names}/last_name.py +2 -3
- endoreg_db/models/administration/person/patient/__init__.py +5 -0
- endoreg_db/models/administration/person/patient/patient.py +460 -0
- endoreg_db/models/administration/person/profession/__init__.py +24 -0
- endoreg_db/models/administration/person/user/__init__.py +5 -0
- endoreg_db/models/administration/person/user/portal_user_information.py +37 -0
- endoreg_db/models/administration/product/__init__.py +14 -0
- endoreg_db/models/administration/product/product.py +97 -0
- endoreg_db/models/administration/product/product_group.py +39 -0
- endoreg_db/models/administration/product/product_material.py +54 -0
- endoreg_db/models/{product → administration/product}/product_weight.py +21 -0
- endoreg_db/models/{product → administration/product}/reference_product.py +44 -13
- endoreg_db/models/administration/qualification/__init__.py +7 -0
- endoreg_db/models/administration/qualification/qualification.py +37 -0
- endoreg_db/models/administration/qualification/qualification_type.py +35 -0
- endoreg_db/models/administration/shift/__init__.py +9 -0
- endoreg_db/models/administration/shift/scheduled_days.py +69 -0
- endoreg_db/models/administration/shift/shift.py +51 -0
- endoreg_db/models/administration/shift/shift_type.py +108 -0
- endoreg_db/models/label/__init__.py +24 -1
- endoreg_db/models/label/annotation/__init__.py +12 -0
- endoreg_db/models/label/annotation/image_classification.py +84 -0
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +66 -0
- endoreg_db/models/label/label.py +53 -54
- endoreg_db/models/label/label_set.py +53 -0
- endoreg_db/models/label/label_type.py +29 -0
- endoreg_db/models/label/label_video_segment/__init__.py +3 -0
- endoreg_db/models/label/label_video_segment/_create_from_video.py +41 -0
- endoreg_db/models/label/label_video_segment/label_video_segment.py +511 -0
- endoreg_db/models/label/video_segmentation_label.py +31 -0
- endoreg_db/models/label/video_segmentation_labelset.py +27 -0
- endoreg_db/models/media/__init__.py +16 -0
- endoreg_db/models/media/frame/__init__.py +3 -0
- endoreg_db/models/media/frame/frame.py +111 -0
- endoreg_db/models/media/pdf/__init__.py +11 -0
- endoreg_db/models/media/pdf/raw_pdf.py +757 -0
- endoreg_db/models/media/pdf/report_file.py +162 -0
- endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +77 -0
- endoreg_db/models/media/video/__init__.py +8 -0
- endoreg_db/models/media/video/create_from_file.py +358 -0
- endoreg_db/models/media/video/pipe_1.py +213 -0
- endoreg_db/models/media/video/pipe_2.py +105 -0
- endoreg_db/models/media/video/refactor_plan.md +0 -0
- endoreg_db/models/media/video/video_file.py +825 -0
- endoreg_db/models/media/video/video_file_ai.py +443 -0
- endoreg_db/models/media/video/video_file_anonymize.py +349 -0
- endoreg_db/models/media/video/video_file_frames/__init__.py +47 -0
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +22 -0
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +104 -0
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +174 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +28 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +27 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +20 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +27 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +34 -0
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +27 -0
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +129 -0
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +141 -0
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +65 -0
- endoreg_db/models/media/video/video_file_frames.py +0 -0
- endoreg_db/models/media/video/video_file_io.py +168 -0
- endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +45 -0
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +39 -0
- endoreg_db/models/media/video/video_file_meta/get_fps.py +147 -0
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +143 -0
- endoreg_db/models/media/video/video_file_meta/text_meta.py +134 -0
- endoreg_db/models/media/video/video_file_meta/video_meta.py +70 -0
- endoreg_db/models/media/video/video_file_segments.py +209 -0
- endoreg_db/models/media/video/video_metadata.py +65 -0
- endoreg_db/models/media/video/video_processing.py +152 -0
- endoreg_db/models/medical/__init__.py +146 -0
- endoreg_db/models/medical/contraindication/__init__.py +17 -0
- endoreg_db/models/medical/disease.py +156 -0
- endoreg_db/models/medical/event.py +137 -0
- endoreg_db/models/medical/examination/__init__.py +9 -0
- endoreg_db/models/medical/examination/examination.py +148 -0
- endoreg_db/models/medical/examination/examination_indication.py +278 -0
- endoreg_db/models/medical/examination/examination_time.py +49 -0
- endoreg_db/models/medical/examination/examination_time_type.py +41 -0
- endoreg_db/models/medical/examination/examination_type.py +48 -0
- endoreg_db/models/medical/finding/__init__.py +18 -0
- endoreg_db/models/medical/finding/finding.py +96 -0
- endoreg_db/models/medical/finding/finding_classification.py +142 -0
- endoreg_db/models/medical/finding/finding_intervention.py +52 -0
- endoreg_db/models/medical/finding/finding_type.py +35 -0
- endoreg_db/models/medical/hardware/__init__.py +8 -0
- endoreg_db/models/medical/hardware/endoscope.py +65 -0
- endoreg_db/models/{hardware → medical/hardware}/endoscopy_processor.py +68 -29
- endoreg_db/models/medical/laboratory/__init__.py +5 -0
- endoreg_db/models/medical/laboratory/lab_value.py +419 -0
- endoreg_db/models/medical/medication/__init__.py +19 -0
- endoreg_db/models/medical/medication/medication.py +31 -0
- endoreg_db/models/medical/medication/medication_indication.py +50 -0
- endoreg_db/models/medical/medication/medication_indication_type.py +39 -0
- endoreg_db/models/medical/medication/medication_intake_time.py +44 -0
- endoreg_db/models/medical/medication/medication_schedule.py +45 -0
- endoreg_db/models/medical/organ/__init__.py +35 -0
- endoreg_db/models/medical/patient/__init__.py +56 -0
- endoreg_db/models/medical/patient/medication_examples.py +38 -0
- endoreg_db/models/medical/patient/patient_disease.py +63 -0
- endoreg_db/models/medical/patient/patient_event.py +75 -0
- endoreg_db/models/medical/patient/patient_examination.py +249 -0
- endoreg_db/models/medical/patient/patient_examination_indication.py +44 -0
- endoreg_db/models/medical/patient/patient_finding.py +357 -0
- endoreg_db/models/medical/patient/patient_finding_classification.py +207 -0
- endoreg_db/models/medical/patient/patient_finding_intervention.py +40 -0
- endoreg_db/models/medical/patient/patient_lab_sample.py +148 -0
- endoreg_db/models/{persons → medical}/patient/patient_lab_value.py +68 -22
- endoreg_db/models/medical/patient/patient_medication.py +104 -0
- endoreg_db/models/medical/patient/patient_medication_schedule.py +136 -0
- endoreg_db/models/medical/risk/__init__.py +7 -0
- endoreg_db/models/medical/risk/risk.py +72 -0
- endoreg_db/models/medical/risk/risk_type.py +51 -0
- endoreg_db/models/metadata/__init__.py +19 -0
- endoreg_db/models/metadata/frame_ocr_result.py +0 -0
- endoreg_db/models/metadata/model_meta.py +206 -0
- endoreg_db/models/metadata/model_meta_logic.py +343 -0
- endoreg_db/models/{data_file/metadata → metadata}/pdf_meta.py +32 -13
- endoreg_db/models/metadata/sensitive_meta.py +288 -0
- endoreg_db/models/metadata/sensitive_meta_logic.py +1048 -0
- endoreg_db/models/metadata/video_meta.py +332 -0
- endoreg_db/models/metadata/video_prediction_logic.py +190 -0
- endoreg_db/models/metadata/video_prediction_meta.py +270 -0
- endoreg_db/models/other/__init__.py +36 -1
- endoreg_db/models/other/distribution/__init__.py +44 -0
- endoreg_db/models/other/distribution/base_value_distribution.py +20 -0
- endoreg_db/models/other/distribution/date_value_distribution.py +89 -0
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +32 -0
- endoreg_db/models/other/distribution/numeric_value_distribution.py +125 -0
- endoreg_db/models/other/distribution/single_categorical_value_distribution.py +22 -0
- endoreg_db/models/other/emission/__init__.py +5 -0
- endoreg_db/models/other/emission/emission_factor.py +94 -0
- endoreg_db/models/{persons → other}/gender.py +8 -3
- endoreg_db/models/other/information_source.py +159 -0
- endoreg_db/models/other/material.py +14 -2
- endoreg_db/models/other/resource.py +6 -2
- endoreg_db/models/other/tag.py +27 -0
- endoreg_db/models/other/transport_route.py +15 -3
- endoreg_db/models/{unit.py → other/unit.py} +16 -6
- endoreg_db/models/other/waste.py +10 -3
- endoreg_db/models/requirement/__init__.py +11 -0
- endoreg_db/models/requirement/requirement.py +767 -0
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
- endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +9 -0
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +95 -0
- endoreg_db/models/requirement/requirement_operator.py +176 -0
- endoreg_db/models/requirement/requirement_set.py +287 -0
- endoreg_db/models/rule/__init__.py +13 -0
- endoreg_db/models/{rules → rule}/rule.py +6 -3
- endoreg_db/models/{rules → rule}/rule_attribute_dtype.py +0 -2
- endoreg_db/models/{rules → rule}/rule_type.py +0 -2
- endoreg_db/models/{rules → rule}/ruleset.py +0 -2
- endoreg_db/models/state/__init__.py +12 -0
- endoreg_db/models/state/abstract.py +11 -0
- endoreg_db/models/state/audit_ledger.py +150 -0
- endoreg_db/models/state/label_video_segment.py +22 -0
- endoreg_db/models/state/raw_pdf.py +187 -0
- endoreg_db/models/state/sensitive_meta.py +46 -0
- endoreg_db/models/state/video.py +232 -0
- endoreg_db/models/upload_job.py +99 -0
- endoreg_db/models/utils.py +135 -0
- endoreg_db/renames.yml +8 -0
- endoreg_db/root_urls.py +9 -0
- endoreg_db/schemas/__init__.py +0 -0
- endoreg_db/schemas/examination_evaluation.py +27 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +775 -0
- endoreg_db/serializers/__init__.py +118 -10
- endoreg_db/serializers/_old/raw_pdf_meta_validation.py +223 -0
- endoreg_db/serializers/_old/raw_video_meta_validation.py +179 -0
- endoreg_db/serializers/_old/video.py +71 -0
- endoreg_db/serializers/administration/__init__.py +14 -0
- endoreg_db/serializers/administration/ai/__init__.py +10 -0
- endoreg_db/serializers/administration/ai/active_model.py +10 -0
- endoreg_db/serializers/administration/ai/ai_model.py +18 -0
- endoreg_db/serializers/administration/ai/model_type.py +10 -0
- endoreg_db/serializers/administration/center.py +9 -0
- endoreg_db/serializers/administration/gender.py +9 -0
- endoreg_db/serializers/anonymization.py +69 -0
- endoreg_db/serializers/evaluation/examination_evaluation.py +1 -0
- endoreg_db/serializers/examination/__init__.py +10 -0
- endoreg_db/serializers/examination/base.py +46 -0
- endoreg_db/serializers/examination/dropdown.py +21 -0
- endoreg_db/serializers/examination_serializer.py +12 -0
- endoreg_db/serializers/finding/__init__.py +5 -0
- endoreg_db/serializers/finding/finding.py +54 -0
- endoreg_db/serializers/finding_classification/__init__.py +7 -0
- endoreg_db/serializers/finding_classification/choice.py +19 -0
- endoreg_db/serializers/finding_classification/classification.py +13 -0
- endoreg_db/serializers/label/__init__.py +7 -0
- endoreg_db/serializers/label/image_classification_annotation.py +62 -0
- endoreg_db/serializers/label/label.py +15 -0
- endoreg_db/serializers/label_video_segment/__init__.py +7 -0
- endoreg_db/serializers/label_video_segment/_lvs_create.py +149 -0
- endoreg_db/serializers/label_video_segment/_lvs_update.py +138 -0
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +149 -0
- endoreg_db/serializers/label_video_segment/label_video_segment.py +344 -0
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +99 -0
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +163 -0
- endoreg_db/serializers/meta/__init__.py +19 -0
- endoreg_db/serializers/meta/pdf_file_meta_extraction.py +115 -0
- endoreg_db/serializers/meta/report_meta.py +53 -0
- endoreg_db/serializers/meta/sensitive_meta_detail.py +162 -0
- endoreg_db/serializers/meta/sensitive_meta_update.py +148 -0
- endoreg_db/serializers/meta/sensitive_meta_verification.py +59 -0
- endoreg_db/serializers/meta/video_meta.py +39 -0
- endoreg_db/serializers/misc/__init__.py +14 -0
- endoreg_db/serializers/misc/file_overview.py +182 -0
- endoreg_db/serializers/misc/sensitive_patient_data.py +120 -0
- endoreg_db/serializers/misc/stats.py +33 -0
- endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
- endoreg_db/serializers/misc/upload_job.py +71 -0
- endoreg_db/serializers/patient/__init__.py +11 -0
- endoreg_db/serializers/patient/patient.py +86 -0
- endoreg_db/serializers/patient/patient_dropdown.py +27 -0
- endoreg_db/serializers/patient_examination/__init__.py +7 -0
- endoreg_db/serializers/patient_examination/patient_examination.py +141 -0
- endoreg_db/serializers/patient_finding/__init__.py +15 -0
- endoreg_db/serializers/patient_finding/patient_finding.py +31 -0
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +39 -0
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +53 -0
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +26 -0
- endoreg_db/serializers/patient_finding/patient_finding_list.py +41 -0
- endoreg_db/serializers/patient_finding/patient_finding_write.py +126 -0
- endoreg_db/serializers/pdf/__init__.py +5 -0
- endoreg_db/serializers/pdf/anony_text_validation.py +85 -0
- endoreg_db/serializers/report/__init__.py +9 -0
- endoreg_db/serializers/report/mixins.py +45 -0
- endoreg_db/serializers/report/report.py +105 -0
- endoreg_db/serializers/report/report_list.py +22 -0
- endoreg_db/serializers/report/secure_file_url.py +26 -0
- endoreg_db/serializers/requirements/requirement_schema.py +25 -0
- endoreg_db/serializers/requirements/requirement_sets.py +29 -0
- endoreg_db/serializers/sensitive_meta_serializer.py +282 -0
- endoreg_db/serializers/video/__init__.py +7 -0
- endoreg_db/serializers/video/segmentation.py +263 -0
- endoreg_db/serializers/video/video_file_brief.py +10 -0
- endoreg_db/serializers/video/video_file_detail.py +83 -0
- endoreg_db/serializers/video/video_file_list.py +67 -0
- endoreg_db/serializers/video/video_metadata.py +105 -0
- endoreg_db/serializers/video/video_processing_history.py +153 -0
- endoreg_db/serializers/video_examination.py +198 -0
- endoreg_db/services/__init__.py +5 -0
- endoreg_db/services/anonymization.py +223 -0
- endoreg_db/services/examination_evaluation.py +149 -0
- endoreg_db/services/finding_description_service.py +0 -0
- endoreg_db/services/lookup_service.py +411 -0
- endoreg_db/services/lookup_store.py +266 -0
- endoreg_db/services/pdf_import.py +1382 -0
- endoreg_db/services/polling_coordinator.py +288 -0
- endoreg_db/services/pseudonym_service.py +89 -0
- endoreg_db/services/requirements_object.py +147 -0
- endoreg_db/services/segment_sync.py +155 -0
- endoreg_db/services/storage_aware_video_processor.py +344 -0
- endoreg_db/services/video_import.py +1259 -0
- endoreg_db/tasks/upload_tasks.py +207 -0
- endoreg_db/tasks/video_ingest.py +157 -0
- endoreg_db/tasks/video_processing_tasks.py +327 -0
- endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
- endoreg_db/templates/admin/start_examination.html +12 -0
- endoreg_db/templates/timeline.html +176 -0
- endoreg_db/urls/__init__.py +83 -0
- endoreg_db/urls/anonymization.py +32 -0
- endoreg_db/urls/auth.py +16 -0
- endoreg_db/urls/classification.py +39 -0
- endoreg_db/urls/examination.py +54 -0
- endoreg_db/urls/files.py +6 -0
- endoreg_db/urls/label_video_segment_validate.py +33 -0
- endoreg_db/urls/label_video_segments.py +46 -0
- endoreg_db/urls/media.py +227 -0
- endoreg_db/urls/patient.py +19 -0
- endoreg_db/urls/report.py +48 -0
- endoreg_db/urls/requirements.py +13 -0
- endoreg_db/urls/sensitive_meta.py +0 -0
- endoreg_db/urls/stats.py +46 -0
- endoreg_db/urls/upload.py +20 -0
- endoreg_db/urls/video.py +61 -0
- endoreg_db/urls.py +9 -0
- endoreg_db/utils/__init__.py +88 -1
- endoreg_db/utils/ai/__init__.py +9 -0
- endoreg_db/{models/ai_model/utils.py → utils/ai/get.py} +1 -4
- endoreg_db/utils/ai/inference_dataset.py +52 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +159 -0
- endoreg_db/utils/ai/postprocess.py +63 -0
- endoreg_db/utils/ai/predict.py +291 -0
- endoreg_db/utils/ai/preprocess.py +68 -0
- endoreg_db/utils/calc_duration_seconds.py +24 -0
- endoreg_db/utils/case_generator/__init__.py +0 -0
- endoreg_db/utils/case_generator/case_generator.py +159 -0
- endoreg_db/utils/case_generator/lab_sample_factory.py +33 -0
- endoreg_db/utils/case_generator/utils.py +30 -0
- endoreg_db/utils/check_video_files.py +148 -0
- endoreg_db/utils/dataloader.py +118 -35
- endoreg_db/utils/dates.py +60 -0
- endoreg_db/utils/env.py +33 -0
- endoreg_db/utils/extract_specific_frames.py +72 -0
- endoreg_db/utils/file_operations.py +29 -1
- endoreg_db/utils/fix_video_path_direct.py +141 -0
- endoreg_db/utils/frame_anonymization_utils.py +463 -0
- endoreg_db/utils/hashs.py +123 -4
- endoreg_db/utils/links/__init__.py +0 -0
- endoreg_db/utils/links/requirement_link.py +193 -0
- endoreg_db/utils/mime_types.py +0 -0
- endoreg_db/utils/names.py +76 -0
- endoreg_db/utils/parse_and_generate_yaml.py +46 -0
- endoreg_db/utils/paths.py +95 -0
- endoreg_db/utils/permissions.py +143 -0
- endoreg_db/utils/pipelines/Readme.md +235 -0
- endoreg_db/utils/pipelines/__init__.py +0 -0
- endoreg_db/utils/pipelines/process_video_dir.py +120 -0
- endoreg_db/utils/product/__init__.py +0 -0
- endoreg_db/utils/product/sum_emissions.py +20 -0
- endoreg_db/utils/product/sum_weights.py +18 -0
- endoreg_db/utils/pydantic_models/__init__.py +6 -0
- endoreg_db/utils/pydantic_models/db_config.py +57 -0
- endoreg_db/utils/requirement_helpers.py +0 -0
- endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
- endoreg_db/utils/requirement_operator_logic/lab_value_operators.py +578 -0
- endoreg_db/utils/requirement_operator_logic/model_evaluators.py +368 -0
- endoreg_db/utils/setup_config.py +177 -0
- endoreg_db/utils/translation.py +27 -0
- endoreg_db/utils/validate_endo_roi.py +19 -0
- endoreg_db/utils/validate_subcategory_dict.py +91 -0
- endoreg_db/utils/validate_video_detailed.py +357 -0
- endoreg_db/utils/video/__init__.py +26 -0
- endoreg_db/utils/video/extract_frames.py +88 -0
- endoreg_db/utils/video/ffmpeg_wrapper.py +835 -0
- endoreg_db/utils/video/names.py +42 -0
- endoreg_db/utils/video/streaming_processor.py +312 -0
- endoreg_db/utils/video/video_splitter.py +94 -0
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +238 -0
- endoreg_db/views/__init__.py +274 -0
- endoreg_db/views/anonymization/__init__.py +27 -0
- endoreg_db/views/anonymization/media_management.py +454 -0
- endoreg_db/views/anonymization/overview.py +216 -0
- endoreg_db/views/anonymization/validate.py +107 -0
- endoreg_db/views/auth/__init__.py +13 -0
- endoreg_db/views/auth/keycloak.py +113 -0
- endoreg_db/views/examination/__init__.py +33 -0
- endoreg_db/views/examination/examination.py +37 -0
- endoreg_db/views/examination/examination_manifest_cache.py +26 -0
- endoreg_db/views/examination/get_finding_classification_choices.py +59 -0
- endoreg_db/views/examination/get_finding_classifications.py +36 -0
- endoreg_db/views/examination/get_findings.py +41 -0
- endoreg_db/views/examination/get_instruments.py +18 -0
- endoreg_db/views/examination/get_interventions.py +14 -0
- endoreg_db/views/finding/__init__.py +9 -0
- endoreg_db/views/finding/finding.py +112 -0
- endoreg_db/views/finding/get_classifications.py +14 -0
- endoreg_db/views/finding/get_interventions.py +17 -0
- endoreg_db/views/finding_classification/__init__.py +13 -0
- endoreg_db/views/finding_classification/base.py +0 -0
- endoreg_db/views/finding_classification/finding_classification.py +42 -0
- endoreg_db/views/finding_classification/get_classification_choices.py +55 -0
- endoreg_db/views/label/__init__.py +5 -0
- endoreg_db/views/label/label.py +15 -0
- endoreg_db/views/label_video_segment/__init__.py +16 -0
- endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +44 -0
- endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +50 -0
- endoreg_db/views/label_video_segment/label_video_segment.py +77 -0
- endoreg_db/views/label_video_segment/label_video_segment_by_label.py +174 -0
- endoreg_db/views/label_video_segment/label_video_segment_detail.py +73 -0
- endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +46 -0
- endoreg_db/views/label_video_segment/validate.py +226 -0
- endoreg_db/views/media/__init__.py +45 -0
- endoreg_db/views/media/pdf_media.py +388 -0
- endoreg_db/views/media/segments.py +71 -0
- endoreg_db/views/media/sensitive_metadata.py +314 -0
- endoreg_db/views/media/video_media.py +272 -0
- endoreg_db/views/media/video_segments.py +524 -0
- endoreg_db/views/meta/__init__.py +15 -0
- endoreg_db/views/meta/available_files_list.py +146 -0
- endoreg_db/views/meta/report_meta.py +53 -0
- endoreg_db/views/meta/sensitive_meta_detail.py +148 -0
- endoreg_db/views/meta/sensitive_meta_list.py +104 -0
- endoreg_db/views/meta/sensitive_meta_verification.py +71 -0
- endoreg_db/views/misc/__init__.py +63 -0
- endoreg_db/views/misc/center.py +13 -0
- endoreg_db/views/misc/csrf.py +7 -0
- endoreg_db/views/misc/gender.py +14 -0
- endoreg_db/views/misc/secure_file_serving_view.py +80 -0
- endoreg_db/views/misc/secure_file_url_view.py +84 -0
- endoreg_db/views/misc/secure_url_validate.py +79 -0
- endoreg_db/views/misc/stats.py +220 -0
- endoreg_db/views/misc/translation.py +182 -0
- endoreg_db/views/misc/upload_views.py +240 -0
- endoreg_db/views/patient/__init__.py +5 -0
- endoreg_db/views/patient/patient.py +210 -0
- endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +164 -0
- endoreg_db/views/patient_examination/__init__.py +11 -0
- endoreg_db/views/patient_examination/patient_examination.py +140 -0
- endoreg_db/views/patient_examination/patient_examination_create.py +63 -0
- endoreg_db/views/patient_examination/patient_examination_detail.py +66 -0
- endoreg_db/views/patient_examination/patient_examination_list.py +68 -0
- endoreg_db/views/patient_examination/video.py +194 -0
- endoreg_db/views/patient_finding/__init__.py +7 -0
- endoreg_db/views/patient_finding/base.py +0 -0
- endoreg_db/views/patient_finding/patient_finding.py +64 -0
- endoreg_db/views/patient_finding/patient_finding_optimized.py +259 -0
- endoreg_db/views/patient_finding_classification/__init__.py +5 -0
- endoreg_db/views/patient_finding_classification/pfc_create.py +67 -0
- endoreg_db/views/patient_finding_location/__init__.py +5 -0
- endoreg_db/views/patient_finding_location/pfl_create.py +70 -0
- endoreg_db/views/patient_finding_morphology/__init__.py +5 -0
- endoreg_db/views/patient_finding_morphology/pfm_create.py +70 -0
- endoreg_db/views/pdf/__init__.py +8 -0
- endoreg_db/views/pdf/pdf_stream.py +187 -0
- endoreg_db/views/pdf/reimport.py +177 -0
- endoreg_db/views/report/__init__.py +9 -0
- endoreg_db/views/report/report_list.py +112 -0
- endoreg_db/views/report/report_with_secure_url.py +28 -0
- endoreg_db/views/report/start_examination.py +7 -0
- endoreg_db/views/requirement/__init__.py +10 -0
- endoreg_db/views/requirement/evaluate.py +279 -0
- endoreg_db/views/requirement/lookup.py +367 -0
- endoreg_db/views/requirement/lookup_store.py +252 -0
- endoreg_db/views/requirement_lookup/lookup.py +0 -0
- endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
- endoreg_db/views/stats/__init__.py +13 -0
- endoreg_db/views/stats/stats_views.py +229 -0
- endoreg_db/views/video/__init__.py +59 -0
- endoreg_db/views/video/correction.py +530 -0
- endoreg_db/views/video/reimport.py +195 -0
- endoreg_db/views/video/segmentation.py +274 -0
- endoreg_db/views/video/task_status.py +49 -0
- endoreg_db/views/video/timeline.py +46 -0
- endoreg_db/views/video/video_analyze.py +52 -0
- endoreg_db/views/video/video_apply_mask.py +48 -0
- endoreg_db/views/video/video_correction.py +21 -0
- endoreg_db/views/video/video_download_processed.py +58 -0
- endoreg_db/views/video/video_examination_viewset.py +242 -0
- endoreg_db/views/video/video_meta.py +29 -0
- endoreg_db/views/video/video_processing_history.py +24 -0
- endoreg_db/views/video/video_remove_frames.py +48 -0
- endoreg_db/views/video/video_stream.py +306 -0
- endoreg_db/views.py +0 -3
- endoreg_db-0.8.6.3.dist-info/METADATA +383 -0
- endoreg_db-0.8.6.3.dist-info/RECORD +793 -0
- {endoreg_db-0.4.5.dist-info → endoreg_db-0.8.6.3.dist-info}/WHEEL +1 -1
- endoreg_db/data/active_model/data.yaml +0 -3
- endoreg_db/data/agl_service/data.yaml +0 -19
- endoreg_db/data/label/label-set/data.yaml +0 -18
- endoreg_db/management/commands/_load_model_template.py +0 -41
- endoreg_db/management/commands/delete_all.py +0 -18
- endoreg_db/management/commands/delete_legacy_images.py +0 -19
- endoreg_db/management/commands/delete_legacy_videos.py +0 -17
- endoreg_db/management/commands/extract_legacy_video_frames.py +0 -18
- endoreg_db/management/commands/fetch_legacy_image_dataset.py +0 -32
- endoreg_db/management/commands/fix_auth_permission.py +0 -20
- endoreg_db/management/commands/import_legacy_images.py +0 -94
- endoreg_db/management/commands/import_legacy_videos.py +0 -76
- endoreg_db/management/commands/load_active_model_data.py +0 -45
- endoreg_db/management/commands/load_endoscopy_processor_data.py +0 -45
- endoreg_db/management/commands/load_g_play_data.py +0 -113
- endoreg_db/management/commands/load_label_data.py +0 -67
- endoreg_db/management/commands/load_medication_indication_data.py +0 -63
- endoreg_db/management/commands/load_medication_indication_type_data.py +0 -41
- endoreg_db/management/commands/load_medication_schedule_data.py +0 -55
- endoreg_db/management/commands/load_network_data.py +0 -57
- endoreg_db/migrations/0002_anonymizedimagelabel_anonymousimageannotation_and_more.py +0 -55
- endoreg_db/migrations/0003_anonymousimageannotation_original_image_url_and_more.py +0 -39
- endoreg_db/migrations/0004_alter_rawpdffile_file.py +0 -20
- endoreg_db/migrations/0005_uploadedfile_alter_rawpdffile_file_anonymizedfile.py +0 -40
- endoreg_db/migrations/0006_alter_rawpdffile_file.py +0 -20
- endoreg_db/migrations/0007_networkdevicelogentry_datetime_and_more.py +0 -43
- endoreg_db/models/ai_model/__init__.py +0 -3
- endoreg_db/models/ai_model/active_model.py +0 -9
- endoreg_db/models/ai_model/model_meta.py +0 -24
- endoreg_db/models/annotation/__init__.py +0 -3
- endoreg_db/models/annotation/anonymized_image_annotation.py +0 -60
- endoreg_db/models/annotation/binary_classification_annotation_task.py +0 -80
- endoreg_db/models/annotation/image_classification.py +0 -27
- endoreg_db/models/center/center.py +0 -25
- endoreg_db/models/center/center_product.py +0 -34
- endoreg_db/models/center/center_resource.py +0 -19
- endoreg_db/models/center/center_waste.py +0 -11
- endoreg_db/models/data_file/__init__.py +0 -6
- endoreg_db/models/data_file/base_classes/__init__.py +0 -2
- endoreg_db/models/data_file/base_classes/abstract_frame.py +0 -51
- endoreg_db/models/data_file/base_classes/abstract_video.py +0 -201
- endoreg_db/models/data_file/frame.py +0 -45
- endoreg_db/models/data_file/import_classes/__init__.py +0 -32
- endoreg_db/models/data_file/import_classes/processing_functions/__init__.py +0 -35
- endoreg_db/models/data_file/import_classes/processing_functions/pdf.py +0 -28
- endoreg_db/models/data_file/import_classes/processing_functions/video.py +0 -260
- endoreg_db/models/data_file/import_classes/raw_pdf.py +0 -188
- endoreg_db/models/data_file/import_classes/raw_video.py +0 -343
- endoreg_db/models/data_file/metadata/__init__.py +0 -3
- endoreg_db/models/data_file/metadata/sensitive_meta.py +0 -31
- endoreg_db/models/data_file/metadata/video_meta.py +0 -133
- endoreg_db/models/data_file/report_file.py +0 -89
- endoreg_db/models/data_file/video/__init__.py +0 -7
- endoreg_db/models/data_file/video/import_meta.py +0 -25
- endoreg_db/models/data_file/video/video.py +0 -25
- endoreg_db/models/data_file/video_segment.py +0 -107
- endoreg_db/models/disease.py +0 -56
- endoreg_db/models/emission/__init__.py +0 -1
- endoreg_db/models/emission/emission_factor.py +0 -20
- endoreg_db/models/event.py +0 -22
- endoreg_db/models/examination/__init__.py +0 -4
- endoreg_db/models/examination/examination.py +0 -26
- endoreg_db/models/examination/examination_time.py +0 -27
- endoreg_db/models/examination/examination_time_type.py +0 -24
- endoreg_db/models/examination/examination_type.py +0 -18
- endoreg_db/models/hardware/__init__.py +0 -2
- endoreg_db/models/hardware/endoscope.py +0 -44
- endoreg_db/models/information_source.py +0 -29
- endoreg_db/models/laboratory/__init__.py +0 -1
- endoreg_db/models/laboratory/lab_value.py +0 -102
- endoreg_db/models/legacy_data/__init__.py +0 -3
- endoreg_db/models/legacy_data/image.py +0 -34
- endoreg_db/models/logging/__init__.py +0 -4
- endoreg_db/models/logging/agl_service.py +0 -19
- endoreg_db/models/logging/base.py +0 -22
- endoreg_db/models/logging/log_type.py +0 -23
- endoreg_db/models/logging/network_device.py +0 -24
- endoreg_db/models/medication/__init__.py +0 -1
- endoreg_db/models/medication/medication.py +0 -148
- endoreg_db/models/network/__init__.py +0 -3
- endoreg_db/models/network/agl_service.py +0 -38
- endoreg_db/models/network/network_device.py +0 -53
- endoreg_db/models/network/network_device_type.py +0 -23
- endoreg_db/models/other/distribution.py +0 -215
- endoreg_db/models/patient_examination/__init__.py +0 -35
- endoreg_db/models/permissions/__init__.py +0 -44
- endoreg_db/models/persons/__init__.py +0 -7
- endoreg_db/models/persons/examiner/__init__.py +0 -2
- endoreg_db/models/persons/examiner/examiner.py +0 -16
- endoreg_db/models/persons/examiner/examiner_type.py +0 -2
- endoreg_db/models/persons/patient/__init__.py +0 -8
- endoreg_db/models/persons/patient/case/case.py +0 -30
- endoreg_db/models/persons/patient/patient.py +0 -216
- endoreg_db/models/persons/patient/patient_disease.py +0 -16
- endoreg_db/models/persons/patient/patient_event.py +0 -22
- endoreg_db/models/persons/patient/patient_lab_sample.py +0 -106
- endoreg_db/models/persons/patient/patient_medication.py +0 -44
- endoreg_db/models/persons/patient/patient_medication_schedule.py +0 -28
- endoreg_db/models/persons/portal_user_information.py +0 -27
- endoreg_db/models/prediction/__init__.py +0 -2
- endoreg_db/models/prediction/image_classification.py +0 -37
- endoreg_db/models/prediction/video_prediction_meta.py +0 -244
- endoreg_db/models/product/__init__.py +0 -5
- endoreg_db/models/product/product.py +0 -97
- endoreg_db/models/product/product_group.py +0 -19
- endoreg_db/models/product/product_material.py +0 -24
- endoreg_db/models/questionnaires/__init__.py +0 -114
- endoreg_db/models/quiz/__init__.py +0 -2
- endoreg_db/models/quiz/quiz_answer.py +0 -41
- endoreg_db/models/quiz/quiz_question.py +0 -54
- endoreg_db/models/report_reader/__init__.py +0 -2
- endoreg_db/models/report_reader/report_reader_config.py +0 -53
- endoreg_db/models/rules/__init__.py +0 -5
- endoreg_db/queries/get/__init__.py +0 -6
- endoreg_db/queries/get/center.py +0 -42
- endoreg_db/queries/get/model.py +0 -13
- endoreg_db/queries/get/patient.py +0 -14
- endoreg_db/queries/get/patient_examination.py +0 -20
- endoreg_db/queries/get/report_file.py +0 -33
- endoreg_db/queries/get/video.py +0 -31
- endoreg_db/serializers/ai_model.py +0 -19
- endoreg_db/serializers/annotation.py +0 -17
- endoreg_db/serializers/center.py +0 -11
- endoreg_db/serializers/examination.py +0 -33
- endoreg_db/serializers/frame.py +0 -13
- endoreg_db/serializers/hardware.py +0 -21
- endoreg_db/serializers/label.py +0 -22
- endoreg_db/serializers/patient.py +0 -10
- endoreg_db/serializers/prediction.py +0 -15
- endoreg_db/serializers/report_file.py +0 -7
- endoreg_db/serializers/video.py +0 -27
- endoreg_db/tests.py +0 -3
- endoreg_db/utils/legacy_ocr.py +0 -201
- endoreg_db/utils/video_metadata.py +0 -87
- endoreg_db-0.4.5.dist-info/METADATA +0 -34
- endoreg_db-0.4.5.dist-info/RECORD +0 -316
- /endoreg_db/{data/distribution/numeric/.init → api/serializers/finding_descriptions.py} +0 -0
- /endoreg_db/{models/persons/patient/case/__init__.py → api/views/finding_descriptions.py} +0 -0
- /endoreg_db/{queries/get/annotation.py → config/__init__.py} +0 -0
- /endoreg_db/data/{label → ai_model_label}/label-type/data.yaml +0 -0
- /endoreg_db/data/{model_type → ai_model_type}/data.yaml +0 -0
- /endoreg_db/{queries/get/prediction.py → data/shift/m2.yaml} +0 -0
- /endoreg_db/{queries/get/video_import_meta.py → factories/__init__.py} +0 -0
- /endoreg_db/{queries/get/video_prediction_meta.py → helpers/__init__.py} +0 -0
- /endoreg_db/management/commands/{load_report_reader_flag.py → load_report_reader_flag_data.py} +0 -0
- /endoreg_db/models/{persons → administration/person}/person.py +0 -0
- /endoreg_db/models/{report_reader → media/pdf/report_reader}/report_reader_flag.py +0 -0
- /endoreg_db/models/{rules → rule}/rule_applicator.py +0 -0
- {endoreg_db-0.4.5.dist-info → endoreg_db-0.8.6.3.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import logging
|
|
3
|
+
from typing import Optional
|
|
4
|
+
from celery_app import shared_task
|
|
5
|
+
from django.core.files.storage import default_storage
|
|
6
|
+
|
|
7
|
+
from endoreg_db.models.upload_job import UploadJob
|
|
8
|
+
from endoreg_db.models import SensitiveMeta, VideoFile
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@shared_task(bind=True)
|
|
15
|
+
def process_upload_job(self, job_id: str) -> dict:
|
|
16
|
+
"""
|
|
17
|
+
Process an uploaded file (PDF or video) asynchronously.
|
|
18
|
+
|
|
19
|
+
This task:
|
|
20
|
+
1. Updates job status to 'processing'
|
|
21
|
+
2. Determines file type and routes to appropriate processor
|
|
22
|
+
3. For PDFs: Uses ReportReader to extract metadata and create SensitiveMeta
|
|
23
|
+
4. For videos: Uses VideoFile.create_from_file_initialized() and processing pipeline
|
|
24
|
+
5. Links the created SensitiveMeta to the job and marks as 'anonymized'
|
|
25
|
+
6. On error: marks job as 'error' with details
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
job_id: UUID string of the UploadJob to process
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
dict: Processing result with status and details
|
|
32
|
+
"""
|
|
33
|
+
try:
|
|
34
|
+
# Get the upload job
|
|
35
|
+
try:
|
|
36
|
+
upload_job = UploadJob.objects.get(id=job_id)
|
|
37
|
+
except UploadJob.DoesNotExist:
|
|
38
|
+
error_msg = f"UploadJob with id {job_id} not found"
|
|
39
|
+
logger.error(error_msg)
|
|
40
|
+
return {'status': 'error', 'error': error_msg}
|
|
41
|
+
|
|
42
|
+
# Mark as processing
|
|
43
|
+
upload_job.mark_processing()
|
|
44
|
+
logger.info(f"Started processing upload job {job_id}")
|
|
45
|
+
|
|
46
|
+
# Get file path
|
|
47
|
+
file_path = upload_job.file.path
|
|
48
|
+
if not os.path.exists(file_path):
|
|
49
|
+
error_msg = f"Uploaded file not found at {file_path}"
|
|
50
|
+
upload_job.mark_error(error_msg)
|
|
51
|
+
logger.error(error_msg)
|
|
52
|
+
return {'status': 'error', 'error': error_msg}
|
|
53
|
+
|
|
54
|
+
# Route to appropriate processor based on content type
|
|
55
|
+
sensitive_meta = None
|
|
56
|
+
|
|
57
|
+
if upload_job.content_type == 'application/pdf':
|
|
58
|
+
sensitive_meta = _process_pdf_file(upload_job, file_path)
|
|
59
|
+
elif upload_job.content_type.startswith('video/'):
|
|
60
|
+
sensitive_meta = _process_video_file(upload_job, file_path)
|
|
61
|
+
else:
|
|
62
|
+
error_msg = f"Unsupported content type: {upload_job.content_type}"
|
|
63
|
+
upload_job.mark_error(error_msg)
|
|
64
|
+
logger.error(error_msg)
|
|
65
|
+
return {'status': 'error', 'error': error_msg}
|
|
66
|
+
|
|
67
|
+
if sensitive_meta:
|
|
68
|
+
# Mark job as completed with the created SensitiveMeta
|
|
69
|
+
upload_job.mark_completed(sensitive_meta)
|
|
70
|
+
logger.info(f"Successfully processed upload job {job_id}, created SensitiveMeta {sensitive_meta.id}")
|
|
71
|
+
return {
|
|
72
|
+
'status': 'anonymized',
|
|
73
|
+
'sensitive_meta_id': sensitive_meta.id,
|
|
74
|
+
'job_id': job_id
|
|
75
|
+
}
|
|
76
|
+
else:
|
|
77
|
+
error_msg = "Processing completed but no SensitiveMeta was created"
|
|
78
|
+
upload_job.mark_error(error_msg)
|
|
79
|
+
logger.error(f"Upload job {job_id}: {error_msg}")
|
|
80
|
+
return {'status': 'error', 'error': error_msg}
|
|
81
|
+
|
|
82
|
+
except Exception as e:
|
|
83
|
+
error_msg = f"Unexpected error processing upload job {job_id}: {str(e)}"
|
|
84
|
+
logger.exception(error_msg)
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
upload_job = UploadJob.objects.get(id=job_id)
|
|
88
|
+
upload_job.mark_error(error_msg)
|
|
89
|
+
except Exception as inner_e:
|
|
90
|
+
logger.error(f"Failed to update job status after error: {inner_e}")
|
|
91
|
+
|
|
92
|
+
return {'status': 'error', 'error': error_msg}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _process_pdf_file(upload_job: UploadJob, file_path: str) -> Optional[SensitiveMeta]:
|
|
96
|
+
"""
|
|
97
|
+
Process a PDF file using ReportReader.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
upload_job: The UploadJob instance
|
|
101
|
+
file_path: Path to the uploaded PDF file
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
SensitiveMeta instance if successful, None otherwise
|
|
105
|
+
"""
|
|
106
|
+
try:
|
|
107
|
+
# Import ReportReader (lazy import to avoid startup issues)
|
|
108
|
+
try:
|
|
109
|
+
from lx_anonymizer.report_reader import ReportReader
|
|
110
|
+
except ImportError as e:
|
|
111
|
+
raise ImportError(f"Could not import ReportReader: {e}. Make sure lx-anonymizer is available.")
|
|
112
|
+
|
|
113
|
+
logger.info(f"Processing PDF file: {file_path}")
|
|
114
|
+
|
|
115
|
+
# Create ReportReader instance
|
|
116
|
+
reader = ReportReader(
|
|
117
|
+
locale="de_DE",
|
|
118
|
+
text_date_format="%d.%m.%Y"
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
# Process the report
|
|
122
|
+
original_text, anonymized_text, metadata = reader.process_report(
|
|
123
|
+
pdf_path=file_path,
|
|
124
|
+
use_ensemble=False, # Can be made configurable
|
|
125
|
+
verbose=False,
|
|
126
|
+
use_llm_extractor=None # Use default extractor
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
if not metadata:
|
|
130
|
+
raise ValueError("No metadata extracted from PDF")
|
|
131
|
+
|
|
132
|
+
# Create SensitiveMeta from extracted metadata
|
|
133
|
+
sensitive_meta = SensitiveMeta.create_from_dict(metadata)
|
|
134
|
+
sensitive_meta.save()
|
|
135
|
+
|
|
136
|
+
logger.info(f"Created SensitiveMeta {sensitive_meta.id} from PDF processing")
|
|
137
|
+
return sensitive_meta
|
|
138
|
+
|
|
139
|
+
except Exception as e:
|
|
140
|
+
logger.exception(f"Error processing PDF file {file_path}: {e}")
|
|
141
|
+
raise
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _process_video_file(upload_job: UploadJob, file_path: str) -> Optional[SensitiveMeta]:
|
|
145
|
+
"""
|
|
146
|
+
Process a video file using VideoFile creation and processing pipeline.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
upload_job: The UploadJob instance
|
|
150
|
+
file_path: Path to the uploaded video file
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
SensitiveMeta instance if successful, None otherwise
|
|
154
|
+
"""
|
|
155
|
+
try:
|
|
156
|
+
logger.info(f"Processing video file: {file_path}")
|
|
157
|
+
|
|
158
|
+
# Get the relative path for storage
|
|
159
|
+
relative_path = os.path.relpath(file_path, default_storage.location)
|
|
160
|
+
|
|
161
|
+
# Create VideoFile instance using the existing method
|
|
162
|
+
video_file = VideoFile.create_from_file_initialized(
|
|
163
|
+
file_path=relative_path,
|
|
164
|
+
skip_frame_extraction=False, # Extract frames for processing
|
|
165
|
+
skip_meta_extraction=False, # Extract metadata
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
if not video_file:
|
|
169
|
+
raise ValueError("Failed to create VideoFile instance")
|
|
170
|
+
|
|
171
|
+
# Run the video processing pipeline (pipe_1)
|
|
172
|
+
try:
|
|
173
|
+
# Import the processing function
|
|
174
|
+
from endoreg_db.management.commands.import_video import pipe_1
|
|
175
|
+
|
|
176
|
+
# Run processing pipeline
|
|
177
|
+
pipe_1(
|
|
178
|
+
video_file=video_file,
|
|
179
|
+
skip_frame_extraction=False,
|
|
180
|
+
skip_meta_extraction=False,
|
|
181
|
+
skip_anonymization=False
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
except Exception as e:
|
|
185
|
+
logger.warning(f"Video processing pipeline failed, but VideoFile was created: {e}")
|
|
186
|
+
# Continue anyway - we might still have usable metadata
|
|
187
|
+
|
|
188
|
+
# Get the associated SensitiveMeta
|
|
189
|
+
if hasattr(video_file, 'sensitive_meta') and video_file.sensitive_meta:
|
|
190
|
+
sensitive_meta = video_file.sensitive_meta
|
|
191
|
+
logger.info(f"Found SensitiveMeta {sensitive_meta.id} from video processing")
|
|
192
|
+
return sensitive_meta
|
|
193
|
+
else:
|
|
194
|
+
# Create a basic SensitiveMeta if none exists
|
|
195
|
+
sensitive_meta = SensitiveMeta.objects.create()
|
|
196
|
+
|
|
197
|
+
# Link it to the video if possible
|
|
198
|
+
if hasattr(video_file, 'sensitive_meta'):
|
|
199
|
+
video_file.sensitive_meta = sensitive_meta
|
|
200
|
+
video_file.save(update_fields=['sensitive_meta'])
|
|
201
|
+
|
|
202
|
+
logger.info(f"Created basic SensitiveMeta {sensitive_meta.id} for video file")
|
|
203
|
+
return sensitive_meta
|
|
204
|
+
|
|
205
|
+
except Exception as e:
|
|
206
|
+
logger.exception(f"Error processing video file {file_path}: {e}")
|
|
207
|
+
raise
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Video ingestion tasks using RQ (Redis Queue).
|
|
3
|
+
|
|
4
|
+
This module provides background task processing for video file imports,
|
|
5
|
+
allowing asynchronous processing of video files dropped into the inbox directory.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
import os
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
import rq
|
|
14
|
+
import django
|
|
15
|
+
from django.conf import settings
|
|
16
|
+
from django.core.management import call_command
|
|
17
|
+
from redis import Redis
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
# Redis connection configuration
|
|
22
|
+
REDIS_URL = os.getenv('RQ_REDIS_URL', 'redis://localhost:6379/0')
|
|
23
|
+
|
|
24
|
+
def get_redis_connection():
|
|
25
|
+
"""Get Redis connection for RQ."""
|
|
26
|
+
try:
|
|
27
|
+
return Redis.from_url(REDIS_URL)
|
|
28
|
+
except Exception as e:
|
|
29
|
+
logger.error(f"Failed to connect to Redis at {REDIS_URL}: {e}")
|
|
30
|
+
raise
|
|
31
|
+
|
|
32
|
+
def get_queue():
|
|
33
|
+
"""Get the 'anonymize' RQ queue."""
|
|
34
|
+
redis_conn = get_redis_connection()
|
|
35
|
+
return rq.Queue('anonymize', connection=redis_conn)
|
|
36
|
+
|
|
37
|
+
def run_import_video(file_path: str, center_name: Optional[str] = None,
|
|
38
|
+
processor_name: Optional[str] = None) -> dict:
|
|
39
|
+
"""
|
|
40
|
+
RQ task to import a video file using the video import service.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
file_path: Path to the video file to import
|
|
44
|
+
center_name: Optional center name (defaults to settings.DEFAULT_CENTER)
|
|
45
|
+
processor_name: Optional processor name (defaults to settings.DEFAULT_PROCESSOR)
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Dict containing task result status and metadata
|
|
49
|
+
"""
|
|
50
|
+
try:
|
|
51
|
+
# Ensure Django is properly set up
|
|
52
|
+
if not django.apps.apps.ready:
|
|
53
|
+
django.setup()
|
|
54
|
+
|
|
55
|
+
file_path = Path(file_path)
|
|
56
|
+
if not file_path.exists():
|
|
57
|
+
error_msg = f"Video file not found: {file_path}"
|
|
58
|
+
logger.error(error_msg)
|
|
59
|
+
return {
|
|
60
|
+
'status': 'error',
|
|
61
|
+
'message': error_msg,
|
|
62
|
+
'file_path': str(file_path)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
# Use settings defaults if not provided
|
|
66
|
+
center_name = center_name or getattr(settings, 'DEFAULT_CENTER', 'university_hospital_wuerzburg')
|
|
67
|
+
processor_name = processor_name or getattr(settings, 'DEFAULT_PROCESSOR', 'olympus_cv_1500')
|
|
68
|
+
|
|
69
|
+
logger.info(f"Starting import for video: {file_path}")
|
|
70
|
+
logger.info(f"Using center: {center_name}, processor: {processor_name}")
|
|
71
|
+
|
|
72
|
+
# Try to use the service first
|
|
73
|
+
try:
|
|
74
|
+
from endoreg_db.services import video_import
|
|
75
|
+
|
|
76
|
+
video_file_obj = video_import.import_and_anonymize(
|
|
77
|
+
file_path=file_path,
|
|
78
|
+
center_name=center_name,
|
|
79
|
+
processor_name=processor_name,
|
|
80
|
+
save_video=True,
|
|
81
|
+
delete_source=True
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
success_msg = f"Successfully imported video: {file_path.name} (UUID: {video_file_obj.uuid})"
|
|
85
|
+
logger.info(success_msg)
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
'status': 'success',
|
|
89
|
+
'message': success_msg,
|
|
90
|
+
'file_path': str(file_path),
|
|
91
|
+
'video_uuid': str(video_file_obj.uuid),
|
|
92
|
+
'center_name': center_name,
|
|
93
|
+
'processor_name': processor_name
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
except Exception as service_error:
|
|
97
|
+
logger.warning(f"Service import failed: {service_error}, falling back to management command")
|
|
98
|
+
|
|
99
|
+
# Fallback to Django management command
|
|
100
|
+
call_command(
|
|
101
|
+
"import_video",
|
|
102
|
+
str(file_path),
|
|
103
|
+
"--center_name", center_name,
|
|
104
|
+
"--processor_name", processor_name,
|
|
105
|
+
"--save_video_file",
|
|
106
|
+
"--delete_source"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
success_msg = f"Successfully imported video via fallback: {file_path.name}"
|
|
110
|
+
logger.info(success_msg)
|
|
111
|
+
|
|
112
|
+
return {
|
|
113
|
+
'status': 'success',
|
|
114
|
+
'message': success_msg,
|
|
115
|
+
'file_path': str(file_path),
|
|
116
|
+
'center_name': center_name,
|
|
117
|
+
'processor_name': processor_name,
|
|
118
|
+
'method': 'fallback_command'
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
except Exception as e:
|
|
122
|
+
error_msg = f"Failed to import video {file_path}: {str(e)}"
|
|
123
|
+
logger.error(error_msg, exc_info=True)
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
'status': 'error',
|
|
127
|
+
'message': error_msg,
|
|
128
|
+
'file_path': str(file_path),
|
|
129
|
+
'exception': str(e)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
def enqueue_video_import(file_path: str, center_name: Optional[str] = None,
|
|
133
|
+
processor_name: Optional[str] = None) -> rq.Job:
|
|
134
|
+
"""
|
|
135
|
+
Enqueue a video import task.
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
file_path: Path to the video file to import
|
|
139
|
+
center_name: Optional center name
|
|
140
|
+
processor_name: Optional processor name
|
|
141
|
+
|
|
142
|
+
Returns:
|
|
143
|
+
RQ Job instance
|
|
144
|
+
"""
|
|
145
|
+
queue = get_queue()
|
|
146
|
+
|
|
147
|
+
job = queue.enqueue(
|
|
148
|
+
run_import_video,
|
|
149
|
+
file_path,
|
|
150
|
+
center_name,
|
|
151
|
+
processor_name,
|
|
152
|
+
job_timeout='30m', # 30 minute timeout for video processing
|
|
153
|
+
result_ttl=86400 # Keep results for 24 hours
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
logger.info(f"Enqueued video import job {job.id} for file: {file_path}")
|
|
157
|
+
return job
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Video processing tasks for Celery-based asynchronous video processing.
|
|
3
|
+
|
|
4
|
+
This module provides Celery tasks for:
|
|
5
|
+
- Video masking with streaming FFmpeg processing
|
|
6
|
+
- Frame removal with streaming optimization
|
|
7
|
+
- Video reprocessing workflows
|
|
8
|
+
- Progress tracking and status updates
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Dict, Any, List, Optional
|
|
13
|
+
from celery import shared_task
|
|
14
|
+
from celery.utils.log import get_task_logger
|
|
15
|
+
from django.shortcuts import get_object_or_404
|
|
16
|
+
|
|
17
|
+
logger = get_task_logger(__name__)
|
|
18
|
+
|
|
19
|
+
@shared_task(bind=True)
|
|
20
|
+
def apply_video_mask_task(self, video_id: int, mask_type: str = 'device_default',
|
|
21
|
+
device_name: str = 'olympus_cv_1500', use_streaming: bool = True,
|
|
22
|
+
custom_mask: Optional[Dict[str, Any]] = None):
|
|
23
|
+
"""
|
|
24
|
+
Apply mask to video using streaming processing.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
video_id: ID of the VideoFile to process
|
|
28
|
+
mask_type: Type of mask to apply ('device_default', 'roi_based', 'custom')
|
|
29
|
+
device_name: Device name for default mask
|
|
30
|
+
use_streaming: Whether to use streaming processing (recommended)
|
|
31
|
+
custom_mask: Custom mask configuration (if mask_type='custom')
|
|
32
|
+
"""
|
|
33
|
+
try:
|
|
34
|
+
# Update task progress
|
|
35
|
+
self.update_state(state='PROGRESS', meta={'progress': 0, 'message': 'Initializing mask application...'})
|
|
36
|
+
|
|
37
|
+
# Import models here to avoid circular imports
|
|
38
|
+
from endoreg_db.models import VideoFile
|
|
39
|
+
from lx_anonymizer.frame_cleaner import FrameCleaner
|
|
40
|
+
|
|
41
|
+
# Get video file
|
|
42
|
+
video = get_object_or_404(VideoFile, pk=video_id)
|
|
43
|
+
video_path = Path(video.file.path)
|
|
44
|
+
|
|
45
|
+
if not video_path.exists():
|
|
46
|
+
raise FileNotFoundError(f"Video file not found: {video_path}")
|
|
47
|
+
|
|
48
|
+
# Create output path
|
|
49
|
+
output_dir = video_path.parent / "processed"
|
|
50
|
+
output_dir.mkdir(exist_ok=True)
|
|
51
|
+
output_path = output_dir / f"{video_path.stem}_masked{video_path.suffix}"
|
|
52
|
+
|
|
53
|
+
self.update_state(state='PROGRESS', meta={'progress': 10, 'message': 'Setting up FrameCleaner...'})
|
|
54
|
+
|
|
55
|
+
# Initialize FrameCleaner
|
|
56
|
+
cleaner = FrameCleaner()
|
|
57
|
+
|
|
58
|
+
# Determine mask configuration
|
|
59
|
+
if mask_type == 'custom' and custom_mask:
|
|
60
|
+
mask_config = custom_mask
|
|
61
|
+
elif mask_type == 'roi_based':
|
|
62
|
+
# Would need ROI data from video metadata
|
|
63
|
+
mask_config = cleaner._load_mask(device_name) # Fallback for now
|
|
64
|
+
else: # device_default
|
|
65
|
+
mask_config = cleaner._load_mask(device_name)
|
|
66
|
+
|
|
67
|
+
self.update_state(state='PROGRESS', meta={'progress': 20, 'message': f'Applying {mask_type} mask...'})
|
|
68
|
+
|
|
69
|
+
# Apply mask using streaming
|
|
70
|
+
if use_streaming:
|
|
71
|
+
success = cleaner._mask_video_streaming(
|
|
72
|
+
video_path, mask_config, output_path, use_named_pipe=True
|
|
73
|
+
)
|
|
74
|
+
else:
|
|
75
|
+
success = cleaner._mask_video(video_path, mask_config, output_path)
|
|
76
|
+
|
|
77
|
+
if not success:
|
|
78
|
+
raise RuntimeError("Video masking failed")
|
|
79
|
+
|
|
80
|
+
self.update_state(state='PROGRESS', meta={'progress': 90, 'message': 'Finalizing...'})
|
|
81
|
+
|
|
82
|
+
# Verify output
|
|
83
|
+
if not output_path.exists() or output_path.stat().st_size == 0:
|
|
84
|
+
raise RuntimeError("Output video is empty or missing")
|
|
85
|
+
|
|
86
|
+
result = {
|
|
87
|
+
'status': 'completed',
|
|
88
|
+
'output_path': str(output_path),
|
|
89
|
+
'mask_type': mask_type,
|
|
90
|
+
'device_name': device_name,
|
|
91
|
+
'use_streaming': use_streaming,
|
|
92
|
+
'output_size': output_path.stat().st_size,
|
|
93
|
+
'message': 'Video masking completed successfully'
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
logger.info(f"Video masking completed for video {video_id}: {output_path}")
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
except Exception as e:
|
|
100
|
+
logger.error(f"Video masking failed for video {video_id}: {e}")
|
|
101
|
+
self.update_state(
|
|
102
|
+
state='FAILURE',
|
|
103
|
+
meta={'error': str(e), 'message': f'Video masking failed: {e}'}
|
|
104
|
+
)
|
|
105
|
+
raise
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _setup_frame_removal(video_id: int, detection_engine: str):
|
|
109
|
+
from endoreg_db.models import VideoFile
|
|
110
|
+
from lx_anonymizer.frame_cleaner import FrameCleaner
|
|
111
|
+
from django.shortcuts import get_object_or_404
|
|
112
|
+
video = get_object_or_404(VideoFile, pk=video_id)
|
|
113
|
+
video_path = Path(video.raw_file.path)
|
|
114
|
+
if not video_path.exists():
|
|
115
|
+
raise FileNotFoundError(f"Video file not found: {video_path}")
|
|
116
|
+
output_dir = video_path.parent / "processed"
|
|
117
|
+
output_dir.mkdir(exist_ok=True)
|
|
118
|
+
output_path = output_dir / f"{video_path.stem}_cleaned{video_path.suffix}"
|
|
119
|
+
use_minicpm = detection_engine == 'minicpm'
|
|
120
|
+
cleaner = FrameCleaner()
|
|
121
|
+
return video, video_path, output_path, cleaner
|
|
122
|
+
|
|
123
|
+
def _detect_sensitive_frames(self, cleaner, video_path, selection_method, manual_frames, total_frames):
|
|
124
|
+
frames_to_remove = []
|
|
125
|
+
if selection_method == 'manual' and manual_frames:
|
|
126
|
+
frames_to_remove = manual_frames
|
|
127
|
+
self.update_state(state='PROGRESS', meta={'progress': 50, 'message': f'Using manual frame selection: {len(frames_to_remove)} frames'})
|
|
128
|
+
elif selection_method == 'automatic':
|
|
129
|
+
self.update_state(state='PROGRESS', meta={'progress': 20, 'message': 'Detecting sensitive frames...'})
|
|
130
|
+
if total_frames > 10000:
|
|
131
|
+
self.update_state(state='PROGRESS', meta={'progress': 30, 'message': 'Using statistical sampling for large video...'})
|
|
132
|
+
sensitive_frames, estimated_ratio, early_stopped = cleaner._sample_frames_coroutine(
|
|
133
|
+
video_path, total_frames, max_samples=500
|
|
134
|
+
)
|
|
135
|
+
frames_to_remove = sensitive_frames
|
|
136
|
+
else:
|
|
137
|
+
self.update_state(state='PROGRESS', meta={'progress': 30, 'message': 'Analyzing all frames...'})
|
|
138
|
+
for abs_i, gray_frame, skip in cleaner._iter_video(video_path, total_frames):
|
|
139
|
+
progress = 30 + (abs_i / total_frames) * 40 # 30-70% for detection
|
|
140
|
+
if abs_i % 100 == 0:
|
|
141
|
+
self.update_state(state='PROGRESS', meta={
|
|
142
|
+
'progress': int(progress),
|
|
143
|
+
'message': f'Analyzing frame {abs_i}/{total_frames}...'
|
|
144
|
+
})
|
|
145
|
+
ocr_text, avg_conf, _ = cleaner.frame_ocr.extract_text_from_frame(
|
|
146
|
+
gray_frame, roi=None, high_quality=True
|
|
147
|
+
)
|
|
148
|
+
if ocr_text:
|
|
149
|
+
frame_metadata = cleaner.extract_metadata_deepseek(ocr_text)
|
|
150
|
+
if not frame_metadata:
|
|
151
|
+
frame_metadata = cleaner.frame_metadata_extractor.extract_metadata_from_frame_text(ocr_text)
|
|
152
|
+
has_sensitive = cleaner.frame_metadata_extractor.is_sensitive_content(frame_metadata)
|
|
153
|
+
if has_sensitive:
|
|
154
|
+
frames_to_remove.append(abs_i)
|
|
155
|
+
return frames_to_remove
|
|
156
|
+
|
|
157
|
+
def _remove_frames_from_video(cleaner, video_path, frames_to_remove, output_path, use_streaming):
|
|
158
|
+
if use_streaming:
|
|
159
|
+
return cleaner.remove_frames_from_video_streaming(
|
|
160
|
+
video_path, frames_to_remove, output_path, use_named_pipe=True
|
|
161
|
+
)
|
|
162
|
+
else:
|
|
163
|
+
return cleaner.remove_frames_from_video(
|
|
164
|
+
video_path, frames_to_remove, output_path
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
def _verify_frame_removal_output(output_path):
|
|
168
|
+
if not output_path.exists() or output_path.stat().st_size == 0:
|
|
169
|
+
raise RuntimeError("Output video is empty or missing")
|
|
170
|
+
|
|
171
|
+
@shared_task(bind=True)
|
|
172
|
+
def remove_video_frames_task(self, video_id: int, selection_method: str = 'automatic',
|
|
173
|
+
detection_engine: str = 'minicpm', use_streaming: bool = True,
|
|
174
|
+
manual_frames: Optional[List[int]] = None):
|
|
175
|
+
"""
|
|
176
|
+
Remove frames from video using streaming processing.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
video_id: ID of the VideoFile to process
|
|
180
|
+
selection_method: 'automatic' or 'manual'
|
|
181
|
+
detection_engine: 'minicpm' or 'traditional'
|
|
182
|
+
use_streaming: Whether to use streaming processing
|
|
183
|
+
manual_frames: List of frame indices to remove (for manual selection)
|
|
184
|
+
"""
|
|
185
|
+
try:
|
|
186
|
+
# Update task progress
|
|
187
|
+
self.update_state(state='PROGRESS', meta={'progress': 0, 'message': 'Initializing frame removal...'})
|
|
188
|
+
|
|
189
|
+
video, video_path, output_path, cleaner = _setup_frame_removal(video_id, detection_engine)
|
|
190
|
+
|
|
191
|
+
self.update_state(state='PROGRESS', meta={'progress': 10, 'message': 'Setting up FrameCleaner...'})
|
|
192
|
+
|
|
193
|
+
# Get total frame count
|
|
194
|
+
total_frames = cleaner._get_total_frames()
|
|
195
|
+
|
|
196
|
+
frames_to_remove = _detect_sensitive_frames(
|
|
197
|
+
self, cleaner, video_path, selection_method, manual_frames, total_frames
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
self.update_state(state='PROGRESS', meta={'progress': 70, 'message': f'Removing {len(frames_to_remove)} frames...'})
|
|
201
|
+
|
|
202
|
+
# Remove frames using streaming
|
|
203
|
+
success = _remove_frames_from_video(cleaner, video_path, frames_to_remove, output_path, use_streaming)
|
|
204
|
+
|
|
205
|
+
if not success:
|
|
206
|
+
raise RuntimeError("Frame removal failed")
|
|
207
|
+
|
|
208
|
+
self.update_state(state='PROGRESS', meta={'progress': 90, 'message': 'Finalizing...'})
|
|
209
|
+
|
|
210
|
+
_verify_frame_removal_output(output_path)
|
|
211
|
+
|
|
212
|
+
result = {
|
|
213
|
+
'status': 'completed',
|
|
214
|
+
'output_path': str(output_path),
|
|
215
|
+
'frames_removed': len(frames_to_remove),
|
|
216
|
+
'selection_method': selection_method,
|
|
217
|
+
'detection_engine': detection_engine,
|
|
218
|
+
'use_streaming': use_streaming,
|
|
219
|
+
'output_size': output_path.stat().st_size,
|
|
220
|
+
'message': f'Successfully removed {len(frames_to_remove)} frames'
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
logger.info(f"Frame removal completed for video {video_id}: {len(frames_to_remove)} frames removed")
|
|
224
|
+
return result
|
|
225
|
+
|
|
226
|
+
except Exception as e:
|
|
227
|
+
logger.error(f"Frame removal failed for video {video_id}: {e}")
|
|
228
|
+
self.update_state(
|
|
229
|
+
state='FAILURE',
|
|
230
|
+
meta={'error': str(e), 'message': f'Frame removal failed: {e}'}
|
|
231
|
+
)
|
|
232
|
+
raise
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
@shared_task(bind=True)
|
|
236
|
+
def reprocess_video_task(self, video_id: int):
|
|
237
|
+
"""
|
|
238
|
+
Reprocess video with updated settings.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
video_id: ID of the VideoFile to reprocess
|
|
242
|
+
"""
|
|
243
|
+
try:
|
|
244
|
+
self.update_state(state='PROGRESS', meta={'progress': 0, 'message': 'Starting video reprocessing...'})
|
|
245
|
+
|
|
246
|
+
# Import models here to avoid circular imports
|
|
247
|
+
from endoreg_db.models import VideoFile
|
|
248
|
+
from lx_anonymizer.frame_cleaner import FrameCleaner
|
|
249
|
+
|
|
250
|
+
# Get video file
|
|
251
|
+
video = get_object_or_404(VideoFile, pk=video_id)
|
|
252
|
+
video_path = Path(video.file.path)
|
|
253
|
+
|
|
254
|
+
if not video_path.exists():
|
|
255
|
+
raise FileNotFoundError(f"Video file not found: {video_path}")
|
|
256
|
+
|
|
257
|
+
self.update_state(state='PROGRESS', meta={'progress': 20, 'message': 'Initializing FrameCleaner...'})
|
|
258
|
+
|
|
259
|
+
# Initialize FrameCleaner with optimal settings
|
|
260
|
+
cleaner = FrameCleaner()
|
|
261
|
+
|
|
262
|
+
# Create output path
|
|
263
|
+
output_dir = video_path.parent / "processed"
|
|
264
|
+
output_dir.mkdir(exist_ok=True)
|
|
265
|
+
output_path = output_dir / f"{video_path.stem}_reprocessed{video_path.suffix}"
|
|
266
|
+
|
|
267
|
+
self.update_state(state='PROGRESS', meta={'progress': 30, 'message': 'Analyzing video...'})
|
|
268
|
+
|
|
269
|
+
# Perform full analysis
|
|
270
|
+
analysis_result = cleaner.analyze_video_sensitivity()
|
|
271
|
+
|
|
272
|
+
self.update_state(state='PROGRESS', meta={'progress': 60, 'message': f'Processing with {analysis_result["recommended_method"]}...'})
|
|
273
|
+
|
|
274
|
+
# Process based on analysis recommendation
|
|
275
|
+
if analysis_result["recommended_method"] == "masking":
|
|
276
|
+
# Apply device-specific masking
|
|
277
|
+
mask_config = cleaner._load_mask("olympus_cv_1500") # Default device
|
|
278
|
+
success = cleaner._mask_video_streaming(
|
|
279
|
+
video_path, mask_config, output_path, use_named_pipe=True
|
|
280
|
+
)
|
|
281
|
+
else:
|
|
282
|
+
# Remove sensitive frames
|
|
283
|
+
sensitive_frames = analysis_result.get("sensitive_frame_list", [])
|
|
284
|
+
# Filter out string elements like "...truncated"
|
|
285
|
+
sensitive_frames = [f for f in sensitive_frames if isinstance(f, int)]
|
|
286
|
+
|
|
287
|
+
success = cleaner.remove_frames_from_video_streaming(
|
|
288
|
+
video_path, sensitive_frames, output_path, use_named_pipe=True
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
if not success:
|
|
292
|
+
raise RuntimeError("Video reprocessing failed")
|
|
293
|
+
|
|
294
|
+
self.update_state(state='PROGRESS', meta={'progress': 90, 'message': 'Finalizing...'})
|
|
295
|
+
|
|
296
|
+
# Update video metadata with analysis results
|
|
297
|
+
if hasattr(video, 'sensitive_frame_count'):
|
|
298
|
+
video.sensitive_frame_count = analysis_result.get('sensitive_frames', 0)
|
|
299
|
+
if hasattr(video, 'total_frames'):
|
|
300
|
+
video.total_frames = analysis_result.get('total_frames', 0)
|
|
301
|
+
if hasattr(video, 'sensitive_ratio'):
|
|
302
|
+
video.sensitive_ratio = analysis_result.get('sensitivity_ratio', 0.0)
|
|
303
|
+
|
|
304
|
+
try:
|
|
305
|
+
video.save()
|
|
306
|
+
except Exception as e:
|
|
307
|
+
logger.warning(f"Could not save video metadata: {e}")
|
|
308
|
+
|
|
309
|
+
result = {
|
|
310
|
+
'status': 'completed',
|
|
311
|
+
'output_path': str(output_path),
|
|
312
|
+
'analysis_result': analysis_result,
|
|
313
|
+
'processing_method': analysis_result["recommended_method"],
|
|
314
|
+
'output_size': output_path.stat().st_size if output_path.exists() else 0,
|
|
315
|
+
'message': 'Video reprocessing completed successfully'
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
logger.info(f"Video reprocessing completed for video {video_id}")
|
|
319
|
+
return result
|
|
320
|
+
|
|
321
|
+
except Exception as e:
|
|
322
|
+
logger.error(f"Video reprocessing failed for video {video_id}: {e}")
|
|
323
|
+
self.update_state(
|
|
324
|
+
state='FAILURE',
|
|
325
|
+
meta={'error': str(e), 'message': f'Video reprocessing failed: {e}'}
|
|
326
|
+
)
|
|
327
|
+
raise
|