endoreg-db 0.8.9.32__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/__init__.py +0 -0
- endoreg_db/_version.py +34 -0
- endoreg_db/admin.py +97 -0
- endoreg_db/api/serializers/finding_descriptions.py +0 -0
- endoreg_db/api/views/finding_descriptions.py +0 -0
- endoreg_db/api_urls.py +4 -0
- endoreg_db/apps.py +17 -0
- endoreg_db/assets/dummy_model.ckpt +1 -0
- endoreg_db/authz/auth.py +78 -0
- endoreg_db/authz/backends.py +168 -0
- endoreg_db/authz/management/commands/list_routes.py +20 -0
- endoreg_db/authz/middleware.py +84 -0
- endoreg_db/authz/permissions.py +138 -0
- endoreg_db/authz/policy.py +224 -0
- endoreg_db/authz/settings.py +64 -0
- endoreg_db/authz/views_auth.py +70 -0
- endoreg_db/codemods/readme.md +88 -0
- endoreg_db/codemods/rename_datetime_fields.py +99 -0
- endoreg_db/config/__init__.py +0 -0
- endoreg_db/config/env.py +106 -0
- endoreg_db/config/settings/__init__.py +6 -0
- endoreg_db/config/settings/base.py +148 -0
- endoreg_db/config/settings/case_gen.py +32 -0
- endoreg_db/config/settings/dev.py +108 -0
- endoreg_db/config/settings/keycloak.py +177 -0
- endoreg_db/config/settings/prod.py +66 -0
- endoreg_db/config/settings/test.py +72 -0
- endoreg_db/data/__init__.py +135 -0
- endoreg_db/data/ai_model/data.yaml +7 -0
- endoreg_db/data/ai_model_label/label/data.yaml +88 -0
- endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
- endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
- endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
- endoreg_db/data/ai_model_label/label-type/data.yaml +7 -0
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
- endoreg_db/data/ai_model_type/data.yaml +7 -0
- endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
- endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
- endoreg_db/data/case_template/rule/00_patient_lab_sample_add_default_value.yaml +167 -0
- endoreg_db/data/case_template/rule/01_patient-set-age.yaml +8 -0
- endoreg_db/data/case_template/rule/01_patient-set-gender.yaml +9 -0
- endoreg_db/data/case_template/rule/11_create_patient_lab_sample.yaml +23 -0
- endoreg_db/data/case_template/rule/12_create-patient_medication-anticoagulation.yaml +19 -0
- endoreg_db/data/case_template/rule/13_create-patient_medication_schedule-anticoagulation.yaml +19 -0
- endoreg_db/data/case_template/rule/19_create_patient.yaml +17 -0
- endoreg_db/data/case_template/rule_type/base_types.yaml +35 -0
- endoreg_db/data/case_template/rule_value/.init +0 -0
- endoreg_db/data/case_template/rule_value_type/base_types.yaml +59 -0
- endoreg_db/data/case_template/template/base.yaml +8 -0
- endoreg_db/data/case_template/template_type/pre_endoscopy.yaml +3 -0
- endoreg_db/data/case_template/tmp/_rule_value +13 -0
- endoreg_db/data/case_template/tmp/rule/01_atrial_fibrillation.yaml +21 -0
- endoreg_db/data/case_template/tmp/rule/02_create_object.yaml +10 -0
- endoreg_db/data/case_template/tmp/template/atrial_fibrillation_low_risk.yaml +7 -0
- endoreg_db/data/center/data.yaml +99 -0
- endoreg_db/data/center_resource/green_endoscopy_dashboard_CenterResource.yaml +144 -0
- endoreg_db/data/center_shift/ukw.yaml +9 -0
- endoreg_db/data/center_waste/green_endoscopy_dashboard_CenterWaste.yaml +48 -0
- endoreg_db/data/contraindication/bleeding.yaml +11 -0
- endoreg_db/data/db_summary.csv +58 -0
- endoreg_db/data/db_summary.xlsx +0 -0
- endoreg_db/data/disease/cardiovascular.yaml +37 -0
- endoreg_db/data/disease/hepatology.yaml +5 -0
- endoreg_db/data/disease/misc.yaml +5 -0
- endoreg_db/data/disease/renal.yaml +5 -0
- endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +6 -0
- endoreg_db/data/disease_classification/coronary_vessel_disease.yaml +6 -0
- endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +41 -0
- endoreg_db/data/disease_classification_choice/coronary_vessel_disease.yaml +20 -0
- endoreg_db/data/distribution/date/patient.yaml +7 -0
- endoreg_db/data/distribution/multiple_categorical/.init +0 -0
- endoreg_db/data/distribution/numeric/data.yaml +14 -0
- endoreg_db/data/distribution/single_categorical/patient.yaml +7 -0
- endoreg_db/data/emission_factor/green_endoscopy_dashboard_EmissionFactor.yaml +132 -0
- endoreg_db/data/endoscope/data.yaml +93 -0
- endoreg_db/data/endoscope_type/data.yaml +11 -0
- endoreg_db/data/endoscopy_processor/data.yaml +50 -0
- endoreg_db/data/event/cardiology.yaml +15 -0
- endoreg_db/data/event/neurology.yaml +14 -0
- endoreg_db/data/event/surgery.yaml +13 -0
- endoreg_db/data/event/thrombembolism.yaml +20 -0
- endoreg_db/data/event_classification/data.yaml +4 -0
- endoreg_db/data/event_classification_choice/data.yaml +9 -0
- endoreg_db/data/examination/examinations/data.yaml +172 -0
- endoreg_db/data/examination/time/data.yaml +48 -0
- endoreg_db/data/examination/time-type/data.yaml +5 -0
- endoreg_db/data/examination/type/data.yaml +17 -0
- endoreg_db/data/examination_indication/endoscopy.yaml +359 -0
- endoreg_db/data/examination_indication_classification/endoscopy.yaml +90 -0
- endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +97 -0
- endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
- endoreg_db/data/finding/00_generic.yaml +35 -0
- endoreg_db/data/finding/00_generic_complication.yaml +9 -0
- endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
- endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
- endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
- endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
- endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
- endoreg_db/data/finding_classification/00_generic.yaml +44 -0
- endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
- endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
- endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
- endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
- endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
- endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
- endoreg_db/data/finding_classification_choice/00_generic.yaml +15 -0
- endoreg_db/data/finding_classification_choice/00_generic_baseline.yaml +23 -0
- endoreg_db/data/finding_classification_choice/00_generic_complication.yaml +15 -0
- endoreg_db/data/finding_classification_choice/00_generic_histology.yaml +21 -0
- endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_bowel_preparation.yaml +49 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_generic.yaml +19 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_histology.yaml +20 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_location.yaml +248 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_morphology.yaml +75 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
- endoreg_db/data/finding_classification_type/00_generic.yaml +53 -0
- endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
- endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
- endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
- endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
- endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
- endoreg_db/data/finding_type/data.yaml +39 -0
- endoreg_db/data/gender/data.yaml +42 -0
- endoreg_db/data/information_source/annotation.yaml +6 -0
- endoreg_db/data/information_source/data.yaml +30 -0
- endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
- endoreg_db/data/information_source/medication.yaml +6 -0
- endoreg_db/data/information_source/prediction.yaml +7 -0
- endoreg_db/data/information_source_type/data.yaml +8 -0
- endoreg_db/data/lab_value/cardiac_enzymes.yaml +37 -0
- endoreg_db/data/lab_value/coagulation.yaml +54 -0
- endoreg_db/data/lab_value/electrolytes.yaml +228 -0
- endoreg_db/data/lab_value/gastrointestinal_function.yaml +133 -0
- endoreg_db/data/lab_value/hematology.yaml +184 -0
- endoreg_db/data/lab_value/hormones.yaml +59 -0
- endoreg_db/data/lab_value/lipids.yaml +53 -0
- endoreg_db/data/lab_value/misc.yaml +76 -0
- endoreg_db/data/lab_value/renal_function.yaml +12 -0
- endoreg_db/data/log_type/data.yaml +57 -0
- endoreg_db/data/lx_client_tag/base.yaml +54 -0
- endoreg_db/data/lx_client_type/base.yaml +30 -0
- endoreg_db/data/lx_permission/base.yaml +24 -0
- endoreg_db/data/lx_permission/endoreg.yaml +52 -0
- endoreg_db/data/material/material.yaml +91 -0
- endoreg_db/data/medication/anticoagulation.yaml +65 -0
- endoreg_db/data/medication/tah.yaml +70 -0
- endoreg_db/data/medication_indication/anticoagulation.yaml +115 -0
- endoreg_db/data/medication_indication_type/data.yaml +11 -0
- endoreg_db/data/medication_indication_type/thrombembolism.yaml +41 -0
- endoreg_db/data/medication_intake_time/base.yaml +31 -0
- endoreg_db/data/medication_schedule/apixaban.yaml +95 -0
- endoreg_db/data/medication_schedule/ass.yaml +12 -0
- endoreg_db/data/medication_schedule/enoxaparin.yaml +26 -0
- endoreg_db/data/names_first/first_names.yaml +54 -0
- endoreg_db/data/names_last/last_names.yaml +51 -0
- endoreg_db/data/network_device/data.yaml +59 -0
- endoreg_db/data/network_device_type/data.yaml +12 -0
- endoreg_db/data/organ/data.yaml +29 -0
- endoreg_db/data/patient_lab_sample_type/generic.yaml +6 -0
- endoreg_db/data/pdf_type/data.yaml +46 -0
- endoreg_db/data/product/green_endoscopy_dashboard_Product.yaml +66 -0
- endoreg_db/data/product_group/green_endoscopy_dashboard_ProductGroup.yaml +33 -0
- endoreg_db/data/product_material/green_endoscopy_dashboard_ProductMaterial.yaml +308 -0
- endoreg_db/data/product_weight/green_endoscopy_dashboard_ProductWeight.yaml +88 -0
- endoreg_db/data/profession/data.yaml +70 -0
- endoreg_db/data/qualification/endoscopy.yaml +36 -0
- endoreg_db/data/qualification/m2.yaml +39 -0
- endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
- endoreg_db/data/qualification/sonography.yaml +36 -0
- endoreg_db/data/qualification_type/base.yaml +29 -0
- endoreg_db/data/reference_product/green_endoscopy_dashboard_ReferenceProduct.yaml +55 -0
- endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
- endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +30 -0
- endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +24 -0
- endoreg_db/data/requirement/01_patient_data.yaml +93 -0
- endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
- endoreg_db/data/requirement/old/colonoscopy_baseline_austria.yaml +45 -0
- endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
- endoreg_db/data/requirement/old/disease_cardiovascular.yaml +79 -0
- endoreg_db/data/requirement/old/disease_classification_choice_cardiovascular.yaml +41 -0
- endoreg_db/data/requirement/old/disease_hepatology.yaml +12 -0
- endoreg_db/data/requirement/old/disease_misc.yaml +12 -0
- endoreg_db/data/requirement/old/disease_renal.yaml +96 -0
- endoreg_db/data/requirement/old/endoscopy_bleeding_risk.yaml +59 -0
- endoreg_db/data/requirement/old/event_cardiology.yaml +251 -0
- endoreg_db/data/requirement/old/event_requirements.yaml +145 -0
- endoreg_db/data/requirement/old/finding_colon_polyp.yaml +50 -0
- endoreg_db/data/requirement/old/gender.yaml +0 -0
- endoreg_db/data/requirement/old/lab_value.yaml +441 -0
- endoreg_db/data/requirement/old/medication.yaml +93 -0
- endoreg_db/data/requirement_operator/_old/age.yaml +13 -0
- endoreg_db/data/requirement_operator/_old/lab_operators.yaml +129 -0
- endoreg_db/data/requirement_operator/_old/model_operators.yaml +96 -0
- endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +65 -0
- endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
- endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
- endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
- endoreg_db/data/requirement_set/_old_ +109 -0
- endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
- endoreg_db/data/requirement_set_type/data.yaml +41 -0
- endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
- endoreg_db/data/resource/green_endoscopy_dashboard_Resource.yaml +15 -0
- endoreg_db/data/risk/bleeding.yaml +26 -0
- endoreg_db/data/risk/thrombosis.yaml +37 -0
- endoreg_db/data/risk_type/data.yaml +27 -0
- endoreg_db/data/setup_config.yaml +38 -0
- endoreg_db/data/shift/endoscopy.yaml +21 -0
- endoreg_db/data/shift/m2.yaml +0 -0
- endoreg_db/data/shift_type/base.yaml +35 -0
- endoreg_db/data/tag/requirement_set_tags.yaml +32 -0
- endoreg_db/data/tmp/chronic_kidney_disease.yaml +0 -0
- endoreg_db/data/tmp/congestive_heart_failure.yaml +0 -0
- endoreg_db/data/transport_route/green_endoscopy_dashboard_TransportRoute.yaml +12 -0
- endoreg_db/data/unit/concentration.yaml +115 -0
- endoreg_db/data/unit/data.yaml +17 -0
- endoreg_db/data/unit/length.yaml +31 -0
- endoreg_db/data/unit/misc.yaml +20 -0
- endoreg_db/data/unit/rate.yaml +6 -0
- endoreg_db/data/unit/time.yaml +48 -0
- endoreg_db/data/unit/volume.yaml +35 -0
- endoreg_db/data/unit/weight.yaml +38 -0
- endoreg_db/data/waste/data.yaml +12 -0
- endoreg_db/exceptions.py +24 -0
- endoreg_db/export/frames/export.py +6 -0
- endoreg_db/export/frames/export_frames_with_labels.py +616 -0
- endoreg_db/factories/__init__.py +0 -0
- endoreg_db/forms/__init__.py +4 -0
- endoreg_db/forms/examination_form.py +12 -0
- endoreg_db/forms/patient_finding_intervention_form.py +40 -0
- endoreg_db/forms/patient_form.py +23 -0
- endoreg_db/forms/questionnaires/__init__.py +1 -0
- endoreg_db/forms/questionnaires/tto_questionnaire.py +23 -0
- endoreg_db/forms/settings/__init__.py +11 -0
- endoreg_db/forms/unit.py +7 -0
- endoreg_db/helpers/__init__.py +0 -0
- endoreg_db/helpers/count_db.py +48 -0
- endoreg_db/helpers/data_loader.py +280 -0
- endoreg_db/helpers/default_objects.py +414 -0
- endoreg_db/helpers/download_segmentation_model.py +32 -0
- endoreg_db/helpers/interact.py +1 -0
- endoreg_db/helpers/test_video_helper.py +127 -0
- endoreg_db/import_files/__init__.py +27 -0
- endoreg_db/import_files/context/__init__.py +7 -0
- endoreg_db/import_files/context/default_sensitive_meta.py +83 -0
- endoreg_db/import_files/context/ensure_center.py +17 -0
- endoreg_db/import_files/context/file_lock.py +66 -0
- endoreg_db/import_files/context/import_context.py +42 -0
- endoreg_db/import_files/context/validate_directories.py +57 -0
- endoreg_db/import_files/file_storage/__init__.py +15 -0
- endoreg_db/import_files/file_storage/create_report_file.py +99 -0
- endoreg_db/import_files/file_storage/create_video_file.py +104 -0
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +42 -0
- endoreg_db/import_files/file_storage/state_management.py +463 -0
- endoreg_db/import_files/file_storage/storage.py +42 -0
- endoreg_db/import_files/import_service.md +26 -0
- endoreg_db/import_files/processing/__init__.py +11 -0
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +99 -0
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
- endoreg_db/import_files/pseudonymization/__init__.py +0 -0
- endoreg_db/import_files/pseudonymization/fake.py +52 -0
- endoreg_db/import_files/pseudonymization/k_anonymity.py +181 -0
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +139 -0
- endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
- endoreg_db/import_files/report_import_service.py +147 -0
- endoreg_db/import_files/video_import_service.py +154 -0
- endoreg_db/logger_conf.py +156 -0
- endoreg_db/management/__init__.py +1 -0
- endoreg_db/management/commands/__init__.py +1 -0
- endoreg_db/management/commands/anonymize_video.py +0 -0
- endoreg_db/management/commands/check_auth.py +132 -0
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +177 -0
- endoreg_db/management/commands/create_multilabel_model_meta.py +419 -0
- endoreg_db/management/commands/export_frame_annot.py +196 -0
- endoreg_db/management/commands/fix_missing_patient_data.py +206 -0
- endoreg_db/management/commands/fix_video_paths.py +186 -0
- endoreg_db/management/commands/import_report.py +361 -0
- endoreg_db/management/commands/list_routes.py +20 -0
- endoreg_db/management/commands/load_ai_model_data.py +83 -0
- endoreg_db/management/commands/load_ai_model_label_data.py +60 -0
- endoreg_db/management/commands/load_base_db_data.py +63 -0
- endoreg_db/management/commands/load_center_data.py +68 -0
- endoreg_db/management/commands/load_contraindication_data.py +39 -0
- endoreg_db/management/commands/load_disease_classification_choices_data.py +38 -0
- endoreg_db/management/commands/load_disease_classification_data.py +38 -0
- endoreg_db/management/commands/load_disease_data.py +59 -0
- endoreg_db/management/commands/load_distribution_data.py +63 -0
- endoreg_db/management/commands/load_endoscope_data.py +58 -0
- endoreg_db/management/commands/load_event_data.py +39 -0
- endoreg_db/management/commands/load_examination_data.py +78 -0
- endoreg_db/management/commands/load_examination_indication_data.py +85 -0
- endoreg_db/management/commands/load_finding_data.py +115 -0
- endoreg_db/management/commands/load_gender_data.py +37 -0
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +142 -0
- endoreg_db/management/commands/load_information_source.py +46 -0
- endoreg_db/management/commands/load_lab_value_data.py +52 -0
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_medication_data.py +104 -0
- endoreg_db/management/commands/load_name_data.py +36 -0
- endoreg_db/management/commands/load_organ_data.py +39 -0
- endoreg_db/management/commands/load_pdf_type_data.py +58 -0
- endoreg_db/management/commands/load_profession_data.py +40 -0
- endoreg_db/management/commands/load_qualification_data.py +56 -0
- endoreg_db/management/commands/load_report_reader_flag_data.py +40 -0
- endoreg_db/management/commands/load_requirement_data.py +207 -0
- endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
- endoreg_db/management/commands/load_risk_data.py +57 -0
- endoreg_db/management/commands/load_shift_data.py +57 -0
- endoreg_db/management/commands/load_tag_data.py +54 -0
- endoreg_db/management/commands/load_unit_data.py +40 -0
- endoreg_db/management/commands/load_user_groups.py +26 -0
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +70 -0
- endoreg_db/management/commands/setup_endoreg_db.py +459 -0
- endoreg_db/management/commands/start_filewatcher.py +115 -0
- endoreg_db/management/commands/storage_management.py +622 -0
- endoreg_db/management/commands/summarize_db_content.py +280 -0
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +189 -0
- endoreg_db/management/commands/video_validation.py +20 -0
- endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
- endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
- endoreg_db/mermaid/binary_classification_annotation.md +50 -0
- endoreg_db/mermaid/classification.md +8 -0
- endoreg_db/mermaid/examination.md +8 -0
- endoreg_db/mermaid/findings.md +7 -0
- endoreg_db/mermaid/image_classification.md +28 -0
- endoreg_db/mermaid/interventions.md +8 -0
- endoreg_db/mermaid/morphology.md +8 -0
- endoreg_db/mermaid/patient_creation.md +14 -0
- endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
- endoreg_db/migrations/0001_initial.py +1953 -0
- endoreg_db/migrations/__init__.py +0 -0
- endoreg_db/models/__init__.py +322 -0
- endoreg_db/models/administration/__init__.py +95 -0
- endoreg_db/models/administration/ai/__init__.py +9 -0
- endoreg_db/models/administration/ai/active_model.py +35 -0
- endoreg_db/models/administration/ai/ai_model.py +180 -0
- endoreg_db/models/administration/ai/model_type.py +42 -0
- endoreg_db/models/administration/case/__init__.py +5 -0
- endoreg_db/models/administration/case/case.py +114 -0
- endoreg_db/models/administration/case/case_template/__init__.py +3 -0
- endoreg_db/models/administration/case/case_template/case_template.py +3 -0
- endoreg_db/models/administration/case/case_template/case_template_rule.py +3 -0
- endoreg_db/models/administration/case/case_template/case_template_rule_value.py +3 -0
- endoreg_db/models/administration/case/case_template/case_template_type.py +3 -0
- endoreg_db/models/administration/center/__init__.py +13 -0
- endoreg_db/models/administration/center/center.py +85 -0
- endoreg_db/models/administration/center/center_product.py +67 -0
- endoreg_db/models/administration/center/center_resource.py +69 -0
- endoreg_db/models/administration/center/center_shift.py +94 -0
- endoreg_db/models/administration/center/center_waste.py +42 -0
- endoreg_db/models/administration/person/__init__.py +26 -0
- endoreg_db/models/administration/person/employee/__init__.py +3 -0
- endoreg_db/models/administration/person/employee/employee.py +40 -0
- endoreg_db/models/administration/person/employee/employee_qualification.py +44 -0
- endoreg_db/models/administration/person/employee/employee_type.py +50 -0
- endoreg_db/models/administration/person/examiner/__init__.py +4 -0
- endoreg_db/models/administration/person/examiner/examiner.py +64 -0
- endoreg_db/models/administration/person/names/__init__.py +0 -0
- endoreg_db/models/administration/person/names/first_name.py +20 -0
- endoreg_db/models/administration/person/names/last_name.py +20 -0
- endoreg_db/models/administration/person/patient/__init__.py +7 -0
- endoreg_db/models/administration/person/patient/patient.py +488 -0
- endoreg_db/models/administration/person/patient/patient_external_id.py +36 -0
- endoreg_db/models/administration/person/person.py +35 -0
- endoreg_db/models/administration/person/profession/__init__.py +28 -0
- endoreg_db/models/administration/person/user/__init__.py +5 -0
- endoreg_db/models/administration/person/user/portal_user_information.py +41 -0
- endoreg_db/models/administration/product/__init__.py +15 -0
- endoreg_db/models/administration/product/product.py +106 -0
- endoreg_db/models/administration/product/product_group.py +41 -0
- endoreg_db/models/administration/product/product_material.py +60 -0
- endoreg_db/models/administration/product/product_weight.py +51 -0
- endoreg_db/models/administration/product/reference_product.py +147 -0
- endoreg_db/models/administration/qualification/__init__.py +7 -0
- endoreg_db/models/administration/qualification/qualification.py +43 -0
- endoreg_db/models/administration/qualification/qualification_type.py +39 -0
- endoreg_db/models/administration/shift/__init__.py +9 -0
- endoreg_db/models/administration/shift/scheduled_days.py +72 -0
- endoreg_db/models/administration/shift/shift.py +57 -0
- endoreg_db/models/administration/shift/shift_type.py +108 -0
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +23 -0
- endoreg_db/models/label/annotation/__init__.py +12 -0
- endoreg_db/models/label/annotation/image_classification.py +85 -0
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +61 -0
- endoreg_db/models/label/label.py +91 -0
- endoreg_db/models/label/label_set.py +68 -0
- endoreg_db/models/label/label_type.py +29 -0
- endoreg_db/models/label/label_video_segment/__init__.py +3 -0
- endoreg_db/models/label/label_video_segment/_create_from_video.py +42 -0
- endoreg_db/models/label/label_video_segment/label_video_segment.py +611 -0
- endoreg_db/models/label/video_segmentation_label.py +35 -0
- endoreg_db/models/label/video_segmentation_labelset.py +28 -0
- endoreg_db/models/media/__init__.py +23 -0
- endoreg_db/models/media/frame/__init__.py +3 -0
- endoreg_db/models/media/frame/frame.py +137 -0
- endoreg_db/models/media/pdf/__init__.py +12 -0
- endoreg_db/models/media/pdf/raw_pdf.py +764 -0
- endoreg_db/models/media/pdf/report_file.py +162 -0
- endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +85 -0
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +46 -0
- endoreg_db/models/media/video/__init__.py +9 -0
- endoreg_db/models/media/video/create_from_file.py +402 -0
- endoreg_db/models/media/video/pipe_1.py +258 -0
- endoreg_db/models/media/video/pipe_2.py +129 -0
- endoreg_db/models/media/video/video_file.py +907 -0
- endoreg_db/models/media/video/video_file_ai.py +828 -0
- endoreg_db/models/media/video/video_file_anonymize.py +524 -0
- endoreg_db/models/media/video/video_file_frames/__init__.py +49 -0
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +25 -0
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +126 -0
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +233 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +36 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +13 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +24 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +40 -0
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +44 -0
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +30 -0
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +205 -0
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +228 -0
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +107 -0
- endoreg_db/models/media/video/video_file_io.py +272 -0
- endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +58 -0
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +62 -0
- endoreg_db/models/media/video/video_file_meta/get_fps.py +183 -0
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +198 -0
- endoreg_db/models/media/video/video_file_meta/text_meta.py +178 -0
- endoreg_db/models/media/video/video_file_meta/video_meta.py +105 -0
- endoreg_db/models/media/video/video_file_segments.py +317 -0
- endoreg_db/models/media/video/video_metadata.py +67 -0
- endoreg_db/models/media/video/video_processing.py +192 -0
- endoreg_db/models/medical/__init__.py +136 -0
- endoreg_db/models/medical/contraindication/README.md +1 -0
- endoreg_db/models/medical/contraindication/__init__.py +29 -0
- endoreg_db/models/medical/disease.py +174 -0
- endoreg_db/models/medical/event.py +154 -0
- endoreg_db/models/medical/examination/__init__.py +20 -0
- endoreg_db/models/medical/examination/examination.py +183 -0
- endoreg_db/models/medical/examination/examination_indication.py +229 -0
- endoreg_db/models/medical/examination/examination_time.py +68 -0
- endoreg_db/models/medical/examination/examination_time_type.py +44 -0
- endoreg_db/models/medical/examination/examination_type.py +47 -0
- endoreg_db/models/medical/finding/__init__.py +20 -0
- endoreg_db/models/medical/finding/finding.py +113 -0
- endoreg_db/models/medical/finding/finding_classification.py +131 -0
- endoreg_db/models/medical/finding/finding_intervention.py +68 -0
- endoreg_db/models/medical/finding/finding_type.py +38 -0
- endoreg_db/models/medical/hardware/__init__.py +8 -0
- endoreg_db/models/medical/hardware/endoscope.py +77 -0
- endoreg_db/models/medical/hardware/endoscopy_processor.py +182 -0
- endoreg_db/models/medical/laboratory/__init__.py +5 -0
- endoreg_db/models/medical/laboratory/lab_value.py +490 -0
- endoreg_db/models/medical/medication/__init__.py +23 -0
- endoreg_db/models/medical/medication/medication.py +45 -0
- endoreg_db/models/medical/medication/medication_indication.py +78 -0
- endoreg_db/models/medical/medication/medication_indication_type.py +58 -0
- endoreg_db/models/medical/medication/medication_intake_time.py +58 -0
- endoreg_db/models/medical/medication/medication_schedule.py +58 -0
- endoreg_db/models/medical/organ/__init__.py +38 -0
- endoreg_db/models/medical/patient/__init__.py +48 -0
- endoreg_db/models/medical/patient/medication_examples.py +56 -0
- endoreg_db/models/medical/patient/patient_disease.py +72 -0
- endoreg_db/models/medical/patient/patient_event.py +80 -0
- endoreg_db/models/medical/patient/patient_examination.py +280 -0
- endoreg_db/models/medical/patient/patient_examination_indication.py +57 -0
- endoreg_db/models/medical/patient/patient_finding.py +416 -0
- endoreg_db/models/medical/patient/patient_finding_classification.py +231 -0
- endoreg_db/models/medical/patient/patient_finding_intervention.py +37 -0
- endoreg_db/models/medical/patient/patient_lab_sample.py +157 -0
- endoreg_db/models/medical/patient/patient_lab_value.py +247 -0
- endoreg_db/models/medical/patient/patient_medication.py +111 -0
- endoreg_db/models/medical/patient/patient_medication_schedule.py +152 -0
- endoreg_db/models/medical/risk/__init__.py +7 -0
- endoreg_db/models/medical/risk/risk.py +73 -0
- endoreg_db/models/medical/risk/risk_type.py +54 -0
- endoreg_db/models/metadata/__init__.py +19 -0
- endoreg_db/models/metadata/model_meta.py +266 -0
- endoreg_db/models/metadata/model_meta_logic.py +485 -0
- endoreg_db/models/metadata/pdf_meta.py +96 -0
- endoreg_db/models/metadata/sensitive_meta.py +345 -0
- endoreg_db/models/metadata/sensitive_meta_logic.py +1161 -0
- endoreg_db/models/metadata/video_meta.py +459 -0
- endoreg_db/models/metadata/video_prediction_logic.py +232 -0
- endoreg_db/models/metadata/video_prediction_meta.py +319 -0
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +40 -0
- endoreg_db/models/other/distribution/__init__.py +46 -0
- endoreg_db/models/other/distribution/base_value_distribution.py +22 -0
- endoreg_db/models/other/distribution/date_value_distribution.py +163 -0
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +50 -0
- endoreg_db/models/other/distribution/numeric_value_distribution.py +211 -0
- endoreg_db/models/other/distribution/single_categorical_value_distribution.py +23 -0
- endoreg_db/models/other/emission/__init__.py +5 -0
- endoreg_db/models/other/emission/emission_factor.py +110 -0
- endoreg_db/models/other/gender.py +32 -0
- endoreg_db/models/other/information_source.py +190 -0
- endoreg_db/models/other/material.py +34 -0
- endoreg_db/models/other/resource.py +24 -0
- endoreg_db/models/other/tag.py +32 -0
- endoreg_db/models/other/transport_route.py +40 -0
- endoreg_db/models/other/unit.py +40 -0
- endoreg_db/models/other/waste.py +28 -0
- endoreg_db/models/report/__init__.py +0 -0
- endoreg_db/models/report/images.py +0 -0
- endoreg_db/models/report/report.py +5 -0
- endoreg_db/models/requirement/__init__.py +11 -0
- endoreg_db/models/requirement/requirement.py +792 -0
- endoreg_db/models/requirement/requirement_error.py +84 -0
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
- endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +6 -0
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +137 -0
- endoreg_db/models/requirement/requirement_operator.py +187 -0
- endoreg_db/models/requirement/requirement_set.py +327 -0
- endoreg_db/models/state/__init__.py +13 -0
- endoreg_db/models/state/abstract.py +11 -0
- endoreg_db/models/state/anonymization.py +30 -0
- endoreg_db/models/state/audit_ledger.py +155 -0
- endoreg_db/models/state/label_video_segment.py +31 -0
- endoreg_db/models/state/processing_history/__init__.py +3 -0
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +219 -0
- endoreg_db/models/state/sensitive_meta.py +50 -0
- endoreg_db/models/state/video.py +251 -0
- endoreg_db/models/upload_job.py +100 -0
- endoreg_db/models/utils.py +138 -0
- endoreg_db/queries/__init__.py +3 -0
- endoreg_db/queries/annotations/__init__.py +1 -0
- endoreg_db/queries/annotations/legacy.py +169 -0
- endoreg_db/queries/sanity/__init_.py +0 -0
- endoreg_db/root_urls.py +27 -0
- endoreg_db/schemas/__init__.py +0 -0
- endoreg_db/schemas/examination_evaluation.py +30 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +861 -0
- endoreg_db/serializers/__init__.py +104 -0
- endoreg_db/serializers/administration/__init__.py +13 -0
- endoreg_db/serializers/administration/ai/__init__.py +9 -0
- endoreg_db/serializers/administration/ai/active_model.py +12 -0
- endoreg_db/serializers/administration/ai/ai_model.py +20 -0
- endoreg_db/serializers/administration/ai/model_type.py +12 -0
- endoreg_db/serializers/administration/center.py +14 -0
- endoreg_db/serializers/administration/gender.py +11 -0
- endoreg_db/serializers/anonymization.py +77 -0
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -0
- endoreg_db/serializers/examination/__init__.py +10 -0
- endoreg_db/serializers/examination/base.py +45 -0
- endoreg_db/serializers/examination/dropdown.py +20 -0
- endoreg_db/serializers/examination_serializer.py +9 -0
- endoreg_db/serializers/finding/__init__.py +5 -0
- endoreg_db/serializers/finding/finding.py +61 -0
- endoreg_db/serializers/finding_classification/__init__.py +7 -0
- endoreg_db/serializers/finding_classification/choice.py +19 -0
- endoreg_db/serializers/finding_classification/classification.py +11 -0
- endoreg_db/serializers/label_video_segment/__init__.py +9 -0
- endoreg_db/serializers/label_video_segment/image_classification_annotation.py +62 -0
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/label_video_segment/label/label.py +15 -0
- endoreg_db/serializers/label_video_segment/label_video_segment.py +427 -0
- endoreg_db/serializers/meta/__init__.py +13 -0
- endoreg_db/serializers/meta/sensitive_meta_detail.py +122 -0
- endoreg_db/serializers/meta/sensitive_meta_update.py +153 -0
- endoreg_db/serializers/meta/sensitive_meta_verification.py +62 -0
- endoreg_db/serializers/meta/video_meta.py +39 -0
- endoreg_db/serializers/misc/__init__.py +14 -0
- endoreg_db/serializers/misc/file_overview.py +72 -0
- endoreg_db/serializers/misc/sensitive_patient_data.py +144 -0
- endoreg_db/serializers/misc/stats.py +35 -0
- endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
- endoreg_db/serializers/misc/upload_job.py +74 -0
- endoreg_db/serializers/patient/__init__.py +12 -0
- endoreg_db/serializers/patient/patient.py +103 -0
- endoreg_db/serializers/patient/patient_dropdown.py +35 -0
- endoreg_db/serializers/patient_examination/__init__.py +7 -0
- endoreg_db/serializers/patient_examination/patient_examination.py +168 -0
- endoreg_db/serializers/patient_finding/__init__.py +15 -0
- endoreg_db/serializers/patient_finding/patient_finding.py +32 -0
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +47 -0
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +62 -0
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +28 -0
- endoreg_db/serializers/patient_finding/patient_finding_list.py +40 -0
- endoreg_db/serializers/patient_finding/patient_finding_write.py +135 -0
- endoreg_db/serializers/pdf/__init__.py +3 -0
- endoreg_db/serializers/pdf/anony_text_validation.py +101 -0
- endoreg_db/serializers/requirements/requirement_schema.py +20 -0
- endoreg_db/serializers/requirements/requirement_sets.py +99 -0
- endoreg_db/serializers/sensitive_meta_serializer.py +301 -0
- endoreg_db/serializers/video/__init__.py +7 -0
- endoreg_db/serializers/video/video_file.py +283 -0
- endoreg_db/serializers/video/video_file_brief.py +14 -0
- endoreg_db/serializers/video/video_file_detail.py +96 -0
- endoreg_db/serializers/video/video_file_list.py +100 -0
- endoreg_db/serializers/video/video_processing_history.py +172 -0
- endoreg_db/serializers/video_examination.py +198 -0
- endoreg_db/services/__init__.py +5 -0
- endoreg_db/services/anonymization.py +274 -0
- endoreg_db/services/examination_evaluation.py +172 -0
- endoreg_db/services/finding_description_service.py +0 -0
- endoreg_db/services/lookup_service.py +424 -0
- endoreg_db/services/lookup_store.py +266 -0
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/pdf_import.py +0 -0
- endoreg_db/services/polling_coordinator.py +319 -0
- endoreg_db/services/pseudonym_service.py +94 -0
- endoreg_db/services/report_import.py +13 -0
- endoreg_db/services/segment_sync.py +171 -0
- endoreg_db/services/video_import.py +9 -0
- endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
- endoreg_db/templates/admin/start_examination.html +12 -0
- endoreg_db/templates/timeline.html +176 -0
- endoreg_db/urls/__init__.py +56 -0
- endoreg_db/urls/ai.py +14 -0
- endoreg_db/urls/anonymization.py +78 -0
- endoreg_db/urls/auth.py +16 -0
- endoreg_db/urls/classification.py +34 -0
- endoreg_db/urls/examination.py +63 -0
- endoreg_db/urls/media.py +251 -0
- endoreg_db/urls/patient.py +23 -0
- endoreg_db/urls/requirements.py +15 -0
- endoreg_db/urls/root_urls.py +28 -0
- endoreg_db/urls/stats.py +54 -0
- endoreg_db/urls/upload.py +12 -0
- endoreg_db/urls.py +9 -0
- endoreg_db/utils/__init__.py +97 -0
- endoreg_db/utils/ai/__init__.py +9 -0
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +6 -0
- endoreg_db/utils/ai/inference_dataset.py +51 -0
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +270 -0
- endoreg_db/utils/ai/postprocess.py +63 -0
- endoreg_db/utils/ai/predict.py +293 -0
- endoreg_db/utils/ai/preprocess.py +76 -0
- endoreg_db/utils/calc_duration_seconds.py +24 -0
- endoreg_db/utils/case_generator/__init__.py +3 -0
- endoreg_db/utils/case_generator/lab_sample_factory.py +32 -0
- endoreg_db/utils/check_video_files.py +175 -0
- endoreg_db/utils/cropping.py +30 -0
- endoreg_db/utils/dataloader.py +285 -0
- endoreg_db/utils/dates.py +59 -0
- endoreg_db/utils/defaults/set_default_center.py +33 -0
- endoreg_db/utils/env.py +37 -0
- endoreg_db/utils/extract_specific_frames.py +87 -0
- endoreg_db/utils/file_operations.py +70 -0
- endoreg_db/utils/fix_video_path_direct.py +157 -0
- endoreg_db/utils/frame_anonymization_utils.py +463 -0
- endoreg_db/utils/hashs.py +138 -0
- endoreg_db/utils/links/__init__.py +0 -0
- endoreg_db/utils/links/requirement_link.py +237 -0
- endoreg_db/utils/mime_types.py +0 -0
- endoreg_db/utils/names.py +82 -0
- endoreg_db/utils/ocr.py +195 -0
- endoreg_db/utils/operation_log.py +87 -0
- endoreg_db/utils/parse_and_generate_yaml.py +45 -0
- endoreg_db/utils/paths.py +159 -0
- endoreg_db/utils/permissions.py +160 -0
- endoreg_db/utils/pipelines/Readme.md +235 -0
- endoreg_db/utils/pipelines/__init__.py +0 -0
- endoreg_db/utils/pipelines/process_video_dir.py +144 -0
- endoreg_db/utils/product/__init__.py +0 -0
- endoreg_db/utils/product/sum_emissions.py +22 -0
- endoreg_db/utils/product/sum_weights.py +20 -0
- endoreg_db/utils/pydantic_models/__init__.py +5 -0
- endoreg_db/utils/pydantic_models/db_config.py +57 -0
- endoreg_db/utils/requirement_helpers.py +0 -0
- endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +678 -0
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +842 -0
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +114 -0
- endoreg_db/utils/setup_config.py +196 -0
- endoreg_db/utils/storage.py +117 -0
- endoreg_db/utils/translation.py +31 -0
- endoreg_db/utils/uuid.py +5 -0
- endoreg_db/utils/validate_endo_roi.py +33 -0
- endoreg_db/utils/validate_subcategory_dict.py +93 -0
- endoreg_db/utils/validate_video_detailed.py +415 -0
- endoreg_db/utils/video/__init__.py +30 -0
- endoreg_db/utils/video/extract_frames.py +100 -0
- endoreg_db/utils/video/ffmpeg_wrapper.py +996 -0
- endoreg_db/utils/video/names.py +47 -0
- endoreg_db/utils/video/streaming_processor.py +386 -0
- endoreg_db/utils/video/video_splitter.py +105 -0
- endoreg_db/versioning.md +79 -0
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +247 -0
- endoreg_db/views/__init__.py +157 -0
- endoreg_db/views/anonymization/__init__.py +31 -0
- endoreg_db/views/anonymization/media_management.py +486 -0
- endoreg_db/views/anonymization/overview.py +307 -0
- endoreg_db/views/anonymization/validate.py +310 -0
- endoreg_db/views/auth/__init__.py +13 -0
- endoreg_db/views/auth/keycloak.py +146 -0
- endoreg_db/views/examination/__init__.py +30 -0
- endoreg_db/views/examination/examination.py +37 -0
- endoreg_db/views/examination/examination_manifest_cache.py +26 -0
- endoreg_db/views/examination/get_finding_classification_choices.py +62 -0
- endoreg_db/views/examination/get_finding_classifications.py +38 -0
- endoreg_db/views/examination/get_findings.py +39 -0
- endoreg_db/views/examination/get_instruments.py +19 -0
- endoreg_db/views/examination/get_interventions.py +14 -0
- endoreg_db/views/finding/__init__.py +9 -0
- endoreg_db/views/finding/finding.py +116 -0
- endoreg_db/views/finding/get_classifications.py +14 -0
- endoreg_db/views/finding/get_interventions.py +17 -0
- endoreg_db/views/finding_classification/__init__.py +13 -0
- endoreg_db/views/finding_classification/base.py +0 -0
- endoreg_db/views/finding_classification/finding_classification.py +41 -0
- endoreg_db/views/finding_classification/get_classification_choices.py +54 -0
- endoreg_db/views/media/__init__.py +32 -0
- endoreg_db/views/media/pdf_media.py +411 -0
- endoreg_db/views/media/sensitive_metadata.py +372 -0
- endoreg_db/views/media/video_media.py +275 -0
- endoreg_db/views/meta/__init__.py +7 -0
- endoreg_db/views/meta/sensitive_meta_list.py +102 -0
- endoreg_db/views/meta/sensitive_meta_verification.py +74 -0
- endoreg_db/views/misc/__init__.py +29 -0
- endoreg_db/views/misc/center.py +14 -0
- endoreg_db/views/misc/csrf.py +8 -0
- endoreg_db/views/misc/gender.py +15 -0
- endoreg_db/views/misc/stats.py +255 -0
- endoreg_db/views/misc/upload_views.py +241 -0
- endoreg_db/views/patient/__init__.py +3 -0
- endoreg_db/views/patient/patient.py +253 -0
- endoreg_db/views/patient_examination/__init__.py +11 -0
- endoreg_db/views/patient_examination/patient_examination.py +141 -0
- endoreg_db/views/patient_examination/patient_examination_create.py +58 -0
- endoreg_db/views/patient_examination/patient_examination_detail.py +63 -0
- endoreg_db/views/patient_examination/patient_examination_list.py +72 -0
- endoreg_db/views/patient_examination/video.py +228 -0
- endoreg_db/views/patient_finding/__init__.py +7 -0
- endoreg_db/views/patient_finding/base.py +0 -0
- endoreg_db/views/patient_finding/patient_finding.py +71 -0
- endoreg_db/views/patient_finding/patient_finding_optimized.py +291 -0
- endoreg_db/views/patient_finding_classification/__init__.py +5 -0
- endoreg_db/views/patient_finding_classification/pfc_create.py +75 -0
- endoreg_db/views/report/__init__.py +7 -0
- endoreg_db/views/report/reimport.py +177 -0
- endoreg_db/views/report/report_stream.py +191 -0
- endoreg_db/views/requirement/__init__.py +11 -0
- endoreg_db/views/requirement/evaluate.py +278 -0
- endoreg_db/views/requirement/lookup.py +380 -0
- endoreg_db/views/requirement/lookup_store.py +183 -0
- endoreg_db/views/requirement/requirement_utils.py +87 -0
- endoreg_db/views/requirement_lookup/lookup.py +0 -0
- endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
- endoreg_db/views/stats/__init__.py +13 -0
- endoreg_db/views/stats/stats_views.py +266 -0
- endoreg_db/views/video/__init__.py +49 -0
- endoreg_db/views/video/ai/__init__.py +8 -0
- endoreg_db/views/video/ai/label.py +159 -0
- endoreg_db/views/video/correction.py +529 -0
- endoreg_db/views/video/reimport.py +230 -0
- endoreg_db/views/video/segments_crud.py +709 -0
- endoreg_db/views/video/video_apply_mask.py +49 -0
- endoreg_db/views/video/video_correction.py +22 -0
- endoreg_db/views/video/video_download_processed.py +58 -0
- endoreg_db/views/video/video_examination_viewset.py +242 -0
- endoreg_db/views/video/video_metadata.py +101 -0
- endoreg_db/views/video/video_processing_history.py +25 -0
- endoreg_db/views/video/video_remove_frames.py +49 -0
- endoreg_db/views/video/video_stream.py +334 -0
- endoreg_db-0.8.9.32.dist-info/METADATA +404 -0
- endoreg_db-0.8.9.32.dist-info/RECORD +787 -0
- endoreg_db-0.8.9.32.dist-info/WHEEL +4 -0
- endoreg_db-0.8.9.32.dist-info/licenses/LICENSE +674 -0
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
# endoreg_db/utils/ai/data_loader_for_model_training.py
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Dict, List, Optional, TypedDict
|
|
8
|
+
|
|
9
|
+
from django.db import models
|
|
10
|
+
|
|
11
|
+
from endoreg_db.models import (
|
|
12
|
+
AIDataSet,
|
|
13
|
+
Frame,
|
|
14
|
+
ImageClassificationAnnotation,
|
|
15
|
+
Label,
|
|
16
|
+
LabelSet,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ImageMultilabelDataset(TypedDict):
|
|
21
|
+
"""
|
|
22
|
+
In-memory representation of an image multi-label training dataset.
|
|
23
|
+
|
|
24
|
+
All lists are aligned by index:
|
|
25
|
+
|
|
26
|
+
image_paths[i] -> path to image file for sample i
|
|
27
|
+
label_vectors[i] -> list[int|None] of length == len(labels)
|
|
28
|
+
label_masks[i] -> list[int] of length == len(labels)
|
|
29
|
+
|
|
30
|
+
Where:
|
|
31
|
+
- label_vectors[i][j] is:
|
|
32
|
+
1 -> positive annotation (value=True)
|
|
33
|
+
0 -> negative annotation (value=False)
|
|
34
|
+
None -> UNKNOWN (no annotation for that (frame, label))
|
|
35
|
+
|
|
36
|
+
- label_masks[i][j] is:
|
|
37
|
+
1 -> this entry participates in the loss (0 or 1 is known)
|
|
38
|
+
0 -> IGNORE in the loss (value was None)
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
# type description of the returned dict.
|
|
42
|
+
|
|
43
|
+
image_paths: List[str]
|
|
44
|
+
label_vectors: List[List[Optional[int]]]
|
|
45
|
+
label_masks: List[List[int]]
|
|
46
|
+
labels: List[Label]
|
|
47
|
+
labelset: LabelSet
|
|
48
|
+
|
|
49
|
+
# New: keep track of which DB rows were used, and their legacy exam ids
|
|
50
|
+
frame_ids: List[int] # Frame.pk for each sample
|
|
51
|
+
old_examination_ids: List[Optional[int]] # may be None if not set
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _infer_labelset_from_annotations(
|
|
55
|
+
annotations_qs: models.QuerySet[ImageClassificationAnnotation],
|
|
56
|
+
) -> LabelSet:
|
|
57
|
+
"""
|
|
58
|
+
Try to infer a unique LabelSet from the labels used in the annotations.
|
|
59
|
+
|
|
60
|
+
Strategy:
|
|
61
|
+
1. Collect all distinct label_ids from the annotations.
|
|
62
|
+
2. Fetch all Label objects + their label_sets.
|
|
63
|
+
3. Compute the intersection of all label_sets across all labels.
|
|
64
|
+
4. If there is exactly ONE common LabelSet, return it.
|
|
65
|
+
Otherwise, raise NotImplementedError for now.
|
|
66
|
+
"""
|
|
67
|
+
label_ids = list(annotations_qs.values_list("label_id", flat=True).distinct())
|
|
68
|
+
if not label_ids:
|
|
69
|
+
raise ValueError("Cannot infer LabelSet: annotations queryset has no labels.")
|
|
70
|
+
|
|
71
|
+
labels_qs = Label.objects.filter(id__in=label_ids).prefetch_related("label_sets")
|
|
72
|
+
labelsets_for_each_label = []
|
|
73
|
+
|
|
74
|
+
for lbl in labels_qs:
|
|
75
|
+
# lbl.label_sets is the reverse of LabelSet.labels M2M
|
|
76
|
+
ls_ids = list(lbl.label_sets.values_list("id", flat=True))
|
|
77
|
+
if not ls_ids:
|
|
78
|
+
# This label is not part of any LabelSet -> ambiguous
|
|
79
|
+
raise NotImplementedError(
|
|
80
|
+
f"Label id={lbl.id}, name='{lbl.name}' is not part of any LabelSet. "
|
|
81
|
+
"Explicit LabelSet selection is required."
|
|
82
|
+
)
|
|
83
|
+
labelsets_for_each_label.append(set(ls_ids))
|
|
84
|
+
|
|
85
|
+
# Intersection of all labelset id sets
|
|
86
|
+
common_ids = set.intersection(*labelsets_for_each_label)
|
|
87
|
+
if not common_ids:
|
|
88
|
+
raise NotImplementedError(
|
|
89
|
+
"No common LabelSet across all labels in this AIDataSet. "
|
|
90
|
+
"Please specify a LabelSet explicitly."
|
|
91
|
+
)
|
|
92
|
+
if len(common_ids) > 1:
|
|
93
|
+
raise NotImplementedError(
|
|
94
|
+
"More than one common LabelSet found for the labels in this AIDataSet. "
|
|
95
|
+
"Please specify a LabelSet explicitly to disambiguate."
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
ls_id = next(iter(common_ids))
|
|
99
|
+
return LabelSet.objects.get(id=ls_id)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def build_image_multilabel_dataset_from_db(
|
|
103
|
+
dataset: AIDataSet,
|
|
104
|
+
labelset: Optional[LabelSet] = None,
|
|
105
|
+
) -> ImageMultilabelDataset:
|
|
106
|
+
"""
|
|
107
|
+
Build an in-memory multilabel dataset for an IMAGE-based AIDataSet.
|
|
108
|
+
|
|
109
|
+
Steps:
|
|
110
|
+
1. Take all ImageClassificationAnnotation rows linked to this AIDataSet
|
|
111
|
+
(via dataset.image_annotations M2M).
|
|
112
|
+
2. Determine the LabelSet (either explicitly given or inferred).
|
|
113
|
+
3. For each used Frame, build:
|
|
114
|
+
- an image path
|
|
115
|
+
- a label vector (1, 0, or None for each label in LabelSet)
|
|
116
|
+
- a mask vector (1 where known, 0 where unknown)
|
|
117
|
+
4. Return a dict that can be wrapped in a torch/tf Dataset.
|
|
118
|
+
|
|
119
|
+
NOTE:
|
|
120
|
+
- This function does NOT write anything to the DB.
|
|
121
|
+
- It only reads DB rows and returns Python structures.
|
|
122
|
+
"""
|
|
123
|
+
if dataset.dataset_type != AIDataSet.DATASET_TYPE_IMAGE:
|
|
124
|
+
raise ValueError(
|
|
125
|
+
f"build_image_multilabel_dataset_from_db expected dataset_type='image', "
|
|
126
|
+
f"got '{dataset.dataset_type}' for AIDataSet id={dataset.id}."
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Get the annotation relation dynamically (for future video/text types)
|
|
130
|
+
annotations_qs = dataset.get_annotations_queryset().select_related("frame", "label")
|
|
131
|
+
|
|
132
|
+
if annotations_qs.count() == 0:
|
|
133
|
+
raise ValueError(
|
|
134
|
+
f"AIDataSet id={dataset.id} has no annotations attached. "
|
|
135
|
+
"Make sure your import script populated image_annotations."
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Decide which LabelSet to use
|
|
139
|
+
if labelset is None:
|
|
140
|
+
labelset = _infer_labelset_from_annotations(annotations_qs)
|
|
141
|
+
|
|
142
|
+
# Fixed label order (= fixed column order for the label vectors)
|
|
143
|
+
labels_in_order: List[Label] = labelset.get_labels_in_order()
|
|
144
|
+
if not labels_in_order:
|
|
145
|
+
raise ValueError(
|
|
146
|
+
f"LabelSet id={labelset.id}, name='{labelset.name}' has no labels."
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
num_labels = len(labels_in_order)
|
|
150
|
+
label_index: Dict[int, int] = {
|
|
151
|
+
lbl.id: idx for idx, lbl in enumerate(labels_in_order)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
# Group annotations by frame
|
|
155
|
+
anns_by_frame: Dict[int, List[ImageClassificationAnnotation]] = defaultdict(list)
|
|
156
|
+
frames_order: List[int] = []
|
|
157
|
+
|
|
158
|
+
for ann in annotations_qs:
|
|
159
|
+
frame_id = ann.frame_id
|
|
160
|
+
if frame_id not in anns_by_frame:
|
|
161
|
+
frames_order.append(frame_id)
|
|
162
|
+
anns_by_frame[frame_id].append(ann)
|
|
163
|
+
|
|
164
|
+
# Build vectors
|
|
165
|
+
image_paths: List[str] = []
|
|
166
|
+
label_vectors: List[List[Optional[int]]] = []
|
|
167
|
+
label_masks: List[List[int]] = []
|
|
168
|
+
|
|
169
|
+
# New: id tracking for splitting / logging
|
|
170
|
+
frame_ids: List[int] = []
|
|
171
|
+
old_examination_ids: List[Optional[int]] = []
|
|
172
|
+
|
|
173
|
+
# Cache frames to avoid repeated DB hits
|
|
174
|
+
frame_obj_by_id: Dict[int, Frame] = {}
|
|
175
|
+
|
|
176
|
+
for frame_id in frames_order:
|
|
177
|
+
frame_annotations = anns_by_frame[frame_id]
|
|
178
|
+
|
|
179
|
+
# Resolve frame object (from first annotation of this frame)
|
|
180
|
+
frame = frame_obj_by_id.get(frame_id)
|
|
181
|
+
if frame is None:
|
|
182
|
+
frame = frame_annotations[0].frame
|
|
183
|
+
frame_obj_by_id[frame_id] = frame
|
|
184
|
+
|
|
185
|
+
# New: remember DB ids for this sample
|
|
186
|
+
frame_ids.append(frame_id)
|
|
187
|
+
old_examination_ids.append(getattr(frame, "old_examination_id", None))
|
|
188
|
+
|
|
189
|
+
# Start with unknown for all labels
|
|
190
|
+
vec: List[Optional[int]] = [None] * num_labels
|
|
191
|
+
|
|
192
|
+
# Fill with 1/0 where we have annotations
|
|
193
|
+
for ann in frame_annotations:
|
|
194
|
+
idx = label_index.get(ann.label_id)
|
|
195
|
+
if idx is None:
|
|
196
|
+
# Label not part of this LabelSet: ignore
|
|
197
|
+
continue
|
|
198
|
+
vec[idx] = 1 if ann.value else 0
|
|
199
|
+
|
|
200
|
+
# Build mask: 1 where vec is known, 0 where unknown
|
|
201
|
+
mask: List[int] = [0 if v is None else 1 for v in vec]
|
|
202
|
+
|
|
203
|
+
# Resolve absolute image path from the Frame model
|
|
204
|
+
file_path: Path = frame.file_path
|
|
205
|
+
image_paths.append(str(file_path))
|
|
206
|
+
label_vectors.append(vec)
|
|
207
|
+
label_masks.append(mask)
|
|
208
|
+
|
|
209
|
+
return ImageMultilabelDataset(
|
|
210
|
+
image_paths=image_paths,
|
|
211
|
+
label_vectors=label_vectors,
|
|
212
|
+
label_masks=label_masks,
|
|
213
|
+
labels=labels_in_order,
|
|
214
|
+
labelset=labelset,
|
|
215
|
+
frame_ids=frame_ids,
|
|
216
|
+
old_examination_ids=old_examination_ids,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def build_dataset_for_training(
|
|
221
|
+
dataset: AIDataSet,
|
|
222
|
+
labelset: Optional[LabelSet] = None,
|
|
223
|
+
):
|
|
224
|
+
"""
|
|
225
|
+
High-level entry point to build a training dataset from an AIDataSet row.
|
|
226
|
+
|
|
227
|
+
It inspects:
|
|
228
|
+
- dataset.dataset_type
|
|
229
|
+
- dataset.ai_model_type
|
|
230
|
+
|
|
231
|
+
and dispatches to the appropriate builder.
|
|
232
|
+
|
|
233
|
+
For now, we support:
|
|
234
|
+
- dataset_type = "image"
|
|
235
|
+
- ai_model_type = "image_multilabel_classification"
|
|
236
|
+
|
|
237
|
+
Later, you can extend this to:
|
|
238
|
+
- video segmentation
|
|
239
|
+
- text classification
|
|
240
|
+
etc.
|
|
241
|
+
"""
|
|
242
|
+
# IMAGE MULTILABEL CASE
|
|
243
|
+
if (
|
|
244
|
+
dataset.dataset_type == AIDataSet.DATASET_TYPE_IMAGE
|
|
245
|
+
and dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_IMAGE_MULTILABEL
|
|
246
|
+
):
|
|
247
|
+
return build_image_multilabel_dataset_from_db(dataset, labelset=labelset)
|
|
248
|
+
|
|
249
|
+
# FUTURE EXTENSIONS (example structure, not yet implemented):
|
|
250
|
+
# if dataset.dataset_type == AIDataSet.DATASET_TYPE_VIDEO and \
|
|
251
|
+
# dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_VIDEO_SEGMENTATION:
|
|
252
|
+
# return build_video_segmentation_dataset_from_db(dataset, labelset=labelset)
|
|
253
|
+
#
|
|
254
|
+
# if dataset.dataset_type == AIDataSet.DATASET_TYPE_TEXT and \
|
|
255
|
+
# dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_TEXT_CLASSIFICATION:
|
|
256
|
+
# return build_text_classification_dataset_from_db(dataset, labelset=labelset)
|
|
257
|
+
|
|
258
|
+
raise NotImplementedError(
|
|
259
|
+
f"No dataset builder implemented for "
|
|
260
|
+
f"dataset_type='{dataset.dataset_type}', "
|
|
261
|
+
f"ai_model_type='{dataset.ai_model_type}'."
|
|
262
|
+
)
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
# endoreg_db/utils/ai/data_loader_for_model_training.py
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Dict, List, Optional, TypedDict
|
|
8
|
+
|
|
9
|
+
from django.db import models
|
|
10
|
+
|
|
11
|
+
from endoreg_db.models import (
|
|
12
|
+
AIDataSet,
|
|
13
|
+
Frame,
|
|
14
|
+
ImageClassificationAnnotation,
|
|
15
|
+
Label,
|
|
16
|
+
LabelSet,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ImageMultilabelDataset(TypedDict):
|
|
21
|
+
"""
|
|
22
|
+
In-memory representation of an image multi-label training dataset.
|
|
23
|
+
|
|
24
|
+
All lists are aligned by index:
|
|
25
|
+
|
|
26
|
+
image_paths[i] -> path to image file for sample i
|
|
27
|
+
label_vectors[i] -> list[int|None] of length == len(labels)
|
|
28
|
+
label_masks[i] -> list[int] of length == len(labels)
|
|
29
|
+
|
|
30
|
+
Where:
|
|
31
|
+
- label_vectors[i][j] is:
|
|
32
|
+
1 -> positive annotation (value=True)
|
|
33
|
+
0 -> negative annotation (value=False)
|
|
34
|
+
None -> UNKNOWN (no annotation for that (frame, label))
|
|
35
|
+
|
|
36
|
+
- label_masks[i][j] is:
|
|
37
|
+
1 -> this entry participates in the loss (0 or 1 is known)
|
|
38
|
+
0 -> IGNORE in the loss (value was None)
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
# type description of the returned dict.
|
|
42
|
+
|
|
43
|
+
image_paths: List[str]
|
|
44
|
+
label_vectors: List[List[Optional[int]]]
|
|
45
|
+
label_masks: List[List[int]]
|
|
46
|
+
labels: List[Label]
|
|
47
|
+
labelset: LabelSet
|
|
48
|
+
|
|
49
|
+
# New: keep track of which DB rows were used, and their legacy exam ids
|
|
50
|
+
frame_ids: List[int] # Frame.pk for each sample
|
|
51
|
+
old_examination_ids: List[Optional[int]] # may be None if not set
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _infer_labelset_from_annotations(
|
|
55
|
+
annotations_qs: models.QuerySet[ImageClassificationAnnotation],
|
|
56
|
+
) -> LabelSet:
|
|
57
|
+
"""
|
|
58
|
+
Try to infer a unique LabelSet from the labels used in the annotations.
|
|
59
|
+
|
|
60
|
+
Strategy:
|
|
61
|
+
1. Collect all distinct label_ids from the annotations.
|
|
62
|
+
2. Fetch all Label objects + their label_sets.
|
|
63
|
+
3. Compute the intersection of all label_sets across all labels.
|
|
64
|
+
4. If there is exactly ONE common LabelSet, return it.
|
|
65
|
+
Otherwise, raise NotImplementedError for now.
|
|
66
|
+
"""
|
|
67
|
+
label_ids = list(annotations_qs.values_list("label_id", flat=True).distinct())
|
|
68
|
+
if not label_ids:
|
|
69
|
+
raise ValueError("Cannot infer LabelSet: annotations queryset has no labels.")
|
|
70
|
+
|
|
71
|
+
labels_qs = Label.objects.filter(id__in=label_ids).prefetch_related("label_sets")
|
|
72
|
+
labelsets_for_each_label = []
|
|
73
|
+
|
|
74
|
+
for lbl in labels_qs:
|
|
75
|
+
# lbl.label_sets is the reverse of LabelSet.labels M2M
|
|
76
|
+
ls_ids = list(lbl.label_sets.values_list("id", flat=True))
|
|
77
|
+
if not ls_ids:
|
|
78
|
+
# This label is not part of any LabelSet -> ambiguous
|
|
79
|
+
raise NotImplementedError(
|
|
80
|
+
f"Label id={lbl.id}, name='{lbl.name}' is not part of any LabelSet. "
|
|
81
|
+
"Explicit LabelSet selection is required."
|
|
82
|
+
)
|
|
83
|
+
labelsets_for_each_label.append(set(ls_ids))
|
|
84
|
+
|
|
85
|
+
# Intersection of all labelset id sets
|
|
86
|
+
common_ids = set.intersection(*labelsets_for_each_label)
|
|
87
|
+
if not common_ids:
|
|
88
|
+
raise NotImplementedError(
|
|
89
|
+
"No common LabelSet across all labels in this AIDataSet. "
|
|
90
|
+
"Please specify a LabelSet explicitly."
|
|
91
|
+
)
|
|
92
|
+
if len(common_ids) > 1:
|
|
93
|
+
raise NotImplementedError(
|
|
94
|
+
"More than one common LabelSet found for the labels in this AIDataSet. "
|
|
95
|
+
"Please specify a LabelSet explicitly to disambiguate."
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
ls_id = next(iter(common_ids))
|
|
99
|
+
return LabelSet.objects.get(id=ls_id)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def build_image_multilabel_dataset_from_db(
|
|
103
|
+
dataset: AIDataSet,
|
|
104
|
+
labelset: Optional[LabelSet] = None,
|
|
105
|
+
) -> ImageMultilabelDataset:
|
|
106
|
+
"""
|
|
107
|
+
Build an in-memory multilabel dataset for an IMAGE-based AIDataSet.
|
|
108
|
+
|
|
109
|
+
Steps:
|
|
110
|
+
1. Take all ImageClassificationAnnotation rows linked to this AIDataSet
|
|
111
|
+
(via dataset.image_annotations M2M).
|
|
112
|
+
2. Determine the LabelSet (either explicitly given or inferred).
|
|
113
|
+
3. For each used Frame, build:
|
|
114
|
+
- an image path
|
|
115
|
+
- a label vector (1, 0, or None for each label in LabelSet)
|
|
116
|
+
- a mask vector (1 where known, 0 where unknown)
|
|
117
|
+
4. Return a dict that can be wrapped in a torch/tf Dataset.
|
|
118
|
+
|
|
119
|
+
NOTE:
|
|
120
|
+
- This function does NOT write anything to the DB.
|
|
121
|
+
- It only reads DB rows and returns Python structures.
|
|
122
|
+
"""
|
|
123
|
+
if dataset.dataset_type != AIDataSet.DATASET_TYPE_IMAGE:
|
|
124
|
+
raise ValueError(
|
|
125
|
+
f"build_image_multilabel_dataset_from_db expected dataset_type='image', "
|
|
126
|
+
f"got '{dataset.dataset_type}' for AIDataSet id={dataset.id}."
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Get the annotation relation dynamically (for future video/text types)
|
|
130
|
+
annotations_qs = dataset.get_annotations_queryset().select_related("frame", "label")
|
|
131
|
+
|
|
132
|
+
if annotations_qs.count() == 0:
|
|
133
|
+
raise ValueError(
|
|
134
|
+
f"AIDataSet id={dataset.id} has no annotations attached. "
|
|
135
|
+
"Make sure your import script populated image_annotations."
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Decide which LabelSet to use
|
|
139
|
+
if labelset is None:
|
|
140
|
+
labelset = _infer_labelset_from_annotations(annotations_qs)
|
|
141
|
+
|
|
142
|
+
# Fixed label order (= fixed column order for the label vectors)
|
|
143
|
+
labels_in_order: List[Label] = labelset.get_labels_in_order()
|
|
144
|
+
if not labels_in_order:
|
|
145
|
+
raise ValueError(
|
|
146
|
+
f"LabelSet id={labelset.id}, name='{labelset.name}' has no labels."
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
num_labels = len(labels_in_order)
|
|
150
|
+
label_index: Dict[int, int] = {
|
|
151
|
+
lbl.id: idx for idx, lbl in enumerate(labels_in_order)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
# Group annotations by frame
|
|
155
|
+
anns_by_frame: Dict[int, List[ImageClassificationAnnotation]] = defaultdict(list)
|
|
156
|
+
frames_order: List[int] = []
|
|
157
|
+
|
|
158
|
+
for ann in annotations_qs:
|
|
159
|
+
frame_id = ann.frame_id
|
|
160
|
+
if frame_id not in anns_by_frame:
|
|
161
|
+
frames_order.append(frame_id)
|
|
162
|
+
anns_by_frame[frame_id].append(ann)
|
|
163
|
+
|
|
164
|
+
# Build vectors
|
|
165
|
+
image_paths: List[str] = []
|
|
166
|
+
label_vectors: List[List[Optional[int]]] = []
|
|
167
|
+
label_masks: List[List[int]] = []
|
|
168
|
+
|
|
169
|
+
# New: id tracking for splitting / logging
|
|
170
|
+
frame_ids: List[int] = []
|
|
171
|
+
old_examination_ids: List[Optional[int]] = []
|
|
172
|
+
|
|
173
|
+
# Cache frames to avoid repeated DB hits
|
|
174
|
+
frame_obj_by_id: Dict[int, Frame] = {}
|
|
175
|
+
|
|
176
|
+
for frame_id in frames_order:
|
|
177
|
+
frame_annotations = anns_by_frame[frame_id]
|
|
178
|
+
|
|
179
|
+
# Resolve frame object (from first annotation of this frame)
|
|
180
|
+
frame = frame_obj_by_id.get(frame_id)
|
|
181
|
+
if frame is None:
|
|
182
|
+
frame = frame_annotations[0].frame
|
|
183
|
+
frame_obj_by_id[frame_id] = frame
|
|
184
|
+
|
|
185
|
+
# New: remember DB ids for this sample
|
|
186
|
+
frame_ids.append(frame_id)
|
|
187
|
+
old_examination_ids.append(getattr(frame, "old_examination_id", None))
|
|
188
|
+
|
|
189
|
+
# Start with unknown for all labels
|
|
190
|
+
vec: List[Optional[int]] = [None] * num_labels
|
|
191
|
+
|
|
192
|
+
# Fill with 1/0 where we have annotations
|
|
193
|
+
for ann in frame_annotations:
|
|
194
|
+
idx = label_index.get(ann.label_id)
|
|
195
|
+
if idx is None:
|
|
196
|
+
# Label not part of this LabelSet: ignore
|
|
197
|
+
continue
|
|
198
|
+
vec[idx] = 1 if ann.value else 0
|
|
199
|
+
|
|
200
|
+
# Build mask: 1 where vec is known, 0 where unknown
|
|
201
|
+
mask: List[int] = [0 if v is None else 1 for v in vec]
|
|
202
|
+
|
|
203
|
+
# Resolve absolute image path from the Frame model
|
|
204
|
+
file_path: Path = frame.file_path
|
|
205
|
+
image_paths.append(str(file_path))
|
|
206
|
+
label_vectors.append(vec)
|
|
207
|
+
label_masks.append(mask)
|
|
208
|
+
|
|
209
|
+
return ImageMultilabelDataset(
|
|
210
|
+
image_paths=image_paths,
|
|
211
|
+
label_vectors=label_vectors,
|
|
212
|
+
label_masks=label_masks,
|
|
213
|
+
labels=labels_in_order,
|
|
214
|
+
labelset=labelset,
|
|
215
|
+
frame_ids=frame_ids,
|
|
216
|
+
old_examination_ids=old_examination_ids,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def build_dataset_for_training(
|
|
221
|
+
dataset: AIDataSet,
|
|
222
|
+
labelset: Optional[LabelSet] = None,
|
|
223
|
+
):
|
|
224
|
+
"""
|
|
225
|
+
High-level entry point to build a training dataset from an AIDataSet row.
|
|
226
|
+
|
|
227
|
+
It inspects:
|
|
228
|
+
- dataset.dataset_type
|
|
229
|
+
- dataset.ai_model_type
|
|
230
|
+
|
|
231
|
+
and dispatches to the appropriate builder.
|
|
232
|
+
|
|
233
|
+
For now, we support:
|
|
234
|
+
- dataset_type = "image"
|
|
235
|
+
- ai_model_type = "image_multilabel_classification"
|
|
236
|
+
|
|
237
|
+
Later, you can extend this to:
|
|
238
|
+
- video segmentation
|
|
239
|
+
- text classification
|
|
240
|
+
etc.
|
|
241
|
+
"""
|
|
242
|
+
# IMAGE MULTILABEL CASE
|
|
243
|
+
if (
|
|
244
|
+
dataset.dataset_type == AIDataSet.DATASET_TYPE_IMAGE
|
|
245
|
+
and dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_IMAGE_MULTILABEL
|
|
246
|
+
):
|
|
247
|
+
return build_image_multilabel_dataset_from_db(dataset, labelset=labelset)
|
|
248
|
+
|
|
249
|
+
# FUTURE EXTENSIONS (example structure, not yet implemented):
|
|
250
|
+
# if dataset.dataset_type == AIDataSet.DATASET_TYPE_VIDEO and \
|
|
251
|
+
# dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_VIDEO_SEGMENTATION:
|
|
252
|
+
# return build_video_segmentation_dataset_from_db(dataset, labelset=labelset)
|
|
253
|
+
#
|
|
254
|
+
# if dataset.dataset_type == AIDataSet.DATASET_TYPE_TEXT and \
|
|
255
|
+
# dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_TEXT_CLASSIFICATION:
|
|
256
|
+
# return build_text_classification_dataset_from_db(dataset, labelset=labelset)
|
|
257
|
+
|
|
258
|
+
raise NotImplementedError(
|
|
259
|
+
f"No dataset builder implemented for "
|
|
260
|
+
f"dataset_type='{dataset.dataset_type}', "
|
|
261
|
+
f"ai_model_type='{dataset.ai_model_type}'."
|
|
262
|
+
)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from torch.utils.data import Dataset
|
|
2
|
+
import numpy as np
|
|
3
|
+
from PIL import Image
|
|
4
|
+
from torchvision import transforms
|
|
5
|
+
from .preprocess import Cropper
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class InferenceDataset(Dataset):
|
|
9
|
+
def __init__(self, paths, crops, config):
|
|
10
|
+
self.paths = paths
|
|
11
|
+
self.crops = crops
|
|
12
|
+
self.cropper = Cropper() # Assuming Cropper can work with NumPy arrays
|
|
13
|
+
self.config = config
|
|
14
|
+
|
|
15
|
+
# Initialize the image transformations using torchvision
|
|
16
|
+
self.transforms = transforms.Compose(
|
|
17
|
+
[
|
|
18
|
+
# Convert PIL image to PyTorch tensor
|
|
19
|
+
transforms.ToTensor(),
|
|
20
|
+
# Normalize the image using the provided mean and std
|
|
21
|
+
transforms.Normalize(mean=self.config["mean"], std=self.config["std"]),
|
|
22
|
+
]
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
def __len__(self):
|
|
26
|
+
# Returns the total number of samples
|
|
27
|
+
return len(self.paths)
|
|
28
|
+
|
|
29
|
+
def __getitem__(self, idx):
|
|
30
|
+
# Open the image with Pillow
|
|
31
|
+
with Image.open(self.paths[idx]) as pil_image:
|
|
32
|
+
# Convert the image to RGB to ensure 3 channels
|
|
33
|
+
pil_image = pil_image.convert("RGB")
|
|
34
|
+
|
|
35
|
+
# Get the corresponding crop for the current image
|
|
36
|
+
crop = self.crops[idx]
|
|
37
|
+
|
|
38
|
+
# Crop the image based on the provided crop parameters and convert to numpy for cropping
|
|
39
|
+
cropped = self.cropper(
|
|
40
|
+
np.array(pil_image), # Convert PIL image to numpy array for cropping
|
|
41
|
+
crop,
|
|
42
|
+
scale=[self.config["size_x"], self.config["size_y"]],
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
# Convert cropped numpy array back to PIL image for torchvision transforms
|
|
46
|
+
cropped_pil = Image.fromarray(cropped.astype("uint8"), "RGB")
|
|
47
|
+
|
|
48
|
+
# Apply the transformations
|
|
49
|
+
img = self.transforms(cropped_pil)
|
|
50
|
+
|
|
51
|
+
return img
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# endoreg_db/utils/ai/model_training/config.py
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
from django.conf import settings
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# ---------------------------------------------------------------------
|
|
13
|
+
# PATHS
|
|
14
|
+
# ---------------------------------------------------------------------
|
|
15
|
+
|
|
16
|
+
# Base project directory (e.g. /home/admin/dev/endoreg-db)
|
|
17
|
+
BASE_DIR = Path(getattr(settings, "BASE_DIR", Path(__file__).resolve().parents[4]))
|
|
18
|
+
|
|
19
|
+
# All training artifacts go here:
|
|
20
|
+
# /home/admin/dev/endoreg-db/data/model_training/
|
|
21
|
+
TRAINING_ROOT = BASE_DIR / "data" / "model_training"
|
|
22
|
+
CHECKPOINTS_DIR = TRAINING_ROOT / "checkpoints"
|
|
23
|
+
RUNS_DIR = TRAINING_ROOT / "runs"
|
|
24
|
+
|
|
25
|
+
for d in (TRAINING_ROOT, CHECKPOINTS_DIR, RUNS_DIR):
|
|
26
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
|
|
28
|
+
# Which LabelSet.version we train on (for label filtering)
|
|
29
|
+
DEFAULT_LABELSET_VERSION_TO_TRAIN: int = 2
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# ---------------------------------------------------------------------
|
|
33
|
+
# TRAINING CONFIG
|
|
34
|
+
# ---------------------------------------------------------------------
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class TrainingConfig:
|
|
39
|
+
"""
|
|
40
|
+
Configuration for GastroNet multi-label training.
|
|
41
|
+
|
|
42
|
+
Most important knobs:
|
|
43
|
+
- dataset_id: which AIDataSet row to use from the database
|
|
44
|
+
- labelset_version_to_train: only labels belonging to LabelSet.version == this
|
|
45
|
+
are used for training (e.g. 2).
|
|
46
|
+
- treat_unlabeled_as_negative:
|
|
47
|
+
True -> Option A: for v2 labels, if not annotated in a frame, we
|
|
48
|
+
assume "absent" (0) and include it in the loss.
|
|
49
|
+
False -> keep "unknown" semantics (mask = 0, ignored in loss/metrics).
|
|
50
|
+
|
|
51
|
+
Learning rate schedule:
|
|
52
|
+
- lr_head / lr_backbone: base learning rates
|
|
53
|
+
- use_scheduler: if True, we use warm-up + cosine decay
|
|
54
|
+
- warmup_epochs: how many epochs to linearly increase LR from 0 → base LR
|
|
55
|
+
- min_lr: lowest LR reached at the end of cosine schedule
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
# --- WHAT TO TRAIN ON -------------------------------------------------
|
|
59
|
+
dataset_id: int
|
|
60
|
+
|
|
61
|
+
# Train only on labels belonging to ANY LabelSet with this version.
|
|
62
|
+
labelset_version_to_train: int = DEFAULT_LABELSET_VERSION_TO_TRAIN
|
|
63
|
+
|
|
64
|
+
# Path to GastroNet RN50 checkpoint (.pth); if None, backbone is random.
|
|
65
|
+
backbone_checkpoint: Optional[str] = None
|
|
66
|
+
|
|
67
|
+
# --- EPOCHS / BATCHING -----------------------------------------------
|
|
68
|
+
num_epochs: int = 5
|
|
69
|
+
batch_size: int = 32
|
|
70
|
+
|
|
71
|
+
# Split ratios (by colonoscopy exam groups, not by individual frames)
|
|
72
|
+
val_split: float = 0.2
|
|
73
|
+
test_split: float = 0.1
|
|
74
|
+
|
|
75
|
+
# --- LEARNING RATES --------------------------------------------------
|
|
76
|
+
# Base learning rates for classifier head and backbone.
|
|
77
|
+
lr_head: float = 1e-3 # usually larger (newly initialized layer)
|
|
78
|
+
lr_backbone: float = 1e-4 # smaller (pretrained GastroNet backbone)
|
|
79
|
+
|
|
80
|
+
# --- FOCAL LOSS HYPERPARAMETERS -------------------------------------
|
|
81
|
+
gamma_focal: float = 2.0 # how strongly to focus on hard examples
|
|
82
|
+
alpha_focal: float = 0.25 # weight for positives vs negatives
|
|
83
|
+
|
|
84
|
+
# --- DEVICE & SEED ---------------------------------------------------
|
|
85
|
+
device: str = "auto" # "auto", "cpu", or "cuda"
|
|
86
|
+
random_seed: int = 42
|
|
87
|
+
|
|
88
|
+
# --- LABEL SEMANTICS -------------------------------------------------
|
|
89
|
+
# For the filtered labels (LabelSet.version == labelset_version_to_train):
|
|
90
|
+
# True -> Option A: unlabeled => negative (0) and mask=1 (supervised)
|
|
91
|
+
# False -> keep unlabeled as unknown (mask=0, ignored)
|
|
92
|
+
treat_unlabeled_as_negative: bool = True
|
|
93
|
+
|
|
94
|
+
# --- LR SCHEDULER: WARM-UP + COSINE DECAY ----------------------------
|
|
95
|
+
# If True, we apply:
|
|
96
|
+
# - linear warm-up for 'warmup_epochs'
|
|
97
|
+
# - then CosineAnnealingLR for the remaining epochs
|
|
98
|
+
use_scheduler: bool = True
|
|
99
|
+
|
|
100
|
+
# Number of warm-up epochs (can be 0 for "no warm-up").
|
|
101
|
+
warmup_epochs: int = 3
|
|
102
|
+
|
|
103
|
+
# Minimum learning rate at the end of cosine decay for all param groups.
|
|
104
|
+
# (Both head and backbone decay towards this value.)
|
|
105
|
+
min_lr: float = 1e-6
|
|
106
|
+
|
|
107
|
+
# which CNN backbone / weights to use
|
|
108
|
+
# "gastro_rn50" → current behavior (ResNet50 + GastroNet checkpoint)
|
|
109
|
+
# "resnet50_imagenet" → ResNet50 with ImageNet weights
|
|
110
|
+
# "resnet50_random" → ResNet50 with random initialization
|
|
111
|
+
# (later) "efficientnet_b0_imagenet", etc.
|
|
112
|
+
backbone_name: str = "gastro_rn50"
|
|
113
|
+
|
|
114
|
+
# whether to freeze backbone (feature extractor)
|
|
115
|
+
freeze_backbone: bool = True
|
|
116
|
+
|
|
117
|
+
# backbone_name: str = "gastro_rn50"
|