endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.8.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/data/__init__.py +22 -8
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +0 -1
- endoreg_db/data/examination/examinations/data.yaml +114 -14
- endoreg_db/data/examination/time-type/data.yaml +0 -3
- endoreg_db/data/examination_indication/endoscopy.yaml +108 -173
- endoreg_db/data/examination_indication_classification/endoscopy.yaml +0 -70
- endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +33 -37
- endoreg_db/data/finding/00_generic.yaml +35 -0
- endoreg_db/data/finding/00_generic_complication.yaml +9 -0
- endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
- endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
- endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
- endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
- endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
- endoreg_db/data/finding_classification/00_generic.yaml +44 -0
- endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
- endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
- endoreg_db/data/finding_classification/{colonoscopy_bowel_preparation.yaml → 02_colonoscopy_baseline.yaml} +35 -20
- endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
- endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
- endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
- endoreg_db/data/finding_classification_choice/{yes_no_na.yaml → 00_generic.yaml} +5 -1
- endoreg_db/data/finding_classification_choice/{examination_setting_generic_types.yaml → 00_generic_baseline.yaml} +10 -2
- endoreg_db/data/finding_classification_choice/{complication_generic_types.yaml → 00_generic_complication.yaml} +1 -1
- endoreg_db/data/finding_classification_choice/{histology.yaml → 00_generic_histology.yaml} +1 -4
- endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
- endoreg_db/data/finding_classification_choice/{bowel_preparation.yaml → 02_colonoscopy_bowel_preparation.yaml} +1 -30
- endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml → finding_classification_choice/02_colonoscopy_generic.yaml} +1 -1
- endoreg_db/data/finding_classification_choice/{histology_polyp.yaml → 02_colonoscopy_histology.yaml} +1 -1
- endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_location.yaml → finding_classification_choice/02_colonoscopy_location.yaml} +23 -4
- endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
- endoreg_db/data/{_examples/finding_classification_choice/colon_lesion_paris.yaml → finding_classification_choice/02_colonoscopy_polyp_morphology.yaml} +26 -8
- endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
- endoreg_db/data/finding_classification_type/{colonoscopy_basic.yaml → 00_generic.yaml} +18 -13
- endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
- endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
- endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
- endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
- endoreg_db/data/finding_type/data.yaml +8 -12
- endoreg_db/data/requirement/01_patient_data.yaml +93 -0
- endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +0 -2
- endoreg_db/data/requirement_set/90_coloreg.yaml +20 -8
- endoreg_db/exceptions.py +0 -1
- endoreg_db/forms/examination_form.py +1 -1
- endoreg_db/helpers/data_loader.py +124 -52
- endoreg_db/helpers/default_objects.py +116 -81
- endoreg_db/import_files/__init__.py +27 -0
- endoreg_db/import_files/context/__init__.py +7 -0
- endoreg_db/import_files/context/default_sensitive_meta.py +81 -0
- endoreg_db/import_files/context/ensure_center.py +17 -0
- endoreg_db/import_files/context/file_lock.py +66 -0
- endoreg_db/import_files/context/import_context.py +43 -0
- endoreg_db/import_files/context/validate_directories.py +56 -0
- endoreg_db/import_files/file_storage/__init__.py +15 -0
- endoreg_db/import_files/file_storage/create_report_file.py +76 -0
- endoreg_db/import_files/file_storage/create_video_file.py +75 -0
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +39 -0
- endoreg_db/import_files/file_storage/state_management.py +400 -0
- endoreg_db/import_files/file_storage/storage.py +36 -0
- endoreg_db/import_files/import_service.md +26 -0
- endoreg_db/import_files/processing/__init__.py +11 -0
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +94 -0
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
- endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +119 -0
- endoreg_db/import_files/pseudonymization/fake.py +52 -0
- endoreg_db/import_files/pseudonymization/k_anonymity.py +182 -0
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +128 -0
- endoreg_db/import_files/report_import_service.py +141 -0
- endoreg_db/import_files/video_import_service.py +150 -0
- endoreg_db/management/commands/import_report.py +130 -65
- endoreg_db/management/commands/import_video_with_classification.py +1 -1
- endoreg_db/management/commands/load_ai_model_data.py +5 -5
- endoreg_db/management/commands/load_ai_model_label_data.py +9 -7
- endoreg_db/management/commands/load_base_db_data.py +5 -134
- endoreg_db/management/commands/load_contraindication_data.py +14 -16
- endoreg_db/management/commands/load_disease_classification_choices_data.py +15 -18
- endoreg_db/management/commands/load_disease_classification_data.py +15 -18
- endoreg_db/management/commands/load_disease_data.py +25 -28
- endoreg_db/management/commands/load_endoscope_data.py +20 -27
- endoreg_db/management/commands/load_event_data.py +14 -16
- endoreg_db/management/commands/load_examination_data.py +31 -44
- endoreg_db/management/commands/load_examination_indication_data.py +20 -21
- endoreg_db/management/commands/load_finding_data.py +52 -80
- endoreg_db/management/commands/load_information_source.py +21 -23
- endoreg_db/management/commands/load_lab_value_data.py +17 -26
- endoreg_db/management/commands/load_medication_data.py +13 -12
- endoreg_db/management/commands/load_organ_data.py +15 -19
- endoreg_db/management/commands/load_pdf_type_data.py +19 -18
- endoreg_db/management/commands/load_profession_data.py +14 -17
- endoreg_db/management/commands/load_qualification_data.py +20 -23
- endoreg_db/management/commands/load_report_reader_flag_data.py +17 -19
- endoreg_db/management/commands/load_requirement_data.py +14 -20
- endoreg_db/management/commands/load_risk_data.py +7 -6
- endoreg_db/management/commands/load_shift_data.py +20 -23
- endoreg_db/management/commands/load_tag_data.py +8 -11
- endoreg_db/management/commands/load_unit_data.py +17 -19
- endoreg_db/management/commands/start_filewatcher.py +46 -37
- endoreg_db/management/commands/validate_video_files.py +1 -5
- endoreg_db/migrations/0001_initial.py +1360 -1812
- endoreg_db/models/administration/person/patient/patient.py +72 -46
- endoreg_db/models/label/__init__.py +2 -2
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +18 -26
- endoreg_db/models/label/label_video_segment/label_video_segment.py +23 -1
- endoreg_db/models/media/pdf/raw_pdf.py +136 -64
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +34 -10
- endoreg_db/models/media/processing_history/__init__.py +5 -0
- endoreg_db/models/media/processing_history/processing_history.py +96 -0
- endoreg_db/models/media/video/create_from_file.py +101 -31
- endoreg_db/models/media/video/video_file.py +125 -105
- endoreg_db/models/media/video/video_file_io.py +31 -26
- endoreg_db/models/medical/contraindication/README.md +1 -0
- endoreg_db/models/medical/examination/examination.py +28 -8
- endoreg_db/models/medical/examination/examination_indication.py +13 -79
- endoreg_db/models/medical/examination/examination_time.py +8 -3
- endoreg_db/models/medical/finding/finding.py +5 -12
- endoreg_db/models/medical/finding/finding_classification.py +18 -37
- endoreg_db/models/medical/finding/finding_intervention.py +7 -9
- endoreg_db/models/medical/hardware/endoscope.py +6 -0
- endoreg_db/models/medical/patient/medication_examples.py +5 -1
- endoreg_db/models/medical/patient/patient_finding.py +1 -1
- endoreg_db/models/metadata/pdf_meta.py +22 -10
- endoreg_db/models/metadata/sensitive_meta.py +3 -0
- endoreg_db/models/metadata/sensitive_meta_logic.py +200 -124
- endoreg_db/models/other/information_source.py +27 -6
- endoreg_db/models/report/__init__.py +0 -0
- endoreg_db/models/report/images.py +0 -0
- endoreg_db/models/report/report.py +6 -0
- endoreg_db/models/requirement/requirement.py +59 -399
- endoreg_db/models/requirement/requirement_operator.py +86 -98
- endoreg_db/models/state/audit_ledger.py +4 -5
- endoreg_db/models/state/raw_pdf.py +69 -30
- endoreg_db/models/state/video.py +64 -49
- endoreg_db/models/upload_job.py +33 -9
- endoreg_db/models/utils.py +27 -23
- endoreg_db/queries/__init__.py +3 -1
- endoreg_db/schemas/examination_evaluation.py +1 -1
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/label_video_segment/label_video_segment.py +2 -29
- endoreg_db/serializers/meta/__init__.py +1 -6
- endoreg_db/serializers/misc/sensitive_patient_data.py +50 -26
- endoreg_db/serializers/patient_examination/patient_examination.py +3 -3
- endoreg_db/serializers/pdf/anony_text_validation.py +39 -23
- endoreg_db/serializers/video/video_file_list.py +65 -34
- endoreg_db/services/__old/pdf_import.py +1487 -0
- endoreg_db/services/__old/video_import.py +1306 -0
- endoreg_db/services/anonymization.py +63 -26
- endoreg_db/services/lookup_service.py +28 -28
- endoreg_db/services/lookup_store.py +2 -2
- endoreg_db/services/pdf_import.py +0 -1480
- endoreg_db/services/report_import.py +10 -0
- endoreg_db/services/video_import.py +6 -1165
- endoreg_db/tasks/upload_tasks.py +79 -70
- endoreg_db/tasks/video_ingest.py +8 -4
- endoreg_db/urls/__init__.py +0 -14
- endoreg_db/urls/ai.py +32 -0
- endoreg_db/urls/media.py +21 -24
- endoreg_db/utils/dataloader.py +87 -57
- endoreg_db/utils/paths.py +110 -46
- endoreg_db/utils/pipelines/Readme.md +1 -1
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +97 -0
- endoreg_db/views/__init__.py +85 -173
- endoreg_db/views/ai/__init__.py +8 -0
- endoreg_db/views/ai/label.py +155 -0
- endoreg_db/views/anonymization/media_management.py +8 -7
- endoreg_db/views/anonymization/overview.py +97 -68
- endoreg_db/views/anonymization/validate.py +25 -21
- endoreg_db/views/media/__init__.py +5 -20
- endoreg_db/views/media/pdf_media.py +109 -65
- endoreg_db/views/media/sensitive_metadata.py +163 -148
- endoreg_db/views/meta/__init__.py +0 -8
- endoreg_db/views/misc/__init__.py +1 -7
- endoreg_db/views/misc/upload_views.py +94 -93
- endoreg_db/views/report/__init__.py +7 -0
- endoreg_db/views/{pdf → report}/reimport.py +45 -24
- endoreg_db/views/{pdf/pdf_stream.py → report/report_stream.py} +40 -32
- endoreg_db/views/requirement/lookup_store.py +22 -90
- endoreg_db/views/video/__init__.py +23 -22
- endoreg_db/views/video/correction.py +201 -172
- endoreg_db/views/video/reimport.py +1 -1
- endoreg_db/views/{media/video_segments.py → video/segments_crud.py} +75 -37
- endoreg_db/views/video/{video_meta.py → video_meta_stats.py} +2 -2
- endoreg_db/views/video/video_stream.py +7 -8
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/METADATA +2 -2
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/RECORD +217 -335
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/WHEEL +1 -1
- endoreg_db/data/_examples/disease.yaml +0 -55
- endoreg_db/data/_examples/disease_classification.yaml +0 -13
- endoreg_db/data/_examples/disease_classification_choice.yaml +0 -62
- endoreg_db/data/_examples/event.yaml +0 -64
- endoreg_db/data/_examples/examination.yaml +0 -72
- endoreg_db/data/_examples/finding/anatomy_colon.yaml +0 -128
- endoreg_db/data/_examples/finding/colonoscopy.yaml +0 -40
- endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +0 -56
- endoreg_db/data/_examples/finding/complication.yaml +0 -16
- endoreg_db/data/_examples/finding/data.yaml +0 -105
- endoreg_db/data/_examples/finding/examination_setting.yaml +0 -16
- endoreg_db/data/_examples/finding/medication_related.yaml +0 -18
- endoreg_db/data/_examples/finding/outcome.yaml +0 -12
- endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +0 -68
- endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +0 -22
- endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +0 -25
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +0 -68
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +0 -80
- endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +0 -21
- endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +0 -26
- endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +0 -22
- endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +0 -53
- endoreg_db/data/_examples/finding_classification/complication_generic.yaml +0 -25
- endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +0 -40
- endoreg_db/data/_examples/finding_classification/histology_colo.yaml +0 -51
- endoreg_db/data/_examples/finding_classification/intervention_required.yaml +0 -26
- endoreg_db/data/_examples/finding_classification/medication_related.yaml +0 -23
- endoreg_db/data/_examples/finding_classification/visualized.yaml +0 -33
- endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +0 -78
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +0 -17
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +0 -14
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +0 -82
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/histology.yaml +0 -24
- endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +0 -20
- endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +0 -19
- endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +0 -11
- endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +0 -48
- endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +0 -43
- endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
- endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +0 -128
- endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +0 -32
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +0 -9
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +0 -36
- endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +0 -15
- endoreg_db/data/_examples/finding_type/data.yaml +0 -43
- endoreg_db/data/_examples/requirement/age.yaml +0 -26
- endoreg_db/data/_examples/requirement/gender.yaml +0 -25
- endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +0 -48
- endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +0 -57
- endoreg_db/data/_examples/requirement_set/endoscopy_bleeding_risk.yaml +0 -52
- endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
- endoreg_db/data/finding/anatomy_colon.yaml +0 -128
- endoreg_db/data/finding/colonoscopy.yaml +0 -40
- endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +0 -56
- endoreg_db/data/finding/complication.yaml +0 -16
- endoreg_db/data/finding/data.yaml +0 -105
- endoreg_db/data/finding/examination_setting.yaml +0 -16
- endoreg_db/data/finding/medication_related.yaml +0 -18
- endoreg_db/data/finding/outcome.yaml +0 -12
- endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +0 -22
- endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +0 -25
- endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +0 -38
- endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +0 -49
- endoreg_db/data/finding_classification/colonoscopy_lst.yaml +0 -21
- endoreg_db/data/finding_classification/colonoscopy_nice.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_paris.yaml +0 -26
- endoreg_db/data/finding_classification/colonoscopy_sano.yaml +0 -22
- endoreg_db/data/finding_classification/colonoscopy_summary.yaml +0 -53
- endoreg_db/data/finding_classification/complication_generic.yaml +0 -25
- endoreg_db/data/finding_classification/examination_setting_generic.yaml +0 -40
- endoreg_db/data/finding_classification/histology_colo.yaml +0 -43
- endoreg_db/data/finding_classification/intervention_required.yaml +0 -26
- endoreg_db/data/finding_classification/medication_related.yaml +0 -23
- endoreg_db/data/finding_classification/visualized.yaml +0 -33
- endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
- endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
- endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
- endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +0 -15
- endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +0 -17
- endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +0 -57
- endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
- endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +0 -14
- endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
- endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +0 -229
- endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +0 -19
- endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +0 -82
- endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
- endoreg_db/data/finding_classification_choice/outcome.yaml +0 -19
- endoreg_db/data/finding_intervention/endoscopy.yaml +0 -43
- endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
- endoreg_db/data/finding_intervention/endoscopy_egd.yaml +0 -128
- endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +0 -32
- endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +0 -9
- endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +0 -36
- endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +0 -79
- endoreg_db/data/requirement/age.yaml +0 -26
- endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +0 -45
- endoreg_db/data/requirement/disease_cardiovascular.yaml +0 -79
- endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +0 -41
- endoreg_db/data/requirement/disease_hepatology.yaml +0 -12
- endoreg_db/data/requirement/disease_misc.yaml +0 -12
- endoreg_db/data/requirement/disease_renal.yaml +0 -96
- endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +0 -59
- endoreg_db/data/requirement/event_cardiology.yaml +0 -251
- endoreg_db/data/requirement/event_requirements.yaml +0 -145
- endoreg_db/data/requirement/finding_colon_polyp.yaml +0 -50
- endoreg_db/data/requirement/gender.yaml +0 -25
- endoreg_db/data/requirement/lab_value.yaml +0 -441
- endoreg_db/data/requirement/medication.yaml +0 -93
- endoreg_db/data/requirement_operator/age.yaml +0 -13
- endoreg_db/data/requirement_operator/lab_operators.yaml +0 -129
- endoreg_db/data/requirement_operator/model_operators.yaml +0 -96
- endoreg_db/management/commands/init_default_ai_model.py +0 -112
- endoreg_db/management/commands/reset_celery_schedule.py +0 -9
- endoreg_db/management/commands/validate_video.py +0 -204
- endoreg_db/migrations/0002_requirementset_depends_on.py +0 -18
- endoreg_db/migrations/_old/0001_initial.py +0 -1857
- endoreg_db/migrations/_old/0002_add_video_correction_models.py +0 -52
- endoreg_db/migrations/_old/0003_add_center_display_name.py +0 -30
- endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +0 -68
- endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +0 -77
- endoreg_db/migrations/_old/0005_merge_20251111_1003.py +0 -14
- endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +0 -68
- endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +0 -89
- endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +0 -27
- endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +0 -21
- endoreg_db/renames.yml +0 -8
- endoreg_db/serializers/_old/raw_pdf_meta_validation.py +0 -223
- endoreg_db/serializers/_old/raw_video_meta_validation.py +0 -179
- endoreg_db/serializers/_old/video.py +0 -71
- endoreg_db/serializers/meta/pdf_file_meta_extraction.py +0 -115
- endoreg_db/serializers/meta/report_meta.py +0 -53
- endoreg_db/serializers/report/__init__.py +0 -9
- endoreg_db/serializers/report/mixins.py +0 -45
- endoreg_db/serializers/report/report.py +0 -105
- endoreg_db/serializers/report/report_list.py +0 -22
- endoreg_db/serializers/report/secure_file_url.py +0 -26
- endoreg_db/services/requirements_object.py +0 -147
- endoreg_db/services/storage_aware_video_processor.py +0 -370
- endoreg_db/urls/files.py +0 -6
- endoreg_db/urls/label_video_segment_validate.py +0 -33
- endoreg_db/urls/label_video_segments.py +0 -46
- endoreg_db/views/label/__init__.py +0 -5
- endoreg_db/views/label/label.py +0 -15
- endoreg_db/views/label_video_segment/__init__.py +0 -16
- endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +0 -44
- endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +0 -50
- endoreg_db/views/label_video_segment/label_video_segment.py +0 -77
- endoreg_db/views/label_video_segment/label_video_segment_by_label.py +0 -174
- endoreg_db/views/label_video_segment/label_video_segment_detail.py +0 -73
- endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +0 -46
- endoreg_db/views/label_video_segment/validate.py +0 -226
- endoreg_db/views/media/segments.py +0 -71
- endoreg_db/views/meta/available_files_list.py +0 -146
- endoreg_db/views/meta/report_meta.py +0 -53
- endoreg_db/views/meta/sensitive_meta_detail.py +0 -85
- endoreg_db/views/misc/secure_file_serving_view.py +0 -80
- endoreg_db/views/misc/secure_file_url_view.py +0 -84
- endoreg_db/views/misc/secure_url_validate.py +0 -79
- endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +0 -164
- endoreg_db/views/patient_finding_location/__init__.py +0 -5
- endoreg_db/views/patient_finding_location/pfl_create.py +0 -70
- endoreg_db/views/patient_finding_morphology/__init__.py +0 -5
- endoreg_db/views/patient_finding_morphology/pfm_create.py +0 -70
- endoreg_db/views/pdf/__init__.py +0 -8
- endoreg_db/views/video/segmentation.py +0 -274
- endoreg_db/views/video/task_status.py +0 -49
- endoreg_db/views/video/timeline.py +0 -46
- endoreg_db/views/video/video_analyze.py +0 -52
- /endoreg_db/data/requirement/{colon_polyp_intervention.yaml → old/colon_polyp_intervention.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/colonoscopy_baseline_austria.yaml +0 -0
- /endoreg_db/data/requirement/{coloreg_colon_polyp.yaml → old/coloreg_colon_polyp.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_cardiovascular.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_classification_choice_cardiovascular.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_hepatology.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_misc.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_renal.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/endoscopy_bleeding_risk.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/event_cardiology.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/event_requirements.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/finding_colon_polyp.yaml +0 -0
- /endoreg_db/{migrations/__init__.py → data/requirement/old/gender.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/lab_value.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/medication.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/age.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/lab_operators.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/model_operators.yaml +0 -0
- /endoreg_db/{urls/sensitive_meta.py → import_files/pseudonymization/__init__.py} +0 -0
- /endoreg_db/{views/pdf/pdf_stream_views.py → import_files/pseudonymization/pseudonymize.py} +0 -0
- /endoreg_db/utils/requirement_operator_logic/{lab_value_operators.py → _old/lab_value_operators.py} +0 -0
- /endoreg_db/utils/requirement_operator_logic/{model_evaluators.py → _old/model_evaluators.py} +0 -0
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,27 +1,28 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
1
|
import os
|
|
3
2
|
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
4
5
|
from django.core.management import BaseCommand
|
|
5
|
-
from endoreg_db.models import (
|
|
6
|
-
RawPdfFile,
|
|
7
|
-
PdfType,
|
|
8
|
-
)
|
|
9
|
-
from endoreg_db.helpers.data_loader import (
|
|
10
|
-
load_data
|
|
11
|
-
)
|
|
12
6
|
from icecream import ic
|
|
7
|
+
|
|
8
|
+
from endoreg_db.helpers.data_loader import load_data
|
|
9
|
+
from endoreg_db.models import PdfType, RawPdfFile
|
|
10
|
+
|
|
11
|
+
|
|
13
12
|
# python manage.py import_report tests/assets/lux-gastro-report.pdf --verbose --start_ollama
|
|
14
13
|
# Dynamic import path manipulation to ensure local development version is used
|
|
15
14
|
def ensure_local_lx_anonymizer():
|
|
16
15
|
"""
|
|
17
16
|
Checks for a local development version of the lx-anonymizer package and adds it to sys.path if available.
|
|
18
|
-
|
|
17
|
+
|
|
19
18
|
Returns:
|
|
20
19
|
True if the local lx-anonymizer directory was found and added to sys.path; False otherwise.
|
|
21
20
|
"""
|
|
22
|
-
script_dir = Path(
|
|
21
|
+
script_dir = Path(
|
|
22
|
+
__file__
|
|
23
|
+
).parent.parent.parent.parent.parent # /home/admin/dev/lx-annotate/endoreg-db
|
|
23
24
|
local_lx_anonymizer_path = script_dir / "lx-anonymizer"
|
|
24
|
-
|
|
25
|
+
|
|
25
26
|
if local_lx_anonymizer_path.exists() and local_lx_anonymizer_path.is_dir():
|
|
26
27
|
# Add the directory containing lx_anonymizer to the Python path
|
|
27
28
|
if str(local_lx_anonymizer_path) not in sys.path:
|
|
@@ -30,11 +31,11 @@ def ensure_local_lx_anonymizer():
|
|
|
30
31
|
return True
|
|
31
32
|
return False
|
|
32
33
|
|
|
34
|
+
|
|
33
35
|
# Try to use local version, fall back to installed version
|
|
34
36
|
local_version_available = ensure_local_lx_anonymizer()
|
|
35
37
|
|
|
36
38
|
|
|
37
|
-
|
|
38
39
|
# Now import from lx_anonymizer
|
|
39
40
|
try:
|
|
40
41
|
from lx_anonymizer.ollama_service import ollama_service
|
|
@@ -42,6 +43,7 @@ except ImportError:
|
|
|
42
43
|
print("Could not import init_ollama_service from local or installed lx_anonymizer")
|
|
43
44
|
raise
|
|
44
45
|
|
|
46
|
+
|
|
45
47
|
class Command(BaseCommand):
|
|
46
48
|
"""Management Command to import a report file to the database"""
|
|
47
49
|
|
|
@@ -53,7 +55,7 @@ class Command(BaseCommand):
|
|
|
53
55
|
def add_arguments(self, parser):
|
|
54
56
|
"""
|
|
55
57
|
Defines command-line arguments for the import_report management command.
|
|
56
|
-
|
|
58
|
+
|
|
57
59
|
Adds options for specifying the report file path, center name, report directory, deletion and save behavior, and controls for initializing the Ollama LLM service.
|
|
58
60
|
"""
|
|
59
61
|
parser.add_argument(
|
|
@@ -94,21 +96,21 @@ class Command(BaseCommand):
|
|
|
94
96
|
default=False,
|
|
95
97
|
help="Save the report object to the database",
|
|
96
98
|
)
|
|
97
|
-
|
|
99
|
+
|
|
98
100
|
parser.add_argument(
|
|
99
101
|
"--start_ollama",
|
|
100
102
|
action="store_true",
|
|
101
103
|
default=False,
|
|
102
104
|
help="Start Ollama server for LLM processing",
|
|
103
105
|
)
|
|
104
|
-
|
|
106
|
+
|
|
105
107
|
parser.add_argument(
|
|
106
108
|
"--ollama_debug",
|
|
107
109
|
action="store_true",
|
|
108
110
|
default=False,
|
|
109
111
|
help="Enable debug mode for Ollama",
|
|
110
112
|
)
|
|
111
|
-
|
|
113
|
+
|
|
112
114
|
parser.add_argument(
|
|
113
115
|
"--ollama_timeout",
|
|
114
116
|
type=int,
|
|
@@ -118,9 +120,9 @@ class Command(BaseCommand):
|
|
|
118
120
|
|
|
119
121
|
def handle(self, *args, **options):
|
|
120
122
|
"""
|
|
121
|
-
Handles the import of a
|
|
122
|
-
|
|
123
|
-
This method validates input options, optionally starts the Ollama LLM service, ensures the existence of required files and directories, determines the report type, processes the
|
|
123
|
+
Handles the import of a report report file into the database, with optional LLM service initialization and anonymization.
|
|
124
|
+
|
|
125
|
+
This method validates input options, optionally starts the Ollama LLM service, ensures the existence of required files and directories, determines the report type, processes the report for text and metadata extraction, anonymizes content, and saves the resulting data to the database. Provides verbose output and error handling throughout the process.
|
|
124
126
|
"""
|
|
125
127
|
# Load initial or prerequisite data for the application.
|
|
126
128
|
# This may include loading default values, configurations, or lookup table data
|
|
@@ -132,8 +134,7 @@ class Command(BaseCommand):
|
|
|
132
134
|
self.stdout.write(self.style.ERROR(f"Failed to load initial data: {e}"))
|
|
133
135
|
# Depending on the criticality of load_data(), you might want to exit or handle differently.
|
|
134
136
|
# For now, we'll just log the error and continue.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
+
|
|
137
138
|
verbose = options["verbose"]
|
|
138
139
|
center_name = options["center_name"]
|
|
139
140
|
report_dir_root = options["report_dir_root"]
|
|
@@ -147,11 +148,15 @@ class Command(BaseCommand):
|
|
|
147
148
|
if not isinstance(delete_source, bool):
|
|
148
149
|
raise ValueError("delete_source must be a boolean")
|
|
149
150
|
|
|
150
|
-
self.stdout.write(
|
|
151
|
-
|
|
151
|
+
self.stdout.write(
|
|
152
|
+
self.style.SUCCESS(f"Starting report import for {file_path}...")
|
|
153
|
+
)
|
|
154
|
+
|
|
152
155
|
if local_version_available:
|
|
153
|
-
self.stdout.write(
|
|
154
|
-
|
|
156
|
+
self.stdout.write(
|
|
157
|
+
self.style.SUCCESS("Using local development version of lx-anonymizer")
|
|
158
|
+
)
|
|
159
|
+
|
|
155
160
|
ollama_proc = None # Track Ollama process if started
|
|
156
161
|
try:
|
|
157
162
|
# Initialize Ollama service if requested
|
|
@@ -161,60 +166,101 @@ class Command(BaseCommand):
|
|
|
161
166
|
# Set Ollama environment variables
|
|
162
167
|
os.environ["OLLAMA_MAX_WAIT_TIME"] = str(ollama_timeout)
|
|
163
168
|
os.environ["OLLAMA_DEBUG"] = "true" if ollama_debug else "false"
|
|
164
|
-
|
|
169
|
+
|
|
165
170
|
# Try to find Ollama binary location from env or common paths
|
|
166
171
|
ollama_bin = os.environ.get("OLLAMA_BIN")
|
|
167
172
|
if not ollama_bin:
|
|
168
173
|
# Try common Nix store paths first
|
|
169
|
-
for path in [
|
|
170
|
-
|
|
174
|
+
for path in [
|
|
175
|
+
"/run/current-system/sw/bin/ollama",
|
|
176
|
+
"/nix/store/*/bin/ollama",
|
|
177
|
+
]:
|
|
171
178
|
import glob
|
|
179
|
+
|
|
172
180
|
matches = glob.glob(path)
|
|
173
181
|
if matches:
|
|
174
182
|
ollama_bin = matches[0]
|
|
175
183
|
break
|
|
176
|
-
|
|
184
|
+
|
|
177
185
|
if ollama_bin:
|
|
178
|
-
self.stdout.write(
|
|
186
|
+
self.stdout.write(
|
|
187
|
+
self.style.SUCCESS(f"Using Ollama binary at: {ollama_bin}")
|
|
188
|
+
)
|
|
179
189
|
os.environ["OLLAMA_BIN"] = ollama_bin
|
|
180
|
-
|
|
190
|
+
|
|
181
191
|
# Start Ollama server process if not already running
|
|
182
|
-
import subprocess
|
|
183
192
|
import shutil
|
|
184
|
-
|
|
193
|
+
import subprocess
|
|
194
|
+
|
|
185
195
|
# Check if ollama is already running
|
|
186
196
|
try:
|
|
187
197
|
import requests
|
|
188
|
-
|
|
198
|
+
|
|
199
|
+
resp = requests.get(
|
|
200
|
+
"http://127.0.0.1:11434/api/version", timeout=1
|
|
201
|
+
)
|
|
189
202
|
if resp.status_code == 200:
|
|
190
|
-
self.stdout.write(
|
|
203
|
+
self.stdout.write(
|
|
204
|
+
self.style.SUCCESS("Ollama is already running")
|
|
205
|
+
)
|
|
191
206
|
else:
|
|
192
|
-
self.stdout.write(
|
|
207
|
+
self.stdout.write(
|
|
208
|
+
self.style.WARNING(
|
|
209
|
+
f"Ollama returned status code {resp.status_code}"
|
|
210
|
+
)
|
|
211
|
+
)
|
|
193
212
|
except requests.exceptions.RequestException:
|
|
194
|
-
self.stdout.write(
|
|
213
|
+
self.stdout.write(
|
|
214
|
+
self.style.WARNING(
|
|
215
|
+
"Ollama is not running, attempting to start..."
|
|
216
|
+
)
|
|
217
|
+
)
|
|
195
218
|
# Find ollama binary
|
|
196
219
|
ollama_path = ollama_bin or shutil.which("ollama")
|
|
197
220
|
if ollama_path:
|
|
198
|
-
self.stdout.write(
|
|
221
|
+
self.stdout.write(
|
|
222
|
+
self.style.SUCCESS(
|
|
223
|
+
f"Starting Ollama using {ollama_path}"
|
|
224
|
+
)
|
|
225
|
+
)
|
|
199
226
|
# Start ollama serve in background
|
|
200
|
-
ollama_proc = subprocess.Popen(
|
|
201
|
-
ollama_path, "serve"
|
|
202
|
-
|
|
203
|
-
|
|
227
|
+
ollama_proc = subprocess.Popen(
|
|
228
|
+
[ollama_path, "serve"],
|
|
229
|
+
stdout=subprocess.PIPE,
|
|
230
|
+
stderr=subprocess.STDOUT,
|
|
231
|
+
start_new_session=True,
|
|
232
|
+
)
|
|
233
|
+
self.stdout.write(
|
|
234
|
+
self.style.SUCCESS(
|
|
235
|
+
"Ollama server started in background"
|
|
236
|
+
)
|
|
237
|
+
)
|
|
204
238
|
else:
|
|
205
|
-
self.stdout.write(
|
|
206
|
-
|
|
239
|
+
self.stdout.write(
|
|
240
|
+
self.style.ERROR("Ollama binary not found in PATH")
|
|
241
|
+
)
|
|
242
|
+
|
|
207
243
|
# Start the service with explicit initialization
|
|
208
244
|
ollama_service(auto_start=True)
|
|
209
|
-
self.stdout.write(
|
|
245
|
+
self.stdout.write(
|
|
246
|
+
self.style.SUCCESS("Ollama service initialized successfully")
|
|
247
|
+
)
|
|
210
248
|
except Exception as e:
|
|
211
|
-
self.stdout.write(
|
|
212
|
-
|
|
249
|
+
self.stdout.write(
|
|
250
|
+
self.style.ERROR(f"Failed to initialize Ollama service: {e}")
|
|
251
|
+
)
|
|
252
|
+
self.stdout.write(
|
|
253
|
+
self.style.WARNING(
|
|
254
|
+
"Continuing without Ollama - some features may not work"
|
|
255
|
+
)
|
|
256
|
+
)
|
|
213
257
|
|
|
214
258
|
# Ensure the report file exists
|
|
215
259
|
file_path = Path(file_path).expanduser()
|
|
216
260
|
if not file_path.exists():
|
|
217
|
-
self.stdout.write(
|
|
261
|
+
self.stdout.write(
|
|
262
|
+
self.style.ERROR(f"Report file not found: {file_path}")
|
|
263
|
+
)
|
|
218
264
|
return
|
|
219
265
|
|
|
220
266
|
# Ensure the report directory exists
|
|
@@ -222,7 +268,9 @@ class Command(BaseCommand):
|
|
|
222
268
|
report_dir_root.mkdir(parents=True, exist_ok=True)
|
|
223
269
|
|
|
224
270
|
# Create the report file object
|
|
225
|
-
self.stdout.write(
|
|
271
|
+
self.stdout.write(
|
|
272
|
+
self.style.SUCCESS(f"Creating RawPdfFile object from {file_path}...")
|
|
273
|
+
)
|
|
226
274
|
report_file_obj = RawPdfFile.create_from_file(
|
|
227
275
|
file_path=file_path,
|
|
228
276
|
center_name=center_name,
|
|
@@ -230,11 +278,13 @@ class Command(BaseCommand):
|
|
|
230
278
|
save=save,
|
|
231
279
|
)
|
|
232
280
|
if not report_file_obj:
|
|
233
|
-
self.stdout.write(
|
|
281
|
+
self.stdout.write(
|
|
282
|
+
self.style.ERROR("Failed to create RawPdfFile object.")
|
|
283
|
+
)
|
|
234
284
|
return
|
|
235
|
-
|
|
285
|
+
|
|
236
286
|
report_file_obj.anonymized = False
|
|
237
|
-
|
|
287
|
+
|
|
238
288
|
# Assign pdfType to the report file object
|
|
239
289
|
if "report" in file_path.name:
|
|
240
290
|
pdf_type_name = "ukw-endoscopy-examination-report-generic"
|
|
@@ -247,19 +297,24 @@ class Command(BaseCommand):
|
|
|
247
297
|
else:
|
|
248
298
|
raise ValueError(f"Unknown report type: {file_path.name}")
|
|
249
299
|
|
|
250
|
-
self.stdout.write(self.style.SUCCESS(f"Using
|
|
300
|
+
self.stdout.write(self.style.SUCCESS(f"Using report type: {pdf_type_name}"))
|
|
251
301
|
try:
|
|
252
302
|
pdf_type = PdfType.objects.get(name=pdf_type_name)
|
|
253
303
|
except PdfType.DoesNotExist:
|
|
254
|
-
self.stdout.write(
|
|
304
|
+
self.stdout.write(
|
|
305
|
+
self.style.ERROR(
|
|
306
|
+
f"PdfType with name '{pdf_type_name}' does not exist."
|
|
307
|
+
)
|
|
308
|
+
)
|
|
255
309
|
return
|
|
256
310
|
report_file_obj.pdf_type = pdf_type
|
|
257
311
|
|
|
258
312
|
rr_config = report_file_obj.get_report_reader_config()
|
|
259
313
|
pdf_path = report_file_obj.file.path
|
|
260
|
-
|
|
314
|
+
|
|
261
315
|
# Import at this point to avoid initializing the module too early
|
|
262
316
|
from lx_anonymizer import ReportReader
|
|
317
|
+
|
|
263
318
|
self.stdout.write(self.style.SUCCESS("Creating ReportReader..."))
|
|
264
319
|
rr = ReportReader(**rr_config)
|
|
265
320
|
|
|
@@ -270,29 +325,39 @@ class Command(BaseCommand):
|
|
|
270
325
|
|
|
271
326
|
if verbose:
|
|
272
327
|
ic(text, anonymized_text, report_meta)
|
|
273
|
-
|
|
328
|
+
|
|
274
329
|
self.stdout.write(self.style.SUCCESS("Processing file..."))
|
|
275
|
-
report_file_obj.process_file(
|
|
276
|
-
|
|
330
|
+
report_file_obj.process_file(
|
|
331
|
+
text, anonymized_text, report_meta, verbose=verbose
|
|
332
|
+
)
|
|
333
|
+
|
|
277
334
|
sensitive_meta = report_file_obj.sensitive_meta
|
|
278
335
|
if verbose:
|
|
279
336
|
ic(report_file_obj.sensitive_meta)
|
|
280
|
-
|
|
281
|
-
|
|
337
|
+
|
|
282
338
|
self.stdout.write(self.style.SUCCESS("Saving..."))
|
|
283
339
|
sensitive_meta.save()
|
|
284
340
|
if verbose:
|
|
285
341
|
ic(sensitive_meta)
|
|
286
|
-
|
|
287
|
-
report_file_obj.
|
|
342
|
+
|
|
343
|
+
report_file_obj.state.anonymization_status.mark_anonymized()
|
|
288
344
|
finally:
|
|
289
345
|
# Clean up Ollama process if we started it
|
|
290
346
|
if ollama_proc is not None:
|
|
291
347
|
import signal
|
|
292
|
-
|
|
348
|
+
|
|
349
|
+
self.stdout.write(
|
|
350
|
+
self.style.SUCCESS("Cleaning up Ollama server process...")
|
|
351
|
+
)
|
|
293
352
|
try:
|
|
294
353
|
ollama_proc.terminate()
|
|
295
354
|
ollama_proc.wait(timeout=10)
|
|
296
|
-
self.stdout.write(
|
|
355
|
+
self.stdout.write(
|
|
356
|
+
self.style.SUCCESS("Ollama server process terminated.")
|
|
357
|
+
)
|
|
297
358
|
except Exception as e:
|
|
298
|
-
self.stdout.write(
|
|
359
|
+
self.stdout.write(
|
|
360
|
+
self.style.WARNING(
|
|
361
|
+
f"Failed to terminate Ollama server process: {e}"
|
|
362
|
+
)
|
|
363
|
+
)
|
|
@@ -10,7 +10,7 @@ from endoreg_db.models.administration.center import Center
|
|
|
10
10
|
from endoreg_db.models.medical.hardware import EndoscopyProcessor
|
|
11
11
|
|
|
12
12
|
# TODO Migrate
|
|
13
|
-
from endoreg_db.serializers.
|
|
13
|
+
from endoreg_db.serializers.Frames_NICE_and_PARIS_classifications import (
|
|
14
14
|
ForNiceClassificationSerializer,
|
|
15
15
|
ForParisClassificationSerializer
|
|
16
16
|
)
|
|
@@ -22,13 +22,13 @@ IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
|
|
|
22
22
|
VideoSegmentationLabel.__name__,
|
|
23
23
|
VideoSegmentationLabelSet.__name__,
|
|
24
24
|
AiModel.__name__,
|
|
25
|
-
ModelMeta.__name__, #
|
|
25
|
+
# ModelMeta.__name__, # Disable automatic model meta loading
|
|
26
26
|
]
|
|
27
27
|
|
|
28
28
|
IMPORT_METADATA = {
|
|
29
29
|
ModelType.__name__: {
|
|
30
30
|
"dir": MODEL_TYPE_DATA_DIR, # e.g. "intervention_types"
|
|
31
|
-
"model": ModelType,
|
|
31
|
+
"model": ModelType,
|
|
32
32
|
"foreign_keys": [], # e.g. ["interventions"]
|
|
33
33
|
"foreign_key_models": [], # e.g. [Intervention]
|
|
34
34
|
},
|
|
@@ -40,19 +40,19 @@ IMPORT_METADATA = {
|
|
|
40
40
|
},
|
|
41
41
|
VideoSegmentationLabel.__name__: {
|
|
42
42
|
"dir": VIDEO_SEGMENTATION_LABEL_DATA_DIR, # e.g. "interventions"
|
|
43
|
-
"model": VideoSegmentationLabel,
|
|
43
|
+
"model": VideoSegmentationLabel,
|
|
44
44
|
"foreign_keys": [], # e.g. ["intervention_types"]
|
|
45
45
|
"foreign_key_models": [], # e.g. [InterventionType]
|
|
46
46
|
},
|
|
47
47
|
VideoSegmentationLabelSet.__name__: {
|
|
48
48
|
"dir": VIDEO_SEGMENTATION_LABELSET_DATA_DIR, # e.g. "interventions"
|
|
49
|
-
"model": VideoSegmentationLabelSet,
|
|
49
|
+
"model": VideoSegmentationLabelSet,
|
|
50
50
|
"foreign_keys": ["labels"], # e.g. ["intervention_types"]
|
|
51
51
|
"foreign_key_models": [VideoSegmentationLabel], # e.g. [Intervention]
|
|
52
52
|
},
|
|
53
53
|
AiModel.__name__: {
|
|
54
54
|
"dir": AI_MODEL_DATA_DIR, # e.g. "intervention_types"
|
|
55
|
-
"model": AiModel,
|
|
55
|
+
"model": AiModel,
|
|
56
56
|
"foreign_keys": ["video_segmentation_labelset", "model_type"], # e.g. ["interventions"]
|
|
57
57
|
"foreign_key_models": [VideoSegmentationLabelSet, ModelType], # e.g. [Intervention]
|
|
58
58
|
},
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
1
3
|
from django.conf import settings
|
|
2
4
|
from django.core.management.base import BaseCommand
|
|
3
|
-
|
|
4
|
-
import os
|
|
5
|
-
from ...utils import load_model_data_from_yaml
|
|
5
|
+
|
|
6
6
|
from ...data import LABEL_DATA_DIR
|
|
7
|
+
from ...models import Label, LabelSet, LabelType
|
|
8
|
+
from ...utils import load_model_data_from_yaml
|
|
7
9
|
|
|
8
10
|
SOURCE_DIR = LABEL_DATA_DIR
|
|
9
11
|
|
|
@@ -16,25 +18,25 @@ IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
|
|
|
16
18
|
IMPORT_METADATA = {
|
|
17
19
|
# "": { # same as model name in "import models", e.g. "Intervention"
|
|
18
20
|
# "subdir": os.path.join(SOURCE_DIR,""), # e.g. "interventions"
|
|
19
|
-
# "model": None,
|
|
21
|
+
# "model": None,
|
|
20
22
|
# "foreign_keys": [], # e.g. ["intervention_types"]
|
|
21
23
|
# "foreign_key_models": [] # e.g. [InterventionType]
|
|
22
24
|
# },
|
|
23
25
|
"LabelType": {
|
|
24
26
|
"dir": os.path.join(SOURCE_DIR, "label-type"), # e.g. "interventions"
|
|
25
|
-
"model": LabelType,
|
|
27
|
+
"model": LabelType,
|
|
26
28
|
"foreign_keys": [], # e.g. ["intervention_types"]
|
|
27
29
|
"foreign_key_models": [], # e.g. [InterventionType]
|
|
28
30
|
},
|
|
29
31
|
"Label": {
|
|
30
32
|
"dir": os.path.join(SOURCE_DIR, "label"), # e.g. "interventions"
|
|
31
|
-
"model": Label,
|
|
33
|
+
"model": Label,
|
|
32
34
|
"foreign_keys": ["label_type"], # e.g. ["intervention_types"]
|
|
33
35
|
"foreign_key_models": [LabelType], # e.g. [InterventionType]
|
|
34
36
|
},
|
|
35
37
|
"LabelSet": {
|
|
36
38
|
"dir": os.path.join(SOURCE_DIR, "label-set"), # e.g. "interventions"
|
|
37
|
-
"model": LabelSet,
|
|
39
|
+
"model": LabelSet,
|
|
38
40
|
"foreign_keys": ["labels"], # e.g. ["intervention_types"]
|
|
39
41
|
"foreign_key_models": [Label], # e.g. [InterventionType]
|
|
40
42
|
},
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from django.core.management import BaseCommand, call_command
|
|
2
|
-
from io import StringIO
|
|
3
2
|
|
|
4
3
|
|
|
5
4
|
class Command(BaseCommand):
|
|
@@ -16,9 +15,9 @@ class Command(BaseCommand):
|
|
|
16
15
|
# verbose = options['verbose']
|
|
17
16
|
"""
|
|
18
17
|
Orchestrates the sequential execution of data loading commands to populate base database models.
|
|
19
|
-
|
|
20
|
-
This management command displays an initial message and then runs a series of data loading routines
|
|
21
|
-
(via call_command) in a specified order. It ignores any verbose setting from the command-line options
|
|
18
|
+
|
|
19
|
+
This management command displays an initial message and then runs a series of data loading routines
|
|
20
|
+
(via call_command) in a specified order. It ignores any verbose setting from the command-line options
|
|
22
21
|
and forces verbose output. A final success message is printed after all commands complete.
|
|
23
22
|
"""
|
|
24
23
|
verbose = True
|
|
@@ -45,10 +44,10 @@ class Command(BaseCommand):
|
|
|
45
44
|
call_command("load_event_data", stdout=out, verbose=verbose)
|
|
46
45
|
call_command("load_organ_data", stdout=out, verbose=verbose)
|
|
47
46
|
call_command("load_contraindication_data", stdout=out, verbose=verbose)
|
|
48
|
-
call_command("load_examination_data", stdout=out, verbose=verbose)
|
|
49
|
-
call_command("load_lab_value_data", stdout=out, verbose=verbose)
|
|
50
47
|
call_command("load_finding_data", stdout=out, verbose=verbose)
|
|
51
48
|
call_command("load_examination_indication_data", stdout=out, verbose=verbose)
|
|
49
|
+
call_command("load_examination_data", stdout=out, verbose=verbose)
|
|
50
|
+
call_command("load_lab_value_data", stdout=out, verbose=verbose)
|
|
52
51
|
call_command("load_medication_data", stdout=out, verbose=verbose)
|
|
53
52
|
|
|
54
53
|
call_command("load_requirement_data", stdout=out, verbose=verbose)
|
|
@@ -62,131 +61,3 @@ class Command(BaseCommand):
|
|
|
62
61
|
"All data loading commands executed successfully."
|
|
63
62
|
)
|
|
64
63
|
)
|
|
65
|
-
|
|
66
|
-
# # Run the load_network_data command
|
|
67
|
-
# self.stdout.write(self.style.SUCCESS("Running load_network_data..."))
|
|
68
|
-
# call_command('load_network_data', verbose=verbose)
|
|
69
|
-
# Run the load_network_data command
|
|
70
|
-
# self.stdout.write(self.style.SUCCESS("Running load_network_data..."))
|
|
71
|
-
# call_command("load_network_data", verbose=verbose)
|
|
72
|
-
|
|
73
|
-
# # Run the load_logging_data command
|
|
74
|
-
# self.stdout.write(self.style.SUCCESS("Running load_logging_data..."))
|
|
75
|
-
# call_command("load_logging_data", verbose=verbose)
|
|
76
|
-
|
|
77
|
-
# # Run the load_profession_data command
|
|
78
|
-
# self.stdout.write(self.style.SUCCESS("Running load_profession_data..."))
|
|
79
|
-
# call_command("load_profession_data", verbose=verbose)
|
|
80
|
-
|
|
81
|
-
# # run the load_gender_data command
|
|
82
|
-
# self.stdout.write(self.style.SUCCESS("Running load_gender_data..."))
|
|
83
|
-
# call_command("load_gender_data", verbose=verbose)
|
|
84
|
-
|
|
85
|
-
# # Run the load_examination_data command
|
|
86
|
-
# self.stdout.write(self.style.SUCCESS("Running load_examination_data..."))
|
|
87
|
-
# call_command("load_examination_data", verbose=verbose)
|
|
88
|
-
|
|
89
|
-
# # run the load_finding_data command
|
|
90
|
-
# self.stdout.write(self.style.SUCCESS("Running load_finding_data..."))
|
|
91
|
-
# call_command("load_finding_data", verbose=verbose)
|
|
92
|
-
|
|
93
|
-
# # Run the load_disease data command
|
|
94
|
-
# self.stdout.write(self.style.SUCCESS("Running load_disease_data..."))
|
|
95
|
-
# call_command("load_disease_data", verbose=verbose)
|
|
96
|
-
|
|
97
|
-
# # # Run the load_disease_classification data command
|
|
98
|
-
# # self.stdout.write(self.style.SUCCESS("Running load_disease_classification_data..."))
|
|
99
|
-
# # call_command('load_disease_classification_data', verbose=verbose)
|
|
100
|
-
|
|
101
|
-
# # # Run the load_disease_classification_choices data command
|
|
102
|
-
# # self.stdout.write(self.style.SUCCESS("Running load_disease_classification_choices_data..."))
|
|
103
|
-
# # call_command('load_disease_classification_choices_data', verbose=verbose)
|
|
104
|
-
|
|
105
|
-
# # rund the load_user_groups command
|
|
106
|
-
# self.stdout.write(self.style.SUCCESS("Running load_user_groups..."))
|
|
107
|
-
# call_command("load_user_groups", verbose=verbose)
|
|
108
|
-
|
|
109
|
-
# # run the load_report_reader_flag command
|
|
110
|
-
# self.stdout.write(self.style.SUCCESS("Running load_report_reader_flag..."))
|
|
111
|
-
# call_command("load_report_reader_flag", verbose=verbose)
|
|
112
|
-
|
|
113
|
-
# # Run the load_pdf_type_data command
|
|
114
|
-
# self.stdout.write(self.style.SUCCESS("Running load_pdf_type_data..."))
|
|
115
|
-
# call_command("load_pdf_type_data", verbose=verbose)
|
|
116
|
-
|
|
117
|
-
# # Run the load_endoscopy_processor_data command
|
|
118
|
-
# self.stdout.write(
|
|
119
|
-
# self.style.SUCCESS("Running load_endoscopy_processor_data...")
|
|
120
|
-
# )
|
|
121
|
-
# call_command("load_endoscopy_processor_data", verbose=verbose)
|
|
122
|
-
|
|
123
|
-
# # Run the load_endoscope_type_data command
|
|
124
|
-
# self.stdout.write(self.style.SUCCESS("Running load_endoscope_type_data..."))
|
|
125
|
-
# call_command("load_endoscope_type_data", verbose=verbose)
|
|
126
|
-
|
|
127
|
-
# # Run the load_unit_data command
|
|
128
|
-
# self.stdout.write(self.style.SUCCESS("Running load_unit_data..."))
|
|
129
|
-
# call_command("load_unit_data", verbose=verbose)
|
|
130
|
-
|
|
131
|
-
# # Run the load_lab_value_data command
|
|
132
|
-
# self.stdout.write(self.style.SUCCESS("Running load_lab_value_data..."))
|
|
133
|
-
# call_command("load_lab_value_data", verbose=verbose)
|
|
134
|
-
|
|
135
|
-
# # Run the load_information_source command
|
|
136
|
-
# self.stdout.write(self.style.SUCCESS("Running load_information_source..."))
|
|
137
|
-
# call_command("load_information_source", verbose=verbose)
|
|
138
|
-
|
|
139
|
-
# # Run the load_center_data command
|
|
140
|
-
# self.stdout.write(self.style.SUCCESS("Running load_center_data..."))
|
|
141
|
-
# call_command("load_center_data", verbose=verbose)
|
|
142
|
-
|
|
143
|
-
# # Run the load_label_data command
|
|
144
|
-
# self.stdout.write(self.style.SUCCESS("Running load_label_data..."))
|
|
145
|
-
# call_command("load_label_data", verbose=verbose)
|
|
146
|
-
|
|
147
|
-
# # Run the load_ai_model_data command
|
|
148
|
-
# self.stdout.write(self.style.SUCCESS("Running load_ai_model_data..."))
|
|
149
|
-
# call_command("load_ai_model_data", verbose=verbose)
|
|
150
|
-
|
|
151
|
-
# # Run the load_event_data command
|
|
152
|
-
# self.stdout.write(self.style.SUCCESS("Running load_event_data..."))
|
|
153
|
-
# call_command("load_event_data", verbose=verbose)
|
|
154
|
-
|
|
155
|
-
# # run the load_medication_data command
|
|
156
|
-
# self.stdout.write(self.style.SUCCESS("Running load_medication_data..."))
|
|
157
|
-
# call_command("load_medication_data", verbose=verbose)
|
|
158
|
-
|
|
159
|
-
# # # Run the load_medication_indication_type_data command
|
|
160
|
-
# # self.stdout.write(self.style.SUCCESS("Running load_medication_indication_type_data..."))
|
|
161
|
-
# # call_command('load_medication_indication_type_data', verbose=verbose)
|
|
162
|
-
|
|
163
|
-
# # # Run the load_medication_intake_time_data command
|
|
164
|
-
# # self.stdout.write(self.style.SUCCESS("Running load_medication_intake_time_data..."))
|
|
165
|
-
# # call_command('load_medication_intake_time_data', verbose=verbose)
|
|
166
|
-
|
|
167
|
-
# # # Run the load_medication_schedule_data command
|
|
168
|
-
# # self.stdout.write(self.style.SUCCESS("Running load_medication_schedule_data..."))
|
|
169
|
-
# # call_command('load_medication_schedule_data', verbose=verbose)
|
|
170
|
-
|
|
171
|
-
# # # Run the load_medication_indication_data command
|
|
172
|
-
# # self.stdout.write(self.style.SUCCESS("Running load_medication_indication_data..."))
|
|
173
|
-
# # call_command('load_medication_indication_data', verbose=verbose)
|
|
174
|
-
|
|
175
|
-
# # Run the load_green_endoscopy_wuerzburg_data command
|
|
176
|
-
# self.stdout.write(
|
|
177
|
-
# self.style.SUCCESS("Running load_green_endoscopy_wuerzburg_data...")
|
|
178
|
-
# )
|
|
179
|
-
# call_command("load_green_endoscopy_wuerzburg_data", verbose=verbose)
|
|
180
|
-
|
|
181
|
-
# # Load G-Play Data
|
|
182
|
-
# # run the load_distribution_data command
|
|
183
|
-
# self.stdout.write(self.style.SUCCESS("Running load_distribution_data..."))
|
|
184
|
-
# call_command("load_distribution_data", verbose=verbose)
|
|
185
|
-
# # Run the load_g_play_data command
|
|
186
|
-
# self.stdout.write(self.style.SUCCESS("Running load_g_play_data..."))
|
|
187
|
-
# call_command("load_g_play_data", verbose=verbose)
|
|
188
|
-
|
|
189
|
-
# # Necessary? Migrate? FIXME
|
|
190
|
-
# # # Run the load_active_model_data command
|
|
191
|
-
# # self.stdout.write(self.style.SUCCESS("Running load_active_model_data..."))
|
|
192
|
-
# # call_command('load_active_model_data', verbose=verbose)
|