endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/data/__init__.py +22 -8
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +0 -1
- endoreg_db/data/examination/examinations/data.yaml +114 -14
- endoreg_db/data/examination/time-type/data.yaml +0 -3
- endoreg_db/data/examination_indication/endoscopy.yaml +108 -173
- endoreg_db/data/examination_indication_classification/endoscopy.yaml +0 -70
- endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +33 -37
- endoreg_db/data/finding/00_generic.yaml +35 -0
- endoreg_db/data/finding/00_generic_complication.yaml +9 -0
- endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
- endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
- endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
- endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
- endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
- endoreg_db/data/finding_classification/00_generic.yaml +44 -0
- endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
- endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
- endoreg_db/data/finding_classification/{colonoscopy_bowel_preparation.yaml → 02_colonoscopy_baseline.yaml} +35 -20
- endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
- endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
- endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
- endoreg_db/data/finding_classification_choice/{yes_no_na.yaml → 00_generic.yaml} +5 -1
- endoreg_db/data/finding_classification_choice/{examination_setting_generic_types.yaml → 00_generic_baseline.yaml} +10 -2
- endoreg_db/data/finding_classification_choice/{complication_generic_types.yaml → 00_generic_complication.yaml} +1 -1
- endoreg_db/data/finding_classification_choice/{histology.yaml → 00_generic_histology.yaml} +1 -4
- endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
- endoreg_db/data/finding_classification_choice/{bowel_preparation.yaml → 02_colonoscopy_bowel_preparation.yaml} +1 -30
- endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml → finding_classification_choice/02_colonoscopy_generic.yaml} +1 -1
- endoreg_db/data/finding_classification_choice/{histology_polyp.yaml → 02_colonoscopy_histology.yaml} +1 -1
- endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_location.yaml → finding_classification_choice/02_colonoscopy_location.yaml} +23 -4
- endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
- endoreg_db/data/{_examples/finding_classification_choice/colon_lesion_paris.yaml → finding_classification_choice/02_colonoscopy_polyp_morphology.yaml} +26 -8
- endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
- endoreg_db/data/finding_classification_type/{colonoscopy_basic.yaml → 00_generic.yaml} +18 -13
- endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
- endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
- endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
- endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
- endoreg_db/data/finding_type/data.yaml +8 -12
- endoreg_db/data/requirement/01_patient_data.yaml +93 -0
- endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +0 -2
- endoreg_db/data/requirement_set/90_coloreg.yaml +20 -8
- endoreg_db/exceptions.py +0 -1
- endoreg_db/forms/examination_form.py +1 -1
- endoreg_db/helpers/data_loader.py +124 -52
- endoreg_db/helpers/default_objects.py +116 -81
- endoreg_db/import_files/__init__.py +27 -0
- endoreg_db/import_files/context/__init__.py +7 -0
- endoreg_db/import_files/context/default_sensitive_meta.py +81 -0
- endoreg_db/import_files/context/ensure_center.py +17 -0
- endoreg_db/import_files/context/file_lock.py +66 -0
- endoreg_db/import_files/context/import_context.py +43 -0
- endoreg_db/import_files/context/validate_directories.py +56 -0
- endoreg_db/import_files/file_storage/__init__.py +15 -0
- endoreg_db/import_files/file_storage/create_report_file.py +76 -0
- endoreg_db/import_files/file_storage/create_video_file.py +75 -0
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +39 -0
- endoreg_db/import_files/file_storage/state_management.py +496 -0
- endoreg_db/import_files/file_storage/storage.py +36 -0
- endoreg_db/import_files/import_service.md +26 -0
- endoreg_db/import_files/processing/__init__.py +11 -0
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +94 -0
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
- endoreg_db/import_files/pseudonymization/fake.py +52 -0
- endoreg_db/import_files/pseudonymization/k_anonymity.py +182 -0
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +128 -0
- endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
- endoreg_db/import_files/report_import_service.py +141 -0
- endoreg_db/import_files/video_import_service.py +150 -0
- endoreg_db/management/commands/import_report.py +130 -65
- endoreg_db/management/commands/import_video_with_classification.py +1 -1
- endoreg_db/management/commands/load_ai_model_data.py +5 -5
- endoreg_db/management/commands/load_ai_model_label_data.py +9 -7
- endoreg_db/management/commands/load_base_db_data.py +5 -134
- endoreg_db/management/commands/load_contraindication_data.py +14 -16
- endoreg_db/management/commands/load_disease_classification_choices_data.py +15 -18
- endoreg_db/management/commands/load_disease_classification_data.py +15 -18
- endoreg_db/management/commands/load_disease_data.py +25 -28
- endoreg_db/management/commands/load_endoscope_data.py +20 -27
- endoreg_db/management/commands/load_event_data.py +14 -16
- endoreg_db/management/commands/load_examination_data.py +31 -44
- endoreg_db/management/commands/load_examination_indication_data.py +20 -21
- endoreg_db/management/commands/load_finding_data.py +52 -80
- endoreg_db/management/commands/load_information_source.py +21 -23
- endoreg_db/management/commands/load_lab_value_data.py +17 -26
- endoreg_db/management/commands/load_medication_data.py +13 -12
- endoreg_db/management/commands/load_organ_data.py +15 -19
- endoreg_db/management/commands/load_pdf_type_data.py +19 -18
- endoreg_db/management/commands/load_profession_data.py +14 -17
- endoreg_db/management/commands/load_qualification_data.py +20 -23
- endoreg_db/management/commands/load_report_reader_flag_data.py +17 -19
- endoreg_db/management/commands/load_requirement_data.py +14 -20
- endoreg_db/management/commands/load_risk_data.py +7 -6
- endoreg_db/management/commands/load_shift_data.py +20 -23
- endoreg_db/management/commands/load_tag_data.py +8 -11
- endoreg_db/management/commands/load_unit_data.py +17 -19
- endoreg_db/management/commands/start_filewatcher.py +46 -37
- endoreg_db/management/commands/validate_video_files.py +1 -5
- endoreg_db/migrations/0001_initial.py +1360 -1812
- endoreg_db/models/administration/person/patient/patient.py +72 -46
- endoreg_db/models/label/__init__.py +2 -2
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +18 -26
- endoreg_db/models/label/label_video_segment/label_video_segment.py +23 -1
- endoreg_db/models/media/pdf/raw_pdf.py +136 -64
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +34 -10
- endoreg_db/models/media/processing_history/__init__.py +5 -0
- endoreg_db/models/media/processing_history/processing_history.py +96 -0
- endoreg_db/models/media/video/create_from_file.py +101 -31
- endoreg_db/models/media/video/video_file.py +125 -105
- endoreg_db/models/media/video/video_file_io.py +31 -26
- endoreg_db/models/medical/contraindication/README.md +1 -0
- endoreg_db/models/medical/examination/examination.py +28 -8
- endoreg_db/models/medical/examination/examination_indication.py +13 -79
- endoreg_db/models/medical/examination/examination_time.py +8 -3
- endoreg_db/models/medical/finding/finding.py +5 -12
- endoreg_db/models/medical/finding/finding_classification.py +18 -37
- endoreg_db/models/medical/finding/finding_intervention.py +7 -9
- endoreg_db/models/medical/hardware/endoscope.py +6 -0
- endoreg_db/models/medical/patient/medication_examples.py +5 -1
- endoreg_db/models/medical/patient/patient_finding.py +1 -1
- endoreg_db/models/metadata/pdf_meta.py +22 -10
- endoreg_db/models/metadata/sensitive_meta.py +3 -0
- endoreg_db/models/metadata/sensitive_meta_logic.py +200 -124
- endoreg_db/models/other/information_source.py +27 -6
- endoreg_db/models/report/__init__.py +0 -0
- endoreg_db/models/report/images.py +0 -0
- endoreg_db/models/report/report.py +6 -0
- endoreg_db/models/requirement/requirement.py +59 -399
- endoreg_db/models/requirement/requirement_operator.py +86 -98
- endoreg_db/models/state/audit_ledger.py +4 -5
- endoreg_db/models/state/raw_pdf.py +69 -30
- endoreg_db/models/state/video.py +65 -49
- endoreg_db/models/upload_job.py +33 -9
- endoreg_db/models/utils.py +27 -23
- endoreg_db/queries/__init__.py +3 -1
- endoreg_db/schemas/examination_evaluation.py +1 -1
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/label_video_segment/label_video_segment.py +2 -29
- endoreg_db/serializers/meta/__init__.py +1 -6
- endoreg_db/serializers/misc/sensitive_patient_data.py +50 -26
- endoreg_db/serializers/patient_examination/patient_examination.py +3 -3
- endoreg_db/serializers/pdf/anony_text_validation.py +39 -23
- endoreg_db/serializers/video/video_file_list.py +65 -34
- endoreg_db/services/__old/pdf_import.py +1487 -0
- endoreg_db/services/__old/video_import.py +1306 -0
- endoreg_db/services/anonymization.py +63 -26
- endoreg_db/services/lookup_service.py +28 -28
- endoreg_db/services/lookup_store.py +2 -2
- endoreg_db/services/pdf_import.py +0 -1480
- endoreg_db/services/report_import.py +10 -0
- endoreg_db/services/video_import.py +6 -1165
- endoreg_db/tasks/upload_tasks.py +79 -70
- endoreg_db/tasks/video_ingest.py +8 -4
- endoreg_db/urls/__init__.py +0 -14
- endoreg_db/urls/ai.py +32 -0
- endoreg_db/urls/media.py +21 -24
- endoreg_db/utils/dataloader.py +87 -57
- endoreg_db/utils/paths.py +110 -46
- endoreg_db/utils/pipelines/Readme.md +1 -1
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +97 -0
- endoreg_db/utils/video/ffmpeg_wrapper.py +217 -52
- endoreg_db/views/__init__.py +85 -173
- endoreg_db/views/ai/__init__.py +8 -0
- endoreg_db/views/ai/label.py +155 -0
- endoreg_db/views/anonymization/media_management.py +8 -7
- endoreg_db/views/anonymization/overview.py +97 -68
- endoreg_db/views/anonymization/validate.py +25 -21
- endoreg_db/views/media/__init__.py +5 -20
- endoreg_db/views/media/pdf_media.py +109 -65
- endoreg_db/views/media/sensitive_metadata.py +163 -148
- endoreg_db/views/meta/__init__.py +0 -8
- endoreg_db/views/misc/__init__.py +1 -7
- endoreg_db/views/misc/upload_views.py +94 -93
- endoreg_db/views/report/__init__.py +7 -0
- endoreg_db/views/{pdf → report}/reimport.py +45 -24
- endoreg_db/views/{pdf/pdf_stream.py → report/report_stream.py} +40 -32
- endoreg_db/views/requirement/lookup_store.py +22 -90
- endoreg_db/views/video/__init__.py +23 -22
- endoreg_db/views/video/correction.py +201 -172
- endoreg_db/views/video/reimport.py +1 -1
- endoreg_db/views/{media/video_segments.py → video/segments_crud.py} +75 -37
- endoreg_db/views/video/{video_meta.py → video_meta_stats.py} +2 -2
- endoreg_db/views/video/video_stream.py +7 -8
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/METADATA +2 -2
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/RECORD +217 -335
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/WHEEL +1 -1
- endoreg_db/data/_examples/disease.yaml +0 -55
- endoreg_db/data/_examples/disease_classification.yaml +0 -13
- endoreg_db/data/_examples/disease_classification_choice.yaml +0 -62
- endoreg_db/data/_examples/event.yaml +0 -64
- endoreg_db/data/_examples/examination.yaml +0 -72
- endoreg_db/data/_examples/finding/anatomy_colon.yaml +0 -128
- endoreg_db/data/_examples/finding/colonoscopy.yaml +0 -40
- endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +0 -56
- endoreg_db/data/_examples/finding/complication.yaml +0 -16
- endoreg_db/data/_examples/finding/data.yaml +0 -105
- endoreg_db/data/_examples/finding/examination_setting.yaml +0 -16
- endoreg_db/data/_examples/finding/medication_related.yaml +0 -18
- endoreg_db/data/_examples/finding/outcome.yaml +0 -12
- endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +0 -68
- endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +0 -22
- endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +0 -25
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +0 -68
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +0 -80
- endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +0 -21
- endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +0 -26
- endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +0 -22
- endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +0 -53
- endoreg_db/data/_examples/finding_classification/complication_generic.yaml +0 -25
- endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +0 -40
- endoreg_db/data/_examples/finding_classification/histology_colo.yaml +0 -51
- endoreg_db/data/_examples/finding_classification/intervention_required.yaml +0 -26
- endoreg_db/data/_examples/finding_classification/medication_related.yaml +0 -23
- endoreg_db/data/_examples/finding_classification/visualized.yaml +0 -33
- endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +0 -78
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +0 -17
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +0 -14
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +0 -82
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/histology.yaml +0 -24
- endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +0 -20
- endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +0 -19
- endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +0 -11
- endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +0 -48
- endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +0 -43
- endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
- endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +0 -128
- endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +0 -32
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +0 -9
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +0 -36
- endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +0 -15
- endoreg_db/data/_examples/finding_type/data.yaml +0 -43
- endoreg_db/data/_examples/requirement/age.yaml +0 -26
- endoreg_db/data/_examples/requirement/gender.yaml +0 -25
- endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +0 -48
- endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +0 -57
- endoreg_db/data/_examples/requirement_set/endoscopy_bleeding_risk.yaml +0 -52
- endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
- endoreg_db/data/finding/anatomy_colon.yaml +0 -128
- endoreg_db/data/finding/colonoscopy.yaml +0 -40
- endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +0 -56
- endoreg_db/data/finding/complication.yaml +0 -16
- endoreg_db/data/finding/data.yaml +0 -105
- endoreg_db/data/finding/examination_setting.yaml +0 -16
- endoreg_db/data/finding/medication_related.yaml +0 -18
- endoreg_db/data/finding/outcome.yaml +0 -12
- endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +0 -22
- endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +0 -25
- endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +0 -38
- endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +0 -49
- endoreg_db/data/finding_classification/colonoscopy_lst.yaml +0 -21
- endoreg_db/data/finding_classification/colonoscopy_nice.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_paris.yaml +0 -26
- endoreg_db/data/finding_classification/colonoscopy_sano.yaml +0 -22
- endoreg_db/data/finding_classification/colonoscopy_summary.yaml +0 -53
- endoreg_db/data/finding_classification/complication_generic.yaml +0 -25
- endoreg_db/data/finding_classification/examination_setting_generic.yaml +0 -40
- endoreg_db/data/finding_classification/histology_colo.yaml +0 -43
- endoreg_db/data/finding_classification/intervention_required.yaml +0 -26
- endoreg_db/data/finding_classification/medication_related.yaml +0 -23
- endoreg_db/data/finding_classification/visualized.yaml +0 -33
- endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
- endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
- endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
- endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +0 -15
- endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +0 -17
- endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +0 -57
- endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
- endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +0 -14
- endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
- endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +0 -229
- endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +0 -19
- endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +0 -82
- endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
- endoreg_db/data/finding_classification_choice/outcome.yaml +0 -19
- endoreg_db/data/finding_intervention/endoscopy.yaml +0 -43
- endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
- endoreg_db/data/finding_intervention/endoscopy_egd.yaml +0 -128
- endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +0 -32
- endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +0 -9
- endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +0 -36
- endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +0 -79
- endoreg_db/data/requirement/age.yaml +0 -26
- endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +0 -45
- endoreg_db/data/requirement/disease_cardiovascular.yaml +0 -79
- endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +0 -41
- endoreg_db/data/requirement/disease_hepatology.yaml +0 -12
- endoreg_db/data/requirement/disease_misc.yaml +0 -12
- endoreg_db/data/requirement/disease_renal.yaml +0 -96
- endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +0 -59
- endoreg_db/data/requirement/event_cardiology.yaml +0 -251
- endoreg_db/data/requirement/event_requirements.yaml +0 -145
- endoreg_db/data/requirement/finding_colon_polyp.yaml +0 -50
- endoreg_db/data/requirement/gender.yaml +0 -25
- endoreg_db/data/requirement/lab_value.yaml +0 -441
- endoreg_db/data/requirement/medication.yaml +0 -93
- endoreg_db/data/requirement_operator/age.yaml +0 -13
- endoreg_db/data/requirement_operator/lab_operators.yaml +0 -129
- endoreg_db/data/requirement_operator/model_operators.yaml +0 -96
- endoreg_db/management/commands/init_default_ai_model.py +0 -112
- endoreg_db/management/commands/reset_celery_schedule.py +0 -9
- endoreg_db/management/commands/validate_video.py +0 -204
- endoreg_db/migrations/0002_requirementset_depends_on.py +0 -18
- endoreg_db/migrations/_old/0001_initial.py +0 -1857
- endoreg_db/migrations/_old/0002_add_video_correction_models.py +0 -52
- endoreg_db/migrations/_old/0003_add_center_display_name.py +0 -30
- endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +0 -68
- endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +0 -77
- endoreg_db/migrations/_old/0005_merge_20251111_1003.py +0 -14
- endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +0 -68
- endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +0 -89
- endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +0 -27
- endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +0 -21
- endoreg_db/renames.yml +0 -8
- endoreg_db/serializers/_old/raw_pdf_meta_validation.py +0 -223
- endoreg_db/serializers/_old/raw_video_meta_validation.py +0 -179
- endoreg_db/serializers/_old/video.py +0 -71
- endoreg_db/serializers/meta/pdf_file_meta_extraction.py +0 -115
- endoreg_db/serializers/meta/report_meta.py +0 -53
- endoreg_db/serializers/report/__init__.py +0 -9
- endoreg_db/serializers/report/mixins.py +0 -45
- endoreg_db/serializers/report/report.py +0 -105
- endoreg_db/serializers/report/report_list.py +0 -22
- endoreg_db/serializers/report/secure_file_url.py +0 -26
- endoreg_db/services/requirements_object.py +0 -147
- endoreg_db/services/storage_aware_video_processor.py +0 -370
- endoreg_db/urls/files.py +0 -6
- endoreg_db/urls/label_video_segment_validate.py +0 -33
- endoreg_db/urls/label_video_segments.py +0 -46
- endoreg_db/views/label/__init__.py +0 -5
- endoreg_db/views/label/label.py +0 -15
- endoreg_db/views/label_video_segment/__init__.py +0 -16
- endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +0 -44
- endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +0 -50
- endoreg_db/views/label_video_segment/label_video_segment.py +0 -77
- endoreg_db/views/label_video_segment/label_video_segment_by_label.py +0 -174
- endoreg_db/views/label_video_segment/label_video_segment_detail.py +0 -73
- endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +0 -46
- endoreg_db/views/label_video_segment/validate.py +0 -226
- endoreg_db/views/media/segments.py +0 -71
- endoreg_db/views/meta/available_files_list.py +0 -146
- endoreg_db/views/meta/report_meta.py +0 -53
- endoreg_db/views/meta/sensitive_meta_detail.py +0 -85
- endoreg_db/views/misc/secure_file_serving_view.py +0 -80
- endoreg_db/views/misc/secure_file_url_view.py +0 -84
- endoreg_db/views/misc/secure_url_validate.py +0 -79
- endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +0 -164
- endoreg_db/views/patient_finding_location/__init__.py +0 -5
- endoreg_db/views/patient_finding_location/pfl_create.py +0 -70
- endoreg_db/views/patient_finding_morphology/__init__.py +0 -5
- endoreg_db/views/patient_finding_morphology/pfm_create.py +0 -70
- endoreg_db/views/pdf/__init__.py +0 -8
- endoreg_db/views/video/segmentation.py +0 -274
- endoreg_db/views/video/task_status.py +0 -49
- endoreg_db/views/video/timeline.py +0 -46
- endoreg_db/views/video/video_analyze.py +0 -52
- /endoreg_db/data/requirement/{colon_polyp_intervention.yaml → old/colon_polyp_intervention.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/colonoscopy_baseline_austria.yaml +0 -0
- /endoreg_db/data/requirement/{coloreg_colon_polyp.yaml → old/coloreg_colon_polyp.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_cardiovascular.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_classification_choice_cardiovascular.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_hepatology.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_misc.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_renal.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/endoscopy_bleeding_risk.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/event_cardiology.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/event_requirements.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/finding_colon_polyp.yaml +0 -0
- /endoreg_db/{urls/sensitive_meta.py → data/requirement/old/gender.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/lab_value.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/medication.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/age.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/lab_operators.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/model_operators.yaml +0 -0
- /endoreg_db/{views/pdf/pdf_stream_views.py → import_files/pseudonymization/__init__.py} +0 -0
- /endoreg_db/utils/requirement_operator_logic/{lab_value_operators.py → _old/lab_value_operators.py} +0 -0
- /endoreg_db/utils/requirement_operator_logic/{model_evaluators.py → _old/model_evaluators.py} +0 -0
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -13,7 +13,11 @@ from django.db import models
|
|
|
13
13
|
|
|
14
14
|
from endoreg_db.utils.file_operations import get_uuid_filename
|
|
15
15
|
from endoreg_db.utils.hashs import get_pdf_hash
|
|
16
|
-
from endoreg_db.utils.paths import
|
|
16
|
+
from endoreg_db.utils.paths import (
|
|
17
|
+
ANONYM_REPORT_DIR,
|
|
18
|
+
IMPORT_REPORT_DIR,
|
|
19
|
+
SENSITIVE_REPORT_DIR,
|
|
20
|
+
)
|
|
17
21
|
from endoreg_db.utils.storage import (
|
|
18
22
|
delete_field_file,
|
|
19
23
|
ensure_local_file,
|
|
@@ -26,7 +30,6 @@ if TYPE_CHECKING:
|
|
|
26
30
|
|
|
27
31
|
from endoreg_db.models.state import RawPdfState
|
|
28
32
|
|
|
29
|
-
# setup logging to pdf_import.log
|
|
30
33
|
import logging
|
|
31
34
|
from pathlib import Path
|
|
32
35
|
|
|
@@ -67,14 +70,14 @@ class RawPdfFile(models.Model):
|
|
|
67
70
|
text = models.TextField(blank=True, null=True)
|
|
68
71
|
date_created = models.DateTimeField(auto_now_add=True)
|
|
69
72
|
date_modified = models.DateTimeField(auto_now=True)
|
|
70
|
-
|
|
73
|
+
|
|
71
74
|
file = models.FileField(
|
|
72
|
-
# Use the relative path from the specific
|
|
73
|
-
upload_to=
|
|
75
|
+
# Use the relative path from the specific REPORT_DIR
|
|
76
|
+
upload_to=SENSITIVE_REPORT_DIR.name,
|
|
74
77
|
validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
|
|
75
78
|
)
|
|
76
|
-
|
|
77
|
-
upload_to=
|
|
79
|
+
processed_file = models.FileField(
|
|
80
|
+
upload_to=ANONYM_REPORT_DIR.name,
|
|
78
81
|
validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
|
|
79
82
|
null=True,
|
|
80
83
|
blank=True,
|
|
@@ -130,9 +133,11 @@ class RawPdfFile(models.Model):
|
|
|
130
133
|
state: models.ForeignKey["RawPdfState | None"]
|
|
131
134
|
patient: models.ForeignKey["Patient | None"]
|
|
132
135
|
sensitive_meta: models.ForeignKey["SensitiveMeta | None"]
|
|
133
|
-
anonym_examination_report: models.OneToOneField[
|
|
136
|
+
anonym_examination_report: models.OneToOneField[
|
|
137
|
+
"AnonymExaminationReport | None"
|
|
138
|
+
]
|
|
134
139
|
file = cast(FieldFile, file)
|
|
135
|
-
|
|
140
|
+
processed_file = cast(FieldFile, processed_file)
|
|
136
141
|
|
|
137
142
|
@property
|
|
138
143
|
def uuid(self):
|
|
@@ -147,7 +152,7 @@ class RawPdfFile(models.Model):
|
|
|
147
152
|
@property
|
|
148
153
|
def file_path(self) -> Path | None:
|
|
149
154
|
"""
|
|
150
|
-
Returns the file path of the stored
|
|
155
|
+
Returns the file path of the stored report file if available; otherwise, returns None.
|
|
151
156
|
"""
|
|
152
157
|
from django.db.models.fields.files import FieldFile
|
|
153
158
|
|
|
@@ -162,7 +167,7 @@ class RawPdfFile(models.Model):
|
|
|
162
167
|
|
|
163
168
|
def set_file_path(self, file_path: Path):
|
|
164
169
|
"""
|
|
165
|
-
Sets the file path of the stored
|
|
170
|
+
Sets the file path of the stored report file.
|
|
166
171
|
"""
|
|
167
172
|
if not file_path.exists():
|
|
168
173
|
raise FileNotFoundError(f"File path does not exist: {file_path}")
|
|
@@ -173,30 +178,30 @@ class RawPdfFile(models.Model):
|
|
|
173
178
|
@property
|
|
174
179
|
def anonymized_file_path(self) -> Path | None:
|
|
175
180
|
"""
|
|
176
|
-
Returns the file path of the anonymized
|
|
181
|
+
Returns the file path of the anonymized report file if available; otherwise, returns None.
|
|
177
182
|
"""
|
|
178
|
-
if self.
|
|
183
|
+
if self.processed_file and self.processed_file.name:
|
|
179
184
|
try:
|
|
180
|
-
return Path(self.
|
|
185
|
+
return Path(self.processed_file.path)
|
|
181
186
|
except (ValueError, AttributeError, NotImplementedError):
|
|
182
187
|
return None
|
|
183
188
|
return None
|
|
184
189
|
|
|
185
190
|
def set_anonymized_file_path(self, file_path: Path):
|
|
186
191
|
"""
|
|
187
|
-
Sets the file path of the anonymized
|
|
192
|
+
Sets the file path of the anonymized report file.
|
|
188
193
|
"""
|
|
189
194
|
if not file_path.exists():
|
|
190
195
|
raise FileNotFoundError(f"File path does not exist: {file_path}")
|
|
191
196
|
|
|
192
|
-
save_local_file(self.
|
|
193
|
-
self.save(update_fields=["
|
|
197
|
+
save_local_file(self.processed_file, file_path, name=file_path.name, save=False)
|
|
198
|
+
self.save(update_fields=["processed_file"])
|
|
194
199
|
|
|
195
200
|
def get_raw_file_path(self) -> Optional[Path]:
|
|
196
201
|
"""
|
|
197
|
-
Get the path to the raw
|
|
202
|
+
Get the path to the raw report file, searching common locations.
|
|
198
203
|
|
|
199
|
-
This method attempts to find the original raw
|
|
204
|
+
This method attempts to find the original raw report file by checking:
|
|
200
205
|
1. Direct hash-based path in raw_pdfs/
|
|
201
206
|
2. Scanning raw_pdfs/ directory for files matching the hash
|
|
202
207
|
3. Checking the file field if it exists
|
|
@@ -211,17 +216,17 @@ class RawPdfFile(models.Model):
|
|
|
211
216
|
try:
|
|
212
217
|
file_path = Path(self.file.path)
|
|
213
218
|
if file_path.exists():
|
|
214
|
-
logger.debug(f"Found raw
|
|
219
|
+
logger.debug(f"Found raw report via file field: {file_path}")
|
|
215
220
|
return file_path
|
|
216
221
|
except (ValueError, AttributeError, NotImplementedError):
|
|
217
222
|
pass
|
|
218
223
|
|
|
219
224
|
# Define potential raw directories
|
|
220
225
|
raw_dirs = [
|
|
221
|
-
|
|
222
|
-
Path(settings.BASE_DIR) / "data" / "
|
|
226
|
+
SENSITIVE_REPORT_DIR, # Files might be in sensitive dir
|
|
227
|
+
Path(settings.BASE_DIR) / "data" / "temporary_reports",
|
|
223
228
|
Path(settings.BASE_DIR) / "data" / "pdfs" / "raw",
|
|
224
|
-
|
|
229
|
+
IMPORT_REPORT_DIR, # General report directory
|
|
225
230
|
]
|
|
226
231
|
|
|
227
232
|
# Check direct hash-based name in each directory
|
|
@@ -231,7 +236,7 @@ class RawPdfFile(models.Model):
|
|
|
231
236
|
|
|
232
237
|
hash_path = raw_dir / f"{self.pdf_hash}.pdf"
|
|
233
238
|
if hash_path.exists():
|
|
234
|
-
logger.debug(f"Found raw
|
|
239
|
+
logger.debug(f"Found raw report at: {hash_path}")
|
|
235
240
|
return hash_path
|
|
236
241
|
|
|
237
242
|
# Scan directories for matching hash
|
|
@@ -243,19 +248,19 @@ class RawPdfFile(models.Model):
|
|
|
243
248
|
try:
|
|
244
249
|
file_hash = get_pdf_hash(file_path)
|
|
245
250
|
if file_hash == self.pdf_hash:
|
|
246
|
-
logger.debug(f"Found matching
|
|
251
|
+
logger.debug(f"Found matching report by hash: {file_path}")
|
|
247
252
|
return file_path
|
|
248
253
|
except Exception as e:
|
|
249
254
|
logger.debug(f"Error checking {file_path}: {e}")
|
|
250
255
|
continue
|
|
251
256
|
|
|
252
|
-
logger.warning(f"No raw file found for
|
|
257
|
+
logger.warning(f"No raw file found for report hash: {self.pdf_hash}")
|
|
253
258
|
return None
|
|
254
259
|
|
|
255
260
|
@property
|
|
256
261
|
def file_url(self):
|
|
257
262
|
"""
|
|
258
|
-
Returns the URL of the stored
|
|
263
|
+
Returns the URL of the stored report file if available; otherwise, returns None.
|
|
259
264
|
"""
|
|
260
265
|
try:
|
|
261
266
|
return self.file.url if self.file and self.file.name else None
|
|
@@ -265,16 +270,20 @@ class RawPdfFile(models.Model):
|
|
|
265
270
|
@property
|
|
266
271
|
def anonymized_file_url(self):
|
|
267
272
|
"""
|
|
268
|
-
Returns the URL of the stored
|
|
273
|
+
Returns the URL of the stored report file if available; otherwise, returns None.
|
|
269
274
|
"""
|
|
270
275
|
try:
|
|
271
|
-
return
|
|
276
|
+
return (
|
|
277
|
+
self.processed_file.url
|
|
278
|
+
if self.processed_file and self.processed_file.name
|
|
279
|
+
else None
|
|
280
|
+
)
|
|
272
281
|
except (ValueError, AttributeError):
|
|
273
282
|
return None
|
|
274
283
|
|
|
275
284
|
def __str__(self):
|
|
276
285
|
"""
|
|
277
|
-
Return a string representation of the RawPdfFile, including its
|
|
286
|
+
Return a string representation of the RawPdfFile, including its report hash, type, and center.
|
|
278
287
|
"""
|
|
279
288
|
str_repr = f"{self.pdf_hash} ({self.pdf_type}, {self.center})"
|
|
280
289
|
return str_repr
|
|
@@ -283,19 +292,57 @@ class RawPdfFile(models.Model):
|
|
|
283
292
|
"""
|
|
284
293
|
Deletes the RawPdfFile instance from the database and removes the associated file from storage if it exists.
|
|
285
294
|
|
|
286
|
-
This method ensures that the physical
|
|
295
|
+
This method ensures that the physical report file is deleted from the file system after the database record is removed. Logs warnings or errors if the file cannot be found or deleted.
|
|
287
296
|
"""
|
|
288
297
|
primary_name = self.file.name if self.file and self.file.name else None
|
|
289
|
-
anonymized_name =
|
|
298
|
+
anonymized_name = (
|
|
299
|
+
self.processed_file.name
|
|
300
|
+
if self.processed_file and self.processed_file.name
|
|
301
|
+
else None
|
|
302
|
+
)
|
|
290
303
|
|
|
291
304
|
if delete_field_file(self.file, missing_ok=True, save=False):
|
|
292
305
|
logger.info("Original file removed from storage: %s", primary_name)
|
|
293
|
-
if delete_field_file(self.
|
|
306
|
+
if delete_field_file(self.processed_file, missing_ok=True, save=False):
|
|
294
307
|
logger.info("Anonymized file removed from storage: %s", anonymized_name)
|
|
295
308
|
|
|
296
309
|
super().delete(*args, **kwargs)
|
|
297
310
|
|
|
298
|
-
|
|
311
|
+
# --- Convenience state/meta helpers used in tests and admin workflows ---
|
|
312
|
+
|
|
313
|
+
def mark_sensitive_meta_processed(self, *, save: bool = True) -> "RawPdfFile":
|
|
314
|
+
"""
|
|
315
|
+
Mark this video's processing state as having its sensitive meta fully processed.
|
|
316
|
+
This proxies to the related VideoState and persists by default.
|
|
317
|
+
"""
|
|
318
|
+
sm = self.sensitive_meta
|
|
319
|
+
from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
|
|
320
|
+
|
|
321
|
+
if not isinstance(sm, SensitiveMeta):
|
|
322
|
+
raise AttributeError()
|
|
323
|
+
state = self.get_or_create_state()
|
|
324
|
+
state.mark_sensitive_meta_processed(save=save)
|
|
325
|
+
return self
|
|
326
|
+
|
|
327
|
+
def mark_sensitive_meta_verified(self) -> "RawPdfFile":
|
|
328
|
+
"""
|
|
329
|
+
Mark the associated SensitiveMeta as verified by setting both DOB and names as verified.
|
|
330
|
+
Ensures the SensitiveMeta and its state exist.
|
|
331
|
+
"""
|
|
332
|
+
sm = self.sensitive_meta
|
|
333
|
+
# Use SensitiveMeta methods to update underlying SensitiveMetaState
|
|
334
|
+
from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
|
|
335
|
+
|
|
336
|
+
if not isinstance(sm, SensitiveMeta):
|
|
337
|
+
raise AttributeError()
|
|
338
|
+
|
|
339
|
+
sm.mark_dob_verified()
|
|
340
|
+
sm.mark_names_verified()
|
|
341
|
+
return self
|
|
342
|
+
|
|
343
|
+
def validate_metadata_annotation(
|
|
344
|
+
self, extracted_data_dict: Optional[dict] = None
|
|
345
|
+
) -> bool:
|
|
299
346
|
"""
|
|
300
347
|
Validate the metadata of the RawPdf instance.
|
|
301
348
|
|
|
@@ -303,9 +350,8 @@ class RawPdfFile(models.Model):
|
|
|
303
350
|
It also ensures the video file is properly saved after the metadata update.
|
|
304
351
|
"""
|
|
305
352
|
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
return False
|
|
353
|
+
self.mark_sensitive_meta_processed()
|
|
354
|
+
self.mark_sensitive_meta_verified()
|
|
309
355
|
|
|
310
356
|
if not extracted_data_dict:
|
|
311
357
|
logger.error("No extracted data provided for validation.")
|
|
@@ -322,16 +368,22 @@ class RawPdfFile(models.Model):
|
|
|
322
368
|
# Save the RawPdfFile instance to ensure all changes are saved
|
|
323
369
|
self.save()
|
|
324
370
|
|
|
325
|
-
logger.info(
|
|
371
|
+
logger.info(
|
|
372
|
+
f"Metadata for report {self.pk} validated and updated successfully."
|
|
373
|
+
)
|
|
326
374
|
|
|
327
375
|
deleted_original = delete_field_file(self.file, missing_ok=True, save=False)
|
|
328
|
-
deleted_anonymized = delete_field_file(
|
|
376
|
+
deleted_anonymized = delete_field_file(
|
|
377
|
+
self.processed_file, missing_ok=True, save=False
|
|
378
|
+
)
|
|
329
379
|
self.get_or_create_state().mark_anonymization_validated()
|
|
330
380
|
|
|
331
381
|
if deleted_original or deleted_anonymized:
|
|
332
|
-
self.save(
|
|
382
|
+
self.save(
|
|
383
|
+
update_fields=["file", "processed_file"]
|
|
384
|
+
) # Persist cleared fields
|
|
333
385
|
|
|
334
|
-
logger.info(f"Files for
|
|
386
|
+
logger.info(f"Files for report {self.pk} deleted successfully.")
|
|
335
387
|
return True
|
|
336
388
|
|
|
337
389
|
@classmethod
|
|
@@ -345,8 +397,8 @@ class RawPdfFile(models.Model):
|
|
|
345
397
|
Creates a RawPdfFile instance from a file and center name, ensuring an associated RawPdfState exists.
|
|
346
398
|
|
|
347
399
|
Parameters:
|
|
348
|
-
file_path (Path): Path to the source
|
|
349
|
-
center_name (str): Name of the center to associate with the
|
|
400
|
+
file_path (Path): Path to the source report file.
|
|
401
|
+
center_name (str): Name of the center to associate with the report.
|
|
350
402
|
delete_source (bool): Whether to delete the source file after processing. Defaults to True.
|
|
351
403
|
|
|
352
404
|
Returns:
|
|
@@ -370,12 +422,12 @@ class RawPdfFile(models.Model):
|
|
|
370
422
|
delete_source=True,
|
|
371
423
|
):
|
|
372
424
|
"""
|
|
373
|
-
Creates or retrieves a RawPdfFile instance from a given
|
|
425
|
+
Creates or retrieves a RawPdfFile instance from a given report file path and center name.
|
|
374
426
|
|
|
375
|
-
If a RawPdfFile with the same
|
|
427
|
+
If a RawPdfFile with the same report hash already exists, verifies the file exists in storage and restores it if missing. Otherwise, creates a new RawPdfFile, assigns the file, and saves it to storage. Optionally deletes the source file after processing.
|
|
376
428
|
|
|
377
429
|
Parameters:
|
|
378
|
-
file_path (Path): Path to the source
|
|
430
|
+
file_path (Path): Path to the source report file.
|
|
379
431
|
center_name (str): Name of the center to associate with the file.
|
|
380
432
|
save (bool, optional): Deprecated; saving occurs internally.
|
|
381
433
|
delete_source (bool, optional): Whether to delete the source file after processing (default True).
|
|
@@ -386,7 +438,7 @@ class RawPdfFile(models.Model):
|
|
|
386
438
|
Raises:
|
|
387
439
|
FileNotFoundError: If the source file does not exist.
|
|
388
440
|
Center.DoesNotExist: If the specified center is not found.
|
|
389
|
-
ValueError: If the
|
|
441
|
+
ValueError: If the report hash cannot be calculated.
|
|
390
442
|
IOError: If the file fails to save to storage.
|
|
391
443
|
"""
|
|
392
444
|
from endoreg_db.models.administration import Center
|
|
@@ -427,9 +479,13 @@ class RawPdfFile(models.Model):
|
|
|
427
479
|
)
|
|
428
480
|
# Re-save the file from the source to potentially fix it
|
|
429
481
|
with file_path.open("rb") as f:
|
|
430
|
-
django_file = File(
|
|
482
|
+
django_file = File(
|
|
483
|
+
f, name=Path(_file.name).name
|
|
484
|
+
) # Use existing name if possible
|
|
431
485
|
existing_pdf_file.file = django_file
|
|
432
|
-
existing_pdf_file.save(
|
|
486
|
+
existing_pdf_file.save(
|
|
487
|
+
update_fields=["file"]
|
|
488
|
+
) # Only update file field
|
|
433
489
|
else:
|
|
434
490
|
pass
|
|
435
491
|
# logger.debug("File for existing RawPdfFile %s already exists in storage.", pdf_hash)
|
|
@@ -485,7 +541,9 @@ class RawPdfFile(models.Model):
|
|
|
485
541
|
"File was not saved correctly to storage path %s after model save.",
|
|
486
542
|
_file.name,
|
|
487
543
|
)
|
|
488
|
-
raise IOError(
|
|
544
|
+
raise IOError(
|
|
545
|
+
f"File not found at expected storage path after save: {_file.name}"
|
|
546
|
+
)
|
|
489
547
|
|
|
490
548
|
try:
|
|
491
549
|
logger.info("File saved to absolute path: %s", _file.path)
|
|
@@ -496,14 +554,18 @@ class RawPdfFile(models.Model):
|
|
|
496
554
|
)
|
|
497
555
|
|
|
498
556
|
except Exception as e:
|
|
499
|
-
logger.error(
|
|
557
|
+
logger.error(
|
|
558
|
+
"Error processing or saving file %s for new record: %s", file_path, e
|
|
559
|
+
)
|
|
500
560
|
raise
|
|
501
561
|
|
|
502
562
|
# Delete source file *after* successful save and verification
|
|
503
563
|
if delete_source:
|
|
504
564
|
try:
|
|
505
565
|
file_path.unlink()
|
|
506
|
-
logger.info(
|
|
566
|
+
logger.info(
|
|
567
|
+
"Deleted source file %s after creating new record.", file_path
|
|
568
|
+
)
|
|
507
569
|
except OSError as e:
|
|
508
570
|
logger.error("Error deleting source file %s: %s", file_path, e)
|
|
509
571
|
|
|
@@ -514,15 +576,17 @@ class RawPdfFile(models.Model):
|
|
|
514
576
|
# Ensure hash is calculated before the first save if possible and not already set
|
|
515
577
|
# This is primarily a fallback if instance created manually without using create_from_file
|
|
516
578
|
"""
|
|
517
|
-
Saves the RawPdfFile instance, ensuring the
|
|
579
|
+
Saves the RawPdfFile instance, ensuring the report hash is set and related fields are derived from metadata.
|
|
518
580
|
|
|
519
|
-
If the
|
|
581
|
+
If the report hash is missing, attempts to calculate it from the file before saving. Validates that the file has a `.pdf` extension. If related fields such as patient, examination, center, or examiner are unset but available in the associated sensitive metadata, they are populated accordingly before saving.
|
|
520
582
|
"""
|
|
521
583
|
if not self.pk and not self.pdf_hash and self.file:
|
|
522
584
|
try:
|
|
523
585
|
with ensure_local_file(self.file) as local_path:
|
|
524
586
|
self.pdf_hash = get_pdf_hash(local_path)
|
|
525
|
-
logger.info(
|
|
587
|
+
logger.info(
|
|
588
|
+
"Calculated hash during pre-save for %s", self.file.name
|
|
589
|
+
)
|
|
526
590
|
except Exception as exc:
|
|
527
591
|
logger.warning(
|
|
528
592
|
"Could not calculate hash before initial save for %s: %s",
|
|
@@ -531,7 +595,7 @@ class RawPdfFile(models.Model):
|
|
|
531
595
|
)
|
|
532
596
|
|
|
533
597
|
if self.file and not self.file.name.endswith(".pdf"):
|
|
534
|
-
raise ValidationError("Only
|
|
598
|
+
raise ValidationError("Only report files are allowed")
|
|
535
599
|
|
|
536
600
|
# If hash is still missing after potential creation logic (e.g., direct instantiation)
|
|
537
601
|
# and the file exists in storage, try calculating it from storage path.
|
|
@@ -539,7 +603,9 @@ class RawPdfFile(models.Model):
|
|
|
539
603
|
if not self.pdf_hash and self.pk and self.file and file_exists(self.file):
|
|
540
604
|
try:
|
|
541
605
|
with ensure_local_file(self.file) as local_path:
|
|
542
|
-
logger.warning(
|
|
606
|
+
logger.warning(
|
|
607
|
+
"Hash missing for saved file %s. Recalculating.", self.file.name
|
|
608
|
+
)
|
|
543
609
|
self.pdf_hash = get_pdf_hash(local_path)
|
|
544
610
|
except Exception as exc:
|
|
545
611
|
logger.error(
|
|
@@ -585,7 +651,7 @@ class RawPdfFile(models.Model):
|
|
|
585
651
|
# This method might still be useful if called explicitly, but create_from_file now handles restoration
|
|
586
652
|
# Ensure fallback_file is a Path object.
|
|
587
653
|
"""
|
|
588
|
-
Checks if the stored
|
|
654
|
+
Checks if the stored report file exists in storage and attempts to restore it from a fallback file path if missing.
|
|
589
655
|
|
|
590
656
|
Parameters:
|
|
591
657
|
fallback_file: Path or string representing the fallback file location to restore from if the stored file is missing.
|
|
@@ -597,12 +663,18 @@ class RawPdfFile(models.Model):
|
|
|
597
663
|
assert _file is not None
|
|
598
664
|
try:
|
|
599
665
|
if not _file.field.storage.exists(_file.name):
|
|
600
|
-
logger.warning(
|
|
666
|
+
logger.warning(
|
|
667
|
+
f"File missing at storage path {_file.name}. Attempting copy from fallback {fallback_file}"
|
|
668
|
+
)
|
|
601
669
|
if fallback_file.exists():
|
|
602
670
|
with fallback_file.open("rb") as f:
|
|
603
671
|
# Use save method which handles storage backend
|
|
604
|
-
_file.save(
|
|
605
|
-
|
|
672
|
+
_file.save(
|
|
673
|
+
Path(_file.name).name, File(f), save=True
|
|
674
|
+
) # Re-save the file content
|
|
675
|
+
logger.info(
|
|
676
|
+
f"Successfully restored file from fallback {fallback_file} to {_file.name}"
|
|
677
|
+
)
|
|
606
678
|
else:
|
|
607
679
|
logger.error(f"Fallback file {fallback_file} does not exist.")
|
|
608
680
|
except Exception as e:
|
|
@@ -678,8 +750,8 @@ class RawPdfFile(models.Model):
|
|
|
678
750
|
return settings_dict
|
|
679
751
|
|
|
680
752
|
@staticmethod
|
|
681
|
-
def
|
|
753
|
+
def get_pdf_by_pk(pk: int) -> "RawPdfFile":
|
|
682
754
|
try:
|
|
683
|
-
return RawPdfFile.objects.get(pk=
|
|
755
|
+
return RawPdfFile.objects.get(pk=pk)
|
|
684
756
|
except RawPdfFile.DoesNotExist:
|
|
685
|
-
raise ValueError(f"
|
|
757
|
+
raise ValueError(f"report with ID {pdf_id} does not exist.")
|
|
@@ -12,21 +12,41 @@ if TYPE_CHECKING:
|
|
|
12
12
|
|
|
13
13
|
class ReportReaderConfig(models.Model):
|
|
14
14
|
"""
|
|
15
|
-
Configuration settings for parsing
|
|
15
|
+
Configuration settings for parsing report reports (ReportReader).
|
|
16
16
|
|
|
17
17
|
Stores locale, name lists, date format, and flags used to identify key information lines
|
|
18
18
|
and text sections to ignore.
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
locale = models.CharField(default="de_DE", max_length=10)
|
|
22
|
-
first_names = models.ManyToManyField(
|
|
23
|
-
|
|
22
|
+
first_names = models.ManyToManyField(
|
|
23
|
+
"FirstName", related_name="report_reader_configs"
|
|
24
|
+
)
|
|
25
|
+
last_names = models.ManyToManyField(
|
|
26
|
+
"LastName", related_name="report_reader_configs"
|
|
27
|
+
)
|
|
24
28
|
text_date_format = models.CharField(default="%d.%m.%Y", max_length=10)
|
|
25
|
-
patient_info_line_flag = models.ForeignKey(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
patient_info_line_flag = models.ForeignKey(
|
|
30
|
+
"ReportReaderFlag",
|
|
31
|
+
related_name="report_reader_configs_patient_info_line",
|
|
32
|
+
on_delete=models.CASCADE,
|
|
33
|
+
)
|
|
34
|
+
endoscope_info_line_flag = models.ForeignKey(
|
|
35
|
+
"ReportReaderFlag",
|
|
36
|
+
related_name="report_reader_configs_endoscope_info_line",
|
|
37
|
+
on_delete=models.CASCADE,
|
|
38
|
+
)
|
|
39
|
+
examiner_info_line_flag = models.ForeignKey(
|
|
40
|
+
"ReportReaderFlag",
|
|
41
|
+
related_name="report_reader_configs_examiner_info_line",
|
|
42
|
+
on_delete=models.CASCADE,
|
|
43
|
+
)
|
|
44
|
+
cut_off_below = models.ManyToManyField(
|
|
45
|
+
"ReportReaderFlag", related_name="report_reader_configs_cut_off_below"
|
|
46
|
+
)
|
|
47
|
+
cut_off_above = models.ManyToManyField(
|
|
48
|
+
"ReportReaderFlag", related_name="report_reader_configs_cut_off_above"
|
|
49
|
+
)
|
|
30
50
|
|
|
31
51
|
if TYPE_CHECKING:
|
|
32
52
|
patient_info_line_flag = models.ForeignKey["ReportReaderFlag"]
|
|
@@ -35,8 +55,12 @@ class ReportReaderConfig(models.Model):
|
|
|
35
55
|
|
|
36
56
|
first_names = cast(models.manager.RelatedManager["FirstName"], first_names)
|
|
37
57
|
last_names = cast(models.manager.RelatedManager["LastName"], last_names)
|
|
38
|
-
cut_off_below = cast(
|
|
39
|
-
|
|
58
|
+
cut_off_below = cast(
|
|
59
|
+
models.manager.RelatedManager["ReportReaderFlag"], cut_off_below
|
|
60
|
+
)
|
|
61
|
+
cut_off_above = cast(
|
|
62
|
+
models.manager.RelatedManager["ReportReaderFlag"], cut_off_above
|
|
63
|
+
)
|
|
40
64
|
|
|
41
65
|
def __str__(self):
|
|
42
66
|
"""Returns a string representation including the locale and primary key."""
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from django.contrib.contenttypes.fields import GenericForeignKey
|
|
4
|
+
from django.contrib.contenttypes.models import ContentType
|
|
5
|
+
from django.db import models
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ProcessingHistory(models.Model):
|
|
9
|
+
"""
|
|
10
|
+
Generic processing history for media files (video, pdf, ...).
|
|
11
|
+
|
|
12
|
+
Stores:
|
|
13
|
+
- which object (VideoFile/RawPdfFile/other) this entry belongs to
|
|
14
|
+
- whether processing/anonymization succeeded
|
|
15
|
+
- timestamps
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
created_at = models.DateTimeField(auto_now_add=True)
|
|
19
|
+
success = models.BooleanField(default=False, blank=True)
|
|
20
|
+
|
|
21
|
+
content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
|
|
22
|
+
object_id = models.PositiveBigIntegerField()
|
|
23
|
+
content_object = GenericForeignKey("content_type", "object_id")
|
|
24
|
+
|
|
25
|
+
class Meta:
|
|
26
|
+
ordering = ["-created_at"]
|
|
27
|
+
|
|
28
|
+
def __str__(self) -> str:
|
|
29
|
+
return f"{self.content_type} #{self.object_id}, Success: {self.success}"
|
|
30
|
+
|
|
31
|
+
# --- public helpers that work with *objects* ---
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def get_or_create_for_object(
|
|
35
|
+
cls,
|
|
36
|
+
*,
|
|
37
|
+
obj: models.Model,
|
|
38
|
+
success: Optional[bool],
|
|
39
|
+
) -> "ProcessingHistory":
|
|
40
|
+
"""
|
|
41
|
+
Get or create a ProcessingHistory entry for a concrete model instance.
|
|
42
|
+
|
|
43
|
+
- Returns existing entry for (object_id, content_type) if present
|
|
44
|
+
- Otherwise creates one with the given success flag (default False)
|
|
45
|
+
- If an entry exists and `success` is provided, it updates `success`
|
|
46
|
+
"""
|
|
47
|
+
ct = ContentType.objects.get_for_model(obj, for_concrete_model=False)
|
|
48
|
+
|
|
49
|
+
ph, created = cls.objects.get_or_create(
|
|
50
|
+
object_id=obj.pk,
|
|
51
|
+
content_type=ct,
|
|
52
|
+
defaults={"success": success},
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
if not created and success is not None and ph.success != success:
|
|
56
|
+
ph.success = success
|
|
57
|
+
ph.save(update_fields=["success"])
|
|
58
|
+
|
|
59
|
+
return ph
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def has_history(
|
|
63
|
+
cls,
|
|
64
|
+
*,
|
|
65
|
+
object_id: int,
|
|
66
|
+
content_type: ContentType,
|
|
67
|
+
success: Optional[bool] = None,
|
|
68
|
+
) -> bool:
|
|
69
|
+
"""
|
|
70
|
+
Return True if there is at least one ProcessingHistory entry for the
|
|
71
|
+
given (object_id, content_type) combination.
|
|
72
|
+
|
|
73
|
+
If `success` is not None, only entries matching that success flag are
|
|
74
|
+
considered.
|
|
75
|
+
"""
|
|
76
|
+
qs = cls.objects.filter(object_id=object_id, content_type=content_type)
|
|
77
|
+
if success is not None:
|
|
78
|
+
qs = qs.filter(success=success)
|
|
79
|
+
return qs.exists()
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def has_history_for_object(
|
|
83
|
+
cls,
|
|
84
|
+
*,
|
|
85
|
+
obj: models.Model,
|
|
86
|
+
success: Optional[bool] = None,
|
|
87
|
+
) -> bool:
|
|
88
|
+
"""
|
|
89
|
+
Convenience wrapper for model instances.
|
|
90
|
+
"""
|
|
91
|
+
ct = ContentType.objects.get_for_model(obj, for_concrete_model=False)
|
|
92
|
+
return cls.has_history(
|
|
93
|
+
object_id=obj.pk,
|
|
94
|
+
content_type=ct,
|
|
95
|
+
success=success,
|
|
96
|
+
)
|