endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.8.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/data/__init__.py +22 -8
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +0 -1
- endoreg_db/data/examination/examinations/data.yaml +114 -14
- endoreg_db/data/examination/time-type/data.yaml +0 -3
- endoreg_db/data/examination_indication/endoscopy.yaml +108 -173
- endoreg_db/data/examination_indication_classification/endoscopy.yaml +0 -70
- endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +33 -37
- endoreg_db/data/finding/00_generic.yaml +35 -0
- endoreg_db/data/finding/00_generic_complication.yaml +9 -0
- endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
- endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
- endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
- endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
- endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
- endoreg_db/data/finding_classification/00_generic.yaml +44 -0
- endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
- endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
- endoreg_db/data/finding_classification/{colonoscopy_bowel_preparation.yaml → 02_colonoscopy_baseline.yaml} +35 -20
- endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
- endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
- endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
- endoreg_db/data/finding_classification_choice/{yes_no_na.yaml → 00_generic.yaml} +5 -1
- endoreg_db/data/finding_classification_choice/{examination_setting_generic_types.yaml → 00_generic_baseline.yaml} +10 -2
- endoreg_db/data/finding_classification_choice/{complication_generic_types.yaml → 00_generic_complication.yaml} +1 -1
- endoreg_db/data/finding_classification_choice/{histology.yaml → 00_generic_histology.yaml} +1 -4
- endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
- endoreg_db/data/finding_classification_choice/{bowel_preparation.yaml → 02_colonoscopy_bowel_preparation.yaml} +1 -30
- endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml → finding_classification_choice/02_colonoscopy_generic.yaml} +1 -1
- endoreg_db/data/finding_classification_choice/{histology_polyp.yaml → 02_colonoscopy_histology.yaml} +1 -1
- endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_location.yaml → finding_classification_choice/02_colonoscopy_location.yaml} +23 -4
- endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
- endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
- endoreg_db/data/{_examples/finding_classification_choice/colon_lesion_paris.yaml → finding_classification_choice/02_colonoscopy_polyp_morphology.yaml} +26 -8
- endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
- endoreg_db/data/finding_classification_type/{colonoscopy_basic.yaml → 00_generic.yaml} +18 -13
- endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
- endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
- endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
- endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
- endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
- endoreg_db/data/finding_type/data.yaml +8 -12
- endoreg_db/data/requirement/01_patient_data.yaml +93 -0
- endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +0 -2
- endoreg_db/data/requirement_set/90_coloreg.yaml +20 -8
- endoreg_db/exceptions.py +0 -1
- endoreg_db/forms/examination_form.py +1 -1
- endoreg_db/helpers/data_loader.py +124 -52
- endoreg_db/helpers/default_objects.py +116 -81
- endoreg_db/import_files/__init__.py +27 -0
- endoreg_db/import_files/context/__init__.py +7 -0
- endoreg_db/import_files/context/default_sensitive_meta.py +81 -0
- endoreg_db/import_files/context/ensure_center.py +17 -0
- endoreg_db/import_files/context/file_lock.py +66 -0
- endoreg_db/import_files/context/import_context.py +43 -0
- endoreg_db/import_files/context/validate_directories.py +56 -0
- endoreg_db/import_files/file_storage/__init__.py +15 -0
- endoreg_db/import_files/file_storage/create_report_file.py +76 -0
- endoreg_db/import_files/file_storage/create_video_file.py +75 -0
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +39 -0
- endoreg_db/import_files/file_storage/state_management.py +400 -0
- endoreg_db/import_files/file_storage/storage.py +36 -0
- endoreg_db/import_files/import_service.md +26 -0
- endoreg_db/import_files/processing/__init__.py +11 -0
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +94 -0
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
- endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +119 -0
- endoreg_db/import_files/pseudonymization/fake.py +52 -0
- endoreg_db/import_files/pseudonymization/k_anonymity.py +182 -0
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +128 -0
- endoreg_db/import_files/report_import_service.py +141 -0
- endoreg_db/import_files/video_import_service.py +150 -0
- endoreg_db/management/commands/import_report.py +130 -65
- endoreg_db/management/commands/import_video_with_classification.py +1 -1
- endoreg_db/management/commands/load_ai_model_data.py +5 -5
- endoreg_db/management/commands/load_ai_model_label_data.py +9 -7
- endoreg_db/management/commands/load_base_db_data.py +5 -134
- endoreg_db/management/commands/load_contraindication_data.py +14 -16
- endoreg_db/management/commands/load_disease_classification_choices_data.py +15 -18
- endoreg_db/management/commands/load_disease_classification_data.py +15 -18
- endoreg_db/management/commands/load_disease_data.py +25 -28
- endoreg_db/management/commands/load_endoscope_data.py +20 -27
- endoreg_db/management/commands/load_event_data.py +14 -16
- endoreg_db/management/commands/load_examination_data.py +31 -44
- endoreg_db/management/commands/load_examination_indication_data.py +20 -21
- endoreg_db/management/commands/load_finding_data.py +52 -80
- endoreg_db/management/commands/load_information_source.py +21 -23
- endoreg_db/management/commands/load_lab_value_data.py +17 -26
- endoreg_db/management/commands/load_medication_data.py +13 -12
- endoreg_db/management/commands/load_organ_data.py +15 -19
- endoreg_db/management/commands/load_pdf_type_data.py +19 -18
- endoreg_db/management/commands/load_profession_data.py +14 -17
- endoreg_db/management/commands/load_qualification_data.py +20 -23
- endoreg_db/management/commands/load_report_reader_flag_data.py +17 -19
- endoreg_db/management/commands/load_requirement_data.py +14 -20
- endoreg_db/management/commands/load_risk_data.py +7 -6
- endoreg_db/management/commands/load_shift_data.py +20 -23
- endoreg_db/management/commands/load_tag_data.py +8 -11
- endoreg_db/management/commands/load_unit_data.py +17 -19
- endoreg_db/management/commands/start_filewatcher.py +46 -37
- endoreg_db/management/commands/validate_video_files.py +1 -5
- endoreg_db/migrations/0001_initial.py +1360 -1812
- endoreg_db/models/administration/person/patient/patient.py +72 -46
- endoreg_db/models/label/__init__.py +2 -2
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +18 -26
- endoreg_db/models/label/label_video_segment/label_video_segment.py +23 -1
- endoreg_db/models/media/pdf/raw_pdf.py +136 -64
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +34 -10
- endoreg_db/models/media/processing_history/__init__.py +5 -0
- endoreg_db/models/media/processing_history/processing_history.py +96 -0
- endoreg_db/models/media/video/create_from_file.py +101 -31
- endoreg_db/models/media/video/video_file.py +125 -105
- endoreg_db/models/media/video/video_file_io.py +31 -26
- endoreg_db/models/medical/contraindication/README.md +1 -0
- endoreg_db/models/medical/examination/examination.py +28 -8
- endoreg_db/models/medical/examination/examination_indication.py +13 -79
- endoreg_db/models/medical/examination/examination_time.py +8 -3
- endoreg_db/models/medical/finding/finding.py +5 -12
- endoreg_db/models/medical/finding/finding_classification.py +18 -37
- endoreg_db/models/medical/finding/finding_intervention.py +7 -9
- endoreg_db/models/medical/hardware/endoscope.py +6 -0
- endoreg_db/models/medical/patient/medication_examples.py +5 -1
- endoreg_db/models/medical/patient/patient_finding.py +1 -1
- endoreg_db/models/metadata/pdf_meta.py +22 -10
- endoreg_db/models/metadata/sensitive_meta.py +3 -0
- endoreg_db/models/metadata/sensitive_meta_logic.py +200 -124
- endoreg_db/models/other/information_source.py +27 -6
- endoreg_db/models/report/__init__.py +0 -0
- endoreg_db/models/report/images.py +0 -0
- endoreg_db/models/report/report.py +6 -0
- endoreg_db/models/requirement/requirement.py +59 -399
- endoreg_db/models/requirement/requirement_operator.py +86 -98
- endoreg_db/models/state/audit_ledger.py +4 -5
- endoreg_db/models/state/raw_pdf.py +69 -30
- endoreg_db/models/state/video.py +64 -49
- endoreg_db/models/upload_job.py +33 -9
- endoreg_db/models/utils.py +27 -23
- endoreg_db/queries/__init__.py +3 -1
- endoreg_db/schemas/examination_evaluation.py +1 -1
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/label_video_segment/label_video_segment.py +2 -29
- endoreg_db/serializers/meta/__init__.py +1 -6
- endoreg_db/serializers/misc/sensitive_patient_data.py +50 -26
- endoreg_db/serializers/patient_examination/patient_examination.py +3 -3
- endoreg_db/serializers/pdf/anony_text_validation.py +39 -23
- endoreg_db/serializers/video/video_file_list.py +65 -34
- endoreg_db/services/__old/pdf_import.py +1487 -0
- endoreg_db/services/__old/video_import.py +1306 -0
- endoreg_db/services/anonymization.py +63 -26
- endoreg_db/services/lookup_service.py +28 -28
- endoreg_db/services/lookup_store.py +2 -2
- endoreg_db/services/pdf_import.py +0 -1480
- endoreg_db/services/report_import.py +10 -0
- endoreg_db/services/video_import.py +6 -1165
- endoreg_db/tasks/upload_tasks.py +79 -70
- endoreg_db/tasks/video_ingest.py +8 -4
- endoreg_db/urls/__init__.py +0 -14
- endoreg_db/urls/ai.py +32 -0
- endoreg_db/urls/media.py +21 -24
- endoreg_db/utils/dataloader.py +87 -57
- endoreg_db/utils/paths.py +110 -46
- endoreg_db/utils/pipelines/Readme.md +1 -1
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +97 -0
- endoreg_db/views/__init__.py +85 -173
- endoreg_db/views/ai/__init__.py +8 -0
- endoreg_db/views/ai/label.py +155 -0
- endoreg_db/views/anonymization/media_management.py +8 -7
- endoreg_db/views/anonymization/overview.py +97 -68
- endoreg_db/views/anonymization/validate.py +25 -21
- endoreg_db/views/media/__init__.py +5 -20
- endoreg_db/views/media/pdf_media.py +109 -65
- endoreg_db/views/media/sensitive_metadata.py +163 -148
- endoreg_db/views/meta/__init__.py +0 -8
- endoreg_db/views/misc/__init__.py +1 -7
- endoreg_db/views/misc/upload_views.py +94 -93
- endoreg_db/views/report/__init__.py +7 -0
- endoreg_db/views/{pdf → report}/reimport.py +45 -24
- endoreg_db/views/{pdf/pdf_stream.py → report/report_stream.py} +40 -32
- endoreg_db/views/requirement/lookup_store.py +22 -90
- endoreg_db/views/video/__init__.py +23 -22
- endoreg_db/views/video/correction.py +201 -172
- endoreg_db/views/video/reimport.py +1 -1
- endoreg_db/views/{media/video_segments.py → video/segments_crud.py} +75 -37
- endoreg_db/views/video/{video_meta.py → video_meta_stats.py} +2 -2
- endoreg_db/views/video/video_stream.py +7 -8
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/METADATA +2 -2
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/RECORD +217 -335
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/WHEEL +1 -1
- endoreg_db/data/_examples/disease.yaml +0 -55
- endoreg_db/data/_examples/disease_classification.yaml +0 -13
- endoreg_db/data/_examples/disease_classification_choice.yaml +0 -62
- endoreg_db/data/_examples/event.yaml +0 -64
- endoreg_db/data/_examples/examination.yaml +0 -72
- endoreg_db/data/_examples/finding/anatomy_colon.yaml +0 -128
- endoreg_db/data/_examples/finding/colonoscopy.yaml +0 -40
- endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +0 -56
- endoreg_db/data/_examples/finding/complication.yaml +0 -16
- endoreg_db/data/_examples/finding/data.yaml +0 -105
- endoreg_db/data/_examples/finding/examination_setting.yaml +0 -16
- endoreg_db/data/_examples/finding/medication_related.yaml +0 -18
- endoreg_db/data/_examples/finding/outcome.yaml +0 -12
- endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +0 -68
- endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +0 -22
- endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +0 -25
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +0 -68
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +0 -80
- endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +0 -21
- endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +0 -20
- endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +0 -26
- endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +0 -22
- endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +0 -53
- endoreg_db/data/_examples/finding_classification/complication_generic.yaml +0 -25
- endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +0 -40
- endoreg_db/data/_examples/finding_classification/histology_colo.yaml +0 -51
- endoreg_db/data/_examples/finding_classification/intervention_required.yaml +0 -26
- endoreg_db/data/_examples/finding_classification/medication_related.yaml +0 -23
- endoreg_db/data/_examples/finding_classification/visualized.yaml +0 -33
- endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +0 -78
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +0 -17
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +0 -14
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +0 -82
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +0 -15
- endoreg_db/data/_examples/finding_classification_choice/histology.yaml +0 -24
- endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +0 -20
- endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +0 -19
- endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +0 -11
- endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +0 -48
- endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +0 -43
- endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
- endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +0 -128
- endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +0 -32
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +0 -9
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +0 -36
- endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +0 -15
- endoreg_db/data/_examples/finding_type/data.yaml +0 -43
- endoreg_db/data/_examples/requirement/age.yaml +0 -26
- endoreg_db/data/_examples/requirement/gender.yaml +0 -25
- endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +0 -48
- endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +0 -57
- endoreg_db/data/_examples/requirement_set/endoscopy_bleeding_risk.yaml +0 -52
- endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
- endoreg_db/data/finding/anatomy_colon.yaml +0 -128
- endoreg_db/data/finding/colonoscopy.yaml +0 -40
- endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +0 -56
- endoreg_db/data/finding/complication.yaml +0 -16
- endoreg_db/data/finding/data.yaml +0 -105
- endoreg_db/data/finding/examination_setting.yaml +0 -16
- endoreg_db/data/finding/medication_related.yaml +0 -18
- endoreg_db/data/finding/outcome.yaml +0 -12
- endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +0 -22
- endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +0 -25
- endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +0 -38
- endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +0 -49
- endoreg_db/data/finding_classification/colonoscopy_lst.yaml +0 -21
- endoreg_db/data/finding_classification/colonoscopy_nice.yaml +0 -20
- endoreg_db/data/finding_classification/colonoscopy_paris.yaml +0 -26
- endoreg_db/data/finding_classification/colonoscopy_sano.yaml +0 -22
- endoreg_db/data/finding_classification/colonoscopy_summary.yaml +0 -53
- endoreg_db/data/finding_classification/complication_generic.yaml +0 -25
- endoreg_db/data/finding_classification/examination_setting_generic.yaml +0 -40
- endoreg_db/data/finding_classification/histology_colo.yaml +0 -43
- endoreg_db/data/finding_classification/intervention_required.yaml +0 -26
- endoreg_db/data/finding_classification/medication_related.yaml +0 -23
- endoreg_db/data/finding_classification/visualized.yaml +0 -33
- endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
- endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
- endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
- endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +0 -15
- endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +0 -17
- endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +0 -57
- endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
- endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +0 -14
- endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
- endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +0 -229
- endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +0 -19
- endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +0 -82
- endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
- endoreg_db/data/finding_classification_choice/outcome.yaml +0 -19
- endoreg_db/data/finding_intervention/endoscopy.yaml +0 -43
- endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
- endoreg_db/data/finding_intervention/endoscopy_egd.yaml +0 -128
- endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +0 -32
- endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +0 -9
- endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +0 -36
- endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +0 -79
- endoreg_db/data/requirement/age.yaml +0 -26
- endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +0 -45
- endoreg_db/data/requirement/disease_cardiovascular.yaml +0 -79
- endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +0 -41
- endoreg_db/data/requirement/disease_hepatology.yaml +0 -12
- endoreg_db/data/requirement/disease_misc.yaml +0 -12
- endoreg_db/data/requirement/disease_renal.yaml +0 -96
- endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +0 -59
- endoreg_db/data/requirement/event_cardiology.yaml +0 -251
- endoreg_db/data/requirement/event_requirements.yaml +0 -145
- endoreg_db/data/requirement/finding_colon_polyp.yaml +0 -50
- endoreg_db/data/requirement/gender.yaml +0 -25
- endoreg_db/data/requirement/lab_value.yaml +0 -441
- endoreg_db/data/requirement/medication.yaml +0 -93
- endoreg_db/data/requirement_operator/age.yaml +0 -13
- endoreg_db/data/requirement_operator/lab_operators.yaml +0 -129
- endoreg_db/data/requirement_operator/model_operators.yaml +0 -96
- endoreg_db/management/commands/init_default_ai_model.py +0 -112
- endoreg_db/management/commands/reset_celery_schedule.py +0 -9
- endoreg_db/management/commands/validate_video.py +0 -204
- endoreg_db/migrations/0002_requirementset_depends_on.py +0 -18
- endoreg_db/migrations/_old/0001_initial.py +0 -1857
- endoreg_db/migrations/_old/0002_add_video_correction_models.py +0 -52
- endoreg_db/migrations/_old/0003_add_center_display_name.py +0 -30
- endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +0 -68
- endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +0 -77
- endoreg_db/migrations/_old/0005_merge_20251111_1003.py +0 -14
- endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +0 -68
- endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +0 -89
- endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +0 -27
- endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +0 -21
- endoreg_db/renames.yml +0 -8
- endoreg_db/serializers/_old/raw_pdf_meta_validation.py +0 -223
- endoreg_db/serializers/_old/raw_video_meta_validation.py +0 -179
- endoreg_db/serializers/_old/video.py +0 -71
- endoreg_db/serializers/meta/pdf_file_meta_extraction.py +0 -115
- endoreg_db/serializers/meta/report_meta.py +0 -53
- endoreg_db/serializers/report/__init__.py +0 -9
- endoreg_db/serializers/report/mixins.py +0 -45
- endoreg_db/serializers/report/report.py +0 -105
- endoreg_db/serializers/report/report_list.py +0 -22
- endoreg_db/serializers/report/secure_file_url.py +0 -26
- endoreg_db/services/requirements_object.py +0 -147
- endoreg_db/services/storage_aware_video_processor.py +0 -370
- endoreg_db/urls/files.py +0 -6
- endoreg_db/urls/label_video_segment_validate.py +0 -33
- endoreg_db/urls/label_video_segments.py +0 -46
- endoreg_db/views/label/__init__.py +0 -5
- endoreg_db/views/label/label.py +0 -15
- endoreg_db/views/label_video_segment/__init__.py +0 -16
- endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +0 -44
- endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +0 -50
- endoreg_db/views/label_video_segment/label_video_segment.py +0 -77
- endoreg_db/views/label_video_segment/label_video_segment_by_label.py +0 -174
- endoreg_db/views/label_video_segment/label_video_segment_detail.py +0 -73
- endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +0 -46
- endoreg_db/views/label_video_segment/validate.py +0 -226
- endoreg_db/views/media/segments.py +0 -71
- endoreg_db/views/meta/available_files_list.py +0 -146
- endoreg_db/views/meta/report_meta.py +0 -53
- endoreg_db/views/meta/sensitive_meta_detail.py +0 -85
- endoreg_db/views/misc/secure_file_serving_view.py +0 -80
- endoreg_db/views/misc/secure_file_url_view.py +0 -84
- endoreg_db/views/misc/secure_url_validate.py +0 -79
- endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +0 -164
- endoreg_db/views/patient_finding_location/__init__.py +0 -5
- endoreg_db/views/patient_finding_location/pfl_create.py +0 -70
- endoreg_db/views/patient_finding_morphology/__init__.py +0 -5
- endoreg_db/views/patient_finding_morphology/pfm_create.py +0 -70
- endoreg_db/views/pdf/__init__.py +0 -8
- endoreg_db/views/video/segmentation.py +0 -274
- endoreg_db/views/video/task_status.py +0 -49
- endoreg_db/views/video/timeline.py +0 -46
- endoreg_db/views/video/video_analyze.py +0 -52
- /endoreg_db/data/requirement/{colon_polyp_intervention.yaml → old/colon_polyp_intervention.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/colonoscopy_baseline_austria.yaml +0 -0
- /endoreg_db/data/requirement/{coloreg_colon_polyp.yaml → old/coloreg_colon_polyp.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_cardiovascular.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_classification_choice_cardiovascular.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_hepatology.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_misc.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/disease_renal.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/endoscopy_bleeding_risk.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/event_cardiology.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/event_requirements.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/finding_colon_polyp.yaml +0 -0
- /endoreg_db/{migrations/__init__.py → data/requirement/old/gender.yaml} +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/lab_value.yaml +0 -0
- /endoreg_db/data/{_examples/requirement → requirement/old}/medication.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/age.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/lab_operators.yaml +0 -0
- /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/model_operators.yaml +0 -0
- /endoreg_db/{urls/sensitive_meta.py → import_files/pseudonymization/__init__.py} +0 -0
- /endoreg_db/{views/pdf/pdf_stream_views.py → import_files/pseudonymization/pseudonymize.py} +0 -0
- /endoreg_db/utils/requirement_operator_logic/{lab_value_operators.py → _old/lab_value_operators.py} +0 -0
- /endoreg_db/utils/requirement_operator_logic/{model_evaluators.py → _old/model_evaluators.py} +0 -0
- {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.8.9.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,64 +1,66 @@
|
|
|
1
1
|
import mimetypes
|
|
2
|
+
|
|
2
3
|
from django.http import Http404
|
|
3
4
|
from django.urls import reverse
|
|
5
|
+
from django.utils.decorators import method_decorator
|
|
6
|
+
from django.views.decorators.csrf import csrf_exempt
|
|
4
7
|
from rest_framework import status
|
|
5
|
-
from rest_framework.
|
|
6
|
-
from rest_framework.response import Response
|
|
7
|
-
from rest_framework.parsers import MultiPartParser, FormParser
|
|
8
|
+
from rest_framework.parsers import FormParser, MultiPartParser
|
|
8
9
|
from rest_framework.permissions import AllowAny
|
|
9
|
-
from
|
|
10
|
-
from
|
|
10
|
+
from rest_framework.response import Response
|
|
11
|
+
from rest_framework.views import APIView
|
|
11
12
|
|
|
12
13
|
# Try to import python-magic, but provide fallback if not available
|
|
13
14
|
try:
|
|
14
15
|
import magic
|
|
16
|
+
|
|
15
17
|
MAGIC_AVAILABLE = True
|
|
16
18
|
except ImportError:
|
|
17
19
|
MAGIC_AVAILABLE = False
|
|
18
20
|
|
|
19
21
|
from endoreg_db.models.upload_job import UploadJob
|
|
20
|
-
from endoreg_db.serializers.misc.upload_job import
|
|
21
|
-
UploadJobStatusSerializer,
|
|
22
|
-
)
|
|
22
|
+
from endoreg_db.serializers.misc.upload_job import UploadJobStatusSerializer
|
|
23
23
|
|
|
24
24
|
# Try to import celery task, but provide fallback
|
|
25
25
|
try:
|
|
26
26
|
from endoreg_db.tasks.upload_tasks import process_upload_job
|
|
27
|
+
|
|
27
28
|
CELERY_AVAILABLE = True
|
|
28
29
|
except ImportError:
|
|
29
30
|
CELERY_AVAILABLE = False
|
|
31
|
+
|
|
30
32
|
# Define a dummy function for development
|
|
31
33
|
def process_upload_job(job_id):
|
|
32
34
|
pass
|
|
33
35
|
|
|
34
36
|
|
|
35
|
-
@method_decorator(csrf_exempt, name=
|
|
37
|
+
@method_decorator(csrf_exempt, name="dispatch")
|
|
36
38
|
class UploadFileView(APIView):
|
|
37
39
|
"""
|
|
38
40
|
Handle file uploads (POST /api/upload/).
|
|
39
|
-
|
|
40
|
-
Accepts multipart/form-data with a 'file' field containing
|
|
41
|
+
|
|
42
|
+
Accepts multipart/form-data with a 'file' field containing report or video files.
|
|
41
43
|
Creates an UploadJob and starts asynchronous processing.
|
|
42
|
-
|
|
44
|
+
|
|
43
45
|
Returns:
|
|
44
46
|
201 Created: {"upload_id": "<uuid>", "status_url": "/api/upload/<uuid>/status/"}
|
|
45
47
|
400 Bad Request: File validation errors
|
|
46
48
|
"""
|
|
47
|
-
|
|
49
|
+
|
|
48
50
|
parser_classes = [MultiPartParser, FormParser]
|
|
49
51
|
permission_classes = [AllowAny] # Adjust based on your auth requirements
|
|
50
|
-
|
|
52
|
+
|
|
51
53
|
# Maximum file size (1 GiB)
|
|
52
54
|
MAX_FILE_SIZE = 1024 * 1024 * 1024 # 1 GiB in bytes
|
|
53
|
-
|
|
55
|
+
|
|
54
56
|
# Allowed MIME types
|
|
55
57
|
ALLOWED_MIME_TYPES = {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
"application/pdf",
|
|
59
|
+
"video/mp4",
|
|
60
|
+
"video/avi",
|
|
61
|
+
"video/quicktime",
|
|
62
|
+
"video/x-msvideo",
|
|
63
|
+
"video/x-ms-wmv",
|
|
62
64
|
}
|
|
63
65
|
|
|
64
66
|
def post(self, request, *args, **kwargs):
|
|
@@ -66,93 +68,95 @@ class UploadFileView(APIView):
|
|
|
66
68
|
Handle file upload and create processing job.
|
|
67
69
|
"""
|
|
68
70
|
# Validate file presence
|
|
69
|
-
if
|
|
71
|
+
if "file" not in request.FILES:
|
|
70
72
|
return Response(
|
|
71
|
-
{
|
|
72
|
-
|
|
73
|
+
{
|
|
74
|
+
"error": 'No file provided. Please include a file in the "file" field.'
|
|
75
|
+
},
|
|
76
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
73
77
|
)
|
|
74
|
-
|
|
75
|
-
uploaded_file = request.FILES[
|
|
76
|
-
|
|
78
|
+
|
|
79
|
+
uploaded_file = request.FILES["file"]
|
|
80
|
+
|
|
77
81
|
# Validate file is not empty
|
|
78
82
|
if not uploaded_file or uploaded_file.size == 0:
|
|
79
83
|
return Response(
|
|
80
|
-
{
|
|
81
|
-
status=status.HTTP_400_BAD_REQUEST
|
|
84
|
+
{"error": "Uploaded file is empty. Please select a valid file."},
|
|
85
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
82
86
|
)
|
|
83
|
-
|
|
87
|
+
|
|
84
88
|
# Validate file size
|
|
85
89
|
if uploaded_file.size > self.MAX_FILE_SIZE:
|
|
86
90
|
return Response(
|
|
87
|
-
{
|
|
88
|
-
|
|
91
|
+
{
|
|
92
|
+
"error": f"File too large. Maximum size is {self.MAX_FILE_SIZE // (1024**3)} GB."
|
|
93
|
+
},
|
|
94
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
89
95
|
)
|
|
90
|
-
|
|
96
|
+
|
|
91
97
|
# Validate filename
|
|
92
|
-
if not uploaded_file.name or uploaded_file.name.strip() ==
|
|
98
|
+
if not uploaded_file.name or uploaded_file.name.strip() == "":
|
|
93
99
|
return Response(
|
|
94
|
-
{
|
|
95
|
-
status=status.HTTP_400_BAD_REQUEST
|
|
100
|
+
{"error": "Invalid filename. Please ensure the file has a valid name."},
|
|
101
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
96
102
|
)
|
|
97
|
-
|
|
103
|
+
|
|
98
104
|
# Detect MIME type
|
|
99
105
|
try:
|
|
100
106
|
content_type = self._detect_mime_type(uploaded_file)
|
|
101
107
|
except Exception as e:
|
|
102
108
|
return Response(
|
|
103
|
-
{
|
|
104
|
-
status=status.HTTP_400_BAD_REQUEST
|
|
109
|
+
{"error": f"Could not determine file type: {str(e)}"},
|
|
110
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
105
111
|
)
|
|
106
|
-
|
|
112
|
+
|
|
107
113
|
# Validate MIME type
|
|
108
114
|
if content_type not in self.ALLOWED_MIME_TYPES:
|
|
109
115
|
return Response(
|
|
110
|
-
{
|
|
111
|
-
|
|
116
|
+
{
|
|
117
|
+
"error": f"Unsupported file type: {content_type}. Allowed types: report, MP4, AVI, MOV, WMV."
|
|
118
|
+
},
|
|
119
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
112
120
|
)
|
|
113
|
-
|
|
121
|
+
|
|
114
122
|
try:
|
|
115
123
|
# Create upload job
|
|
116
124
|
upload_job = UploadJob.objects.create(
|
|
117
|
-
file=uploaded_file,
|
|
118
|
-
content_type=content_type
|
|
125
|
+
file=uploaded_file, content_type=content_type
|
|
119
126
|
)
|
|
120
|
-
|
|
127
|
+
|
|
121
128
|
# Start asynchronous processing if Celery is available
|
|
122
129
|
if CELERY_AVAILABLE:
|
|
123
130
|
try:
|
|
124
131
|
process_upload_job.delay(str(upload_job.id))
|
|
125
132
|
except Exception as e:
|
|
126
133
|
# If Celery task fails to start, mark job as failed
|
|
127
|
-
upload_job.mark_failed(f
|
|
134
|
+
upload_job.mark_failed(f"Failed to start processing: {str(e)}")
|
|
128
135
|
return Response(
|
|
129
|
-
{
|
|
130
|
-
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
|
136
|
+
{"error": f"Failed to start processing: {str(e)}"},
|
|
137
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
131
138
|
)
|
|
132
139
|
else:
|
|
133
140
|
# For development without Celery, mark as processing immediately
|
|
134
141
|
upload_job.mark_processing()
|
|
135
142
|
# In production, this would be handled by Celery
|
|
136
143
|
# For now, just leave it in processing state
|
|
137
|
-
|
|
144
|
+
|
|
138
145
|
# Prepare response
|
|
139
|
-
status_url = reverse(
|
|
146
|
+
status_url = reverse("upload_status", kwargs={"id": upload_job.id})
|
|
140
147
|
response_data = {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
148
|
+
"upload_id": str(upload_job.id), # Ensure UUID is converted to string
|
|
149
|
+
"status_url": status_url,
|
|
150
|
+
"message": "Upload job created successfully",
|
|
144
151
|
}
|
|
145
|
-
|
|
152
|
+
|
|
146
153
|
# Return the response data directly since serializer fields are read-only
|
|
147
|
-
return Response(
|
|
148
|
-
|
|
149
|
-
status=status.HTTP_201_CREATED
|
|
150
|
-
)
|
|
151
|
-
|
|
154
|
+
return Response(response_data, status=status.HTTP_201_CREATED)
|
|
155
|
+
|
|
152
156
|
except Exception as e:
|
|
153
157
|
return Response(
|
|
154
|
-
{
|
|
155
|
-
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
|
158
|
+
{"error": f"Failed to create upload job: {str(e)}"},
|
|
159
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
156
160
|
)
|
|
157
161
|
|
|
158
162
|
def _detect_mime_type(self, uploaded_file) -> str:
|
|
@@ -163,39 +167,39 @@ class UploadFileView(APIView):
|
|
|
163
167
|
try:
|
|
164
168
|
# Reset file pointer
|
|
165
169
|
uploaded_file.seek(0)
|
|
166
|
-
|
|
170
|
+
|
|
167
171
|
# Try python-magic first (more reliable) if available
|
|
168
172
|
if MAGIC_AVAILABLE:
|
|
169
173
|
try:
|
|
170
174
|
# Read first chunk for magic detection
|
|
171
175
|
chunk = uploaded_file.read(2048)
|
|
172
176
|
uploaded_file.seek(0) # Reset again
|
|
173
|
-
|
|
177
|
+
|
|
174
178
|
mime_type = magic.from_buffer(chunk, mime=True)
|
|
175
|
-
if mime_type and mime_type !=
|
|
179
|
+
if mime_type and mime_type != "application/octet-stream":
|
|
176
180
|
return mime_type
|
|
177
181
|
except Exception:
|
|
178
182
|
pass # Fall back to mimetypes
|
|
179
|
-
|
|
183
|
+
|
|
180
184
|
# Fallback to mimetypes module
|
|
181
185
|
mime_type, _ = mimetypes.guess_type(uploaded_file.name)
|
|
182
186
|
if mime_type:
|
|
183
187
|
return mime_type
|
|
184
|
-
|
|
188
|
+
|
|
185
189
|
# Last resort - check file extension
|
|
186
|
-
if uploaded_file.name.lower().endswith(
|
|
187
|
-
return
|
|
188
|
-
elif uploaded_file.name.lower().endswith((
|
|
189
|
-
return
|
|
190
|
-
elif uploaded_file.name.lower().endswith(
|
|
191
|
-
return
|
|
192
|
-
elif uploaded_file.name.lower().endswith((
|
|
193
|
-
return
|
|
194
|
-
elif uploaded_file.name.lower().endswith(
|
|
195
|
-
return
|
|
196
|
-
|
|
190
|
+
if uploaded_file.name.lower().endswith(".pdf"):
|
|
191
|
+
return "application/pdf"
|
|
192
|
+
elif uploaded_file.name.lower().endswith((".mp4", ".m4v")):
|
|
193
|
+
return "video/mp4"
|
|
194
|
+
elif uploaded_file.name.lower().endswith(".avi"):
|
|
195
|
+
return "video/avi"
|
|
196
|
+
elif uploaded_file.name.lower().endswith((".mov", ".qt")):
|
|
197
|
+
return "video/quicktime"
|
|
198
|
+
elif uploaded_file.name.lower().endswith(".wmv"):
|
|
199
|
+
return "video/x-ms-wmv"
|
|
200
|
+
|
|
197
201
|
raise ValueError("Could not determine file type")
|
|
198
|
-
|
|
202
|
+
|
|
199
203
|
finally:
|
|
200
204
|
# Ensure file pointer is reset
|
|
201
205
|
uploaded_file.seek(0)
|
|
@@ -204,15 +208,15 @@ class UploadFileView(APIView):
|
|
|
204
208
|
class UploadStatusView(APIView):
|
|
205
209
|
"""
|
|
206
210
|
Get upload job status (GET /api/upload/<uuid>/status/).
|
|
207
|
-
|
|
211
|
+
|
|
208
212
|
Returns current processing status and relevant metadata.
|
|
209
213
|
Should be polled every 2 seconds by the frontend.
|
|
210
|
-
|
|
214
|
+
|
|
211
215
|
Returns:
|
|
212
216
|
200 OK: Status information
|
|
213
217
|
404 Not Found: Upload job not found
|
|
214
218
|
"""
|
|
215
|
-
|
|
219
|
+
|
|
216
220
|
permission_classes = [AllowAny] # Adjust based on your auth requirements
|
|
217
221
|
|
|
218
222
|
def get(self, request, id, *args, **kwargs):
|
|
@@ -221,20 +225,17 @@ class UploadStatusView(APIView):
|
|
|
221
225
|
"""
|
|
222
226
|
try:
|
|
223
227
|
# Look up upload job by UUID
|
|
224
|
-
upload_job = UploadJob.objects.select_related(
|
|
225
|
-
|
|
228
|
+
upload_job = UploadJob.objects.select_related("sensitive_meta").get(id=id)
|
|
229
|
+
|
|
226
230
|
# Serialize the response
|
|
227
231
|
serializer = UploadJobStatusSerializer(upload_job)
|
|
228
|
-
|
|
229
|
-
return Response(
|
|
230
|
-
|
|
231
|
-
status=status.HTTP_200_OK
|
|
232
|
-
)
|
|
233
|
-
|
|
232
|
+
|
|
233
|
+
return Response(serializer.data, status=status.HTTP_200_OK)
|
|
234
|
+
|
|
234
235
|
except UploadJob.DoesNotExist:
|
|
235
236
|
raise Http404("Upload job not found")
|
|
236
237
|
except Exception as e:
|
|
237
238
|
return Response(
|
|
238
|
-
{
|
|
239
|
-
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
|
240
|
-
)
|
|
239
|
+
{"error": f"Failed to get upload status: {str(e)}"},
|
|
240
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
241
|
+
)
|
|
@@ -6,12 +6,12 @@ from rest_framework.response import Response
|
|
|
6
6
|
from rest_framework.views import APIView
|
|
7
7
|
|
|
8
8
|
from ...models import RawPdfFile, SensitiveMeta
|
|
9
|
-
from
|
|
9
|
+
from endoreg_db.services.report_import import ReportImportService
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
class
|
|
14
|
+
class ReportReimportView(APIView):
|
|
15
15
|
"""
|
|
16
16
|
API endpoint to re-import a pdf file and regenerate metadata.
|
|
17
17
|
This is useful when OCR failed or metadata is incomplete.
|
|
@@ -19,7 +19,7 @@ class PdfReimportView(APIView):
|
|
|
19
19
|
|
|
20
20
|
def __init__(self, **kwargs):
|
|
21
21
|
super().__init__(**kwargs)
|
|
22
|
-
self.pdf_service =
|
|
22
|
+
self.pdf_service = ReportImportService()
|
|
23
23
|
|
|
24
24
|
def post(self, request, pk):
|
|
25
25
|
"""
|
|
@@ -28,24 +28,24 @@ class PdfReimportView(APIView):
|
|
|
28
28
|
|
|
29
29
|
Args:
|
|
30
30
|
request: HTTP request object
|
|
31
|
-
pk:
|
|
31
|
+
pk: report primary key (ID)
|
|
32
32
|
"""
|
|
33
33
|
pdf_id = pk # Align with media framework naming convention
|
|
34
34
|
|
|
35
35
|
# Validate pdf_id parameter
|
|
36
36
|
if not pdf_id or not isinstance(pdf_id, int):
|
|
37
37
|
return Response(
|
|
38
|
-
{"error": "Invalid
|
|
38
|
+
{"error": "Invalid report ID provided."},
|
|
39
39
|
status=status.HTTP_400_BAD_REQUEST,
|
|
40
40
|
)
|
|
41
41
|
|
|
42
42
|
try:
|
|
43
43
|
pdf = RawPdfFile.objects.get(id=pdf_id)
|
|
44
|
-
logger.info(f"Found
|
|
44
|
+
logger.info(f"Found report {pdf.pdf_hash} (ID: {pdf_id}) for re-import")
|
|
45
45
|
except RawPdfFile.DoesNotExist:
|
|
46
|
-
logger.warning(f"
|
|
46
|
+
logger.warning(f"report with ID {pdf_id} not found")
|
|
47
47
|
return Response(
|
|
48
|
-
{"error": f"
|
|
48
|
+
{"error": f"report with ID {pdf_id} not found."},
|
|
49
49
|
status=status.HTTP_404_NOT_FOUND,
|
|
50
50
|
)
|
|
51
51
|
|
|
@@ -53,29 +53,35 @@ class PdfReimportView(APIView):
|
|
|
53
53
|
raw_file_path = pdf.get_raw_file_path()
|
|
54
54
|
|
|
55
55
|
if not raw_file_path or not raw_file_path.exists():
|
|
56
|
-
logger.error(
|
|
56
|
+
logger.error(
|
|
57
|
+
f"Raw report file not found for hash {pdf.pdf_hash}: {raw_file_path}"
|
|
58
|
+
)
|
|
57
59
|
return Response(
|
|
58
|
-
{
|
|
60
|
+
{
|
|
61
|
+
"error": f"Raw report file not found for report {pdf.pdf_hash}. Please upload the original file again."
|
|
62
|
+
},
|
|
59
63
|
status=status.HTTP_404_NOT_FOUND,
|
|
60
64
|
)
|
|
61
65
|
|
|
62
|
-
# Check if
|
|
66
|
+
# Check if report has required relationships
|
|
63
67
|
if not pdf.center:
|
|
64
|
-
logger.warning(f"
|
|
68
|
+
logger.warning(f"report {pdf.pdf_hash} has no associated center")
|
|
65
69
|
return Response(
|
|
66
|
-
{"error": "
|
|
70
|
+
{"error": "report has no associated center."},
|
|
67
71
|
status=status.HTTP_400_BAD_REQUEST,
|
|
68
72
|
)
|
|
69
73
|
|
|
70
74
|
try:
|
|
71
|
-
logger.info(f"Starting re-import for
|
|
75
|
+
logger.info(f"Starting re-import for report {pdf.pdf_hash} (ID: {pdf_id})")
|
|
72
76
|
|
|
73
77
|
with transaction.atomic():
|
|
74
78
|
# Clear existing metadata to force regeneration
|
|
75
79
|
old_meta_id = None
|
|
76
80
|
if pdf.sensitive_meta:
|
|
77
81
|
old_meta_id = pdf.sensitive_meta.pk
|
|
78
|
-
logger.info(
|
|
82
|
+
logger.info(
|
|
83
|
+
f"Clearing existing SensitiveMeta {old_meta_id} for report {pdf.pdf_hash}"
|
|
84
|
+
)
|
|
79
85
|
pdf.sensitive_meta = None
|
|
80
86
|
pdf.save(update_fields=["sensitive_meta"])
|
|
81
87
|
|
|
@@ -84,11 +90,15 @@ class PdfReimportView(APIView):
|
|
|
84
90
|
SensitiveMeta.objects.filter(pk=old_meta_id).delete()
|
|
85
91
|
logger.info(f"Deleted old SensitiveMeta {old_meta_id}")
|
|
86
92
|
except Exception as e:
|
|
87
|
-
logger.warning(
|
|
93
|
+
logger.warning(
|
|
94
|
+
f"Could not delete old SensitiveMeta {old_meta_id}: {e}"
|
|
95
|
+
)
|
|
88
96
|
|
|
89
|
-
# Use
|
|
97
|
+
# Use ReportImportService for reprocessing
|
|
90
98
|
try:
|
|
91
|
-
logger.info(
|
|
99
|
+
logger.info(
|
|
100
|
+
f"Starting reprocessing using ReportImportService for {pdf.pdf_hash}"
|
|
101
|
+
)
|
|
92
102
|
self.pdf_service.import_and_anonymize(
|
|
93
103
|
file_path=raw_file_path,
|
|
94
104
|
center_name=pdf.center.name,
|
|
@@ -96,18 +106,22 @@ class PdfReimportView(APIView):
|
|
|
96
106
|
retry=True, # Mark as retry attempt
|
|
97
107
|
)
|
|
98
108
|
|
|
99
|
-
logger.info(
|
|
109
|
+
logger.info(
|
|
110
|
+
f"ReportImportService reprocessing completed for {pdf.pdf_hash}"
|
|
111
|
+
)
|
|
100
112
|
|
|
101
113
|
# Refresh to get updated state
|
|
102
114
|
pdf.refresh_from_db()
|
|
103
115
|
|
|
104
116
|
return Response(
|
|
105
117
|
{
|
|
106
|
-
"message": "
|
|
118
|
+
"message": "report re-import completed successfully.",
|
|
107
119
|
"pdf_id": pdf_id,
|
|
108
120
|
"pdf_hash": str(pdf.pdf_hash),
|
|
109
121
|
"sensitive_meta_created": pdf.sensitive_meta is not None,
|
|
110
|
-
"sensitive_meta_id": pdf.sensitive_meta.pk
|
|
122
|
+
"sensitive_meta_id": pdf.sensitive_meta.pk
|
|
123
|
+
if pdf.sensitive_meta
|
|
124
|
+
else None,
|
|
111
125
|
"text_extracted": bool(pdf.text),
|
|
112
126
|
"anonymized": pdf.anonymized,
|
|
113
127
|
"status": "done",
|
|
@@ -116,7 +130,9 @@ class PdfReimportView(APIView):
|
|
|
116
130
|
)
|
|
117
131
|
|
|
118
132
|
except Exception as e:
|
|
119
|
-
logger.exception(
|
|
133
|
+
logger.exception(
|
|
134
|
+
f"ReportImportService reprocessing failed for report {pdf.pdf_hash}: {e}"
|
|
135
|
+
)
|
|
120
136
|
return Response(
|
|
121
137
|
{
|
|
122
138
|
"error": f"Reprocessing failed: {str(e)}",
|
|
@@ -128,11 +144,16 @@ class PdfReimportView(APIView):
|
|
|
128
144
|
)
|
|
129
145
|
|
|
130
146
|
except Exception as e:
|
|
131
|
-
logger.error(
|
|
147
|
+
logger.error(
|
|
148
|
+
f"Failed to re-import report {pdf.pdf_hash}: {str(e)}", exc_info=True
|
|
149
|
+
)
|
|
132
150
|
|
|
133
151
|
# Handle specific error types
|
|
134
152
|
error_msg = str(e)
|
|
135
|
-
if any(
|
|
153
|
+
if any(
|
|
154
|
+
phrase in error_msg.lower()
|
|
155
|
+
for phrase in ["insufficient storage", "no space left", "disk full"]
|
|
156
|
+
):
|
|
136
157
|
# Storage error - return specific error message
|
|
137
158
|
return Response(
|
|
138
159
|
{
|
|
@@ -3,7 +3,10 @@ import os
|
|
|
3
3
|
import re
|
|
4
4
|
|
|
5
5
|
from django.http import FileResponse, Http404, StreamingHttpResponse
|
|
6
|
-
from django.views.decorators.clickjacking import
|
|
6
|
+
from django.views.decorators.clickjacking import (
|
|
7
|
+
xframe_options_exempt,
|
|
8
|
+
xframe_options_sameorigin,
|
|
9
|
+
)
|
|
7
10
|
from rest_framework.views import APIView
|
|
8
11
|
|
|
9
12
|
from endoreg_db.models import RawPdfFile
|
|
@@ -36,29 +39,30 @@ class ClosingFileWrapper:
|
|
|
36
39
|
self.file_handle.close()
|
|
37
40
|
|
|
38
41
|
|
|
39
|
-
class
|
|
42
|
+
class ReportStreamView(APIView):
|
|
40
43
|
"""
|
|
41
|
-
Streams a
|
|
44
|
+
Streams a report file with correct HTTP range support and proper file handle management.
|
|
42
45
|
|
|
43
|
-
Supports streaming both raw (original) and processed
|
|
46
|
+
Supports streaming both raw (original) and processed report files.
|
|
44
47
|
|
|
45
48
|
Query Parameters:
|
|
46
|
-
type: 'raw' (default) or 'processed' - Selects which
|
|
49
|
+
type: 'raw' (default) or 'processed' - Selects which report file to stream
|
|
47
50
|
|
|
48
51
|
Examples:
|
|
49
|
-
GET /api/media/pdf/1/?type=raw - Stream original raw
|
|
50
|
-
GET /api/media/pdf/1/?type=processed - Stream processed
|
|
52
|
+
GET /api/media/pdf/1/?type=raw - Stream original raw report
|
|
53
|
+
GET /api/media/pdf/1/?type=processed - Stream processed report
|
|
51
54
|
"""
|
|
52
55
|
|
|
53
56
|
permission_classes = [EnvironmentAwarePermission]
|
|
57
|
+
|
|
54
58
|
@xframe_options_exempt
|
|
55
59
|
def get(self, request, pk: int, *args, **kwargs):
|
|
56
60
|
file_type = "raw" # Initialize for error logging
|
|
57
61
|
try:
|
|
58
62
|
pdf_obj = RawPdfFile.objects.filter(pk=pk).first()
|
|
59
63
|
if not pdf_obj:
|
|
60
|
-
logger.warning(f"
|
|
61
|
-
raise Http404("
|
|
64
|
+
logger.warning(f"report not found: ID {pk}")
|
|
65
|
+
raise Http404("report not found")
|
|
62
66
|
|
|
63
67
|
# Parse query parameters to determine which file to stream
|
|
64
68
|
file_type = request.query_params.get("type", "raw").lower()
|
|
@@ -70,29 +74,31 @@ class PdfStreamView(APIView):
|
|
|
70
74
|
if file_type == "raw":
|
|
71
75
|
file_field = pdf_obj.file
|
|
72
76
|
if not file_field:
|
|
73
|
-
logger.warning(f"No raw
|
|
74
|
-
raise Http404("Raw
|
|
77
|
+
logger.warning(f"No raw report file available for report ID {pk}")
|
|
78
|
+
raise Http404("Raw report file not available")
|
|
75
79
|
else: # anonymized
|
|
76
|
-
file_field = pdf_obj.
|
|
80
|
+
file_field = pdf_obj.processed_file
|
|
77
81
|
if not file_field:
|
|
78
82
|
logger.warning(
|
|
79
|
-
f"No processed
|
|
83
|
+
f"No processed report file available for report ID {pk}"
|
|
80
84
|
)
|
|
81
|
-
raise Http404("Processed
|
|
85
|
+
raise Http404("Processed report file not available")
|
|
82
86
|
|
|
83
87
|
# Check if file exists on filesystem
|
|
84
88
|
try:
|
|
85
89
|
file_path = file_field.path
|
|
86
90
|
if not os.path.exists(file_path):
|
|
87
|
-
logger.error(
|
|
91
|
+
logger.error(
|
|
92
|
+
f"report file does not exist on filesystem: {file_path}"
|
|
93
|
+
)
|
|
88
94
|
raise Http404(
|
|
89
|
-
f"{file_type.capitalize()}
|
|
95
|
+
f"{file_type.capitalize()} report file not found on filesystem"
|
|
90
96
|
)
|
|
91
97
|
|
|
92
98
|
file_size = os.path.getsize(file_path)
|
|
93
99
|
except (OSError, IOError, AttributeError) as e:
|
|
94
|
-
logger.error(f"Error accessing {file_type}
|
|
95
|
-
raise Http404(f"{file_type.capitalize()}
|
|
100
|
+
logger.error(f"Error accessing {file_type} report file {pk}: {e}")
|
|
101
|
+
raise Http404(f"{file_type.capitalize()} report file not accessible")
|
|
96
102
|
|
|
97
103
|
# Generate safe filename
|
|
98
104
|
base_filename = (
|
|
@@ -114,7 +120,7 @@ class PdfStreamView(APIView):
|
|
|
114
120
|
range_header = request.headers.get("Range")
|
|
115
121
|
if range_header:
|
|
116
122
|
logger.debug(
|
|
117
|
-
f"Range request for {file_type}
|
|
123
|
+
f"Range request for {file_type} report {pk}: {range_header}"
|
|
118
124
|
)
|
|
119
125
|
match = _RANGE_RE.match(range_header)
|
|
120
126
|
if match:
|
|
@@ -138,7 +144,7 @@ class PdfStreamView(APIView):
|
|
|
138
144
|
file_handle.seek(start)
|
|
139
145
|
|
|
140
146
|
logger.debug(
|
|
141
|
-
f"Serving {file_type}
|
|
147
|
+
f"Serving {file_type} report {pk} range {start}-{end}/{file_size}"
|
|
142
148
|
)
|
|
143
149
|
|
|
144
150
|
response = StreamingHttpResponse(
|
|
@@ -156,31 +162,33 @@ class PdfStreamView(APIView):
|
|
|
156
162
|
return response
|
|
157
163
|
except (OSError, IOError) as e:
|
|
158
164
|
logger.error(
|
|
159
|
-
f"Error opening {file_type}
|
|
165
|
+
f"Error opening {file_type} report file for range request: {e}"
|
|
160
166
|
)
|
|
161
|
-
raise Http404(f"Error accessing {file_type}
|
|
167
|
+
raise Http404(f"Error accessing {file_type} report file")
|
|
162
168
|
else:
|
|
163
169
|
logger.warning(f"Invalid Range header format: {range_header}")
|
|
164
170
|
|
|
165
171
|
# Serve entire file using FileResponse (automatically handles file closing)
|
|
166
|
-
logger.debug(f"Serving full {file_type}
|
|
172
|
+
logger.debug(f"Serving full {file_type} report {pk} ({file_size} bytes)")
|
|
167
173
|
|
|
168
174
|
try:
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
175
|
+
with open(file_path, "rb") as file_handle:
|
|
176
|
+
response = FileResponse(file_handle, content_type="application/pdf")
|
|
177
|
+
response["Content-Length"] = str(file_size)
|
|
178
|
+
response["Accept-Ranges"] = "bytes"
|
|
179
|
+
response["Content-Disposition"] = (
|
|
180
|
+
f'inline; filename="{safe_filename}"'
|
|
181
|
+
)
|
|
174
182
|
|
|
175
183
|
# FileResponse will take ownership of file_handle and close it after response
|
|
176
184
|
return response
|
|
177
185
|
except (OSError, IOError) as e:
|
|
178
|
-
logger.error(f"Error opening {file_type}
|
|
179
|
-
raise Http404(f"Error accessing {file_type}
|
|
186
|
+
logger.error(f"Error opening {file_type} report file: {e}")
|
|
187
|
+
raise Http404(f"Error accessing {file_type} report file")
|
|
180
188
|
|
|
181
189
|
except Exception as e:
|
|
182
190
|
logger.error(
|
|
183
|
-
f"Unexpected error streaming {file_type if 'file_type' in locals() else '
|
|
191
|
+
f"Unexpected error streaming {file_type if 'file_type' in locals() else 'report'} {pk}: {e}",
|
|
184
192
|
exc_info=True,
|
|
185
193
|
)
|
|
186
|
-
raise Http404("Error streaming
|
|
194
|
+
raise Http404("Error streaming report")
|