endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +89 -122
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -4
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
from endoreg_db.models.
|
|
2
|
-
|
|
1
|
+
from endoreg_db.models.state.processing_history.processing_history import (
|
|
2
|
+
ProcessingHistory,
|
|
3
|
+
)
|
|
4
|
+
from endoreg_db.utils.paths import ANONYM_REPORT_DIR, ANONYM_VIDEO_DIR
|
|
3
5
|
|
|
4
|
-
import os
|
|
5
6
|
import logging
|
|
6
7
|
import shutil
|
|
7
8
|
from pathlib import Path
|
|
@@ -13,11 +14,24 @@ from endoreg_db.import_files.context.import_context import ImportContext
|
|
|
13
14
|
from endoreg_db.models.media import RawPdfFile, VideoFile
|
|
14
15
|
from endoreg_db.models.state import RawPdfState, VideoState
|
|
15
16
|
from endoreg_db.utils import paths as path_utils
|
|
17
|
+
from endoreg_db.utils.file_operations import sha256_file
|
|
16
18
|
|
|
17
19
|
logger = logging.getLogger(__name__)
|
|
18
20
|
|
|
19
21
|
|
|
20
|
-
def
|
|
22
|
+
def _get_history_filename(ctx: ImportContext) -> str:
|
|
23
|
+
"""
|
|
24
|
+
Prefer original_path.name if provided, otherwise fall back to file_path.name.
|
|
25
|
+
"""
|
|
26
|
+
if ctx.original_path is not None:
|
|
27
|
+
return ctx.original_path.name
|
|
28
|
+
# ctx.file_path is always present and already a Path in your tests
|
|
29
|
+
return Path(ctx.file_path).name
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _ensure_instance_state(
|
|
33
|
+
instance: Union[VideoFile, RawPdfFile],
|
|
34
|
+
) -> Optional[Union[RawPdfState, VideoState]]:
|
|
21
35
|
"""
|
|
22
36
|
Helper: ensure instance.state exists and return it.
|
|
23
37
|
Mirrors PdfImportService._ensure_state.
|
|
@@ -26,7 +40,7 @@ def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[U
|
|
|
26
40
|
state = getattr(instance, "state", None)
|
|
27
41
|
else:
|
|
28
42
|
state = getattr(instance, "state", None)
|
|
29
|
-
|
|
43
|
+
|
|
30
44
|
if state is not None:
|
|
31
45
|
return state
|
|
32
46
|
|
|
@@ -37,14 +51,15 @@ def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[U
|
|
|
37
51
|
|
|
38
52
|
return None
|
|
39
53
|
|
|
40
|
-
|
|
54
|
+
|
|
55
|
+
def mark_instance_processing_started(
|
|
41
56
|
instance: Union[RawPdfFile, VideoFile],
|
|
42
|
-
ctx: ImportContext,
|
|
57
|
+
ctx: ImportContext,
|
|
58
|
+
):
|
|
43
59
|
state = _ensure_instance_state(instance)
|
|
44
60
|
|
|
45
61
|
with transaction.atomic():
|
|
46
62
|
if state is not None:
|
|
47
|
-
|
|
48
63
|
# In the old code, processing_started was set earlier; we guard here
|
|
49
64
|
if not getattr(state, "processing_started", False) and hasattr(
|
|
50
65
|
state, "mark_processing_started"
|
|
@@ -125,28 +140,12 @@ def finalize_report_success(
|
|
|
125
140
|
if current_name != relative_name:
|
|
126
141
|
instance.processed_file.name = relative_name
|
|
127
142
|
logger.info("Updated processed_file to %s", relative_name)
|
|
128
|
-
try:
|
|
129
|
-
relative_name = str(ctx.anonymized_path)
|
|
130
|
-
except ValueError:
|
|
131
|
-
# Fallback: absolute path if outside STORAGE_DIR
|
|
132
|
-
relative_name = str(final_path)
|
|
133
|
-
|
|
134
|
-
current_name = getattr(instance.processed_file, "name", None)
|
|
135
|
-
if current_name != relative_name:
|
|
136
|
-
instance.processed_file.name = relative_name
|
|
137
|
-
logger.info(
|
|
138
|
-
"Updated processed_file reference to: %s",
|
|
139
|
-
instance.processed_file.name,
|
|
140
|
-
)
|
|
141
|
-
|
|
142
143
|
|
|
143
144
|
# --- Update RawPdfState flags (mirrors _finalize_processing) ---
|
|
144
145
|
state = _ensure_instance_state(instance)
|
|
145
146
|
|
|
146
147
|
with transaction.atomic():
|
|
147
148
|
if state is not None:
|
|
148
|
-
|
|
149
|
-
# In the old code, processing_started was set earlier; we guard here
|
|
150
149
|
if not getattr(state, "processing_started", False) and hasattr(
|
|
151
150
|
state, "mark_processing_started"
|
|
152
151
|
):
|
|
@@ -165,17 +164,20 @@ def finalize_report_success(
|
|
|
165
164
|
# --- ProcessingHistory entry ---
|
|
166
165
|
try:
|
|
167
166
|
with transaction.atomic():
|
|
168
|
-
|
|
167
|
+
if not isinstance(ctx.file_hash, str):
|
|
168
|
+
ctx.file_hash = sha256_file(ctx.file_path)
|
|
169
|
+
ProcessingHistory.get_or_create_for_hash(
|
|
169
170
|
obj=instance,
|
|
171
|
+
file_hash=ctx.file_hash,
|
|
170
172
|
success=True,
|
|
171
173
|
)
|
|
172
174
|
except Exception as e:
|
|
173
175
|
logger.debug(
|
|
174
|
-
"Saving not possible; %
|
|
175
|
-
f"skipping ProcessingHistory.{e}",
|
|
176
|
+
f"Saving not possible; %sskipping ProcessingHistory.{e}",
|
|
176
177
|
instance.pk,
|
|
177
178
|
)
|
|
178
179
|
|
|
180
|
+
|
|
179
181
|
def finalize_video_success(
|
|
180
182
|
ctx: ImportContext,
|
|
181
183
|
) -> None:
|
|
@@ -187,8 +189,7 @@ def finalize_video_success(
|
|
|
187
189
|
- Mark VideoState as anonymized + sensitive_meta_processed
|
|
188
190
|
- Mark ProcessingHistory.success = True
|
|
189
191
|
"""
|
|
190
|
-
|
|
191
|
-
assert(nuke is True)
|
|
192
|
+
|
|
192
193
|
instance = ctx.current_video
|
|
193
194
|
if not isinstance(instance, VideoFile):
|
|
194
195
|
logger.warning("finalize_video_success called with non-VideoFile instance")
|
|
@@ -265,10 +266,13 @@ def finalize_video_success(
|
|
|
265
266
|
instance.processed_file.name = relative_name
|
|
266
267
|
logger.info("Updated video processed_file to %s", relative_name)
|
|
267
268
|
|
|
269
|
+
if not nuke_transcoding_dir():
|
|
270
|
+
logger.warning(
|
|
271
|
+
"Transcoding directory cleanup returned False after finalize_video_success; there may be leftover files."
|
|
272
|
+
)
|
|
273
|
+
|
|
268
274
|
# --- Update VideoState flags (mirrors report) ---
|
|
269
275
|
state = _ensure_instance_state(instance)
|
|
270
|
-
|
|
271
|
-
|
|
272
276
|
|
|
273
277
|
with transaction.atomic():
|
|
274
278
|
if state is not None:
|
|
@@ -289,8 +293,10 @@ def finalize_video_success(
|
|
|
289
293
|
# --- ProcessingHistory entry ---
|
|
290
294
|
try:
|
|
291
295
|
with transaction.atomic():
|
|
292
|
-
|
|
293
|
-
|
|
296
|
+
if not isinstance(ctx.file_hash, str):
|
|
297
|
+
ctx.file_hash = sha256_file(ctx.file_path)
|
|
298
|
+
ProcessingHistory.get_or_create_for_hash(
|
|
299
|
+
file_hash=ctx.file_hash,
|
|
294
300
|
success=True,
|
|
295
301
|
)
|
|
296
302
|
except Exception as e:
|
|
@@ -309,7 +315,9 @@ def finalize_failure(
|
|
|
309
315
|
|
|
310
316
|
- Reset RawPdfState flags to "not processed"
|
|
311
317
|
- Mark ProcessingHistory.success = False
|
|
318
|
+
- Delete all associated files
|
|
312
319
|
"""
|
|
320
|
+
|
|
313
321
|
if ctx.instance is None:
|
|
314
322
|
if isinstance(ctx.current_report, RawPdfFile):
|
|
315
323
|
ctx.instance = ctx.current_report
|
|
@@ -317,6 +325,15 @@ def finalize_failure(
|
|
|
317
325
|
ctx.instance = ctx.current_video
|
|
318
326
|
else:
|
|
319
327
|
raise Exception
|
|
328
|
+
|
|
329
|
+
# History entry with success=False
|
|
330
|
+
if not isinstance(ctx.file_hash, str):
|
|
331
|
+
ctx.file_hash = sha256_file(ctx.file_path)
|
|
332
|
+
ProcessingHistory.get_or_create_for_hash(
|
|
333
|
+
file_hash=ctx.file_hash,
|
|
334
|
+
success=False,
|
|
335
|
+
)
|
|
336
|
+
|
|
320
337
|
# Reset state flags similar to _mark_processing_incomplete / _cleanup_on_error
|
|
321
338
|
state = _ensure_instance_state(ctx.instance)
|
|
322
339
|
|
|
@@ -336,29 +353,17 @@ def finalize_failure(
|
|
|
336
353
|
e,
|
|
337
354
|
)
|
|
338
355
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
# History entry with success=False
|
|
345
|
-
if ctx.file_hash:
|
|
346
|
-
ProcessingHistory.get_or_create_for_object(
|
|
347
|
-
obj=ctx.instance,
|
|
348
|
-
success=False,
|
|
349
|
-
)
|
|
350
|
-
else:
|
|
351
|
-
logger.debug(
|
|
352
|
-
"No file_hash in context for instance %s when finalizing failure; "
|
|
353
|
-
"skipping ProcessingHistory.",
|
|
354
|
-
ctx.instance.pk,
|
|
355
|
-
)
|
|
356
|
+
try:
|
|
357
|
+
delete_associated_files(ctx)
|
|
358
|
+
except Exception as e:
|
|
359
|
+
logger.warning(f"There might be files remaining. {e}")
|
|
356
360
|
|
|
357
361
|
logger.error(
|
|
358
|
-
"
|
|
362
|
+
"File processing failed for %s - state reset, ready for retry.",
|
|
359
363
|
ctx.file_path,
|
|
360
364
|
)
|
|
361
365
|
|
|
366
|
+
|
|
362
367
|
def delete_associated_files(ctx: ImportContext) -> None:
|
|
363
368
|
"""
|
|
364
369
|
Best-effort cleanup of anonymized, sensitive and transcoding artefacts.
|
|
@@ -373,92 +378,46 @@ def delete_associated_files(ctx: ImportContext) -> None:
|
|
|
373
378
|
Only restoration of the original import file is treated as critical.
|
|
374
379
|
"""
|
|
375
380
|
|
|
376
|
-
# ---
|
|
377
|
-
original_missing = not isinstance(ctx.original_path, Path) or not ctx.original_path.exists()
|
|
378
|
-
if original_missing:
|
|
379
|
-
logger.warning(
|
|
380
|
-
"Original file missing in ctx (file_type=%s); "
|
|
381
|
-
"trying to restore from sensitive copy.",
|
|
382
|
-
ctx.file_type,
|
|
383
|
-
)
|
|
384
|
-
|
|
385
|
-
if not isinstance(ctx.sensitive_path, Path) or not ctx.sensitive_path.exists():
|
|
386
|
-
# This is serious: we lost both original and sensitive copy
|
|
387
|
-
msg = (
|
|
388
|
-
f"Cannot restore original file for {ctx.file_type}: "
|
|
389
|
-
"sensitive copy missing as well."
|
|
390
|
-
)
|
|
391
|
-
logger.error(msg)
|
|
392
|
-
raise RuntimeError(msg)
|
|
393
|
-
|
|
394
|
-
try:
|
|
395
|
-
if ctx.file_type == "video":
|
|
396
|
-
target_dir = IMPORT_VIDEO_DIR
|
|
397
|
-
elif ctx.file_type == "report":
|
|
398
|
-
target_dir = IMPORT_REPORT_DIR
|
|
399
|
-
else:
|
|
400
|
-
raise ValueError(f"Unknown file_type in context: {ctx.file_type}")
|
|
401
|
-
|
|
402
|
-
target_dir.mkdir(parents=True, exist_ok=True)
|
|
403
|
-
restored_path = shutil.copy2(ctx.sensitive_path, target_dir)
|
|
404
|
-
ctx.original_path = Path(restored_path)
|
|
405
|
-
logger.info("Restored original file for %s to %s", ctx.file_type, ctx.original_path)
|
|
406
|
-
except Exception as e:
|
|
407
|
-
logger.error("Error during safety copy / restore of original file: %s", e, exc_info=True)
|
|
408
|
-
raise
|
|
409
|
-
|
|
410
|
-
# --- 2. Delete anonymized file (best-effort) ---
|
|
381
|
+
# --- Delete anonymized file (best-effort) ---
|
|
411
382
|
if isinstance(ctx.anonymized_path, Path):
|
|
412
383
|
try:
|
|
413
|
-
if ctx.anonymized_path.exists()
|
|
384
|
+
if ctx.anonymized_path.exists():
|
|
414
385
|
ctx.anonymized_path.unlink()
|
|
415
386
|
logger.info("Deleted anonymized file %s", ctx.anonymized_path)
|
|
416
387
|
except Exception as e:
|
|
417
|
-
logger.error(
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
p = Path(path_utils.data_paths["anonym_report" / ctx.anonymized_path])
|
|
424
|
-
p.unlink()
|
|
425
|
-
if ctx.anonymized_path.exists():
|
|
426
|
-
ctx.anonymized_path.rmdir()
|
|
388
|
+
logger.error(
|
|
389
|
+
"Error when unlinking anonymized path %s: %s",
|
|
390
|
+
ctx.anonymized_path,
|
|
391
|
+
e,
|
|
392
|
+
exc_info=True,
|
|
393
|
+
)
|
|
427
394
|
finally:
|
|
428
|
-
if ctx.anonymized_path.exists():
|
|
429
|
-
raise AssertionError("Anonym file remains after all deletion attempts.")
|
|
430
395
|
ctx.anonymized_path = None
|
|
431
396
|
|
|
432
|
-
# ---
|
|
397
|
+
# --- Nuke transcoding directory (best-effort) ---
|
|
433
398
|
if not nuke_transcoding_dir():
|
|
434
|
-
logger.warning(
|
|
399
|
+
logger.warning(
|
|
400
|
+
"Transcoding directory cleanup returned False; there may be leftover files."
|
|
401
|
+
)
|
|
435
402
|
|
|
436
|
-
# ---
|
|
403
|
+
# --- Delete sensitive file (best-effort) ---
|
|
437
404
|
if isinstance(ctx.sensitive_path, Path):
|
|
438
405
|
try:
|
|
439
406
|
if ctx.sensitive_path.exists():
|
|
440
407
|
ctx.sensitive_path.unlink()
|
|
441
408
|
logger.info("Deleted sensitive file %s", ctx.sensitive_path)
|
|
442
409
|
except Exception as e:
|
|
443
|
-
logger.error(
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
p = Path(path_utils.data_paths["sensitive_report" / ctx.sensitive_path])
|
|
450
|
-
p.unlink()
|
|
451
|
-
if ctx.sensitive_path.exists():
|
|
452
|
-
ctx.sensitive_path.rmdir()
|
|
410
|
+
logger.error(
|
|
411
|
+
"Error when unlinking sensitive path %s: %s",
|
|
412
|
+
ctx.sensitive_path,
|
|
413
|
+
e,
|
|
414
|
+
exc_info=True,
|
|
415
|
+
)
|
|
453
416
|
finally:
|
|
454
|
-
if ctx.sensitive_path.exists():
|
|
455
|
-
raise AssertionError("Sensitive file remains after all deletion attempts.")
|
|
456
417
|
ctx.sensitive_path = None
|
|
457
418
|
|
|
458
|
-
|
|
459
|
-
def nuke_transcoding_dir(
|
|
460
|
-
transcoding_dir: Union[str, Path, None] = None
|
|
461
|
-
) -> bool:
|
|
419
|
+
|
|
420
|
+
def nuke_transcoding_dir(transcoding_dir: Union[str, Path, None] = None) -> bool:
|
|
462
421
|
"""
|
|
463
422
|
Delete all files and subdirectories inside the transcoding directory.
|
|
464
423
|
|
|
@@ -473,11 +432,15 @@ def nuke_transcoding_dir(
|
|
|
473
432
|
transcoding_dir = Path(transcoding_dir)
|
|
474
433
|
|
|
475
434
|
if not transcoding_dir.exists():
|
|
476
|
-
logger.info(
|
|
435
|
+
logger.info(
|
|
436
|
+
"Transcoding dir %s does not exist; nothing to clean.", transcoding_dir
|
|
437
|
+
)
|
|
477
438
|
return True
|
|
478
439
|
|
|
479
440
|
if not transcoding_dir.is_dir():
|
|
480
|
-
logger.error(
|
|
441
|
+
logger.error(
|
|
442
|
+
"Configured transcoding path %s is not a directory.", transcoding_dir
|
|
443
|
+
)
|
|
481
444
|
return False
|
|
482
445
|
|
|
483
446
|
for entry in transcoding_dir.iterdir():
|
|
@@ -487,10 +450,14 @@ def nuke_transcoding_dir(
|
|
|
487
450
|
elif entry.is_dir():
|
|
488
451
|
shutil.rmtree(entry)
|
|
489
452
|
except Exception as e:
|
|
490
|
-
logger.warning(
|
|
453
|
+
logger.warning(
|
|
454
|
+
"Failed to remove entry %s in transcoding dir: %s", entry, e
|
|
455
|
+
)
|
|
491
456
|
# Continue trying to delete other entries
|
|
492
457
|
return True
|
|
493
458
|
|
|
494
459
|
except Exception as e:
|
|
495
|
-
logger.error(
|
|
460
|
+
logger.error(
|
|
461
|
+
"Unexpected error while nuking transcoding dir: %s", e, exc_info=True
|
|
462
|
+
)
|
|
496
463
|
return False
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from endoreg_db.models.media.video.create_from_file import
|
|
3
|
+
from endoreg_db.models.media.video.create_from_file import (
|
|
4
|
+
atomic_copy_with_fallback,
|
|
5
|
+
atomic_move_with_fallback,
|
|
6
|
+
)
|
|
7
|
+
|
|
4
8
|
logger = logging.getLogger(__name__)
|
|
5
9
|
|
|
6
10
|
|
|
@@ -2,13 +2,14 @@ import logging
|
|
|
2
2
|
import os
|
|
3
3
|
import sys
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Any, Callable, Literal, NoReturn
|
|
6
5
|
|
|
7
6
|
from lx_anonymizer import ReportReader
|
|
8
7
|
from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
|
|
9
8
|
|
|
10
9
|
from endoreg_db.import_files.context import ImportContext
|
|
11
|
-
from endoreg_db.import_files.file_storage.sensitive_meta_storage import
|
|
10
|
+
from endoreg_db.import_files.file_storage.sensitive_meta_storage import (
|
|
11
|
+
sensitive_meta_storage,
|
|
12
|
+
)
|
|
12
13
|
from endoreg_db.utils.paths import ANONYM_REPORT_DIR
|
|
13
14
|
|
|
14
15
|
|
|
@@ -20,10 +21,8 @@ class ReportAnonymizer:
|
|
|
20
21
|
self._report_reader_class = None
|
|
21
22
|
self._ensure_report_reading_available()
|
|
22
23
|
self.storage = False
|
|
23
|
-
|
|
24
24
|
|
|
25
25
|
def anonymize_report(self, ctx: ImportContext):
|
|
26
|
-
|
|
27
26
|
# Setup anonymized directory
|
|
28
27
|
anonymized_dir = ANONYM_REPORT_DIR
|
|
29
28
|
anonymized_dir.mkdir(parents=True, exist_ok=True)
|
|
@@ -32,29 +31,35 @@ class ReportAnonymizer:
|
|
|
32
31
|
pdf_hash = ctx.current_report.pdf_hash
|
|
33
32
|
anonymized_output_path = anonymized_dir / f"{pdf_hash}.pdf"
|
|
34
33
|
self._report_reader_class = ReportReader()
|
|
35
|
-
|
|
34
|
+
|
|
36
35
|
assert isinstance(self._report_reader_class, ReportReader)
|
|
37
36
|
|
|
38
37
|
# Process with enhanced process_report method (returns 4-tuple now)
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
38
|
+
(
|
|
39
|
+
ctx.original_text,
|
|
40
|
+
ctx.anonymized_text,
|
|
41
|
+
extracted_metadata,
|
|
42
|
+
ctx.anonymized_path,
|
|
43
|
+
) = self._report_reader_class.process_report(
|
|
44
|
+
pdf_path=ctx.file_path,
|
|
45
|
+
create_anonymized_pdf=True,
|
|
46
|
+
anonymized_pdf_output_path=str(anonymized_output_path),
|
|
47
|
+
)
|
|
48
|
+
|
|
45
49
|
if ctx.anonymized_path:
|
|
46
|
-
logger.info(
|
|
47
|
-
|
|
50
|
+
logger.info(
|
|
51
|
+
"DEBUG: after anonymizer, ctx.anonymized_path=%s (exists=%s)",
|
|
52
|
+
ctx.anonymized_path,
|
|
53
|
+
isinstance(ctx.anonymized_path, str),
|
|
54
|
+
)
|
|
48
55
|
|
|
49
56
|
sm = LxSM()
|
|
50
|
-
sm.safe_update(extracted_metadata)
|
|
51
|
-
|
|
57
|
+
sm.safe_update(extracted_metadata)
|
|
58
|
+
|
|
52
59
|
self.storage = sensitive_meta_storage(sm, ctx.current_report)
|
|
53
60
|
return ctx
|
|
54
|
-
|
|
55
|
-
def _ensure_report_reading_available(
|
|
56
|
-
self
|
|
57
|
-
) -> None:
|
|
61
|
+
|
|
62
|
+
def _ensure_report_reading_available(self) -> None:
|
|
58
63
|
"""
|
|
59
64
|
Ensure report reading modules are available by adding lx-anonymizer to path.
|
|
60
65
|
|
|
@@ -21,10 +21,10 @@ def normalize_lx_sensitive_meta(meta: LxSensitiveMeta) -> Dict[str, Any]:
|
|
|
21
21
|
"file_path",
|
|
22
22
|
"patient_first_name",
|
|
23
23
|
"patient_last_name",
|
|
24
|
-
"patient_dob",
|
|
24
|
+
"patient_dob", # string; logic has parsing
|
|
25
25
|
"casenumber",
|
|
26
|
-
"examination_date",
|
|
27
|
-
"examination_time",
|
|
26
|
+
"examination_date", # string; logic has parsing
|
|
27
|
+
"examination_time", # string "HH:MM" is fine
|
|
28
28
|
"examiner_first_name",
|
|
29
29
|
"examiner_last_name",
|
|
30
30
|
"text",
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
from typing import List, Dict, Any, Tuple, Optional
|
|
2
|
-
|
|
3
1
|
import logging
|
|
2
|
+
|
|
4
3
|
logger = logging.getLogger(__name__)
|
|
5
4
|
|
|
6
5
|
from lx_anonymizer import FrameCleaner
|
|
7
6
|
from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
|
|
8
7
|
|
|
9
|
-
|
|
10
|
-
from endoreg_db.import_files.file_storage.sensitive_meta_storage import sensitive_meta_storage
|
|
11
8
|
from endoreg_db.import_files.context import ImportContext
|
|
12
|
-
from endoreg_db.
|
|
9
|
+
from endoreg_db.import_files.file_storage.sensitive_meta_storage import (
|
|
10
|
+
sensitive_meta_storage,
|
|
11
|
+
)
|
|
13
12
|
from endoreg_db.models import EndoscopyProcessor, VideoFile
|
|
13
|
+
from endoreg_db.utils.paths import ANONYM_VIDEO_DIR
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class VideoAnonymizer:
|
|
@@ -20,35 +20,35 @@ class VideoAnonymizer:
|
|
|
20
20
|
self._frame_cleaning_class = None
|
|
21
21
|
self.storage = False
|
|
22
22
|
|
|
23
|
-
|
|
24
23
|
def anonymize_video(self, ctx: ImportContext):
|
|
25
24
|
# Setup anonymized directory
|
|
26
25
|
anonymized_dir = ANONYM_VIDEO_DIR
|
|
27
26
|
anonymized_dir.mkdir(parents=True, exist_ok=True)
|
|
28
27
|
assert ctx.current_video is not None
|
|
29
28
|
# Generate output path for anonymized report
|
|
30
|
-
|
|
29
|
+
|
|
31
30
|
video_hash = ctx.current_video.video_hash
|
|
32
31
|
anonymized_output_path = anonymized_dir / f"{video_hash}.mp4"
|
|
33
|
-
|
|
32
|
+
|
|
34
33
|
self._frame_cleaning_class = FrameCleaner()
|
|
35
|
-
|
|
34
|
+
|
|
36
35
|
assert isinstance(self._frame_cleaning_class, FrameCleaner)
|
|
37
36
|
endoscope_roi, endoscope_roi_nested = self._get_processor_roi_info(ctx)
|
|
38
37
|
# Process with enhanced process_report method (returns 4-tuple now)
|
|
39
|
-
ctx.anonymized_path, extracted_metadata =
|
|
38
|
+
ctx.anonymized_path, extracted_metadata = (
|
|
39
|
+
self._frame_cleaning_class.clean_video(
|
|
40
40
|
video_path=ctx.file_path,
|
|
41
41
|
endoscope_image_roi=endoscope_roi,
|
|
42
42
|
endoscope_data_roi_nested=endoscope_roi_nested,
|
|
43
|
-
output_path=anonymized_output_path
|
|
44
|
-
|
|
43
|
+
output_path=anonymized_output_path,
|
|
45
44
|
)
|
|
45
|
+
)
|
|
46
46
|
sm = LxSM()
|
|
47
47
|
sm.safe_update(extracted_metadata)
|
|
48
|
-
|
|
48
|
+
|
|
49
49
|
self.storage = sensitive_meta_storage(sm, ctx.current_video)
|
|
50
50
|
return ctx
|
|
51
|
-
|
|
51
|
+
|
|
52
52
|
def _ensure_frame_cleaning_available(self):
|
|
53
53
|
"""
|
|
54
54
|
Ensure frame cleaning modules are available by adding lx-anonymizer to path.
|
|
@@ -68,12 +68,12 @@ class VideoAnonymizer:
|
|
|
68
68
|
self._frame_cleaning_class = FrameCleaner()
|
|
69
69
|
self._frame_cleaning_available = True
|
|
70
70
|
|
|
71
|
-
|
|
72
71
|
def _get_processor_roi_info(
|
|
73
72
|
self,
|
|
74
73
|
ctx: ImportContext,
|
|
75
|
-
) -> tuple[
|
|
76
|
-
|
|
74
|
+
) -> tuple[
|
|
75
|
+
dict[str, int | None] | None, dict[str, dict[str, int | None] | None] | None
|
|
76
|
+
]:
|
|
77
77
|
"""Get processor ROI information for masking and data extraction."""
|
|
78
78
|
endoscope_data_roi_nested = None
|
|
79
79
|
endoscope_image_roi = None
|
|
@@ -98,7 +98,7 @@ class VideoAnonymizer:
|
|
|
98
98
|
else:
|
|
99
99
|
logger.warning(
|
|
100
100
|
"No processor found for video %s, proceeding without ROI masking",
|
|
101
|
-
video.
|
|
101
|
+
video.video_hash,
|
|
102
102
|
)
|
|
103
103
|
except Exception as exc:
|
|
104
104
|
logger.error("Failed to retrieve processor ROI information: %s", exc)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from endoreg_db.models import
|
|
1
|
+
from endoreg_db.models import SensitiveMeta
|
|
2
2
|
import logging
|
|
3
3
|
from datetime import timedelta
|
|
4
4
|
from typing import Tuple
|
|
@@ -6,7 +6,7 @@ from typing import Tuple
|
|
|
6
6
|
from django.db.models import QuerySet
|
|
7
7
|
|
|
8
8
|
from itertools import combinations
|
|
9
|
-
from typing import Dict
|
|
9
|
+
from typing import Dict
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
12
12
|
|
|
@@ -68,7 +68,7 @@ def get_k_anonymity(pk, k=3):
|
|
|
68
68
|
pk (_type_): _description_
|
|
69
69
|
k (int, optional): _description_. Defaults to 3.
|
|
70
70
|
"""
|
|
71
|
-
return get_k_anonymity_for_sensitive_meta(pk=pk, k=k, dob_year_tolerance=1)
|
|
71
|
+
return get_k_anonymity_for_sensitive_meta(pk=pk, k=k, dob_year_tolerance=1)
|
|
72
72
|
|
|
73
73
|
|
|
74
74
|
def _build_sensitive_meta_qi_queryset(
|
|
@@ -99,20 +99,19 @@ def _build_sensitive_meta_qi_queryset(
|
|
|
99
99
|
A Django QuerySet for further aggregation.
|
|
100
100
|
"""
|
|
101
101
|
qs = SensitiveMeta.objects.all()
|
|
102
|
-
|
|
102
|
+
|
|
103
103
|
if use_first_name and instance.patient_first_name is not None:
|
|
104
104
|
qs = qs.filter(patient_first_name=instance.patient_first_name)
|
|
105
|
-
|
|
106
|
-
if use_last_name and instance.
|
|
107
|
-
qs = qs.filter(
|
|
108
|
-
|
|
105
|
+
|
|
106
|
+
if use_last_name and instance.patient_last_name is not None:
|
|
107
|
+
qs = qs.filter(patient_last_name=instance.patient_last_name)
|
|
109
108
|
# --- Center ---
|
|
110
109
|
if use_center and instance.center is not None:
|
|
111
110
|
if instance.center.pk is not None:
|
|
112
111
|
qs = qs.filter(center=instance.center.pk)
|
|
113
112
|
|
|
114
113
|
# --- Gender ---
|
|
115
|
-
if use_gender and instance.patient_gender is not None
|
|
114
|
+
if use_gender and instance.patient_gender is not None:
|
|
116
115
|
if instance.patient_gender.pk is not None:
|
|
117
116
|
qs = qs.filter(patient_gender_id=instance.patient_gender)
|
|
118
117
|
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
from typing import Optional, Tuple
|
|
2
|
+
import logging
|
|
2
3
|
|
|
3
4
|
from datetime import date as Date
|
|
4
|
-
import datetime
|
|
5
5
|
|
|
6
6
|
from .k_anonymity import _build_sensitive_meta_qi_queryset
|
|
7
7
|
from .fake import fake_name_with_similar_dob_and_gender
|
|
8
8
|
|
|
9
9
|
from endoreg_db.models import SensitiveMeta
|
|
10
10
|
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
11
14
|
def k_pseudonymize(
|
|
12
15
|
instance: SensitiveMeta,
|
|
13
16
|
*,
|
|
@@ -44,10 +47,16 @@ def k_pseudonymize(
|
|
|
44
47
|
Returns:
|
|
45
48
|
(instance, k_value_after, is_k_anonymous_after)
|
|
46
49
|
"""
|
|
47
|
-
if qi_subset is None:
|
|
48
|
-
qi_subset = tuple(QI_FLAGS)
|
|
49
50
|
|
|
50
51
|
# --- 1) Compute k for the requested subset BEFORE pseudonymization ---
|
|
52
|
+
if qi_subset is None:
|
|
53
|
+
qi_subset = ("first_name", "last_name", "center", "gender", "dob_band")
|
|
54
|
+
# --- 1) Compute k for the requested subset BEFORE pseudonymization ---
|
|
55
|
+
use_first_name = "first_name" in qi_subset
|
|
56
|
+
use_last_name = "last_name" in qi_subset
|
|
57
|
+
use_center = "center" in qi_subset
|
|
58
|
+
use_gender = "gender" in qi_subset
|
|
59
|
+
use_dob_band = "dob_band" in qi_subset
|
|
51
60
|
use_first_name = "first_name" in qi_subset
|
|
52
61
|
use_last_name = "last_name" in qi_subset
|
|
53
62
|
use_center = "center" in qi_subset
|
|
@@ -95,12 +104,14 @@ def k_pseudonymize(
|
|
|
95
104
|
# Assign to instance (SensitiveMeta.patient_dob is a DateTimeField)
|
|
96
105
|
instance.patient_first_name = first_name
|
|
97
106
|
instance.patient_last_name = last_name
|
|
98
|
-
instance.patient_dob =
|
|
107
|
+
instance.patient_dob = Date(
|
|
99
108
|
fake_dob.year, fake_dob.month, fake_dob.day
|
|
100
109
|
) # naive is usually fine for DOB
|
|
101
110
|
|
|
102
111
|
if save:
|
|
103
|
-
instance.save(
|
|
112
|
+
instance.save(
|
|
113
|
+
update_fields=["patient_first_name", "patient_last_name", "patient_dob"]
|
|
114
|
+
)
|
|
104
115
|
|
|
105
116
|
# --- 3) Recompute k AFTER pseudonymization ---
|
|
106
117
|
qs_after = _build_sensitive_meta_qi_queryset(
|