endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +146 -83
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/migrations/__init__.py +0 -0
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -3
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/ffmpeg_wrapper.py +217 -52
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +436 -433
- endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +0 -119
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,20 +1,23 @@
|
|
|
1
|
-
from celery.events import state
|
|
2
|
-
from rest_framework.views import APIView
|
|
3
|
-
from rest_framework.response import Response
|
|
4
|
-
from rest_framework import status
|
|
5
1
|
import logging
|
|
6
2
|
from pathlib import Path
|
|
3
|
+
|
|
7
4
|
from django.db import transaction
|
|
8
|
-
from
|
|
9
|
-
from
|
|
5
|
+
from rest_framework import status
|
|
6
|
+
from rest_framework.response import Response
|
|
7
|
+
from rest_framework.views import APIView
|
|
8
|
+
|
|
9
|
+
from ...models import SensitiveMeta, VideoFile
|
|
10
|
+
from ...services.video_import import VideoImportService
|
|
11
|
+
|
|
10
12
|
logger = logging.getLogger(__name__)
|
|
11
13
|
|
|
14
|
+
|
|
12
15
|
class VideoReimportView(APIView):
|
|
13
16
|
"""
|
|
14
17
|
API endpoint to re-import a video file and regenerate metadata.
|
|
15
18
|
This is useful when OCR failed or metadata is incomplete.
|
|
16
19
|
"""
|
|
17
|
-
|
|
20
|
+
|
|
18
21
|
def __init__(self, **kwargs):
|
|
19
22
|
super().__init__(**kwargs)
|
|
20
23
|
self.video_service = VideoImportService()
|
|
@@ -23,33 +26,33 @@ class VideoReimportView(APIView):
|
|
|
23
26
|
"""
|
|
24
27
|
Re-import a video file to regenerate SensitiveMeta and other metadata.
|
|
25
28
|
Instead of creating a new video, this updates the existing one.
|
|
26
|
-
|
|
29
|
+
|
|
27
30
|
Args:
|
|
28
31
|
pk (int): Primary key of the VideoFile to reimport
|
|
29
32
|
"""
|
|
30
33
|
# Validate pk parameter
|
|
31
34
|
if not pk or not isinstance(pk, int):
|
|
32
35
|
return Response(
|
|
33
|
-
{"error": "Invalid video ID provided."},
|
|
34
|
-
status=status.HTTP_400_BAD_REQUEST
|
|
36
|
+
{"error": "Invalid video ID provided."},
|
|
37
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
35
38
|
)
|
|
36
39
|
|
|
37
40
|
try:
|
|
38
41
|
video = VideoFile.objects.get(id=pk)
|
|
39
|
-
logger.info(f"Found video {video.
|
|
42
|
+
logger.info(f"Found video {video.video_hash} (ID: {pk}) for re-import")
|
|
40
43
|
except VideoFile.DoesNotExist:
|
|
41
44
|
logger.warning(f"Video with ID {pk} not found")
|
|
42
45
|
return Response(
|
|
43
|
-
{"error": f"Video with ID {pk} not found."},
|
|
44
|
-
status=status.HTTP_404_NOT_FOUND
|
|
46
|
+
{"error": f"Video with ID {pk} not found."},
|
|
47
|
+
status=status.HTTP_404_NOT_FOUND,
|
|
45
48
|
)
|
|
46
49
|
|
|
47
50
|
# Check if video has a raw file
|
|
48
51
|
if not video.raw_file:
|
|
49
|
-
logger.warning(f"Video {video.
|
|
52
|
+
logger.warning(f"Video {video.video_hash} has no raw file")
|
|
50
53
|
return Response(
|
|
51
|
-
{"error": "Video has no raw file to re-import."},
|
|
52
|
-
status=status.HTTP_400_BAD_REQUEST
|
|
54
|
+
{"error": "Video has no raw file to re-import."},
|
|
55
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
53
56
|
)
|
|
54
57
|
|
|
55
58
|
# Check if the raw file actually exists on disk
|
|
@@ -57,139 +60,171 @@ class VideoReimportView(APIView):
|
|
|
57
60
|
if not raw_file_path.exists():
|
|
58
61
|
logger.error(f"Raw file not found on disk: {raw_file_path}")
|
|
59
62
|
return Response(
|
|
60
|
-
{"error": f"Video file not found on server: {raw_file_path.name}"},
|
|
61
|
-
status=status.HTTP_400_BAD_REQUEST
|
|
63
|
+
{"error": f"Video file not found on server: {raw_file_path.name}"},
|
|
64
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
62
65
|
)
|
|
63
66
|
|
|
64
67
|
# Check if video has required relationships
|
|
65
68
|
if not video.center:
|
|
66
|
-
logger.warning(f"Video {video.
|
|
69
|
+
logger.warning(f"Video {video.video_hash} has no associated center")
|
|
67
70
|
return Response(
|
|
68
|
-
{"error": "Video has no associated center."},
|
|
69
|
-
status=status.HTTP_400_BAD_REQUEST
|
|
71
|
+
{"error": "Video has no associated center."},
|
|
72
|
+
status=status.HTTP_400_BAD_REQUEST,
|
|
70
73
|
)
|
|
71
74
|
|
|
72
75
|
try:
|
|
73
|
-
logger.info(
|
|
74
|
-
|
|
76
|
+
logger.info(
|
|
77
|
+
f"Starting in-place re-import for video {video.video_hash} (ID: {pk})"
|
|
78
|
+
)
|
|
79
|
+
|
|
75
80
|
with transaction.atomic():
|
|
76
81
|
# Clear existing metadata to force regeneration
|
|
77
82
|
old_meta_id = None
|
|
78
83
|
if video.sensitive_meta:
|
|
79
84
|
old_meta_id = video.sensitive_meta.id
|
|
80
|
-
logger.info(
|
|
85
|
+
logger.info(
|
|
86
|
+
f"Clearing existing SensitiveMeta {old_meta_id} for video {video.video_hash}"
|
|
87
|
+
)
|
|
81
88
|
video.sensitive_meta = None
|
|
82
|
-
video.save(update_fields=[
|
|
83
|
-
|
|
89
|
+
video.save(update_fields=["sensitive_meta"])
|
|
90
|
+
|
|
84
91
|
# Delete the old SensitiveMeta record
|
|
85
92
|
try:
|
|
86
93
|
SensitiveMeta.objects.filter(id=old_meta_id).delete()
|
|
87
94
|
logger.info(f"Deleted old SensitiveMeta {old_meta_id}")
|
|
88
95
|
except Exception as e:
|
|
89
|
-
logger.warning(
|
|
90
|
-
|
|
96
|
+
logger.warning(
|
|
97
|
+
f"Could not delete old SensitiveMeta {old_meta_id}: {e}"
|
|
98
|
+
)
|
|
99
|
+
|
|
91
100
|
# Re-initialize video specs and frames
|
|
92
|
-
logger.info(f"Re-initializing video specs for {video.
|
|
101
|
+
logger.info(f"Re-initializing video specs for {video.video_hash}")
|
|
93
102
|
video.initialize_video_specs()
|
|
94
103
|
video.initialize_frames()
|
|
95
|
-
|
|
104
|
+
|
|
96
105
|
# Run Pipe 1 for OCR and AI processing
|
|
97
|
-
logger.info(f"Starting Pipe 1 processing for {video.
|
|
98
|
-
|
|
106
|
+
logger.info(f"Starting Pipe 1 processing for {video.video_hash}")
|
|
107
|
+
|
|
99
108
|
try:
|
|
100
109
|
success = video.pipe_1(
|
|
101
110
|
model_name="image_multilabel_classification_colonoscopy_default",
|
|
102
111
|
delete_frames_after=True,
|
|
103
112
|
ocr_frame_fraction=0.01,
|
|
104
|
-
ocr_cap=5
|
|
113
|
+
ocr_cap=5,
|
|
105
114
|
)
|
|
106
115
|
except Exception as e:
|
|
107
|
-
logger.error(
|
|
116
|
+
logger.error(
|
|
117
|
+
f"Pipe 1 processing raised exception for {video.video_hash}: {e}"
|
|
118
|
+
)
|
|
108
119
|
return Response(
|
|
109
120
|
{"error": f"OCR and AI processing failed: {str(e)}"},
|
|
110
|
-
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
|
121
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
111
122
|
)
|
|
112
|
-
|
|
123
|
+
|
|
113
124
|
if not success:
|
|
114
|
-
logger.error(
|
|
125
|
+
logger.error(
|
|
126
|
+
f"Pipe 1 processing failed for video {video.video_hash}"
|
|
127
|
+
)
|
|
115
128
|
return Response(
|
|
116
|
-
{"error": "OCR and AI processing failed during re-import."},
|
|
117
|
-
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
|
129
|
+
{"error": "OCR and AI processing failed during re-import."},
|
|
130
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
118
131
|
)
|
|
119
|
-
|
|
120
|
-
logger.info(f"Pipe 1 processing completed for {video.
|
|
121
|
-
|
|
122
|
-
# Ensure minimum patient data is available
|
|
123
|
-
logger.info(f"Ensuring minimum patient data for {video.uuid}")
|
|
124
|
-
self.video_service._ensure_default_patient_data(video)
|
|
125
|
-
|
|
126
|
-
# Refresh from database to get updated data
|
|
127
|
-
video.refresh_from_db()
|
|
128
|
-
|
|
132
|
+
|
|
133
|
+
logger.info(f"Pipe 1 processing completed for {video.video_hash}")
|
|
134
|
+
|
|
129
135
|
# Use VideoImportService for anonymization
|
|
130
136
|
try:
|
|
131
|
-
processor_name =
|
|
132
|
-
|
|
137
|
+
processor_name = (
|
|
138
|
+
video.video_meta.processor.name
|
|
139
|
+
if video.video_meta and video.video_meta.processor
|
|
140
|
+
else "Unknown"
|
|
141
|
+
)
|
|
142
|
+
logger.info(
|
|
143
|
+
f"Starting anonymization using VideoImportService for {video.video_hash}"
|
|
144
|
+
)
|
|
133
145
|
self.video_service.import_and_anonymize(
|
|
134
146
|
file_path=raw_file_path,
|
|
135
147
|
center_name=video.center.name,
|
|
136
148
|
processor_name=processor_name,
|
|
137
|
-
|
|
138
|
-
delete_source=False
|
|
149
|
+
delete_source=False,
|
|
139
150
|
)
|
|
140
|
-
|
|
141
|
-
logger.info(
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
|
|
152
|
+
logger.info(
|
|
153
|
+
f"VideoImportService anonymization completed for {video.video_hash}"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
return Response(
|
|
157
|
+
{
|
|
158
|
+
"message": "Video re-import with VideoImportService completed successfully.",
|
|
159
|
+
"video_id": pk,
|
|
160
|
+
"uuid": str(video.video_hash),
|
|
161
|
+
"frame_cleaning_applied": True,
|
|
162
|
+
"sensitive_meta_created": video.sensitive_meta is not None,
|
|
163
|
+
"sensitive_meta_id": video.sensitive_meta.id
|
|
164
|
+
if video.sensitive_meta
|
|
165
|
+
else None,
|
|
166
|
+
"updated_in_place": True,
|
|
167
|
+
"status": "done",
|
|
168
|
+
},
|
|
169
|
+
status=status.HTTP_200_OK,
|
|
170
|
+
)
|
|
171
|
+
|
|
155
172
|
except Exception as e:
|
|
156
|
-
logger.exception(
|
|
173
|
+
logger.exception(
|
|
174
|
+
f"VideoImportService anonymization failed for video {video.video_hash}: {e}"
|
|
175
|
+
)
|
|
157
176
|
logger.warning("Continuing without anonymization due to error")
|
|
158
|
-
|
|
159
|
-
state.mark_sensitive_meta_processed(save=True)
|
|
160
|
-
|
|
177
|
+
|
|
161
178
|
# If we reach here, everything was successful
|
|
162
|
-
logger.info(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
179
|
+
logger.info(
|
|
180
|
+
f"Video re-import completed successfully for {video.video_hash}"
|
|
181
|
+
)
|
|
182
|
+
video.save(update_fields=["sensitive_meta", "date_modified"])
|
|
183
|
+
|
|
184
|
+
return Response(
|
|
185
|
+
{
|
|
186
|
+
"message": "Video re-import completed successfully.",
|
|
187
|
+
"video_id": pk,
|
|
188
|
+
"uuid": str(video.video_hash),
|
|
189
|
+
"sensitive_meta_created": video.sensitive_meta is not None,
|
|
190
|
+
"sensitive_meta_id": video.sensitive_meta.id
|
|
191
|
+
if video.sensitive_meta
|
|
192
|
+
else None,
|
|
193
|
+
"updated_in_place": True,
|
|
194
|
+
"status": "done", # ⭐ Add explicit done status
|
|
195
|
+
},
|
|
196
|
+
status=status.HTTP_200_OK,
|
|
197
|
+
)
|
|
174
198
|
|
|
175
199
|
except Exception as e:
|
|
176
|
-
logger.error(
|
|
177
|
-
|
|
200
|
+
logger.error(
|
|
201
|
+
f"Failed to re-import video {video.video_hash}: {str(e)}", exc_info=True
|
|
202
|
+
)
|
|
203
|
+
|
|
178
204
|
# Handle specific error types
|
|
179
205
|
error_msg = str(e)
|
|
180
|
-
if any(
|
|
206
|
+
if any(
|
|
207
|
+
phrase in error_msg.lower()
|
|
208
|
+
for phrase in ["insufficient storage", "no space left", "disk full"]
|
|
209
|
+
):
|
|
181
210
|
# Storage error - return specific error message
|
|
182
|
-
return Response(
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
211
|
+
return Response(
|
|
212
|
+
{
|
|
213
|
+
"error": f"Storage error during re-import: {error_msg}",
|
|
214
|
+
"error_type": "storage_error",
|
|
215
|
+
"video_id": pk,
|
|
216
|
+
"uuid": str(video.video_hash),
|
|
217
|
+
},
|
|
218
|
+
status=status.HTTP_507_INSUFFICIENT_STORAGE,
|
|
219
|
+
)
|
|
188
220
|
else:
|
|
189
221
|
# Other errors
|
|
190
|
-
return Response(
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
222
|
+
return Response(
|
|
223
|
+
{
|
|
224
|
+
"error": f"Re-import failed: {error_msg}",
|
|
225
|
+
"error_type": "processing_error",
|
|
226
|
+
"video_id": pk,
|
|
227
|
+
"uuid": str(video.video_hash),
|
|
228
|
+
},
|
|
229
|
+
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
230
|
+
)
|