endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +146 -83
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/migrations/__init__.py +0 -0
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -3
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/ffmpeg_wrapper.py +217 -52
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +436 -433
- endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +0 -119
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -4,12 +4,13 @@ from endoreg_db.models import Label, LabelVideoSegment
|
|
|
4
4
|
from itertools import combinations
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from django.conf import settings
|
|
7
|
+
|
|
7
8
|
# === CONFIGURABLE PARAMETERS - ForNiceClassificationSerializer ===
|
|
8
9
|
POLYP_LABEL_NAME = "polyp"
|
|
9
10
|
CHROMO_LABEL_NAMES = ["digital_chromo_endoscopy", "nbi"]
|
|
10
11
|
FPS = 50 # Frames per second- should fetch dynamically from videofile table
|
|
11
12
|
# Sequence-level filtering
|
|
12
|
-
MIN_SEQUENCE_GAP_SECONDS = 0.5
|
|
13
|
+
MIN_SEQUENCE_GAP_SECONDS = 0.5 # Enforce diversity between sequences
|
|
13
14
|
MIN_SEQUENCE_GAP_FRAMES = FPS * MIN_SEQUENCE_GAP_SECONDS # Convert to frame count
|
|
14
15
|
# Minimum length of a segment in seconds
|
|
15
16
|
MIN_SEGMENT_LENGTH_SECONDS = 0.5
|
|
@@ -27,13 +28,20 @@ FRAME_SELECTION_RULES = [
|
|
|
27
28
|
lambda pred: pred.get("low_quality", 1.0) < 0.1,
|
|
28
29
|
lambda pred: pred.get("outside", 1.0) < 0.1,
|
|
29
30
|
lambda pred: pred.get("snare", 1.0) < 0.1,
|
|
30
|
-
|
|
31
31
|
# Add more rules easily here
|
|
32
32
|
]
|
|
33
33
|
|
|
34
|
-
REQUIRED_FRAME_KEYS = [
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
REQUIRED_FRAME_KEYS = [
|
|
35
|
+
"low_quality",
|
|
36
|
+
"outside",
|
|
37
|
+
"snare",
|
|
38
|
+
] # need tp update when adding rules
|
|
39
|
+
|
|
40
|
+
POLYP_CONFIDENCE_THRESHOLDS = [
|
|
41
|
+
0.9,
|
|
42
|
+
0.8,
|
|
43
|
+
0.7,
|
|
44
|
+
] # basically this is the prediction value/score, we are using for frame selection
|
|
37
45
|
INSTRUMENT_LABEL_NAMES = ["instrument", "snare", "needle"] # Add more as needed later
|
|
38
46
|
|
|
39
47
|
|
|
@@ -42,11 +50,11 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
42
50
|
Base class for NICE and PARIS serializers.
|
|
43
51
|
Handles label lookup, chromo/instrument segment filtering, and shared utilities.
|
|
44
52
|
"""
|
|
45
|
-
|
|
46
|
-
#TODO add
|
|
53
|
+
|
|
54
|
+
# TODO add create method
|
|
55
|
+
# TODO add update method
|
|
47
56
|
LABEL_NAME = "polyp" # default (can be overridden)
|
|
48
|
-
INSTRUMENT_LABEL_NAME = "instrument"
|
|
49
|
-
|
|
57
|
+
INSTRUMENT_LABEL_NAME = "instrument" # TODO @Hamzaukw we should define frequently used labels in a utils file
|
|
50
58
|
|
|
51
59
|
def get_label_id_by_name(self, label_name):
|
|
52
60
|
"""
|
|
@@ -57,10 +65,10 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
57
65
|
label = Label.objects.get(name=label_name)
|
|
58
66
|
return label.id
|
|
59
67
|
except Label.DoesNotExist:
|
|
60
|
-
raise serializers.ValidationError(
|
|
61
|
-
"error": f"Label with name '{label_name}' does not exist."
|
|
62
|
-
|
|
63
|
-
|
|
68
|
+
raise serializers.ValidationError(
|
|
69
|
+
{"error": f"Label with name '{label_name}' does not exist."}
|
|
70
|
+
)
|
|
71
|
+
|
|
64
72
|
def get_label_ids_by_names(self, label_names):
|
|
65
73
|
"""
|
|
66
74
|
Get a list of label IDs for a given list of label names.
|
|
@@ -71,12 +79,12 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
71
79
|
|
|
72
80
|
missing = set(label_names) - set(label_map.keys())
|
|
73
81
|
if missing:
|
|
74
|
-
raise serializers.ValidationError(
|
|
75
|
-
"error": f"Labels not found: {', '.join(missing)}"
|
|
76
|
-
|
|
82
|
+
raise serializers.ValidationError(
|
|
83
|
+
{"error": f"Labels not found: {', '.join(missing)}"}
|
|
84
|
+
)
|
|
77
85
|
|
|
78
86
|
return list(label_map.values())
|
|
79
|
-
|
|
87
|
+
|
|
80
88
|
def get_chromo_segments(self, video_id):
|
|
81
89
|
"""
|
|
82
90
|
Fetch all segments that match chromo-like labels (e.g., chromo or NBI).
|
|
@@ -85,10 +93,10 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
85
93
|
- PARIS: chromo overlap is disqualifying
|
|
86
94
|
"""
|
|
87
95
|
chromo_label_ids = self.get_label_ids_by_names(CHROMO_LABEL_NAMES)
|
|
88
|
-
return LabelVideoSegment.objects.filter(
|
|
89
|
-
|
|
96
|
+
return LabelVideoSegment.objects.filter(
|
|
97
|
+
label_id__in=chromo_label_ids, video_file_id=video_id
|
|
98
|
+
)
|
|
90
99
|
|
|
91
|
-
|
|
92
100
|
def get_filtered_polyp_segments(self, video_id):
|
|
93
101
|
"""
|
|
94
102
|
Return polyp segments that do NOT overlap with 'instrument' segments.
|
|
@@ -96,7 +104,12 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
96
104
|
polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
|
|
97
105
|
instrument_label_id = self.get_label_id_by_name(self.INSTRUMENT_LABEL_NAME)
|
|
98
106
|
|
|
99
|
-
print(
|
|
107
|
+
print(
|
|
108
|
+
"polyp label id is :-",
|
|
109
|
+
polyp_label_id,
|
|
110
|
+
"and - instrument_label_id is ,",
|
|
111
|
+
instrument_label_id,
|
|
112
|
+
)
|
|
100
113
|
|
|
101
114
|
polyp_segments = LabelVideoSegment.objects.filter(
|
|
102
115
|
label_id=polyp_label_id, video_file_id=video_id
|
|
@@ -105,39 +118,61 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
105
118
|
instrument_segments = LabelVideoSegment.objects.filter(
|
|
106
119
|
label_id=instrument_label_id, video_file_id=video_id
|
|
107
120
|
)
|
|
108
|
-
print(
|
|
109
|
-
|
|
110
|
-
|
|
121
|
+
print(
|
|
122
|
+
"------------------------ --------------------------------- -------------------------------"
|
|
123
|
+
)
|
|
124
|
+
print(
|
|
125
|
+
"polyp label id is :-",
|
|
126
|
+
polyp_label_id,
|
|
127
|
+
"and - instrument_label_id is ,",
|
|
128
|
+
instrument_label_id,
|
|
129
|
+
)
|
|
130
|
+
print(
|
|
131
|
+
"polyp segments are",
|
|
132
|
+
polyp_segments,
|
|
133
|
+
"instrument_segments are",
|
|
134
|
+
instrument_segments,
|
|
135
|
+
)
|
|
111
136
|
|
|
112
137
|
def overlaps(seg1, seg2):
|
|
113
|
-
return
|
|
138
|
+
return (
|
|
139
|
+
seg1.start_frame_number <= seg2.end_frame_number
|
|
140
|
+
and seg1.end_frame_number >= seg2.start_frame_number
|
|
141
|
+
)
|
|
114
142
|
|
|
115
143
|
filtered_polyp_segments = []
|
|
116
144
|
for polyp_seg in polyp_segments:
|
|
117
|
-
if not any(
|
|
145
|
+
if not any(
|
|
146
|
+
overlaps(polyp_seg, instr_seg) for instr_seg in instrument_segments
|
|
147
|
+
):
|
|
118
148
|
filtered_polyp_segments.append(polyp_seg)
|
|
119
149
|
|
|
120
150
|
return filtered_polyp_segments
|
|
121
151
|
|
|
122
152
|
def get_polyp_segments(self, video_id):
|
|
123
153
|
polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
|
|
124
|
-
return LabelVideoSegment.objects.filter(
|
|
154
|
+
return LabelVideoSegment.objects.filter(
|
|
155
|
+
label_id=polyp_label_id, video_file_id=video_id
|
|
156
|
+
)
|
|
125
157
|
|
|
126
158
|
def apply_sequence_diversity(self, matching_segments):
|
|
127
159
|
if not matching_segments:
|
|
128
160
|
return []
|
|
129
161
|
|
|
130
|
-
matching_segments.sort(key=lambda seg: seg[
|
|
162
|
+
matching_segments.sort(key=lambda seg: seg["polyp"].start_frame_number)
|
|
131
163
|
|
|
132
164
|
valid_segments = [
|
|
133
|
-
seg
|
|
134
|
-
|
|
165
|
+
seg
|
|
166
|
+
for seg in matching_segments
|
|
167
|
+
if (seg["polyp"].end_frame_number - seg["polyp"].start_frame_number)
|
|
168
|
+
>= MIN_SEGMENT_LENGTH_FRAMES
|
|
135
169
|
]
|
|
136
170
|
|
|
137
171
|
if len(valid_segments) < 3:
|
|
138
172
|
valid_segments.sort(
|
|
139
|
-
key=lambda seg: seg[
|
|
140
|
-
|
|
173
|
+
key=lambda seg: seg["polyp"].end_frame_number
|
|
174
|
+
- seg["polyp"].start_frame_number,
|
|
175
|
+
reverse=True,
|
|
141
176
|
)
|
|
142
177
|
return valid_segments[:3]
|
|
143
178
|
|
|
@@ -145,12 +180,14 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
145
180
|
best_gap_sum = -1
|
|
146
181
|
|
|
147
182
|
for combo in combinations(valid_segments, 3):
|
|
148
|
-
sorted_combo = sorted(
|
|
183
|
+
sorted_combo = sorted(
|
|
184
|
+
combo, key=lambda seg: seg["polyp"].start_frame_number
|
|
185
|
+
)
|
|
149
186
|
s1, s2, s3 = sorted_combo
|
|
150
|
-
s1_end = s1[
|
|
151
|
-
s2_start = s2[
|
|
152
|
-
s2_end = s2[
|
|
153
|
-
s3_start = s3[
|
|
187
|
+
s1_end = s1["polyp"].end_frame_number
|
|
188
|
+
s2_start = s2["polyp"].start_frame_number
|
|
189
|
+
s2_end = s2["polyp"].end_frame_number
|
|
190
|
+
s3_start = s3["polyp"].start_frame_number
|
|
154
191
|
gap1 = s2_start - s1_end
|
|
155
192
|
gap2 = s3_start - s2_end
|
|
156
193
|
|
|
@@ -164,18 +201,19 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
164
201
|
return list(best_combo)
|
|
165
202
|
|
|
166
203
|
valid_segments.sort(
|
|
167
|
-
key=lambda seg: seg[
|
|
168
|
-
|
|
204
|
+
key=lambda seg: seg["polyp"].end_frame_number
|
|
205
|
+
- seg["polyp"].start_frame_number,
|
|
206
|
+
reverse=True,
|
|
169
207
|
)
|
|
170
208
|
return valid_segments[:3]
|
|
171
|
-
|
|
209
|
+
|
|
172
210
|
def select_frames_for_sequence(self, sequence):
|
|
173
211
|
"""
|
|
174
212
|
Selects evenly spaced representative frames from a polyp segment.
|
|
175
|
-
|
|
213
|
+
|
|
176
214
|
Frames are chosen from the segment's frame range, ensuring a minimum gap between selected frames. Returns a list of dictionaries containing frame numbers and their corresponding file paths. If the video's frame directory is unavailable, returns an empty list.
|
|
177
215
|
"""
|
|
178
|
-
polyp_sequence = sequence[
|
|
216
|
+
polyp_sequence = sequence["polyp"]
|
|
179
217
|
video = polyp_sequence.video_file
|
|
180
218
|
|
|
181
219
|
start_frame = polyp_sequence.start_frame_number
|
|
@@ -190,28 +228,32 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
190
228
|
segment_frames = [
|
|
191
229
|
{
|
|
192
230
|
"frame_number": idx,
|
|
193
|
-
"frame_path": f"{frame_dir}/frame_{str(idx).zfill(7)}.png"
|
|
231
|
+
"frame_path": f"{frame_dir}/frame_{str(idx).zfill(7)}.png",
|
|
194
232
|
}
|
|
195
233
|
for idx in range(start_frame, end_frame + 1)
|
|
196
234
|
]
|
|
197
235
|
|
|
198
236
|
selected_frames = []
|
|
199
|
-
last_selected_frame = -float(
|
|
237
|
+
last_selected_frame = -float("inf")
|
|
200
238
|
|
|
201
239
|
for frame in segment_frames:
|
|
202
240
|
if len(selected_frames) >= FRAMES_PER_SEQUENCE:
|
|
203
241
|
break
|
|
204
242
|
|
|
205
243
|
if frame["frame_number"] - last_selected_frame >= MIN_FRAME_GAP_IN_SEQUENCE:
|
|
206
|
-
selected_frames.append(
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
244
|
+
selected_frames.append(
|
|
245
|
+
{
|
|
246
|
+
"frame_number": frame["frame_number"],
|
|
247
|
+
"frame_path": frame["frame_path"],
|
|
248
|
+
}
|
|
249
|
+
)
|
|
210
250
|
last_selected_frame = frame["frame_number"]
|
|
211
251
|
|
|
212
252
|
return selected_frames
|
|
213
|
-
|
|
214
|
-
def extract_and_save_selected_frames(
|
|
253
|
+
|
|
254
|
+
def extract_and_save_selected_frames(
|
|
255
|
+
self, video, frame_numbers, classification_type: str
|
|
256
|
+
):
|
|
215
257
|
"""
|
|
216
258
|
Extract specific frames from the original video file and save them
|
|
217
259
|
into a structured folder path based on the classification type.
|
|
@@ -224,7 +266,13 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
224
266
|
# Resolve the path to the original video
|
|
225
267
|
original_path = Path(video.original_file_name)
|
|
226
268
|
if not original_path.is_absolute():
|
|
227
|
-
base_video_dir =
|
|
269
|
+
base_video_dir = (
|
|
270
|
+
settings.BASE_DIR.parent.parent
|
|
271
|
+
/ "production_test"
|
|
272
|
+
/ "endoreg-db"
|
|
273
|
+
/ "data"
|
|
274
|
+
/ "coloreg_first_test_batch"
|
|
275
|
+
)
|
|
228
276
|
original_path = base_video_dir / original_path
|
|
229
277
|
|
|
230
278
|
# Define output directory based on classification and video ID
|
|
@@ -235,11 +283,10 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
235
283
|
video_path=original_path,
|
|
236
284
|
frame_numbers=frame_numbers,
|
|
237
285
|
output_dir=output_path,
|
|
238
|
-
fps=FPS
|
|
286
|
+
fps=FPS,
|
|
239
287
|
)
|
|
240
288
|
|
|
241
|
-
|
|
242
|
-
''' # # If any error occurs in this function, use the commented one below(fallback:select_frames_for_sequence).
|
|
289
|
+
""" # # If any error occurs in this function, use the commented one below(fallback:select_frames_for_sequence).
|
|
243
290
|
def select_frames_for_sequence(self, sequence):
|
|
244
291
|
# Extract the polyp segment from the current sequence
|
|
245
292
|
polyp_sequence = sequence['polyp']
|
|
@@ -318,11 +365,10 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
318
365
|
return selected_frames
|
|
319
366
|
|
|
320
367
|
# If no threshold yielded enough frames, return what was found in the last attempt (could be empty)
|
|
321
|
-
return selected_frames if selected_frames else []
|
|
322
|
-
|
|
368
|
+
return selected_frames if selected_frames else []"""
|
|
323
369
|
|
|
324
370
|
# fallback:select_frames_for_sequence
|
|
325
|
-
|
|
371
|
+
"""
|
|
326
372
|
def select_frames_for_sequence(self, sequence):
|
|
327
373
|
print("----------------------in selected_frames fro sequnces funtion ----------------------------------------")
|
|
328
374
|
polyp_sequence = sequence['polyp']
|
|
@@ -365,7 +411,9 @@ class BaseClassificationSerializer(serializers.Serializer):
|
|
|
365
411
|
})
|
|
366
412
|
last_selected_frame = frame["frame_number"]
|
|
367
413
|
|
|
368
|
-
return selected_frames
|
|
414
|
+
return selected_frames"""
|
|
415
|
+
|
|
416
|
+
|
|
369
417
|
class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
370
418
|
"""
|
|
371
419
|
NICE classification logic:
|
|
@@ -374,6 +422,7 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
|
374
422
|
- Requires overlap with chromo-like labels (chromo/NBI)
|
|
375
423
|
- Selects diverse sequences and representative frames from each
|
|
376
424
|
"""
|
|
425
|
+
|
|
377
426
|
def get_matching_sequences(self, video_id):
|
|
378
427
|
"""
|
|
379
428
|
1. Fetch all polyp segments in a video
|
|
@@ -383,20 +432,28 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
|
383
432
|
"""
|
|
384
433
|
polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
|
|
385
434
|
instrument_label_ids = self.get_label_ids_by_names(INSTRUMENT_LABEL_NAMES)
|
|
386
|
-
#chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
|
|
435
|
+
# chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
|
|
387
436
|
chromo_segments = self.get_chromo_segments(video_id)
|
|
388
437
|
|
|
438
|
+
polyp_segments = LabelVideoSegment.objects.filter(
|
|
439
|
+
label_id=polyp_label_id, video_file_id=video_id
|
|
440
|
+
)
|
|
441
|
+
instrument_segments = LabelVideoSegment.objects.filter(
|
|
442
|
+
label_id__in=instrument_label_ids, video_file_id=video_id
|
|
443
|
+
)
|
|
389
444
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
print("polyp_label_id is", polyp_label_id, "and ployp_segment is ", polyp_segments)
|
|
445
|
+
# chromo_segments = LabelVideoSegment.objects.filter(label_id=chromo_label_id, video_file_id=video_id)
|
|
446
|
+
print(
|
|
447
|
+
"polyp_label_id is", polyp_label_id, "and ployp_segment is ", polyp_segments
|
|
448
|
+
)
|
|
395
449
|
print("instrument_segments is", instrument_segments)
|
|
396
|
-
#print("chromo_segments is ",chromo_segments)
|
|
450
|
+
# print("chromo_segments is ",chromo_segments)
|
|
397
451
|
|
|
398
452
|
def overlaps(seg1, seg2):
|
|
399
|
-
return
|
|
453
|
+
return (
|
|
454
|
+
seg1.start_frame_number <= seg2.end_frame_number
|
|
455
|
+
and seg1.end_frame_number >= seg2.start_frame_number
|
|
456
|
+
)
|
|
400
457
|
|
|
401
458
|
def subtract_overlap(seg, overlaps):
|
|
402
459
|
start = seg.start_frame_number
|
|
@@ -424,7 +481,9 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
|
424
481
|
matching_segments = []
|
|
425
482
|
|
|
426
483
|
for polyp in polyp_segments:
|
|
427
|
-
overlapping_instr = [
|
|
484
|
+
overlapping_instr = [
|
|
485
|
+
seg for seg in instrument_segments if overlaps(polyp, seg)
|
|
486
|
+
]
|
|
428
487
|
safe_ranges = subtract_overlap(polyp, overlapping_instr)
|
|
429
488
|
|
|
430
489
|
for start, end in safe_ranges:
|
|
@@ -432,7 +491,10 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
|
432
491
|
continue
|
|
433
492
|
|
|
434
493
|
for chromo in chromo_segments:
|
|
435
|
-
if
|
|
494
|
+
if (
|
|
495
|
+
chromo.start_frame_number <= end
|
|
496
|
+
and chromo.end_frame_number >= start
|
|
497
|
+
):
|
|
436
498
|
overlap_start = max(start, chromo.start_frame_number)
|
|
437
499
|
overlap_end = min(end, chromo.end_frame_number)
|
|
438
500
|
|
|
@@ -440,9 +502,9 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
|
440
502
|
overlapping_segment = LabelVideoSegment(
|
|
441
503
|
video_file=polyp.video_file,
|
|
442
504
|
start_frame_number=overlap_start,
|
|
443
|
-
end_frame_number=overlap_end
|
|
505
|
+
end_frame_number=overlap_end,
|
|
444
506
|
)
|
|
445
|
-
matching_segments.append({
|
|
507
|
+
matching_segments.append({"polyp": overlapping_segment})
|
|
446
508
|
|
|
447
509
|
return matching_segments
|
|
448
510
|
|
|
@@ -465,18 +527,22 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
|
465
527
|
matching_segments = self.get_matching_sequences(video_id)
|
|
466
528
|
print("matching_segments are", matching_segments)
|
|
467
529
|
if not matching_segments:
|
|
468
|
-
results.append(
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
530
|
+
results.append(
|
|
531
|
+
{
|
|
532
|
+
"video_id": video_id,
|
|
533
|
+
"message": "No valid polyp segments overlapping with chromo and not overlapping with instrument.",
|
|
534
|
+
}
|
|
535
|
+
)
|
|
472
536
|
continue
|
|
473
537
|
|
|
474
538
|
diverse_segments = self.apply_sequence_diversity(matching_segments)
|
|
475
539
|
if not diverse_segments:
|
|
476
|
-
results.append(
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
540
|
+
results.append(
|
|
541
|
+
{
|
|
542
|
+
"video_id": video_id,
|
|
543
|
+
"message": "No diverse NICE sequences found — either too short or too close together.",
|
|
544
|
+
}
|
|
545
|
+
)
|
|
480
546
|
continue
|
|
481
547
|
|
|
482
548
|
# Collect all frame numbers to extract only once per video
|
|
@@ -487,49 +553,54 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
|
|
|
487
553
|
frames = self.select_frames_for_sequence(segment)
|
|
488
554
|
|
|
489
555
|
if not frames:
|
|
490
|
-
segment_results.append(
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
556
|
+
segment_results.append(
|
|
557
|
+
{
|
|
558
|
+
"video_id": video_id,
|
|
559
|
+
"segment_start": segment["polyp"].start_frame_number,
|
|
560
|
+
"segment_end": segment["polyp"].end_frame_number,
|
|
561
|
+
"message": "No valid frames passed filtering rules (low_quality, outside, snare).",
|
|
562
|
+
}
|
|
563
|
+
)
|
|
496
564
|
else:
|
|
497
|
-
segment_results.append(
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
565
|
+
segment_results.append(
|
|
566
|
+
{
|
|
567
|
+
"video_id": video_id,
|
|
568
|
+
"segment_start": segment["polyp"].start_frame_number,
|
|
569
|
+
"segment_end": segment["polyp"].end_frame_number,
|
|
570
|
+
"frames": frames,
|
|
571
|
+
}
|
|
572
|
+
)
|
|
503
573
|
all_frames.extend(frames)
|
|
504
574
|
|
|
505
575
|
# Extract once per video
|
|
506
576
|
if all_frames:
|
|
507
|
-
unique_frame_numbers = sorted(
|
|
508
|
-
|
|
509
|
-
|
|
577
|
+
unique_frame_numbers = sorted(
|
|
578
|
+
{f["frame_number"] for f in all_frames}
|
|
579
|
+
)
|
|
580
|
+
classification_type = (
|
|
581
|
+
"nice"
|
|
582
|
+
if isinstance(self, ForNiceClassificationSerializer)
|
|
583
|
+
else "paris"
|
|
584
|
+
)
|
|
585
|
+
self.extract_and_save_selected_frames(
|
|
586
|
+
video, unique_frame_numbers, classification_type
|
|
587
|
+
)
|
|
510
588
|
|
|
511
589
|
results.extend(segment_results)
|
|
512
590
|
|
|
513
591
|
except Exception as e:
|
|
514
|
-
results.append({
|
|
515
|
-
"video_id": video.id,
|
|
516
|
-
"error": str(e)
|
|
517
|
-
})
|
|
592
|
+
results.append({"video_id": video.id, "error": str(e)})
|
|
518
593
|
|
|
519
594
|
if not results:
|
|
520
|
-
return [{
|
|
521
|
-
"message": "No classification data generated for any videos."
|
|
522
|
-
}]
|
|
595
|
+
return [{"message": "No classification data generated for any videos."}]
|
|
523
596
|
|
|
524
597
|
return {
|
|
525
|
-
"message": "NICE classification data generated."
|
|
598
|
+
"message": "NICE classification data generated."
|
|
599
|
+
if isinstance(self, ForNiceClassificationSerializer)
|
|
526
600
|
else "PARIS classification data generated.",
|
|
527
|
-
"data": results
|
|
601
|
+
"data": results,
|
|
528
602
|
}
|
|
529
603
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
604
|
|
|
534
605
|
class ForParisClassificationSerializer(BaseClassificationSerializer):
|
|
535
606
|
"""
|
|
@@ -540,26 +611,32 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
|
|
|
540
611
|
- Enforces minimum length and sequence diversity
|
|
541
612
|
- Selects frames from each valid segment
|
|
542
613
|
"""
|
|
543
|
-
|
|
614
|
+
|
|
544
615
|
def get_filtered_polyp_segments(self, video_id):
|
|
545
616
|
"""
|
|
546
617
|
1. Fetch all polyp segments
|
|
547
618
|
2. Subtract any overlap with chromo or instrument segments
|
|
548
619
|
3. Keep only trimmed segments longer than the minimum threshold
|
|
549
620
|
"""
|
|
550
|
-
|
|
621
|
+
|
|
551
622
|
polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
|
|
552
|
-
instrument_label_ids = self.get_label_ids_by_names(INSTRUMENT_LABEL_NAMES)
|
|
553
|
-
#chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
|
|
623
|
+
instrument_label_ids = self.get_label_ids_by_names(INSTRUMENT_LABEL_NAMES)
|
|
624
|
+
# chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
|
|
554
625
|
|
|
555
|
-
polyp_segments = LabelVideoSegment.objects.filter(
|
|
556
|
-
|
|
557
|
-
|
|
626
|
+
polyp_segments = LabelVideoSegment.objects.filter(
|
|
627
|
+
label_id=polyp_label_id, video_file_id=video_id
|
|
628
|
+
)
|
|
629
|
+
instrument_segments = LabelVideoSegment.objects.filter(
|
|
630
|
+
label_id__in=instrument_label_ids, video_file_id=video_id
|
|
631
|
+
)
|
|
632
|
+
# chromo_segments = LabelVideoSegment.objects.filter(label_id=chromo_label_id, video_file_id=video_id)
|
|
558
633
|
chromo_segments = self.get_chromo_segments(video_id)
|
|
559
634
|
|
|
560
|
-
|
|
561
635
|
def overlaps(seg1, seg2):
|
|
562
|
-
return
|
|
636
|
+
return (
|
|
637
|
+
seg1.start_frame_number <= seg2.end_frame_number
|
|
638
|
+
and seg1.end_frame_number >= seg2.start_frame_number
|
|
639
|
+
)
|
|
563
640
|
|
|
564
641
|
def subtract_overlap(seg, overlaps):
|
|
565
642
|
"""
|
|
@@ -594,7 +671,8 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
|
|
|
594
671
|
for polyp_seg in polyp_segments:
|
|
595
672
|
# Collect all overlapping regions with instrument or chromo
|
|
596
673
|
overlapping = [
|
|
597
|
-
seg
|
|
674
|
+
seg
|
|
675
|
+
for seg in (list(instrument_segments) + list(chromo_segments))
|
|
598
676
|
if overlaps(polyp_seg, seg)
|
|
599
677
|
]
|
|
600
678
|
|
|
@@ -602,18 +680,23 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
|
|
|
602
680
|
|
|
603
681
|
for start, end in safe_ranges:
|
|
604
682
|
if end - start + 1 >= MIN_SEGMENT_LENGTH_FRAMES:
|
|
605
|
-
trimmed_segments.append(
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
683
|
+
trimmed_segments.append(
|
|
684
|
+
LabelVideoSegment(
|
|
685
|
+
video_file=polyp_seg.video_file,
|
|
686
|
+
start_frame_number=start,
|
|
687
|
+
end_frame_number=end,
|
|
688
|
+
)
|
|
689
|
+
)
|
|
610
690
|
|
|
611
691
|
return trimmed_segments
|
|
612
692
|
|
|
613
693
|
filtered_segments = []
|
|
614
694
|
for polyp_seg in polyp_segments:
|
|
615
|
-
if not any(
|
|
616
|
-
|
|
695
|
+
if not any(
|
|
696
|
+
overlaps(polyp_seg, instr_seg) for instr_seg in instrument_segments
|
|
697
|
+
) and not any(
|
|
698
|
+
overlaps(polyp_seg, chromo_seg) for chromo_seg in chromo_segments
|
|
699
|
+
):
|
|
617
700
|
filtered_segments.append(polyp_seg)
|
|
618
701
|
|
|
619
702
|
return filtered_segments
|
|
@@ -624,9 +707,8 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
|
|
|
624
707
|
This is needed for consistent downstream processing.
|
|
625
708
|
"""
|
|
626
709
|
segments = self.get_filtered_polyp_segments(video_id)
|
|
627
|
-
return [{
|
|
710
|
+
return [{"polyp": seg} for seg in segments]
|
|
628
711
|
|
|
629
|
-
|
|
630
712
|
def to_representation(self, videos):
|
|
631
713
|
"""
|
|
632
714
|
Processes videos for PARIS classification.
|
|
@@ -642,18 +724,22 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
|
|
|
642
724
|
matching_segments = self.get_matching_sequences(video_id)
|
|
643
725
|
|
|
644
726
|
if not matching_segments:
|
|
645
|
-
results.append(
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
727
|
+
results.append(
|
|
728
|
+
{
|
|
729
|
+
"video_id": video_id,
|
|
730
|
+
"message": "No matching polyp segments found — possibly filtered due to overlap with 'instrument' label.",
|
|
731
|
+
}
|
|
732
|
+
)
|
|
649
733
|
continue
|
|
650
734
|
|
|
651
735
|
diverse_segments = self.apply_sequence_diversity(matching_segments)
|
|
652
736
|
if not diverse_segments:
|
|
653
|
-
results.append(
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
737
|
+
results.append(
|
|
738
|
+
{
|
|
739
|
+
"video_id": video_id,
|
|
740
|
+
"message": "No diverse sequences found — either too short or too close to each other.",
|
|
741
|
+
}
|
|
742
|
+
)
|
|
657
743
|
continue
|
|
658
744
|
|
|
659
745
|
all_frames = []
|
|
@@ -663,43 +749,46 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
|
|
|
663
749
|
frames = self.select_frames_for_sequence(segment)
|
|
664
750
|
|
|
665
751
|
if not frames:
|
|
666
|
-
segment_results.append(
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
752
|
+
segment_results.append(
|
|
753
|
+
{
|
|
754
|
+
"video_id": video_id,
|
|
755
|
+
"segment_start": segment["polyp"].start_frame_number,
|
|
756
|
+
"segment_end": segment["polyp"].end_frame_number,
|
|
757
|
+
"message": "No valid frames selected due to prediction rules (e.g., quality, outside, snare).",
|
|
758
|
+
}
|
|
759
|
+
)
|
|
672
760
|
else:
|
|
673
|
-
segment_results.append(
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
761
|
+
segment_results.append(
|
|
762
|
+
{
|
|
763
|
+
"video_id": video_id,
|
|
764
|
+
"segment_start": segment["polyp"].start_frame_number,
|
|
765
|
+
"segment_end": segment["polyp"].end_frame_number,
|
|
766
|
+
"frames": frames,
|
|
767
|
+
}
|
|
768
|
+
)
|
|
679
769
|
all_frames.extend(frames)
|
|
680
770
|
|
|
681
771
|
if all_frames:
|
|
682
|
-
unique_frame_numbers = sorted(
|
|
683
|
-
|
|
772
|
+
unique_frame_numbers = sorted(
|
|
773
|
+
{f["frame_number"] for f in all_frames}
|
|
774
|
+
)
|
|
775
|
+
self.extract_and_save_selected_frames(
|
|
776
|
+
video, unique_frame_numbers, classification_type="paris"
|
|
777
|
+
)
|
|
684
778
|
|
|
685
779
|
results.extend(segment_results)
|
|
686
780
|
|
|
687
781
|
except Exception as e:
|
|
688
|
-
results.append({
|
|
689
|
-
"video_id": video.id,
|
|
690
|
-
"error": str(e)
|
|
691
|
-
})
|
|
782
|
+
results.append({"video_id": video.id, "error": str(e)})
|
|
692
783
|
|
|
693
784
|
if not results:
|
|
694
|
-
return [
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
"message": "PARIS classification data generated.",
|
|
700
|
-
"data": results
|
|
701
|
-
}
|
|
785
|
+
return [
|
|
786
|
+
{
|
|
787
|
+
"message": "No valid classification results could be generated for any video."
|
|
788
|
+
}
|
|
789
|
+
]
|
|
702
790
|
|
|
791
|
+
return {"message": "PARIS classification data generated.", "data": results}
|
|
703
792
|
|
|
704
793
|
|
|
705
794
|
"""
|
|
@@ -750,10 +839,7 @@ Returns:
|
|
|
750
839
|
"""
|
|
751
840
|
|
|
752
841
|
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
'''
|
|
842
|
+
"""
|
|
757
843
|
await import('https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js');
|
|
758
844
|
const fetchNiceClassification = async () => {
|
|
759
845
|
try {
|
|
@@ -772,4 +858,4 @@ const fetchNiceClassification = async () => {
|
|
|
772
858
|
};
|
|
773
859
|
|
|
774
860
|
fetchNiceClassification().then(data => console.log("Final Output:", data));
|
|
775
|
-
|
|
861
|
+
"""
|