endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/admin.py +10 -5
- endoreg_db/apps.py +4 -7
- endoreg_db/authz/auth.py +1 -0
- endoreg_db/authz/backends.py +1 -1
- endoreg_db/authz/management/commands/list_routes.py +2 -0
- endoreg_db/authz/middleware.py +8 -7
- endoreg_db/authz/permissions.py +21 -10
- endoreg_db/authz/policy.py +14 -19
- endoreg_db/authz/views_auth.py +14 -10
- endoreg_db/codemods/rename_datetime_fields.py +8 -1
- endoreg_db/exceptions.py +5 -2
- endoreg_db/forms/__init__.py +0 -1
- endoreg_db/forms/examination_form.py +4 -3
- endoreg_db/forms/patient_finding_intervention_form.py +30 -8
- endoreg_db/forms/patient_form.py +9 -13
- endoreg_db/forms/questionnaires/__init__.py +1 -1
- endoreg_db/forms/settings/__init__.py +4 -1
- endoreg_db/forms/unit.py +2 -1
- endoreg_db/helpers/count_db.py +17 -14
- endoreg_db/helpers/default_objects.py +2 -1
- endoreg_db/helpers/download_segmentation_model.py +4 -3
- endoreg_db/helpers/interact.py +0 -5
- endoreg_db/helpers/test_video_helper.py +33 -25
- endoreg_db/import_files/__init__.py +1 -1
- endoreg_db/import_files/context/__init__.py +1 -1
- endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
- endoreg_db/import_files/context/ensure_center.py +4 -4
- endoreg_db/import_files/context/file_lock.py +3 -3
- endoreg_db/import_files/context/import_context.py +11 -12
- endoreg_db/import_files/context/validate_directories.py +1 -0
- endoreg_db/import_files/file_storage/create_report_file.py +57 -34
- endoreg_db/import_files/file_storage/create_video_file.py +64 -35
- endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
- endoreg_db/import_files/file_storage/state_management.py +146 -83
- endoreg_db/import_files/file_storage/storage.py +5 -1
- endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
- endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
- endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
- endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
- endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
- endoreg_db/import_files/report_import_service.py +36 -30
- endoreg_db/import_files/video_import_service.py +27 -23
- endoreg_db/logger_conf.py +56 -40
- endoreg_db/management/__init__.py +1 -1
- endoreg_db/management/commands/__init__.py +1 -1
- endoreg_db/management/commands/check_auth.py +45 -38
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
- endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
- endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
- endoreg_db/management/commands/fix_video_paths.py +75 -54
- endoreg_db/management/commands/import_report.py +1 -3
- endoreg_db/management/commands/list_routes.py +2 -0
- endoreg_db/management/commands/load_ai_model_data.py +8 -2
- endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_distribution_data.py +35 -38
- endoreg_db/management/commands/load_endoscope_data.py +0 -3
- endoreg_db/management/commands/load_examination_data.py +20 -4
- endoreg_db/management/commands/load_finding_data.py +18 -3
- endoreg_db/management/commands/load_gender_data.py +17 -24
- endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
- endoreg_db/management/commands/load_information_source.py +0 -3
- endoreg_db/management/commands/load_lab_value_data.py +14 -3
- endoreg_db/management/commands/load_legacy_data.py +303 -0
- endoreg_db/management/commands/load_name_data.py +1 -2
- endoreg_db/management/commands/load_pdf_type_data.py +4 -8
- endoreg_db/management/commands/load_profession_data.py +0 -1
- endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
- endoreg_db/management/commands/load_requirement_data.py +6 -2
- endoreg_db/management/commands/load_unit_data.py +0 -4
- endoreg_db/management/commands/load_user_groups.py +5 -7
- endoreg_db/management/commands/model_input.py +169 -0
- endoreg_db/management/commands/register_ai_model.py +22 -16
- endoreg_db/management/commands/setup_endoreg_db.py +110 -32
- endoreg_db/management/commands/storage_management.py +14 -8
- endoreg_db/management/commands/summarize_db_content.py +154 -63
- endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
- endoreg_db/management/commands/validate_video_files.py +82 -50
- endoreg_db/management/commands/video_validation.py +4 -6
- endoreg_db/migrations/0001_initial.py +112 -63
- endoreg_db/migrations/__init__.py +0 -0
- endoreg_db/models/__init__.py +8 -0
- endoreg_db/models/administration/ai/active_model.py +5 -5
- endoreg_db/models/administration/ai/ai_model.py +41 -18
- endoreg_db/models/administration/ai/model_type.py +1 -0
- endoreg_db/models/administration/case/case.py +22 -22
- endoreg_db/models/administration/center/__init__.py +5 -5
- endoreg_db/models/administration/center/center.py +6 -2
- endoreg_db/models/administration/center/center_resource.py +18 -4
- endoreg_db/models/administration/center/center_shift.py +3 -1
- endoreg_db/models/administration/center/center_waste.py +6 -2
- endoreg_db/models/administration/person/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/__init__.py +1 -1
- endoreg_db/models/administration/person/employee/employee_type.py +3 -1
- endoreg_db/models/administration/person/examiner/__init__.py +1 -1
- endoreg_db/models/administration/person/examiner/examiner.py +10 -2
- endoreg_db/models/administration/person/names/first_name.py +6 -4
- endoreg_db/models/administration/person/names/last_name.py +4 -3
- endoreg_db/models/administration/person/patient/__init__.py +1 -1
- endoreg_db/models/administration/person/patient/patient.py +0 -1
- endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
- endoreg_db/models/administration/person/person.py +1 -1
- endoreg_db/models/administration/product/__init__.py +7 -6
- endoreg_db/models/administration/product/product.py +6 -2
- endoreg_db/models/administration/product/product_group.py +9 -7
- endoreg_db/models/administration/product/product_material.py +9 -2
- endoreg_db/models/administration/product/reference_product.py +64 -15
- endoreg_db/models/administration/qualification/qualification.py +3 -1
- endoreg_db/models/administration/shift/shift.py +3 -1
- endoreg_db/models/administration/shift/shift_type.py +12 -4
- endoreg_db/models/aidataset/__init__.py +5 -0
- endoreg_db/models/aidataset/aidataset.py +193 -0
- endoreg_db/models/label/__init__.py +1 -1
- endoreg_db/models/label/label.py +10 -2
- endoreg_db/models/label/label_set.py +3 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
- endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
- endoreg_db/models/media/__init__.py +12 -5
- endoreg_db/models/media/frame/__init__.py +1 -1
- endoreg_db/models/media/frame/frame.py +34 -8
- endoreg_db/models/media/pdf/__init__.py +2 -1
- endoreg_db/models/media/pdf/raw_pdf.py +11 -4
- endoreg_db/models/media/pdf/report_file.py +6 -2
- endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
- endoreg_db/models/media/video/create_from_file.py +20 -41
- endoreg_db/models/media/video/pipe_1.py +75 -30
- endoreg_db/models/media/video/pipe_2.py +37 -12
- endoreg_db/models/media/video/video_file.py +36 -24
- endoreg_db/models/media/video/video_file_ai.py +235 -70
- endoreg_db/models/media/video/video_file_anonymize.py +240 -65
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
- endoreg_db/models/media/video/video_file_io.py +85 -33
- endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
- endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
- endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
- endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
- endoreg_db/models/media/video/video_file_segments.py +118 -27
- endoreg_db/models/media/video/video_metadata.py +25 -6
- endoreg_db/models/media/video/video_processing.py +54 -15
- endoreg_db/models/medical/__init__.py +3 -13
- endoreg_db/models/medical/contraindication/__init__.py +3 -1
- endoreg_db/models/medical/disease.py +18 -6
- endoreg_db/models/medical/event.py +6 -2
- endoreg_db/models/medical/examination/__init__.py +5 -1
- endoreg_db/models/medical/examination/examination.py +22 -6
- endoreg_db/models/medical/examination/examination_indication.py +23 -7
- endoreg_db/models/medical/examination/examination_time.py +6 -2
- endoreg_db/models/medical/finding/__init__.py +3 -1
- endoreg_db/models/medical/finding/finding.py +37 -12
- endoreg_db/models/medical/finding/finding_classification.py +27 -8
- endoreg_db/models/medical/finding/finding_intervention.py +19 -6
- endoreg_db/models/medical/finding/finding_type.py +3 -1
- endoreg_db/models/medical/hardware/__init__.py +1 -1
- endoreg_db/models/medical/hardware/endoscope.py +14 -2
- endoreg_db/models/medical/laboratory/__init__.py +1 -1
- endoreg_db/models/medical/laboratory/lab_value.py +139 -39
- endoreg_db/models/medical/medication/__init__.py +7 -3
- endoreg_db/models/medical/medication/medication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication.py +3 -1
- endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
- endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
- endoreg_db/models/medical/medication/medication_schedule.py +3 -1
- endoreg_db/models/medical/patient/__init__.py +2 -10
- endoreg_db/models/medical/patient/medication_examples.py +3 -14
- endoreg_db/models/medical/patient/patient_disease.py +17 -5
- endoreg_db/models/medical/patient/patient_event.py +12 -4
- endoreg_db/models/medical/patient/patient_examination.py +52 -15
- endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
- endoreg_db/models/medical/patient/patient_finding.py +105 -29
- endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
- endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
- endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
- endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
- endoreg_db/models/medical/patient/patient_medication.py +25 -7
- endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
- endoreg_db/models/metadata/model_meta.py +40 -12
- endoreg_db/models/metadata/model_meta_logic.py +51 -16
- endoreg_db/models/metadata/sensitive_meta.py +65 -28
- endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
- endoreg_db/models/metadata/video_meta.py +146 -39
- endoreg_db/models/metadata/video_prediction_logic.py +70 -21
- endoreg_db/models/metadata/video_prediction_meta.py +80 -27
- endoreg_db/models/operation_log.py +63 -0
- endoreg_db/models/other/__init__.py +10 -10
- endoreg_db/models/other/distribution/__init__.py +9 -7
- endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
- endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
- endoreg_db/models/other/emission/__init__.py +1 -1
- endoreg_db/models/other/emission/emission_factor.py +9 -3
- endoreg_db/models/other/information_source.py +15 -5
- endoreg_db/models/other/material.py +3 -1
- endoreg_db/models/other/transport_route.py +3 -1
- endoreg_db/models/other/unit.py +6 -2
- endoreg_db/models/report/report.py +0 -1
- endoreg_db/models/requirement/requirement.py +84 -27
- endoreg_db/models/requirement/requirement_error.py +5 -6
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
- endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
- endoreg_db/models/requirement/requirement_operator.py +28 -8
- endoreg_db/models/requirement/requirement_set.py +34 -11
- endoreg_db/models/state/__init__.py +1 -0
- endoreg_db/models/state/audit_ledger.py +9 -2
- endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
- endoreg_db/models/state/processing_history/processing_history.py +136 -0
- endoreg_db/models/state/raw_pdf.py +0 -1
- endoreg_db/models/state/video.py +2 -3
- endoreg_db/models/utils.py +4 -2
- endoreg_db/queries/__init__.py +2 -6
- endoreg_db/queries/annotations/__init__.py +1 -3
- endoreg_db/queries/annotations/legacy.py +37 -26
- endoreg_db/root_urls.py +3 -4
- endoreg_db/schemas/examination_evaluation.py +3 -0
- endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
- endoreg_db/serializers/__init__.py +2 -8
- endoreg_db/serializers/administration/__init__.py +1 -2
- endoreg_db/serializers/administration/ai/__init__.py +0 -1
- endoreg_db/serializers/administration/ai/active_model.py +3 -1
- endoreg_db/serializers/administration/ai/ai_model.py +5 -3
- endoreg_db/serializers/administration/ai/model_type.py +3 -1
- endoreg_db/serializers/administration/center.py +7 -2
- endoreg_db/serializers/administration/gender.py +4 -2
- endoreg_db/serializers/anonymization.py +13 -13
- endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
- endoreg_db/serializers/examination/__init__.py +1 -1
- endoreg_db/serializers/examination/base.py +12 -13
- endoreg_db/serializers/examination/dropdown.py +6 -7
- endoreg_db/serializers/examination_serializer.py +3 -6
- endoreg_db/serializers/finding/__init__.py +1 -1
- endoreg_db/serializers/finding/finding.py +14 -7
- endoreg_db/serializers/finding_classification/__init__.py +3 -3
- endoreg_db/serializers/finding_classification/choice.py +3 -3
- endoreg_db/serializers/finding_classification/classification.py +2 -4
- endoreg_db/serializers/label_video_segment/__init__.py +5 -3
- endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
- endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
- endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
- endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
- endoreg_db/serializers/meta/__init__.py +1 -2
- endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
- endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
- endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
- endoreg_db/serializers/misc/__init__.py +2 -2
- endoreg_db/serializers/misc/file_overview.py +11 -7
- endoreg_db/serializers/misc/stats.py +10 -8
- endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
- endoreg_db/serializers/misc/upload_job.py +32 -29
- endoreg_db/serializers/patient/__init__.py +2 -1
- endoreg_db/serializers/patient/patient.py +32 -15
- endoreg_db/serializers/patient/patient_dropdown.py +11 -3
- endoreg_db/serializers/patient_examination/__init__.py +1 -1
- endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
- endoreg_db/serializers/patient_finding/__init__.py +1 -1
- endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
- endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
- endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
- endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
- endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
- endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
- endoreg_db/serializers/pdf/__init__.py +1 -3
- endoreg_db/serializers/requirements/requirement_schema.py +1 -6
- endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
- endoreg_db/serializers/video/__init__.py +2 -2
- endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
- endoreg_db/serializers/video/video_file_brief.py +6 -2
- endoreg_db/serializers/video/video_file_detail.py +36 -23
- endoreg_db/serializers/video/video_file_list.py +4 -2
- endoreg_db/serializers/video/video_processing_history.py +54 -50
- endoreg_db/services/__init__.py +1 -1
- endoreg_db/services/anonymization.py +2 -2
- endoreg_db/services/examination_evaluation.py +40 -17
- endoreg_db/services/model_meta_from_hf.py +76 -0
- endoreg_db/services/polling_coordinator.py +101 -70
- endoreg_db/services/pseudonym_service.py +27 -22
- endoreg_db/services/report_import.py +6 -3
- endoreg_db/services/segment_sync.py +75 -59
- endoreg_db/services/video_import.py +6 -7
- endoreg_db/urls/__init__.py +2 -2
- endoreg_db/urls/ai.py +7 -25
- endoreg_db/urls/anonymization.py +61 -15
- endoreg_db/urls/auth.py +4 -4
- endoreg_db/urls/classification.py +4 -9
- endoreg_db/urls/examination.py +27 -18
- endoreg_db/urls/media.py +27 -34
- endoreg_db/urls/patient.py +11 -7
- endoreg_db/urls/requirements.py +3 -1
- endoreg_db/urls/root_urls.py +2 -3
- endoreg_db/urls/stats.py +24 -16
- endoreg_db/urls/upload.py +3 -11
- endoreg_db/utils/__init__.py +14 -15
- endoreg_db/utils/ai/__init__.py +1 -1
- endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
- endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
- endoreg_db/utils/ai/get.py +2 -1
- endoreg_db/utils/ai/inference_dataset.py +14 -15
- endoreg_db/utils/ai/model_training/config.py +117 -0
- endoreg_db/utils/ai/model_training/dataset.py +74 -0
- endoreg_db/utils/ai/model_training/losses.py +68 -0
- endoreg_db/utils/ai/model_training/metrics.py +78 -0
- endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
- endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
- endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
- endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
- endoreg_db/utils/ai/predict.py +4 -4
- endoreg_db/utils/ai/preprocess.py +19 -11
- endoreg_db/utils/calc_duration_seconds.py +4 -4
- endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
- endoreg_db/utils/check_video_files.py +74 -47
- endoreg_db/utils/cropping.py +10 -9
- endoreg_db/utils/dataloader.py +11 -3
- endoreg_db/utils/dates.py +3 -4
- endoreg_db/utils/defaults/set_default_center.py +7 -6
- endoreg_db/utils/env.py +6 -2
- endoreg_db/utils/extract_specific_frames.py +24 -9
- endoreg_db/utils/file_operations.py +30 -18
- endoreg_db/utils/fix_video_path_direct.py +57 -41
- endoreg_db/utils/frame_anonymization_utils.py +157 -157
- endoreg_db/utils/hashs.py +3 -18
- endoreg_db/utils/links/requirement_link.py +96 -52
- endoreg_db/utils/ocr.py +30 -25
- endoreg_db/utils/operation_log.py +61 -0
- endoreg_db/utils/parse_and_generate_yaml.py +12 -13
- endoreg_db/utils/paths.py +6 -6
- endoreg_db/utils/permissions.py +40 -24
- endoreg_db/utils/pipelines/process_video_dir.py +50 -26
- endoreg_db/utils/product/sum_emissions.py +5 -3
- endoreg_db/utils/product/sum_weights.py +4 -2
- endoreg_db/utils/pydantic_models/__init__.py +3 -4
- endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
- endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
- endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
- endoreg_db/utils/setup_config.py +21 -5
- endoreg_db/utils/storage.py +3 -1
- endoreg_db/utils/translation.py +19 -15
- endoreg_db/utils/uuid.py +1 -0
- endoreg_db/utils/validate_endo_roi.py +12 -4
- endoreg_db/utils/validate_subcategory_dict.py +26 -24
- endoreg_db/utils/validate_video_detailed.py +207 -149
- endoreg_db/utils/video/__init__.py +7 -3
- endoreg_db/utils/video/extract_frames.py +30 -18
- endoreg_db/utils/video/ffmpeg_wrapper.py +217 -52
- endoreg_db/utils/video/names.py +11 -6
- endoreg_db/utils/video/streaming_processor.py +175 -101
- endoreg_db/utils/video/video_splitter.py +30 -19
- endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
- endoreg_db/views/__init__.py +0 -20
- endoreg_db/views/anonymization/__init__.py +6 -2
- endoreg_db/views/anonymization/media_management.py +2 -6
- endoreg_db/views/anonymization/overview.py +34 -1
- endoreg_db/views/anonymization/validate.py +79 -18
- endoreg_db/views/auth/__init__.py +1 -1
- endoreg_db/views/auth/keycloak.py +16 -14
- endoreg_db/views/examination/__init__.py +12 -15
- endoreg_db/views/examination/examination.py +5 -5
- endoreg_db/views/examination/examination_manifest_cache.py +5 -5
- endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
- endoreg_db/views/examination/get_finding_classifications.py +9 -7
- endoreg_db/views/examination/get_findings.py +8 -10
- endoreg_db/views/examination/get_instruments.py +3 -2
- endoreg_db/views/examination/get_interventions.py +1 -1
- endoreg_db/views/finding/__init__.py +2 -2
- endoreg_db/views/finding/finding.py +58 -54
- endoreg_db/views/finding/get_classifications.py +1 -1
- endoreg_db/views/finding/get_interventions.py +1 -1
- endoreg_db/views/finding_classification/__init__.py +5 -5
- endoreg_db/views/finding_classification/finding_classification.py +5 -6
- endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
- endoreg_db/views/media/__init__.py +13 -13
- endoreg_db/views/media/pdf_media.py +9 -9
- endoreg_db/views/media/sensitive_metadata.py +10 -7
- endoreg_db/views/media/video_media.py +4 -4
- endoreg_db/views/meta/__init__.py +1 -1
- endoreg_db/views/meta/sensitive_meta_list.py +20 -22
- endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
- endoreg_db/views/misc/__init__.py +6 -34
- endoreg_db/views/misc/center.py +2 -1
- endoreg_db/views/misc/csrf.py +2 -1
- endoreg_db/views/misc/gender.py +2 -1
- endoreg_db/views/misc/stats.py +141 -106
- endoreg_db/views/patient/__init__.py +1 -3
- endoreg_db/views/patient/patient.py +141 -99
- endoreg_db/views/patient_examination/__init__.py +5 -5
- endoreg_db/views/patient_examination/patient_examination.py +43 -42
- endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
- endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
- endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
- endoreg_db/views/patient_examination/video.py +114 -80
- endoreg_db/views/patient_finding/__init__.py +1 -1
- endoreg_db/views/patient_finding/patient_finding.py +17 -10
- endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
- endoreg_db/views/patient_finding_classification/__init__.py +1 -1
- endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
- endoreg_db/views/report/reimport.py +1 -1
- endoreg_db/views/report/report_stream.py +5 -8
- endoreg_db/views/requirement/__init__.py +2 -1
- endoreg_db/views/requirement/evaluate.py +7 -9
- endoreg_db/views/requirement/lookup.py +2 -3
- endoreg_db/views/requirement/lookup_store.py +0 -1
- endoreg_db/views/requirement/requirement_utils.py +2 -4
- endoreg_db/views/stats/__init__.py +4 -4
- endoreg_db/views/stats/stats_views.py +152 -115
- endoreg_db/views/video/__init__.py +18 -27
- endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
- endoreg_db/views/{ai → video/ai}/label.py +20 -16
- endoreg_db/views/video/correction.py +5 -6
- endoreg_db/views/video/reimport.py +134 -99
- endoreg_db/views/video/segments_crud.py +134 -44
- endoreg_db/views/video/video_apply_mask.py +13 -12
- endoreg_db/views/video/video_correction.py +2 -1
- endoreg_db/views/video/video_download_processed.py +15 -15
- endoreg_db/views/video/video_meta_stats.py +7 -6
- endoreg_db/views/video/video_processing_history.py +3 -2
- endoreg_db/views/video/video_remove_frames.py +13 -12
- endoreg_db/views/video/video_stream.py +110 -82
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +436 -433
- endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +0 -119
- endoreg_db/management/commands/import_fallback_video.py +0 -203
- endoreg_db/management/commands/import_video.py +0 -422
- endoreg_db/management/commands/import_video_with_classification.py +0 -367
- endoreg_db/models/media/processing_history/processing_history.py +0 -96
- endoreg_db/serializers/label/__init__.py +0 -7
- endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
- endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
- endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
- endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
- endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
- endoreg_db/services/__old/pdf_import.py +0 -1487
- endoreg_db/services/__old/video_import.py +0 -1306
- endoreg_db/tasks/upload_tasks.py +0 -216
- endoreg_db/tasks/video_ingest.py +0 -161
- endoreg_db/tasks/video_processing_tasks.py +0 -327
- endoreg_db/views/misc/translation.py +0 -182
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import os
|
|
2
|
-
import shutil
|
|
3
2
|
from pathlib import Path
|
|
4
|
-
from
|
|
5
|
-
|
|
3
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
4
|
+
|
|
6
5
|
from django.db import transaction
|
|
6
|
+
from icecream import ic
|
|
7
7
|
from tqdm import tqdm
|
|
8
|
-
from typing import TYPE_CHECKING, Union, List, Optional
|
|
9
8
|
|
|
10
9
|
if TYPE_CHECKING:
|
|
11
10
|
from ...models.media import VideoFile
|
|
12
11
|
|
|
13
|
-
from django.core.files import File
|
|
14
12
|
import io
|
|
13
|
+
|
|
14
|
+
from django.core.files import File
|
|
15
|
+
|
|
15
16
|
from .ffmpeg_wrapper import extract_frames as ffmpeg_extract_frames
|
|
16
17
|
|
|
17
18
|
|
|
@@ -28,27 +29,33 @@ def prepare_bulk_frames(frame_paths: List[Path]):
|
|
|
28
29
|
yield frame_number, file_obj
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
def extract_frames(
|
|
32
|
+
def extract_frames(
|
|
33
|
+
video_path: Path,
|
|
34
|
+
output_dir: Path,
|
|
35
|
+
quality: int,
|
|
36
|
+
ext: str = "jpg",
|
|
37
|
+
fps: Optional[float] = None,
|
|
38
|
+
) -> List[Path]:
|
|
32
39
|
"""Extracts frames from a video file using ffmpeg_wrapper."""
|
|
33
40
|
# Ensure output directory exists
|
|
34
41
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
35
42
|
return ffmpeg_extract_frames(video_path, output_dir, quality, ext, fps)
|
|
36
43
|
|
|
37
44
|
|
|
38
|
-
def initialize_frame_objects(
|
|
39
|
-
video: "VideoFile", extracted_paths: List[Path]
|
|
40
|
-
):
|
|
45
|
+
def initialize_frame_objects(video: "VideoFile", extracted_paths: List[Path]):
|
|
41
46
|
"""
|
|
42
47
|
Initialize frame objects for the extracted frames and update state.
|
|
43
48
|
"""
|
|
44
49
|
state = video.get_or_create_state()
|
|
45
50
|
# Check state before proceeding
|
|
46
51
|
if state.frames_initialized:
|
|
47
|
-
ic(f"Frames already initialized for video {video.
|
|
52
|
+
ic(f"Frames already initialized for video {video.video_hash}, skipping.")
|
|
48
53
|
return
|
|
49
54
|
|
|
50
55
|
if not extracted_paths:
|
|
51
|
-
ic(
|
|
56
|
+
ic(
|
|
57
|
+
f"No extracted paths provided for video {video.video_hash}, cannot initialize frames."
|
|
58
|
+
)
|
|
52
59
|
return
|
|
53
60
|
|
|
54
61
|
video.frame_count = len(extracted_paths)
|
|
@@ -58,13 +65,19 @@ def initialize_frame_objects(
|
|
|
58
65
|
# Prepare frame data (relative paths for storage)
|
|
59
66
|
frame_dir = video.get_frame_dir_path()
|
|
60
67
|
if not frame_dir:
|
|
61
|
-
|
|
68
|
+
raise ValueError(f"Frame directory not set for video {video.video_hash}")
|
|
62
69
|
|
|
63
|
-
storage_base_path = Path(
|
|
70
|
+
storage_base_path = Path(
|
|
71
|
+
video._meta.get_field("raw_file").storage.location
|
|
72
|
+
) # Get storage root
|
|
64
73
|
|
|
65
74
|
for i, path in tqdm(enumerate(extracted_paths, start=1)):
|
|
66
|
-
frame_number =
|
|
67
|
-
|
|
75
|
+
frame_number = (
|
|
76
|
+
int(path.stem.split("_")[1]) - 1
|
|
77
|
+
) # Assuming frame_0000001.jpg is frame_number 0
|
|
78
|
+
relative_path = path.relative_to(
|
|
79
|
+
storage_base_path
|
|
80
|
+
).as_posix() # Path relative to MEDIA_ROOT
|
|
68
81
|
|
|
69
82
|
# Create Frame instance (without saving yet)
|
|
70
83
|
frame_obj_instance = video.create_frame_object(
|
|
@@ -83,6 +96,5 @@ def initialize_frame_objects(
|
|
|
83
96
|
|
|
84
97
|
# Update state and save VideoFile (to save frame_count)
|
|
85
98
|
state.frames_initialized = True
|
|
86
|
-
state.save(update_fields=[
|
|
87
|
-
video.save(update_fields=[
|
|
88
|
-
|
|
99
|
+
state.save(update_fields=["frames_initialized"])
|
|
100
|
+
video.save(update_fields=["frame_count"]) # Save frame_count on VideoFile
|
|
@@ -31,7 +31,11 @@ def _resolve_ffmpeg_executable() -> Optional[str]:
|
|
|
31
31
|
try:
|
|
32
32
|
from django.conf import settings
|
|
33
33
|
|
|
34
|
-
env_candidates.extend(
|
|
34
|
+
env_candidates.extend(
|
|
35
|
+
getattr(settings, attr)
|
|
36
|
+
for attr in ("FFMPEG_EXECUTABLE", "FFMPEG_BINARY", "FFMPEG_PATH")
|
|
37
|
+
if hasattr(settings, attr)
|
|
38
|
+
)
|
|
35
39
|
except Exception:
|
|
36
40
|
# Django might not be configured for every consumer
|
|
37
41
|
pass
|
|
@@ -83,9 +87,24 @@ def _detect_nvenc_support() -> bool:
|
|
|
83
87
|
"""
|
|
84
88
|
try:
|
|
85
89
|
# Test NVENC availability with a minimal command (minimum size for NVENC)
|
|
86
|
-
cmd = [
|
|
90
|
+
cmd = [
|
|
91
|
+
"ffmpeg",
|
|
92
|
+
"-f",
|
|
93
|
+
"lavfi",
|
|
94
|
+
"-i",
|
|
95
|
+
"testsrc=duration=1:size=256x256:rate=1",
|
|
96
|
+
"-c:v",
|
|
97
|
+
"h264_nvenc",
|
|
98
|
+
"-preset",
|
|
99
|
+
"p1",
|
|
100
|
+
"-f",
|
|
101
|
+
"null",
|
|
102
|
+
"-",
|
|
103
|
+
]
|
|
87
104
|
|
|
88
|
-
result = subprocess.run(
|
|
105
|
+
result = subprocess.run(
|
|
106
|
+
cmd, capture_output=True, text=True, timeout=15, check=False
|
|
107
|
+
)
|
|
89
108
|
|
|
90
109
|
if result.returncode == 0:
|
|
91
110
|
logger.debug("NVENC h264 encoding test successful")
|
|
@@ -141,7 +160,11 @@ def _get_preferred_encoder() -> Dict[str, str]:
|
|
|
141
160
|
return _preferred_encoder
|
|
142
161
|
|
|
143
162
|
|
|
144
|
-
def _build_encoder_args(
|
|
163
|
+
def _build_encoder_args(
|
|
164
|
+
quality_mode: str = "balanced",
|
|
165
|
+
fallback: bool = False,
|
|
166
|
+
custom_crf: Optional[int] = None,
|
|
167
|
+
) -> Tuple[List[str], str]:
|
|
145
168
|
"""
|
|
146
169
|
Build encoder command arguments based on available hardware and quality requirements.
|
|
147
170
|
|
|
@@ -207,7 +230,16 @@ def _build_encoder_args(quality_mode: str = "balanced", fallback: bool = False,
|
|
|
207
230
|
if custom_crf is not None:
|
|
208
231
|
quality = str(custom_crf)
|
|
209
232
|
|
|
210
|
-
return [
|
|
233
|
+
return [
|
|
234
|
+
"-c:v",
|
|
235
|
+
encoder["name"],
|
|
236
|
+
encoder["preset_param"],
|
|
237
|
+
preset,
|
|
238
|
+
encoder["quality_param"],
|
|
239
|
+
quality,
|
|
240
|
+
"-profile:v",
|
|
241
|
+
"high",
|
|
242
|
+
], encoder["type"]
|
|
211
243
|
|
|
212
244
|
|
|
213
245
|
def is_ffmpeg_available() -> bool:
|
|
@@ -231,7 +263,9 @@ def check_ffmpeg_availability():
|
|
|
231
263
|
True if FFmpeg is available.
|
|
232
264
|
"""
|
|
233
265
|
if not is_ffmpeg_available():
|
|
234
|
-
error_msg =
|
|
266
|
+
error_msg = (
|
|
267
|
+
"FFmpeg is not available. Please install it and ensure it's in your PATH."
|
|
268
|
+
)
|
|
235
269
|
logger.error(error_msg)
|
|
236
270
|
raise FileNotFoundError(error_msg)
|
|
237
271
|
# logger.info("FFmpeg is available.") # Caller can log if needed
|
|
@@ -292,9 +326,15 @@ def assemble_video_from_frames( # Renamed from assemble_video
|
|
|
292
326
|
if first_frame is None:
|
|
293
327
|
raise IOError(f"Could not read first frame: {frame_paths[0]}")
|
|
294
328
|
height, width, _ = first_frame.shape
|
|
295
|
-
logger.info(
|
|
329
|
+
logger.info(
|
|
330
|
+
"Determined video dimensions from first frame: %dx%d", width, height
|
|
331
|
+
)
|
|
296
332
|
except Exception as e:
|
|
297
|
-
logger.error(
|
|
333
|
+
logger.error(
|
|
334
|
+
"Error reading first frame to determine dimensions: %s",
|
|
335
|
+
e,
|
|
336
|
+
exc_info=True,
|
|
337
|
+
)
|
|
298
338
|
return None
|
|
299
339
|
|
|
300
340
|
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
|
@@ -305,7 +345,9 @@ def assemble_video_from_frames( # Renamed from assemble_video
|
|
|
305
345
|
logger.error("Could not open video writer for path: %s", output_path)
|
|
306
346
|
return None
|
|
307
347
|
|
|
308
|
-
logger.info(
|
|
348
|
+
logger.info(
|
|
349
|
+
"Assembling video %s from %d frames...", output_path.name, len(frame_paths)
|
|
350
|
+
)
|
|
309
351
|
try:
|
|
310
352
|
for frame_path in tqdm(frame_paths, desc=f"Assembling {output_path.name}"):
|
|
311
353
|
frame = cv2.imread(str(frame_path))
|
|
@@ -314,7 +356,9 @@ def assemble_video_from_frames( # Renamed from assemble_video
|
|
|
314
356
|
continue
|
|
315
357
|
# Ensure frame dimensions match - resize if necessary (or log error)
|
|
316
358
|
if frame.shape[1] != width or frame.shape[0] != height:
|
|
317
|
-
logger.warning(
|
|
359
|
+
logger.warning(
|
|
360
|
+
f"Frame {frame_path} has dimensions {frame.shape[1]}x{frame.shape[0]}, expected {width}x{height}. Resizing."
|
|
361
|
+
)
|
|
318
362
|
frame = cv2.resize(frame, (width, height))
|
|
319
363
|
video_writer.write(frame)
|
|
320
364
|
finally:
|
|
@@ -364,7 +408,9 @@ def transcode_video(
|
|
|
364
408
|
if codec == "auto" or preset == "auto":
|
|
365
409
|
if force_cpu:
|
|
366
410
|
# Force CPU encoding
|
|
367
|
-
encoder_args, encoder_type = _build_encoder_args(
|
|
411
|
+
encoder_args, encoder_type = _build_encoder_args(
|
|
412
|
+
quality_mode, fallback=False, custom_crf=crf
|
|
413
|
+
)
|
|
368
414
|
# Override to use CPU encoder
|
|
369
415
|
encoder_args[1] = "libx264" # Replace encoder name
|
|
370
416
|
encoder_args[3] = "medium" if preset == "auto" else preset # Replace preset
|
|
@@ -372,7 +418,9 @@ def transcode_video(
|
|
|
372
418
|
encoder_args[5] = str(crf) # Replace quality value
|
|
373
419
|
else:
|
|
374
420
|
# Use automatic hardware detection
|
|
375
|
-
encoder_args, encoder_type = _build_encoder_args(
|
|
421
|
+
encoder_args, encoder_type = _build_encoder_args(
|
|
422
|
+
quality_mode, fallback=False, custom_crf=crf
|
|
423
|
+
)
|
|
376
424
|
else:
|
|
377
425
|
# Manual codec/preset specification (backward compatibility)
|
|
378
426
|
encoder_args = [
|
|
@@ -402,11 +450,18 @@ def transcode_video(
|
|
|
402
450
|
command.extend(extra_args)
|
|
403
451
|
command.append(str(output_path))
|
|
404
452
|
|
|
405
|
-
logger.info(
|
|
453
|
+
logger.info(
|
|
454
|
+
"Starting transcoding: %s -> %s (using %s)",
|
|
455
|
+
input_path.name,
|
|
456
|
+
output_path.name,
|
|
457
|
+
encoder_type,
|
|
458
|
+
)
|
|
406
459
|
logger.debug("FFmpeg command: %s", " ".join(command))
|
|
407
460
|
|
|
408
461
|
try:
|
|
409
|
-
process = subprocess.Popen(
|
|
462
|
+
process = subprocess.Popen(
|
|
463
|
+
command, stderr=subprocess.PIPE, text=True, universal_newlines=True
|
|
464
|
+
)
|
|
410
465
|
|
|
411
466
|
# Progress reporting and error handling
|
|
412
467
|
stderr_output = ""
|
|
@@ -420,32 +475,56 @@ def transcode_video(
|
|
|
420
475
|
logger.info("Transcoding finished successfully: %s", output_path)
|
|
421
476
|
return output_path
|
|
422
477
|
else:
|
|
423
|
-
logger.error(
|
|
478
|
+
logger.error(
|
|
479
|
+
"FFmpeg transcoding failed for %s with return code %d.",
|
|
480
|
+
input_path.name,
|
|
481
|
+
process.returncode,
|
|
482
|
+
)
|
|
424
483
|
logger.error("FFmpeg stderr:\n%s", stderr_output)
|
|
425
484
|
|
|
426
485
|
# Try fallback to CPU if NVENC failed
|
|
427
486
|
if encoder_type == "nvenc" and not force_cpu:
|
|
428
487
|
logger.warning("NVENC transcoding failed, trying CPU fallback...")
|
|
429
|
-
return _transcode_video_fallback(
|
|
488
|
+
return _transcode_video_fallback(
|
|
489
|
+
input_path,
|
|
490
|
+
output_path,
|
|
491
|
+
audio_codec,
|
|
492
|
+
audio_bitrate,
|
|
493
|
+
extra_args,
|
|
494
|
+
quality_mode,
|
|
495
|
+
crf,
|
|
496
|
+
)
|
|
430
497
|
|
|
431
498
|
# Clean up potentially corrupted output file
|
|
432
499
|
if output_path.exists():
|
|
433
500
|
try:
|
|
434
501
|
output_path.unlink()
|
|
435
502
|
except OSError as e:
|
|
436
|
-
logger.error(
|
|
503
|
+
logger.error(
|
|
504
|
+
"Failed to delete incomplete output file %s: %s", output_path, e
|
|
505
|
+
)
|
|
437
506
|
return None
|
|
438
507
|
|
|
439
508
|
except FileNotFoundError:
|
|
440
|
-
logger.error(
|
|
509
|
+
logger.error(
|
|
510
|
+
"ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
|
|
511
|
+
)
|
|
441
512
|
return None
|
|
442
513
|
except Exception as e:
|
|
443
|
-
logger.error(
|
|
514
|
+
logger.error(
|
|
515
|
+
"Error during transcoding of %s: %s", input_path.name, e, exc_info=True
|
|
516
|
+
)
|
|
444
517
|
return None
|
|
445
518
|
|
|
446
519
|
|
|
447
520
|
def _transcode_video_fallback(
|
|
448
|
-
input_path: Path,
|
|
521
|
+
input_path: Path,
|
|
522
|
+
output_path: Path,
|
|
523
|
+
audio_codec: str,
|
|
524
|
+
audio_bitrate: str,
|
|
525
|
+
extra_args: Optional[List[str]],
|
|
526
|
+
quality_mode: str,
|
|
527
|
+
custom_crf: Optional[int],
|
|
449
528
|
) -> Optional[Path]:
|
|
450
529
|
"""
|
|
451
530
|
Fallback transcoding using CPU encoding.
|
|
@@ -464,7 +543,9 @@ def _transcode_video_fallback(
|
|
|
464
543
|
"""
|
|
465
544
|
try:
|
|
466
545
|
# Build CPU encoder arguments
|
|
467
|
-
encoder_args, _ = _build_encoder_args(
|
|
546
|
+
encoder_args, _ = _build_encoder_args(
|
|
547
|
+
quality_mode, fallback=True, custom_crf=custom_crf
|
|
548
|
+
)
|
|
468
549
|
# Force CPU encoder
|
|
469
550
|
encoder_args[1] = "libx264"
|
|
470
551
|
|
|
@@ -484,10 +565,14 @@ def _transcode_video_fallback(
|
|
|
484
565
|
command.extend(extra_args)
|
|
485
566
|
command.append(str(output_path))
|
|
486
567
|
|
|
487
|
-
logger.info(
|
|
568
|
+
logger.info(
|
|
569
|
+
"CPU fallback transcoding: %s -> %s", input_path.name, output_path.name
|
|
570
|
+
)
|
|
488
571
|
logger.debug("Fallback FFmpeg command: %s", " ".join(command))
|
|
489
572
|
|
|
490
|
-
process = subprocess.Popen(
|
|
573
|
+
process = subprocess.Popen(
|
|
574
|
+
command, stderr=subprocess.PIPE, text=True, universal_newlines=True
|
|
575
|
+
)
|
|
491
576
|
stderr_output = ""
|
|
492
577
|
if process.stderr:
|
|
493
578
|
for line in process.stderr:
|
|
@@ -511,7 +596,9 @@ def _transcode_video_fallback(
|
|
|
511
596
|
logger.debug("FFmpeg command: %s", " ".join(command))
|
|
512
597
|
|
|
513
598
|
try:
|
|
514
|
-
process = subprocess.Popen(
|
|
599
|
+
process = subprocess.Popen(
|
|
600
|
+
command, stderr=subprocess.PIPE, text=True, universal_newlines=True
|
|
601
|
+
)
|
|
515
602
|
|
|
516
603
|
# Optional: Progress reporting (can be complex to parse ffmpeg output reliably)
|
|
517
604
|
# For simplicity, just wait and check the return code
|
|
@@ -528,21 +615,31 @@ def _transcode_video_fallback(
|
|
|
528
615
|
logger.info("Transcoding finished successfully: %s", output_path)
|
|
529
616
|
return output_path
|
|
530
617
|
else:
|
|
531
|
-
logger.error(
|
|
618
|
+
logger.error(
|
|
619
|
+
"FFmpeg transcoding failed for %s with return code %d.",
|
|
620
|
+
input_path.name,
|
|
621
|
+
process.returncode,
|
|
622
|
+
)
|
|
532
623
|
logger.error("FFmpeg stderr:\n%s", stderr_output)
|
|
533
624
|
# Clean up potentially corrupted output file
|
|
534
625
|
if output_path.exists():
|
|
535
626
|
try:
|
|
536
627
|
output_path.unlink()
|
|
537
628
|
except OSError as e:
|
|
538
|
-
logger.error(
|
|
629
|
+
logger.error(
|
|
630
|
+
"Failed to delete incomplete output file %s: %s", output_path, e
|
|
631
|
+
)
|
|
539
632
|
return None
|
|
540
633
|
|
|
541
634
|
except FileNotFoundError:
|
|
542
|
-
logger.error(
|
|
635
|
+
logger.error(
|
|
636
|
+
"ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
|
|
637
|
+
)
|
|
543
638
|
return None
|
|
544
639
|
except Exception as e:
|
|
545
|
-
logger.error(
|
|
640
|
+
logger.error(
|
|
641
|
+
"Error during transcoding of %s: %s", input_path.name, e, exc_info=True
|
|
642
|
+
)
|
|
546
643
|
return None
|
|
547
644
|
|
|
548
645
|
|
|
@@ -561,10 +658,15 @@ def transcode_videofile_if_required(
|
|
|
561
658
|
"""
|
|
562
659
|
stream_info = get_stream_info(input_path)
|
|
563
660
|
if not stream_info or "streams" not in stream_info:
|
|
564
|
-
logger.error(
|
|
661
|
+
logger.error(
|
|
662
|
+
"Could not get stream info for %s to check if transcoding is required.",
|
|
663
|
+
input_path,
|
|
664
|
+
)
|
|
565
665
|
return None
|
|
566
666
|
|
|
567
|
-
video_stream = next(
|
|
667
|
+
video_stream = next(
|
|
668
|
+
(s for s in stream_info["streams"] if s.get("codec_type") == "video"), None
|
|
669
|
+
)
|
|
568
670
|
|
|
569
671
|
if not video_stream:
|
|
570
672
|
logger.error("No video stream found in %s.", input_path)
|
|
@@ -573,7 +675,9 @@ def transcode_videofile_if_required(
|
|
|
573
675
|
codec_name = video_stream.get("codec_name")
|
|
574
676
|
pixel_format = video_stream.get("pix_fmt")
|
|
575
677
|
# Check color range as well, default is usually 'tv' (limited)
|
|
576
|
-
color_range = video_stream.get(
|
|
678
|
+
color_range = video_stream.get(
|
|
679
|
+
"color_range", "tv"
|
|
680
|
+
) # Default to tv if not specified
|
|
577
681
|
|
|
578
682
|
needs_transcoding = False
|
|
579
683
|
transcode_reason = []
|
|
@@ -583,16 +687,25 @@ def transcode_videofile_if_required(
|
|
|
583
687
|
transcode_reason.append(reason)
|
|
584
688
|
needs_transcoding = True
|
|
585
689
|
# Check both pixel format and color range for yuv420p
|
|
586
|
-
if pixel_format != required_pixel_format or (
|
|
690
|
+
if pixel_format != required_pixel_format or (
|
|
691
|
+
pixel_format == "yuv420p" and color_range != "pc"
|
|
692
|
+
):
|
|
587
693
|
reason = f"Pixel format/color range mismatch (pix_fmt: {pixel_format}, color_range: {color_range} != {required_pixel_format} with color_range=pc)"
|
|
588
694
|
logger.info("%s for %s. Transcoding required.", reason, input_path.name)
|
|
589
695
|
transcode_reason.append(reason)
|
|
590
696
|
needs_transcoding = True
|
|
591
697
|
|
|
592
698
|
if needs_transcoding:
|
|
593
|
-
logger.info(
|
|
699
|
+
logger.info(
|
|
700
|
+
"Transcoding %s to %s due to: %s",
|
|
701
|
+
input_path.name,
|
|
702
|
+
output_path.name,
|
|
703
|
+
"; ".join(transcode_reason),
|
|
704
|
+
)
|
|
594
705
|
# Ensure codec and pixel format are set in options if not already present
|
|
595
|
-
transcode_options.setdefault(
|
|
706
|
+
transcode_options.setdefault(
|
|
707
|
+
"codec", "libx264" if required_codec == "h264" else required_codec
|
|
708
|
+
)
|
|
596
709
|
transcode_options.setdefault("extra_args", [])
|
|
597
710
|
|
|
598
711
|
# Ensure pixel format and color range are correctly set in extra_args
|
|
@@ -604,11 +717,17 @@ def transcode_videofile_if_required(
|
|
|
604
717
|
try:
|
|
605
718
|
pix_fmt_index = extra_args.index("-pix_fmt")
|
|
606
719
|
if extra_args[pix_fmt_index + 1] != required_pixel_format:
|
|
607
|
-
logger.warning(
|
|
720
|
+
logger.warning(
|
|
721
|
+
"Overriding existing -pix_fmt '%s' with '%s'",
|
|
722
|
+
extra_args[pix_fmt_index + 1],
|
|
723
|
+
required_pixel_format,
|
|
724
|
+
)
|
|
608
725
|
extra_args[pix_fmt_index + 1] = required_pixel_format
|
|
609
726
|
except (ValueError, IndexError):
|
|
610
727
|
# Should not happen if '-pix_fmt' is in extra_args, but handle defensively
|
|
611
|
-
logger.error(
|
|
728
|
+
logger.error(
|
|
729
|
+
"Error processing existing -pix_fmt argument. Appending required format."
|
|
730
|
+
)
|
|
612
731
|
extra_args.extend(["-pix_fmt", required_pixel_format])
|
|
613
732
|
|
|
614
733
|
if "-color_range" not in extra_args:
|
|
@@ -619,16 +738,24 @@ def transcode_videofile_if_required(
|
|
|
619
738
|
try:
|
|
620
739
|
color_range_index = extra_args.index("-color_range")
|
|
621
740
|
if extra_args[color_range_index + 1] != "pc":
|
|
622
|
-
logger.warning(
|
|
741
|
+
logger.warning(
|
|
742
|
+
"Overriding existing -color_range '%s' with 'pc'",
|
|
743
|
+
extra_args[color_range_index + 1],
|
|
744
|
+
)
|
|
623
745
|
extra_args[color_range_index + 1] = "pc"
|
|
624
746
|
except (ValueError, IndexError):
|
|
625
|
-
logger.error(
|
|
747
|
+
logger.error(
|
|
748
|
+
"Error processing existing -color_range argument. Appending 'pc'."
|
|
749
|
+
)
|
|
626
750
|
extra_args.extend(["-color_range", "pc"])
|
|
627
751
|
|
|
628
752
|
return transcode_video(input_path, output_path, **transcode_options)
|
|
629
753
|
else:
|
|
630
754
|
logger.info(
|
|
631
|
-
"Video %s already meets requirements (%s, %s, color_range=pc). No transcoding needed.",
|
|
755
|
+
"Video %s already meets requirements (%s, %s, color_range=pc). No transcoding needed.",
|
|
756
|
+
input_path.name,
|
|
757
|
+
required_codec,
|
|
758
|
+
required_pixel_format,
|
|
632
759
|
)
|
|
633
760
|
# If no transcoding is needed, should we copy/link or just return the original path?
|
|
634
761
|
# For simplicity, let's assume the caller handles the file location.
|
|
@@ -638,15 +765,27 @@ def transcode_videofile_if_required(
|
|
|
638
765
|
try:
|
|
639
766
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
640
767
|
shutil.copy2(input_path, output_path)
|
|
641
|
-
logger.info(
|
|
768
|
+
logger.info(
|
|
769
|
+
"Copied %s to %s as it met requirements.",
|
|
770
|
+
input_path.name,
|
|
771
|
+
output_path.name,
|
|
772
|
+
)
|
|
642
773
|
return output_path
|
|
643
774
|
except Exception as e:
|
|
644
|
-
logger.error(
|
|
775
|
+
logger.error(
|
|
776
|
+
"Failed to copy %s to %s: %s", input_path.name, output_path.name, e
|
|
777
|
+
)
|
|
645
778
|
return None
|
|
646
779
|
return input_path # Return original path if no copy needed
|
|
647
780
|
|
|
648
781
|
|
|
649
|
-
def extract_frames(
|
|
782
|
+
def extract_frames(
|
|
783
|
+
video_path: Path,
|
|
784
|
+
output_dir: Path,
|
|
785
|
+
quality: int,
|
|
786
|
+
ext: str = "jpg",
|
|
787
|
+
fps: Optional[float] = None,
|
|
788
|
+
) -> List[Path]:
|
|
650
789
|
"""
|
|
651
790
|
Extracts frames from a video file using FFmpeg.
|
|
652
791
|
|
|
@@ -702,7 +841,9 @@ def extract_frames(video_path: Path, output_dir: Path, quality: int, ext: str =
|
|
|
702
841
|
# Return empty list on error as frames were likely not created correctly
|
|
703
842
|
return []
|
|
704
843
|
except Exception as e:
|
|
705
|
-
logger.error(
|
|
844
|
+
logger.error(
|
|
845
|
+
"An unexpected error occurred during FFmpeg execution: %s", e, exc_info=True
|
|
846
|
+
)
|
|
706
847
|
return []
|
|
707
848
|
|
|
708
849
|
# Collect paths of extracted frames
|
|
@@ -743,7 +884,11 @@ def extract_frame_range(
|
|
|
743
884
|
RuntimeError: If FFmpeg fails to extract the requested frames.
|
|
744
885
|
"""
|
|
745
886
|
if start_frame >= end_frame:
|
|
746
|
-
logger.warning(
|
|
887
|
+
logger.warning(
|
|
888
|
+
"extract_frame_range called with start_frame (%d) >= end_frame (%d). No frames to extract.",
|
|
889
|
+
start_frame,
|
|
890
|
+
end_frame,
|
|
891
|
+
)
|
|
747
892
|
return []
|
|
748
893
|
|
|
749
894
|
ffmpeg_executable = _resolve_ffmpeg_executable()
|
|
@@ -790,18 +935,30 @@ def extract_frame_range(
|
|
|
790
935
|
logger.error("FFmpeg stderr:\n%s", e.stderr)
|
|
791
936
|
logger.error("FFmpeg stdout:\n%s", e.stdout)
|
|
792
937
|
# Clean up potentially partially created files in the target directory within the expected range
|
|
793
|
-
logger.warning(
|
|
938
|
+
logger.warning(
|
|
939
|
+
"Attempting cleanup of potentially incomplete frames in %s", output_dir
|
|
940
|
+
)
|
|
794
941
|
for i in range(start_frame, end_frame):
|
|
795
942
|
potential_file = output_dir / f"frame_{i:07d}.{ext}"
|
|
796
943
|
if potential_file.exists():
|
|
797
944
|
try:
|
|
798
945
|
potential_file.unlink()
|
|
799
946
|
except OSError as unlink_err:
|
|
800
|
-
logger.error(
|
|
801
|
-
|
|
947
|
+
logger.error(
|
|
948
|
+
"Failed to delete potential frame %s during cleanup: %s",
|
|
949
|
+
potential_file,
|
|
950
|
+
unlink_err,
|
|
951
|
+
)
|
|
952
|
+
raise RuntimeError(
|
|
953
|
+
f"FFmpeg frame range extraction failed for {video_path}"
|
|
954
|
+
) from e
|
|
802
955
|
except Exception as e:
|
|
803
|
-
logger.error(
|
|
804
|
-
|
|
956
|
+
logger.error(
|
|
957
|
+
"An unexpected error occurred during FFmpeg execution: %s", e, exc_info=True
|
|
958
|
+
)
|
|
959
|
+
raise RuntimeError(
|
|
960
|
+
f"Unexpected error during FFmpeg frame range extraction for {video_path}"
|
|
961
|
+
) from e
|
|
805
962
|
|
|
806
963
|
# Collect paths of extracted frames matching the pattern and expected range
|
|
807
964
|
# FFmpeg might create files outside the exact range depending on version/flags,
|
|
@@ -813,9 +970,17 @@ def extract_frame_range(
|
|
|
813
970
|
extracted_files.append(frame_file)
|
|
814
971
|
else:
|
|
815
972
|
# This might happen if ffmpeg fails silently for some frames or if the video ends early.
|
|
816
|
-
logger.warning(
|
|
817
|
-
|
|
818
|
-
|
|
973
|
+
logger.warning(
|
|
974
|
+
"Expected frame file %s not found after extraction.", frame_file
|
|
975
|
+
)
|
|
976
|
+
|
|
977
|
+
logger.info(
|
|
978
|
+
"Found %d extracted frame files in range [%d, %d) for video %s.",
|
|
979
|
+
len(extracted_files),
|
|
980
|
+
start_frame,
|
|
981
|
+
end_frame,
|
|
982
|
+
video_path.name,
|
|
983
|
+
)
|
|
819
984
|
return extracted_files
|
|
820
985
|
|
|
821
986
|
|
endoreg_db/utils/video/names.py
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
|
|
4
|
+
def get_video_key(
|
|
5
|
+
examination_alias: str, content: str, is_anonymous: bool = False
|
|
6
|
+
) -> str:
|
|
4
7
|
"""
|
|
5
8
|
Generates a video key based on the examination alias, content, and anonymity status.
|
|
6
9
|
"""
|
|
@@ -8,8 +11,9 @@ def get_video_key(examination_alias:str, content:str, is_anonymous:bool=False) -
|
|
|
8
11
|
return f"{examination_alias}-{content}-anonymous"
|
|
9
12
|
else:
|
|
10
13
|
return f"{examination_alias}-{content}-non_anonymous"
|
|
11
|
-
|
|
12
|
-
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def identify_video_key(video_key: str) -> str:
|
|
13
17
|
"""
|
|
14
18
|
Identifies the video key based on the provided string.
|
|
15
19
|
"""
|
|
@@ -22,7 +26,9 @@ def identify_video_key(video_key:str) -> str:
|
|
|
22
26
|
|
|
23
27
|
|
|
24
28
|
def get_video_key_regex_by_examination_alias(
|
|
25
|
-
examination_alias:Optional[str]=
|
|
29
|
+
examination_alias: Optional[str] = None,
|
|
30
|
+
content: Optional[str] = None,
|
|
31
|
+
is_anonymous: Optional[bool] = None,
|
|
26
32
|
):
|
|
27
33
|
"""
|
|
28
34
|
Generates a regex pattern to match video keys based on examination alias, content, and anonymity status.
|
|
@@ -37,6 +43,5 @@ def get_video_key_regex_by_examination_alias(
|
|
|
37
43
|
pattern += "anonymous" if is_anonymous else "non_anonymous"
|
|
38
44
|
else:
|
|
39
45
|
pattern += "(anonymous|non_anonymous)"
|
|
40
|
-
|
|
41
|
-
return f"^{pattern}$"
|
|
42
46
|
|
|
47
|
+
return f"^{pattern}$"
|