endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/authz/auth.py +74 -0
- endoreg_db/authz/backends.py +168 -0
- endoreg_db/authz/management/commands/list_routes.py +18 -0
- endoreg_db/authz/middleware.py +83 -0
- endoreg_db/authz/permissions.py +127 -0
- endoreg_db/authz/policy.py +218 -0
- endoreg_db/authz/views_auth.py +66 -0
- endoreg_db/config/env.py +13 -8
- endoreg_db/data/__init__.py +8 -31
- endoreg_db/data/_examples/disease.yaml +55 -0
- endoreg_db/data/_examples/disease_classification.yaml +13 -0
- endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
- endoreg_db/data/_examples/event.yaml +64 -0
- endoreg_db/data/_examples/examination.yaml +72 -0
- endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
- endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
- endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
- endoreg_db/data/_examples/finding/complication.yaml +16 -0
- endoreg_db/data/_examples/finding/data.yaml +105 -0
- endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
- endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
- endoreg_db/data/_examples/finding/outcome.yaml +12 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
- endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
- endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
- endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
- endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
- endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
- endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
- endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
- endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
- endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
- endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
- endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
- endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
- endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
- endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
- endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
- endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
- endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
- endoreg_db/data/_examples/finding_type/data.yaml +43 -0
- endoreg_db/data/_examples/requirement/age.yaml +26 -0
- endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
- endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
- endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
- endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
- endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
- endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
- endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
- endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
- endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
- endoreg_db/data/_examples/requirement/gender.yaml +25 -0
- endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
- endoreg_db/data/_examples/requirement/medication.yaml +93 -0
- endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
- endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
- endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
- endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
- endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
- endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
- endoreg_db/data/event_classification/data.yaml +4 -0
- endoreg_db/data/event_classification_choice/data.yaml +9 -0
- endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
- endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
- endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
- endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
- endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
- endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
- endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
- endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
- endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
- endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
- endoreg_db/data/requirement_set/_old_ +109 -0
- endoreg_db/data/requirement_set_type/data.yaml +21 -0
- endoreg_db/data/setup_config.yaml +4 -4
- endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
- endoreg_db/exceptions.py +5 -2
- endoreg_db/helpers/data_loader.py +1 -1
- endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
- endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
- endoreg_db/management/commands/import_video.py +9 -10
- endoreg_db/management/commands/import_video_with_classification.py +1 -1
- endoreg_db/management/commands/init_default_ai_model.py +1 -1
- endoreg_db/management/commands/list_routes.py +18 -0
- endoreg_db/management/commands/load_center_data.py +12 -12
- endoreg_db/management/commands/load_requirement_data.py +60 -31
- endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
- endoreg_db/management/commands/setup_endoreg_db.py +3 -3
- endoreg_db/management/commands/storage_management.py +271 -203
- endoreg_db/migrations/0001_initial.py +1799 -1300
- endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
- endoreg_db/migrations/_old/0001_initial.py +1857 -0
- endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
- endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
- endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
- endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
- endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
- endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
- endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
- endoreg_db/models/__init__.py +78 -123
- endoreg_db/models/administration/__init__.py +21 -42
- endoreg_db/models/administration/ai/active_model.py +2 -2
- endoreg_db/models/administration/ai/ai_model.py +7 -6
- endoreg_db/models/administration/case/__init__.py +1 -15
- endoreg_db/models/administration/case/case.py +3 -3
- endoreg_db/models/administration/case/case_template/__init__.py +2 -14
- endoreg_db/models/administration/case/case_template/case_template.py +2 -124
- endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
- endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
- endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
- endoreg_db/models/administration/center/center.py +33 -19
- endoreg_db/models/administration/center/center_product.py +12 -9
- endoreg_db/models/administration/center/center_resource.py +25 -19
- endoreg_db/models/administration/center/center_shift.py +21 -17
- endoreg_db/models/administration/center/center_waste.py +16 -8
- endoreg_db/models/administration/person/__init__.py +2 -0
- endoreg_db/models/administration/person/employee/employee.py +10 -5
- endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
- endoreg_db/models/administration/person/employee/employee_type.py +12 -6
- endoreg_db/models/administration/person/examiner/examiner.py +13 -11
- endoreg_db/models/administration/person/patient/__init__.py +2 -0
- endoreg_db/models/administration/person/patient/patient.py +103 -100
- endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
- endoreg_db/models/administration/person/person.py +4 -0
- endoreg_db/models/administration/person/profession/__init__.py +8 -4
- endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
- endoreg_db/models/administration/product/product.py +20 -15
- endoreg_db/models/administration/product/product_material.py +17 -18
- endoreg_db/models/administration/product/product_weight.py +12 -8
- endoreg_db/models/administration/product/reference_product.py +23 -55
- endoreg_db/models/administration/qualification/qualification.py +7 -3
- endoreg_db/models/administration/qualification/qualification_type.py +7 -3
- endoreg_db/models/administration/shift/scheduled_days.py +8 -5
- endoreg_db/models/administration/shift/shift.py +16 -12
- endoreg_db/models/administration/shift/shift_type.py +23 -31
- endoreg_db/models/label/__init__.py +7 -8
- endoreg_db/models/label/annotation/image_classification.py +10 -9
- endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
- endoreg_db/models/label/label.py +15 -15
- endoreg_db/models/label/label_set.py +19 -6
- endoreg_db/models/label/label_type.py +1 -1
- endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
- endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
- endoreg_db/models/label/video_segmentation_label.py +4 -0
- endoreg_db/models/label/video_segmentation_labelset.py +4 -3
- endoreg_db/models/media/frame/frame.py +22 -22
- endoreg_db/models/media/pdf/raw_pdf.py +110 -182
- endoreg_db/models/media/pdf/report_file.py +25 -29
- endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
- endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
- endoreg_db/models/media/video/__init__.py +1 -0
- endoreg_db/models/media/video/create_from_file.py +48 -56
- endoreg_db/models/media/video/pipe_2.py +8 -9
- endoreg_db/models/media/video/video_file.py +150 -108
- endoreg_db/models/media/video/video_file_ai.py +288 -74
- endoreg_db/models/media/video/video_file_anonymize.py +38 -38
- endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
- endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
- endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
- endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
- endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
- endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
- endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
- endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
- endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
- endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
- endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
- endoreg_db/models/media/video/video_file_io.py +109 -62
- endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
- endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
- endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
- endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
- endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
- endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
- endoreg_db/models/media/video/video_file_segments.py +24 -17
- endoreg_db/models/media/video/video_metadata.py +19 -35
- endoreg_db/models/media/video/video_processing.py +96 -95
- endoreg_db/models/medical/contraindication/__init__.py +13 -3
- endoreg_db/models/medical/disease.py +22 -16
- endoreg_db/models/medical/event.py +31 -18
- endoreg_db/models/medical/examination/__init__.py +13 -6
- endoreg_db/models/medical/examination/examination.py +17 -18
- endoreg_db/models/medical/examination/examination_indication.py +26 -25
- endoreg_db/models/medical/examination/examination_time.py +16 -6
- endoreg_db/models/medical/examination/examination_time_type.py +9 -6
- endoreg_db/models/medical/examination/examination_type.py +3 -4
- endoreg_db/models/medical/finding/finding.py +38 -39
- endoreg_db/models/medical/finding/finding_classification.py +37 -48
- endoreg_db/models/medical/finding/finding_intervention.py +27 -22
- endoreg_db/models/medical/finding/finding_type.py +13 -12
- endoreg_db/models/medical/hardware/endoscope.py +20 -26
- endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
- endoreg_db/models/medical/laboratory/lab_value.py +62 -91
- endoreg_db/models/medical/medication/medication.py +22 -10
- endoreg_db/models/medical/medication/medication_indication.py +29 -3
- endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
- endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
- endoreg_db/models/medical/medication/medication_schedule.py +27 -16
- endoreg_db/models/medical/organ/__init__.py +15 -12
- endoreg_db/models/medical/patient/medication_examples.py +1 -5
- endoreg_db/models/medical/patient/patient_disease.py +20 -23
- endoreg_db/models/medical/patient/patient_event.py +19 -22
- endoreg_db/models/medical/patient/patient_examination.py +48 -54
- endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
- endoreg_db/models/medical/patient/patient_finding.py +122 -139
- endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
- endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
- endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
- endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
- endoreg_db/models/medical/patient/patient_medication.py +27 -38
- endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
- endoreg_db/models/medical/risk/risk.py +7 -6
- endoreg_db/models/medical/risk/risk_type.py +8 -5
- endoreg_db/models/metadata/model_meta.py +60 -29
- endoreg_db/models/metadata/model_meta_logic.py +125 -18
- endoreg_db/models/metadata/pdf_meta.py +19 -24
- endoreg_db/models/metadata/sensitive_meta.py +102 -85
- endoreg_db/models/metadata/sensitive_meta_logic.py +192 -173
- endoreg_db/models/metadata/video_meta.py +51 -31
- endoreg_db/models/metadata/video_prediction_logic.py +16 -23
- endoreg_db/models/metadata/video_prediction_meta.py +29 -33
- endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
- endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
- endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
- endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
- endoreg_db/models/other/emission/emission_factor.py +18 -8
- endoreg_db/models/other/gender.py +10 -5
- endoreg_db/models/other/information_source.py +25 -25
- endoreg_db/models/other/material.py +9 -5
- endoreg_db/models/other/resource.py +6 -4
- endoreg_db/models/other/tag.py +10 -5
- endoreg_db/models/other/transport_route.py +13 -8
- endoreg_db/models/other/unit.py +10 -6
- endoreg_db/models/other/waste.py +6 -5
- endoreg_db/models/requirement/requirement.py +580 -272
- endoreg_db/models/requirement/requirement_error.py +85 -0
- endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
- endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
- endoreg_db/models/requirement/requirement_operator.py +36 -33
- endoreg_db/models/requirement/requirement_set.py +74 -57
- endoreg_db/models/state/__init__.py +4 -4
- endoreg_db/models/state/abstract.py +2 -2
- endoreg_db/models/state/anonymization.py +12 -0
- endoreg_db/models/state/audit_ledger.py +46 -47
- endoreg_db/models/state/label_video_segment.py +9 -0
- endoreg_db/models/state/raw_pdf.py +40 -46
- endoreg_db/models/state/sensitive_meta.py +6 -2
- endoreg_db/models/state/video.py +58 -53
- endoreg_db/models/upload_job.py +32 -55
- endoreg_db/models/utils.py +1 -2
- endoreg_db/root_urls.py +21 -2
- endoreg_db/serializers/__init__.py +0 -2
- endoreg_db/serializers/anonymization.py +18 -10
- endoreg_db/serializers/meta/report_meta.py +1 -1
- endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
- endoreg_db/serializers/misc/file_overview.py +11 -99
- endoreg_db/serializers/requirements/requirement_sets.py +92 -22
- endoreg_db/serializers/video/segmentation.py +2 -1
- endoreg_db/serializers/video/video_processing_history.py +20 -5
- endoreg_db/services/anonymization.py +75 -73
- endoreg_db/services/lookup_service.py +37 -24
- endoreg_db/services/pdf_import.py +166 -68
- endoreg_db/services/storage_aware_video_processor.py +140 -114
- endoreg_db/services/video_import.py +193 -283
- endoreg_db/urls/__init__.py +7 -20
- endoreg_db/urls/media.py +108 -67
- endoreg_db/urls/root_urls.py +29 -0
- endoreg_db/utils/__init__.py +15 -5
- endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
- endoreg_db/utils/case_generator/__init__.py +3 -0
- endoreg_db/utils/dataloader.py +88 -16
- endoreg_db/utils/defaults/set_default_center.py +32 -0
- endoreg_db/utils/names.py +22 -16
- endoreg_db/utils/permissions.py +2 -1
- endoreg_db/utils/pipelines/process_video_dir.py +1 -1
- endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
- endoreg_db/utils/setup_config.py +8 -5
- endoreg_db/utils/storage.py +115 -0
- endoreg_db/utils/validate_endo_roi.py +8 -2
- endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
- endoreg_db/views/__init__.py +0 -10
- endoreg_db/views/anonymization/media_management.py +198 -163
- endoreg_db/views/anonymization/overview.py +4 -1
- endoreg_db/views/anonymization/validate.py +174 -40
- endoreg_db/views/media/__init__.py +2 -0
- endoreg_db/views/media/pdf_media.py +131 -152
- endoreg_db/views/media/sensitive_metadata.py +46 -6
- endoreg_db/views/media/video_media.py +89 -82
- endoreg_db/views/media/video_segments.py +2 -3
- endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
- endoreg_db/views/patient/patient.py +5 -4
- endoreg_db/views/pdf/pdf_stream.py +20 -21
- endoreg_db/views/pdf/reimport.py +11 -32
- endoreg_db/views/requirement/evaluate.py +188 -187
- endoreg_db/views/requirement/lookup.py +17 -3
- endoreg_db/views/requirement/requirement_utils.py +89 -0
- endoreg_db/views/video/__init__.py +0 -2
- endoreg_db/views/video/correction.py +2 -2
- {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
- {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +341 -245
- endoreg_db/models/administration/permissions/__init__.py +0 -44
- endoreg_db/models/media/video/video_file_frames.py +0 -0
- endoreg_db/models/metadata/frame_ocr_result.py +0 -0
- endoreg_db/models/rule/__init__.py +0 -13
- endoreg_db/models/rule/rule.py +0 -27
- endoreg_db/models/rule/rule_applicator.py +0 -224
- endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
- endoreg_db/models/rule/rule_type.py +0 -20
- endoreg_db/models/rule/ruleset.py +0 -17
- endoreg_db/serializers/video/video_metadata.py +0 -105
- endoreg_db/urls/report.py +0 -48
- endoreg_db/urls/video.py +0 -61
- endoreg_db/utils/case_generator/case_generator.py +0 -159
- endoreg_db/utils/case_generator/utils.py +0 -30
- endoreg_db/views/report/__init__.py +0 -9
- endoreg_db/views/report/report_list.py +0 -112
- endoreg_db/views/report/report_with_secure_url.py +0 -28
- endoreg_db/views/report/start_examination.py +0 -7
- endoreg_db/views.py +0 -0
- /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
- /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
- /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
- /endoreg_db/{models/media/video/refactor_plan.md → views/pdf/pdf_stream_views.py} +0 -0
- {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,114 +1,121 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import os
|
|
2
3
|
import shutil
|
|
3
|
-
import logging
|
|
4
|
-
from pathlib import Path
|
|
5
4
|
from datetime import datetime, timedelta
|
|
6
|
-
from
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
7
|
from django.conf import settings
|
|
8
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
9
|
+
|
|
8
10
|
from endoreg_db.models import VideoFile
|
|
9
11
|
|
|
10
12
|
logger = logging.getLogger(__name__)
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class Command(BaseCommand):
|
|
14
|
-
help =
|
|
16
|
+
help = "Automated storage management and cleanup to prevent disk space issues"
|
|
15
17
|
|
|
16
18
|
def add_arguments(self, parser):
|
|
17
19
|
parser.add_argument(
|
|
18
|
-
|
|
19
|
-
action=
|
|
20
|
-
help=
|
|
20
|
+
"--dry-run",
|
|
21
|
+
action="store_true",
|
|
22
|
+
help="Show what would be cleaned without actually deleting files",
|
|
21
23
|
)
|
|
22
24
|
parser.add_argument(
|
|
23
|
-
|
|
24
|
-
action=
|
|
25
|
-
help=
|
|
25
|
+
"--force",
|
|
26
|
+
action="store_true",
|
|
27
|
+
help="Force cleanup even if not critically low on space",
|
|
26
28
|
)
|
|
27
29
|
parser.add_argument(
|
|
28
|
-
|
|
29
|
-
action=
|
|
30
|
-
help=
|
|
30
|
+
"--cleanup-frames",
|
|
31
|
+
action="store_true",
|
|
32
|
+
help="Clean up extracted video frames after processing",
|
|
31
33
|
)
|
|
32
34
|
parser.add_argument(
|
|
33
|
-
|
|
34
|
-
action=
|
|
35
|
-
help=
|
|
35
|
+
"--cleanup-old-videos",
|
|
36
|
+
action="store_true",
|
|
37
|
+
help="Clean up old processed videos (keep raw files)",
|
|
36
38
|
)
|
|
37
39
|
parser.add_argument(
|
|
38
|
-
|
|
39
|
-
action=
|
|
40
|
-
help=
|
|
40
|
+
"--cleanup-uploads",
|
|
41
|
+
action="store_true",
|
|
42
|
+
help="Clean up old upload cache files",
|
|
41
43
|
)
|
|
42
44
|
parser.add_argument(
|
|
43
|
-
|
|
44
|
-
action=
|
|
45
|
-
help=
|
|
45
|
+
"--cleanup-logs",
|
|
46
|
+
action="store_true",
|
|
47
|
+
help="Clean up old log files",
|
|
46
48
|
)
|
|
47
49
|
parser.add_argument(
|
|
48
|
-
|
|
50
|
+
"--max-age-days",
|
|
49
51
|
type=int,
|
|
50
52
|
default=30,
|
|
51
|
-
help=
|
|
53
|
+
help="Maximum age in days for files to keep (default: 30)",
|
|
52
54
|
)
|
|
53
55
|
parser.add_argument(
|
|
54
|
-
|
|
56
|
+
"--emergency-threshold",
|
|
55
57
|
type=float,
|
|
56
58
|
default=95.0,
|
|
57
|
-
help=
|
|
59
|
+
help="Storage usage percentage that triggers emergency cleanup (default: 95%%)",
|
|
58
60
|
)
|
|
59
61
|
|
|
60
62
|
def handle(self, *args, **options):
|
|
61
63
|
"""Main command handler for storage management."""
|
|
62
|
-
self.dry_run = options[
|
|
63
|
-
self.force = options[
|
|
64
|
-
self.max_age_days = options[
|
|
65
|
-
self.emergency_threshold = options[
|
|
64
|
+
self.dry_run = options["dry_run"]
|
|
65
|
+
self.force = options["force"]
|
|
66
|
+
self.max_age_days = options["max_age_days"]
|
|
67
|
+
self.emergency_threshold = options["emergency_threshold"]
|
|
66
68
|
|
|
67
69
|
# Validate emergency_threshold range
|
|
68
70
|
if not (0 <= self.emergency_threshold <= 100):
|
|
69
|
-
raise CommandError(
|
|
71
|
+
raise CommandError(
|
|
72
|
+
"The --emergency-threshold value must be between 0 and 100 (inclusive)."
|
|
73
|
+
)
|
|
70
74
|
|
|
71
75
|
if self.dry_run:
|
|
72
|
-
self.stdout.write(
|
|
73
|
-
|
|
76
|
+
self.stdout.write(
|
|
77
|
+
self.style.WARNING("DRY RUN MODE - No files will be deleted")
|
|
78
|
+
)
|
|
79
|
+
|
|
74
80
|
try:
|
|
75
81
|
# Check current storage status
|
|
76
82
|
storage_info = self.get_storage_info()
|
|
77
83
|
self.display_storage_status(storage_info)
|
|
78
|
-
|
|
84
|
+
|
|
79
85
|
# Determine if emergency cleanup is needed
|
|
80
86
|
needs_emergency_cleanup = (
|
|
81
|
-
storage_info[
|
|
82
|
-
or self.force
|
|
87
|
+
storage_info["usage_percent"] >= self.emergency_threshold or self.force
|
|
83
88
|
)
|
|
84
|
-
|
|
89
|
+
|
|
85
90
|
if needs_emergency_cleanup:
|
|
86
91
|
self.stdout.write(
|
|
87
92
|
self.style.ERROR(
|
|
88
|
-
f
|
|
93
|
+
f"🚨 EMERGENCY CLEANUP TRIGGERED - Storage at {storage_info['usage_percent']:.1f}%"
|
|
89
94
|
)
|
|
90
95
|
)
|
|
91
96
|
self.emergency_cleanup(options)
|
|
92
97
|
else:
|
|
93
98
|
self.stdout.write(
|
|
94
99
|
self.style.SUCCESS(
|
|
95
|
-
f
|
|
100
|
+
f"✅ Storage usage is acceptable at {storage_info['usage_percent']:.1f}%"
|
|
96
101
|
)
|
|
97
102
|
)
|
|
98
|
-
|
|
103
|
+
|
|
99
104
|
# Always run maintenance cleanup if requested
|
|
100
|
-
if any(
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
105
|
+
if any(
|
|
106
|
+
[
|
|
107
|
+
options["cleanup_frames"],
|
|
108
|
+
options["cleanup_old_videos"],
|
|
109
|
+
options["cleanup_uploads"],
|
|
110
|
+
options["cleanup_logs"],
|
|
111
|
+
]
|
|
112
|
+
):
|
|
106
113
|
self.maintenance_cleanup(options)
|
|
107
|
-
|
|
114
|
+
|
|
108
115
|
# Show final storage status
|
|
109
116
|
final_storage = self.get_storage_info()
|
|
110
117
|
self.display_cleanup_summary(storage_info, final_storage)
|
|
111
|
-
|
|
118
|
+
|
|
112
119
|
except Exception as e:
|
|
113
120
|
logger.error(f"Storage management failed: {e}")
|
|
114
121
|
raise CommandError(f"Storage management failed: {e}")
|
|
@@ -117,21 +124,21 @@ class Command(BaseCommand):
|
|
|
117
124
|
"""Get current storage information."""
|
|
118
125
|
try:
|
|
119
126
|
# Get storage stats for the root filesystem
|
|
120
|
-
total, used, free = shutil.disk_usage(
|
|
127
|
+
total, used, free = shutil.disk_usage("/")
|
|
121
128
|
usage_percent = (used / total) * 100
|
|
122
|
-
|
|
129
|
+
|
|
123
130
|
# Get project-specific storage info
|
|
124
131
|
project_root = Path(settings.BASE_DIR).parent
|
|
125
132
|
project_storage = self.get_directory_size(project_root)
|
|
126
|
-
|
|
133
|
+
|
|
127
134
|
return {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
+
"total_gb": total / (1024**3),
|
|
136
|
+
"used_gb": used / (1024**3),
|
|
137
|
+
"free_gb": free / (1024**3),
|
|
138
|
+
"usage_percent": usage_percent,
|
|
139
|
+
"project_storage_gb": project_storage / (1024**3),
|
|
140
|
+
"critical": usage_percent >= 95.0,
|
|
141
|
+
"warning": usage_percent >= 85.0,
|
|
135
142
|
}
|
|
136
143
|
except Exception as e:
|
|
137
144
|
logger.error(f"Failed to get storage info: {e}")
|
|
@@ -155,350 +162,397 @@ class Command(BaseCommand):
|
|
|
155
162
|
def display_storage_status(self, storage_info):
|
|
156
163
|
"""Display current storage status."""
|
|
157
164
|
status_color = (
|
|
158
|
-
self.style.ERROR
|
|
159
|
-
|
|
160
|
-
self.style.
|
|
165
|
+
self.style.ERROR
|
|
166
|
+
if storage_info["critical"]
|
|
167
|
+
else self.style.WARNING
|
|
168
|
+
if storage_info["warning"]
|
|
169
|
+
else self.style.SUCCESS
|
|
161
170
|
)
|
|
162
|
-
|
|
163
|
-
self.stdout.write("\n" + "="*60)
|
|
171
|
+
|
|
172
|
+
self.stdout.write("\n" + "=" * 60)
|
|
164
173
|
self.stdout.write(status_color("💾 STORAGE STATUS"))
|
|
165
|
-
self.stdout.write("="*60)
|
|
174
|
+
self.stdout.write("=" * 60)
|
|
166
175
|
self.stdout.write(f"Total Space: {storage_info['total_gb']:.1f} GB")
|
|
167
176
|
self.stdout.write(f"Used Space: {storage_info['used_gb']:.1f} GB")
|
|
168
177
|
self.stdout.write(f"Free Space: {storage_info['free_gb']:.1f} GB")
|
|
169
|
-
self.stdout.write(
|
|
178
|
+
self.stdout.write(
|
|
179
|
+
status_color(f"Usage: {storage_info['usage_percent']:.1f}%")
|
|
180
|
+
)
|
|
170
181
|
self.stdout.write(f"Project Size: {storage_info['project_storage_gb']:.1f} GB")
|
|
171
|
-
|
|
172
|
-
if storage_info[
|
|
182
|
+
|
|
183
|
+
if storage_info["critical"]:
|
|
173
184
|
self.stdout.write(self.style.ERROR("🚨 CRITICAL: Storage critically low!"))
|
|
174
|
-
elif storage_info[
|
|
185
|
+
elif storage_info["warning"]:
|
|
175
186
|
self.stdout.write(self.style.WARNING("⚠️ WARNING: Storage getting low"))
|
|
176
187
|
|
|
177
188
|
def emergency_cleanup(self, options):
|
|
178
189
|
"""Perform emergency cleanup to free critical storage space."""
|
|
179
190
|
self.stdout.write(self.style.ERROR("\n🚨 PERFORMING EMERGENCY CLEANUP"))
|
|
180
|
-
|
|
191
|
+
|
|
181
192
|
total_freed = 0
|
|
182
|
-
|
|
193
|
+
|
|
183
194
|
# 1. AGGRESSIVE: Clean up ALL extracted frames (usually largest space saver)
|
|
184
195
|
frames_freed = self.cleanup_all_extracted_frames()
|
|
185
196
|
total_freed += frames_freed
|
|
186
|
-
|
|
197
|
+
|
|
187
198
|
# 2. AGGRESSIVE: Clean up ALL upload cache
|
|
188
199
|
uploads_freed = self.cleanup_all_uploads()
|
|
189
200
|
total_freed += uploads_freed
|
|
190
|
-
|
|
201
|
+
|
|
191
202
|
# 3. Clean up old logs
|
|
192
203
|
logs_freed = self.cleanup_old_logs()
|
|
193
204
|
total_freed += logs_freed
|
|
194
|
-
|
|
205
|
+
|
|
195
206
|
# 4. Clean up temporary files
|
|
196
207
|
temp_freed = self.cleanup_temp_files()
|
|
197
208
|
total_freed += temp_freed
|
|
198
|
-
|
|
209
|
+
|
|
199
210
|
# 5. If still critical, clean up ALL processed videos (more aggressive)
|
|
200
211
|
storage_info = self.get_storage_info()
|
|
201
|
-
if storage_info[
|
|
212
|
+
if storage_info["usage_percent"] >= 90.0:
|
|
202
213
|
# Use 0 days to clean up ALL processed videos
|
|
203
214
|
videos_freed = self.cleanup_all_processed_videos()
|
|
204
215
|
total_freed += videos_freed
|
|
205
|
-
|
|
216
|
+
|
|
206
217
|
self.stdout.write(
|
|
207
|
-
self.style.SUCCESS(
|
|
218
|
+
self.style.SUCCESS(
|
|
219
|
+
f"✅ Emergency cleanup completed: {total_freed / (1024**3):.2f} GB freed"
|
|
220
|
+
)
|
|
208
221
|
)
|
|
209
222
|
|
|
210
223
|
def maintenance_cleanup(self, options):
|
|
211
224
|
"""Perform regular maintenance cleanup."""
|
|
212
225
|
self.stdout.write(self.style.SUCCESS("\n🔧 PERFORMING MAINTENANCE CLEANUP"))
|
|
213
|
-
|
|
226
|
+
|
|
214
227
|
total_freed = 0
|
|
215
|
-
|
|
216
|
-
if options[
|
|
228
|
+
|
|
229
|
+
if options["cleanup_frames"]:
|
|
217
230
|
freed = self.cleanup_extracted_frames()
|
|
218
231
|
total_freed += freed
|
|
219
|
-
|
|
220
|
-
if options[
|
|
232
|
+
|
|
233
|
+
if options["cleanup_old_videos"]:
|
|
221
234
|
freed = self.cleanup_old_processed_videos(self.max_age_days)
|
|
222
235
|
total_freed += freed
|
|
223
|
-
|
|
224
|
-
if options[
|
|
236
|
+
|
|
237
|
+
if options["cleanup_uploads"]:
|
|
225
238
|
freed = self.cleanup_upload_cache()
|
|
226
239
|
total_freed += freed
|
|
227
|
-
|
|
228
|
-
if options[
|
|
240
|
+
|
|
241
|
+
if options["cleanup_logs"]:
|
|
229
242
|
freed = self.cleanup_old_logs()
|
|
230
243
|
total_freed += freed
|
|
231
|
-
|
|
244
|
+
|
|
232
245
|
self.stdout.write(
|
|
233
|
-
self.style.SUCCESS(
|
|
246
|
+
self.style.SUCCESS(
|
|
247
|
+
f"✅ Maintenance cleanup completed: {total_freed / (1024**3):.2f} GB freed"
|
|
248
|
+
)
|
|
234
249
|
)
|
|
235
250
|
|
|
236
251
|
def cleanup_extracted_frames(self):
|
|
237
252
|
"""Clean up extracted video frames that are no longer needed."""
|
|
238
253
|
self.stdout.write("🖼️ Cleaning up extracted video frames...")
|
|
239
|
-
|
|
254
|
+
|
|
240
255
|
total_freed = 0
|
|
241
|
-
frames_dir = Path(settings.BASE_DIR).parent /
|
|
242
|
-
|
|
256
|
+
frames_dir = Path(settings.BASE_DIR).parent / "storage" / "frames"
|
|
257
|
+
|
|
243
258
|
if not frames_dir.exists():
|
|
244
259
|
return 0
|
|
245
|
-
|
|
260
|
+
|
|
246
261
|
# Find videos that have completed processing
|
|
247
262
|
completed_videos = VideoFile.objects.filter(
|
|
248
|
-
anonymization_tasks__status=
|
|
263
|
+
anonymization_tasks__status="done_processing_anonymization"
|
|
249
264
|
).distinct()
|
|
250
|
-
|
|
265
|
+
|
|
251
266
|
for video in completed_videos:
|
|
252
267
|
try:
|
|
253
268
|
# Find frame directories for this video
|
|
254
269
|
video_frame_dirs = list(frames_dir.glob(f"*{video.uuid}*"))
|
|
255
|
-
|
|
270
|
+
|
|
256
271
|
for frame_dir in video_frame_dirs:
|
|
257
272
|
if frame_dir.is_dir():
|
|
258
273
|
dir_size = self.get_directory_size(frame_dir)
|
|
259
|
-
|
|
274
|
+
|
|
260
275
|
if not self.dry_run:
|
|
261
276
|
shutil.rmtree(frame_dir, ignore_errors=True)
|
|
262
|
-
|
|
277
|
+
|
|
263
278
|
total_freed += dir_size
|
|
264
|
-
self.stdout.write(
|
|
265
|
-
|
|
279
|
+
self.stdout.write(
|
|
280
|
+
f" Removed frames for {video.uuid}: {dir_size / (1024**2):.1f} MB"
|
|
281
|
+
)
|
|
282
|
+
|
|
266
283
|
except Exception as e:
|
|
267
284
|
logger.warning(f"Failed to clean frames for video {video.uuid}: {e}")
|
|
268
285
|
continue
|
|
269
|
-
|
|
286
|
+
|
|
270
287
|
self.stdout.write(f"✅ Frames cleanup: {total_freed / (1024**3):.2f} GB freed")
|
|
271
288
|
return total_freed
|
|
272
289
|
|
|
273
290
|
def cleanup_upload_cache(self):
|
|
274
291
|
"""Clean up old upload cache files."""
|
|
275
292
|
self.stdout.write("📤 Cleaning up upload cache...")
|
|
276
|
-
|
|
293
|
+
|
|
277
294
|
total_freed = 0
|
|
278
|
-
uploads_dir = Path(settings.BASE_DIR).parent /
|
|
279
|
-
|
|
295
|
+
uploads_dir = Path(settings.BASE_DIR).parent / "storage" / "uploads"
|
|
296
|
+
|
|
280
297
|
if not uploads_dir.exists():
|
|
281
298
|
return 0
|
|
282
|
-
|
|
299
|
+
|
|
283
300
|
cutoff_date = datetime.now() - timedelta(days=self.max_age_days)
|
|
284
|
-
|
|
285
|
-
for file_path in uploads_dir.rglob(
|
|
301
|
+
|
|
302
|
+
for file_path in uploads_dir.rglob("*"):
|
|
286
303
|
if file_path.is_file():
|
|
287
304
|
try:
|
|
288
305
|
file_mtime = datetime.fromtimestamp(file_path.stat().st_mtime)
|
|
289
|
-
|
|
306
|
+
|
|
290
307
|
if file_mtime < cutoff_date:
|
|
291
308
|
file_size = file_path.stat().st_size
|
|
292
|
-
|
|
309
|
+
|
|
293
310
|
if not self.dry_run:
|
|
294
311
|
file_path.unlink()
|
|
295
|
-
|
|
312
|
+
|
|
296
313
|
total_freed += file_size
|
|
297
|
-
|
|
314
|
+
|
|
298
315
|
except Exception as e:
|
|
299
316
|
logger.warning(f"Failed to clean upload file {file_path}: {e}")
|
|
300
317
|
continue
|
|
301
|
-
|
|
302
|
-
self.stdout.write(
|
|
318
|
+
|
|
319
|
+
self.stdout.write(
|
|
320
|
+
f"✅ Upload cache cleanup: {total_freed / (1024**3):.2f} GB freed"
|
|
321
|
+
)
|
|
303
322
|
return total_freed
|
|
304
323
|
|
|
305
324
|
def cleanup_old_logs(self):
|
|
306
325
|
"""Clean up old log files."""
|
|
307
326
|
self.stdout.write("📋 Cleaning up old log files...")
|
|
308
|
-
|
|
327
|
+
|
|
309
328
|
total_freed = 0
|
|
310
329
|
project_root = Path(settings.BASE_DIR).parent
|
|
311
|
-
|
|
330
|
+
|
|
312
331
|
# Find and clean large log files
|
|
313
|
-
for log_file in project_root.rglob(
|
|
332
|
+
for log_file in project_root.rglob("*.log"):
|
|
314
333
|
try:
|
|
315
334
|
if log_file.stat().st_size > 10 * 1024 * 1024: # Files larger than 10MB
|
|
316
335
|
file_size = log_file.stat().st_size
|
|
317
|
-
|
|
336
|
+
|
|
318
337
|
if not self.dry_run:
|
|
319
338
|
# Truncate instead of delete to preserve file handles
|
|
320
|
-
with open(log_file,
|
|
321
|
-
f.write(
|
|
322
|
-
|
|
339
|
+
with open(log_file, "w") as f:
|
|
340
|
+
f.write("")
|
|
341
|
+
|
|
323
342
|
total_freed += file_size
|
|
324
|
-
self.stdout.write(
|
|
325
|
-
|
|
343
|
+
self.stdout.write(
|
|
344
|
+
f" Truncated {log_file}: {file_size / (1024**2):.1f} MB"
|
|
345
|
+
)
|
|
346
|
+
|
|
326
347
|
except Exception as e:
|
|
327
348
|
logger.warning(f"Failed to clean log file {log_file}: {e}")
|
|
328
349
|
continue
|
|
329
|
-
|
|
350
|
+
|
|
330
351
|
self.stdout.write(f"✅ Log cleanup: {total_freed / (1024**3):.2f} GB freed")
|
|
331
352
|
return total_freed
|
|
332
353
|
|
|
333
354
|
def cleanup_temp_files(self):
|
|
334
355
|
"""Clean up temporary files."""
|
|
335
356
|
self.stdout.write("🗂️ Cleaning up temporary files...")
|
|
336
|
-
|
|
357
|
+
|
|
337
358
|
total_freed = 0
|
|
338
359
|
temp_dirs = [
|
|
339
|
-
|
|
340
|
-
Path(settings.BASE_DIR).parent /
|
|
341
|
-
Path(settings.BASE_DIR).parent /
|
|
360
|
+
"/tmp",
|
|
361
|
+
Path(settings.BASE_DIR).parent / "data" / "tmp",
|
|
362
|
+
Path(settings.BASE_DIR).parent / "storage" / "tmp",
|
|
342
363
|
]
|
|
343
|
-
|
|
364
|
+
|
|
344
365
|
for temp_dir in temp_dirs:
|
|
345
366
|
if not Path(temp_dir).exists():
|
|
346
367
|
continue
|
|
347
|
-
|
|
368
|
+
|
|
348
369
|
try:
|
|
349
|
-
for item in Path(temp_dir).glob(
|
|
350
|
-
if item.is_file() and item.name.startswith((
|
|
370
|
+
for item in Path(temp_dir).glob("*"):
|
|
371
|
+
if item.is_file() and item.name.startswith(("tmp", "temp")):
|
|
351
372
|
file_size = item.stat().st_size
|
|
352
|
-
|
|
373
|
+
|
|
353
374
|
if not self.dry_run:
|
|
354
375
|
item.unlink()
|
|
355
|
-
|
|
376
|
+
|
|
356
377
|
total_freed += file_size
|
|
357
|
-
|
|
378
|
+
|
|
358
379
|
except Exception as e:
|
|
359
380
|
logger.warning(f"Failed to clean temp dir {temp_dir}: {e}")
|
|
360
381
|
continue
|
|
361
|
-
|
|
362
|
-
self.stdout.write(
|
|
382
|
+
|
|
383
|
+
self.stdout.write(
|
|
384
|
+
f"✅ Temp files cleanup: {total_freed / (1024**3):.2f} GB freed"
|
|
385
|
+
)
|
|
363
386
|
return total_freed
|
|
364
387
|
|
|
365
388
|
def cleanup_old_processed_videos(self, max_age_days):
|
|
366
389
|
"""Clean up old processed videos while keeping raw files."""
|
|
367
|
-
self.stdout.write(
|
|
368
|
-
|
|
390
|
+
self.stdout.write(
|
|
391
|
+
f"🎥 Cleaning up processed videos older than {max_age_days} days..."
|
|
392
|
+
)
|
|
393
|
+
|
|
369
394
|
total_freed = 0
|
|
370
395
|
cutoff_date = datetime.now() - timedelta(days=max_age_days)
|
|
371
|
-
|
|
396
|
+
|
|
372
397
|
# Find old processed videos - fix the date filtering
|
|
373
398
|
try:
|
|
374
399
|
old_videos = VideoFile.objects.filter(
|
|
375
400
|
created_at__lt=cutoff_date, # Use created_at instead of date_created
|
|
376
|
-
processed_file__isnull=False
|
|
401
|
+
processed_file__isnull=False,
|
|
377
402
|
).exclude(
|
|
378
|
-
anonymization_tasks__status__in=[
|
|
403
|
+
anonymization_tasks__status__in=[
|
|
404
|
+
"processing_anonymization",
|
|
405
|
+
"extracting_frames",
|
|
406
|
+
]
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
self.stdout.write(
|
|
410
|
+
f"Found {old_videos.count()} processed videos older than {max_age_days} days"
|
|
379
411
|
)
|
|
380
|
-
|
|
381
|
-
self.stdout.write(f"Found {old_videos.count()} processed videos older than {max_age_days} days")
|
|
382
|
-
|
|
412
|
+
|
|
383
413
|
except Exception as e:
|
|
384
414
|
# Fallback: try different date field names
|
|
385
415
|
try:
|
|
386
416
|
old_videos = VideoFile.objects.filter(
|
|
387
417
|
processed_file__isnull=False
|
|
388
418
|
).exclude(
|
|
389
|
-
anonymization_tasks__status__in=[
|
|
419
|
+
anonymization_tasks__status__in=[
|
|
420
|
+
"processing_anonymization",
|
|
421
|
+
"extracting_frames",
|
|
422
|
+
]
|
|
423
|
+
)
|
|
424
|
+
self.stdout.write(
|
|
425
|
+
f"Using fallback filter, found {old_videos.count()} processed videos"
|
|
390
426
|
)
|
|
391
|
-
self.stdout.write(f"Using fallback filter, found {old_videos.count()} processed videos")
|
|
392
427
|
except Exception as e2:
|
|
393
428
|
logger.error(f"Failed to query videos: {e2}")
|
|
394
429
|
return total_freed
|
|
395
|
-
|
|
430
|
+
|
|
396
431
|
for video in old_videos:
|
|
397
432
|
try:
|
|
398
|
-
if video.processed_file and hasattr(video.processed_file,
|
|
433
|
+
if video.processed_file and hasattr(video.processed_file, "path"):
|
|
399
434
|
processed_path = Path(video.processed_file.path)
|
|
400
|
-
|
|
435
|
+
|
|
401
436
|
if processed_path.exists():
|
|
402
437
|
file_size = processed_path.stat().st_size
|
|
403
|
-
|
|
438
|
+
|
|
404
439
|
if not self.dry_run:
|
|
405
440
|
processed_path.unlink()
|
|
406
441
|
video.processed_file = None
|
|
407
|
-
video.save(update_fields=[
|
|
408
|
-
|
|
442
|
+
video.save(update_fields=["processed_file"])
|
|
443
|
+
|
|
409
444
|
total_freed += file_size
|
|
410
|
-
self.stdout.write(
|
|
411
|
-
|
|
445
|
+
self.stdout.write(
|
|
446
|
+
f" Removed processed video {video.uuid}: {file_size / (1024**2):.1f} MB"
|
|
447
|
+
)
|
|
448
|
+
|
|
412
449
|
except Exception as e:
|
|
413
450
|
logger.warning(f"Failed to clean processed video {video.uuid}: {e}")
|
|
414
451
|
continue
|
|
415
|
-
|
|
416
|
-
self.stdout.write(
|
|
452
|
+
|
|
453
|
+
self.stdout.write(
|
|
454
|
+
f"✅ Processed videos cleanup: {total_freed / (1024**3):.2f} GB freed"
|
|
455
|
+
)
|
|
417
456
|
return total_freed
|
|
418
457
|
|
|
419
458
|
def cleanup_all_extracted_frames(self):
|
|
420
459
|
"""More aggressive cleanup - remove ALL extracted frames regardless of status."""
|
|
421
460
|
self.stdout.write("🖼️ AGGRESSIVE: Cleaning up ALL extracted video frames...")
|
|
422
|
-
|
|
461
|
+
|
|
423
462
|
total_freed = 0
|
|
424
|
-
frames_dir = Path(settings.BASE_DIR).parent /
|
|
425
|
-
|
|
463
|
+
frames_dir = Path(settings.BASE_DIR).parent / "storage" / "frames"
|
|
464
|
+
|
|
426
465
|
if not frames_dir.exists():
|
|
427
466
|
self.stdout.write("No frames directory found")
|
|
428
467
|
return 0
|
|
429
|
-
|
|
468
|
+
|
|
430
469
|
try:
|
|
431
470
|
# Get directory size before cleanup
|
|
432
471
|
initial_size = self.get_directory_size(frames_dir)
|
|
433
|
-
self.stdout.write(
|
|
434
|
-
|
|
472
|
+
self.stdout.write(
|
|
473
|
+
f"Found frames directory with {initial_size / (1024**3):.2f} GB"
|
|
474
|
+
)
|
|
475
|
+
|
|
435
476
|
# Remove all frame directories and files
|
|
436
477
|
for item in frames_dir.iterdir():
|
|
437
478
|
if item.is_dir():
|
|
438
479
|
dir_size = self.get_directory_size(item)
|
|
439
|
-
|
|
480
|
+
|
|
440
481
|
if not self.dry_run:
|
|
441
482
|
shutil.rmtree(item, ignore_errors=True)
|
|
442
|
-
|
|
483
|
+
|
|
443
484
|
total_freed += dir_size
|
|
444
|
-
self.stdout.write(
|
|
485
|
+
self.stdout.write(
|
|
486
|
+
f" Removed frame directory {item.name}: {dir_size / (1024**2):.1f} MB"
|
|
487
|
+
)
|
|
445
488
|
elif item.is_file():
|
|
446
489
|
file_size = item.stat().st_size
|
|
447
|
-
|
|
490
|
+
|
|
448
491
|
if not self.dry_run:
|
|
449
492
|
item.unlink()
|
|
450
|
-
|
|
493
|
+
|
|
451
494
|
total_freed += file_size
|
|
452
|
-
self.stdout.write(
|
|
453
|
-
|
|
495
|
+
self.stdout.write(
|
|
496
|
+
f" Removed frame file {item.name}: {file_size / (1024**2):.1f} MB"
|
|
497
|
+
)
|
|
498
|
+
|
|
454
499
|
except Exception as e:
|
|
455
500
|
logger.error(f"Failed to clean frames directory: {e}")
|
|
456
|
-
|
|
457
|
-
self.stdout.write(
|
|
501
|
+
|
|
502
|
+
self.stdout.write(
|
|
503
|
+
f"✅ Aggressive frames cleanup: {total_freed / (1024**3):.2f} GB freed"
|
|
504
|
+
)
|
|
458
505
|
return total_freed
|
|
459
506
|
|
|
460
507
|
def cleanup_all_uploads(self):
|
|
461
508
|
"""More aggressive cleanup - remove ALL upload cache files."""
|
|
462
509
|
self.stdout.write("📤 AGGRESSIVE: Cleaning up ALL upload cache...")
|
|
463
|
-
|
|
510
|
+
|
|
464
511
|
total_freed = 0
|
|
465
|
-
uploads_dir = Path(settings.BASE_DIR).parent /
|
|
466
|
-
|
|
512
|
+
uploads_dir = Path(settings.BASE_DIR).parent / "storage" / "uploads"
|
|
513
|
+
|
|
467
514
|
if not uploads_dir.exists():
|
|
468
515
|
return 0
|
|
469
|
-
|
|
516
|
+
|
|
470
517
|
try:
|
|
471
|
-
for item in uploads_dir.rglob(
|
|
518
|
+
for item in uploads_dir.rglob("*"):
|
|
472
519
|
if item.is_file():
|
|
473
520
|
file_size = item.stat().st_size
|
|
474
|
-
|
|
521
|
+
|
|
475
522
|
if not self.dry_run:
|
|
476
523
|
item.unlink()
|
|
477
|
-
|
|
524
|
+
|
|
478
525
|
total_freed += file_size
|
|
479
|
-
|
|
526
|
+
|
|
480
527
|
except Exception as e:
|
|
481
528
|
logger.error(f"Failed to clean uploads directory: {e}")
|
|
482
|
-
|
|
483
|
-
self.stdout.write(
|
|
529
|
+
|
|
530
|
+
self.stdout.write(
|
|
531
|
+
f"✅ Aggressive upload cleanup: {total_freed / (1024**3):.2f} GB freed"
|
|
532
|
+
)
|
|
484
533
|
return total_freed
|
|
485
534
|
|
|
486
535
|
def cleanup_all_processed_videos(self):
|
|
487
536
|
"""AGGRESSIVE: Clean up ALL processed videos while keeping raw files."""
|
|
488
537
|
self.stdout.write("🎥 AGGRESSIVE: Cleaning up ALL processed videos...")
|
|
489
|
-
|
|
538
|
+
|
|
490
539
|
total_freed = 0
|
|
491
|
-
|
|
540
|
+
|
|
492
541
|
try:
|
|
493
542
|
# Find ALL processed videos
|
|
494
543
|
processed_videos = VideoFile.objects.filter(
|
|
495
544
|
processed_file__isnull=False
|
|
496
545
|
).exclude(
|
|
497
|
-
anonymization_tasks__status__in=[
|
|
546
|
+
anonymization_tasks__status__in=[
|
|
547
|
+
"processing_anonymization",
|
|
548
|
+
"extracting_frames",
|
|
549
|
+
]
|
|
550
|
+
)
|
|
551
|
+
|
|
552
|
+
self.stdout.write(
|
|
553
|
+
f"Found {processed_videos.count()} processed videos to clean"
|
|
498
554
|
)
|
|
499
|
-
|
|
500
|
-
self.stdout.write(f"Found {processed_videos.count()} processed videos to clean")
|
|
501
|
-
|
|
555
|
+
|
|
502
556
|
for video in processed_videos:
|
|
503
557
|
try:
|
|
504
558
|
freed = self._cleanup_processed_video_file(video)
|
|
@@ -506,43 +560,57 @@ class Command(BaseCommand):
|
|
|
506
560
|
except Exception as e:
|
|
507
561
|
logger.warning(f"Failed to clean processed video {video.uuid}: {e}")
|
|
508
562
|
continue
|
|
509
|
-
|
|
563
|
+
|
|
510
564
|
except Exception as e:
|
|
511
565
|
logger.error(f"Failed to query processed videos: {e}")
|
|
512
|
-
|
|
513
|
-
self.stdout.write(
|
|
566
|
+
|
|
567
|
+
self.stdout.write(
|
|
568
|
+
f"✅ Aggressive processed videos cleanup: {total_freed / (1024**3):.2f} GB freed"
|
|
569
|
+
)
|
|
514
570
|
return total_freed
|
|
515
571
|
|
|
516
572
|
def _cleanup_processed_video_file(self, video):
|
|
517
573
|
"""
|
|
518
574
|
Helper to clean up a single processed video file, update DB, and return freed size in bytes.
|
|
519
575
|
"""
|
|
520
|
-
if video.processed_file and hasattr(video.processed_file,
|
|
576
|
+
if video.processed_file and hasattr(video.processed_file, "path"):
|
|
521
577
|
processed_path = Path(video.processed_file.path)
|
|
522
578
|
if processed_path.exists():
|
|
523
579
|
file_size = processed_path.stat().st_size
|
|
524
580
|
if not self.dry_run:
|
|
525
581
|
processed_path.unlink()
|
|
526
582
|
video.processed_file = None
|
|
527
|
-
video.save(update_fields=[
|
|
528
|
-
self.stdout.write(
|
|
583
|
+
video.save(update_fields=["processed_file"])
|
|
584
|
+
self.stdout.write(
|
|
585
|
+
f" Removed processed video {video.uuid}: {file_size / (1024**2):.1f} MB"
|
|
586
|
+
)
|
|
529
587
|
return file_size
|
|
530
588
|
return 0
|
|
531
589
|
|
|
532
590
|
def display_cleanup_summary(self, before, after):
|
|
533
591
|
"""Display cleanup summary."""
|
|
534
|
-
freed_gb = before[
|
|
535
|
-
|
|
536
|
-
self.stdout.write("\n" + "="*60)
|
|
592
|
+
freed_gb = before["used_gb"] - after["used_gb"]
|
|
593
|
+
|
|
594
|
+
self.stdout.write("\n" + "=" * 60)
|
|
537
595
|
self.stdout.write(self.style.SUCCESS("📊 CLEANUP SUMMARY"))
|
|
538
|
-
self.stdout.write("="*60)
|
|
539
|
-
self.stdout.write(
|
|
540
|
-
|
|
596
|
+
self.stdout.write("=" * 60)
|
|
597
|
+
self.stdout.write(
|
|
598
|
+
f"Before: {before['used_gb']:.1f} GB used ({before['usage_percent']:.1f}%)"
|
|
599
|
+
)
|
|
600
|
+
self.stdout.write(
|
|
601
|
+
f"After: {after['used_gb']:.1f} GB used ({after['usage_percent']:.1f}%)"
|
|
602
|
+
)
|
|
541
603
|
self.stdout.write(self.style.SUCCESS(f"Freed: {freed_gb:.2f} GB"))
|
|
542
|
-
|
|
543
|
-
if after[
|
|
604
|
+
|
|
605
|
+
if after["usage_percent"] < 90.0:
|
|
544
606
|
self.stdout.write(self.style.SUCCESS("✅ Storage levels are now healthy!"))
|
|
545
|
-
elif after[
|
|
546
|
-
self.stdout.write(
|
|
607
|
+
elif after["usage_percent"] < 95.0:
|
|
608
|
+
self.stdout.write(
|
|
609
|
+
self.style.WARNING("⚠️ Storage levels improved but still high")
|
|
610
|
+
)
|
|
547
611
|
else:
|
|
548
|
-
self.stdout.write(
|
|
612
|
+
self.stdout.write(
|
|
613
|
+
self.style.ERROR(
|
|
614
|
+
"🚨 Storage levels still critical - manual intervention needed"
|
|
615
|
+
)
|
|
616
|
+
)
|