endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (503) hide show
  1. endoreg_db/authz/auth.py +74 -0
  2. endoreg_db/authz/backends.py +168 -0
  3. endoreg_db/authz/management/commands/list_routes.py +18 -0
  4. endoreg_db/authz/middleware.py +83 -0
  5. endoreg_db/authz/permissions.py +127 -0
  6. endoreg_db/authz/policy.py +218 -0
  7. endoreg_db/authz/views_auth.py +66 -0
  8. endoreg_db/config/env.py +13 -8
  9. endoreg_db/data/__init__.py +2 -11
  10. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +3 -3
  11. endoreg_db/data/event_classification/data.yaml +4 -0
  12. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  13. endoreg_db/data/examination/examinations/data.yaml +114 -14
  14. endoreg_db/data/examination/time-type/data.yaml +0 -3
  15. endoreg_db/data/examination_indication/endoscopy.yaml +108 -173
  16. endoreg_db/data/examination_indication_classification/endoscopy.yaml +0 -70
  17. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +33 -37
  18. endoreg_db/data/finding/00_generic.yaml +35 -0
  19. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  20. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  21. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  22. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  23. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  24. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  25. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  26. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  27. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  28. endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
  29. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  30. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  31. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  32. endoreg_db/data/finding_classification_choice/{yes_no_na.yaml → 00_generic.yaml} +5 -1
  33. endoreg_db/data/finding_classification_choice/{examination_setting_generic_types.yaml → 00_generic_baseline.yaml} +10 -2
  34. endoreg_db/data/finding_classification_choice/{complication_generic_types.yaml → 00_generic_complication.yaml} +1 -1
  35. endoreg_db/data/finding_classification_choice/{histology.yaml → 00_generic_histology.yaml} +1 -4
  36. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  37. endoreg_db/data/finding_classification_choice/{bowel_preparation.yaml → 02_colonoscopy_bowel_preparation.yaml} +1 -30
  38. endoreg_db/data/finding_classification_choice/{colonoscopy_not_complete_reason.yaml → 02_colonoscopy_generic.yaml} +1 -1
  39. endoreg_db/data/finding_classification_choice/{histology_polyp.yaml → 02_colonoscopy_histology.yaml} +1 -1
  40. endoreg_db/data/finding_classification_choice/{colonoscopy_location.yaml → 02_colonoscopy_location.yaml} +23 -4
  41. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  42. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  43. endoreg_db/data/finding_classification_choice/{colon_lesion_paris.yaml → 02_colonoscopy_polyp_morphology.yaml} +26 -8
  44. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  45. endoreg_db/data/finding_classification_type/{colonoscopy_basic.yaml → 00_generic.yaml} +18 -13
  46. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  47. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  48. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  49. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  50. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  51. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  52. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  53. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  54. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  55. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  56. endoreg_db/data/finding_type/data.yaml +8 -12
  57. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  58. endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
  59. endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
  60. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  61. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +29 -12
  62. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  63. endoreg_db/data/requirement_set/{endoscopy_bleeding_risk.yaml → 02_endoscopy_bleeding_risk.yaml} +0 -6
  64. endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
  65. endoreg_db/data/requirement_set/_old_ +109 -0
  66. endoreg_db/data/requirement_set_type/data.yaml +21 -0
  67. endoreg_db/data/setup_config.yaml +4 -4
  68. endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
  69. endoreg_db/exceptions.py +4 -2
  70. endoreg_db/forms/examination_form.py +1 -1
  71. endoreg_db/helpers/data_loader.py +125 -53
  72. endoreg_db/helpers/default_objects.py +116 -81
  73. endoreg_db/import_files/__init__.py +27 -0
  74. endoreg_db/import_files/context/__init__.py +7 -0
  75. endoreg_db/import_files/context/default_sensitive_meta.py +81 -0
  76. endoreg_db/import_files/context/ensure_center.py +17 -0
  77. endoreg_db/import_files/context/file_lock.py +66 -0
  78. endoreg_db/import_files/context/import_context.py +43 -0
  79. endoreg_db/import_files/context/validate_directories.py +56 -0
  80. endoreg_db/import_files/file_storage/__init__.py +15 -0
  81. endoreg_db/import_files/file_storage/create_report_file.py +76 -0
  82. endoreg_db/import_files/file_storage/create_video_file.py +75 -0
  83. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +39 -0
  84. endoreg_db/import_files/file_storage/state_management.py +400 -0
  85. endoreg_db/import_files/file_storage/storage.py +36 -0
  86. endoreg_db/import_files/import_service.md +26 -0
  87. endoreg_db/import_files/processing/__init__.py +11 -0
  88. endoreg_db/import_files/processing/report_processing/report_anonymization.py +94 -0
  89. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  90. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  91. endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +119 -0
  92. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  93. endoreg_db/import_files/pseudonymization/k_anonymity.py +182 -0
  94. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +128 -0
  95. endoreg_db/import_files/report_import_service.py +141 -0
  96. endoreg_db/import_files/video_import_service.py +150 -0
  97. endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
  98. endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
  99. endoreg_db/management/commands/import_report.py +130 -65
  100. endoreg_db/management/commands/import_video.py +9 -10
  101. endoreg_db/management/commands/import_video_with_classification.py +2 -2
  102. endoreg_db/management/commands/list_routes.py +18 -0
  103. endoreg_db/management/commands/load_ai_model_data.py +5 -5
  104. endoreg_db/management/commands/load_ai_model_label_data.py +9 -7
  105. endoreg_db/management/commands/load_base_db_data.py +5 -134
  106. endoreg_db/management/commands/load_center_data.py +12 -12
  107. endoreg_db/management/commands/load_contraindication_data.py +14 -16
  108. endoreg_db/management/commands/load_disease_classification_choices_data.py +15 -18
  109. endoreg_db/management/commands/load_disease_classification_data.py +15 -18
  110. endoreg_db/management/commands/load_disease_data.py +25 -28
  111. endoreg_db/management/commands/load_endoscope_data.py +20 -27
  112. endoreg_db/management/commands/load_event_data.py +14 -16
  113. endoreg_db/management/commands/load_examination_data.py +31 -44
  114. endoreg_db/management/commands/load_examination_indication_data.py +20 -21
  115. endoreg_db/management/commands/load_finding_data.py +52 -80
  116. endoreg_db/management/commands/load_information_source.py +21 -23
  117. endoreg_db/management/commands/load_lab_value_data.py +17 -26
  118. endoreg_db/management/commands/load_medication_data.py +13 -12
  119. endoreg_db/management/commands/load_organ_data.py +15 -19
  120. endoreg_db/management/commands/load_pdf_type_data.py +19 -18
  121. endoreg_db/management/commands/load_profession_data.py +14 -17
  122. endoreg_db/management/commands/load_qualification_data.py +20 -23
  123. endoreg_db/management/commands/load_report_reader_flag_data.py +17 -19
  124. endoreg_db/management/commands/load_requirement_data.py +62 -39
  125. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  126. endoreg_db/management/commands/load_risk_data.py +7 -6
  127. endoreg_db/management/commands/load_shift_data.py +20 -23
  128. endoreg_db/management/commands/load_tag_data.py +8 -11
  129. endoreg_db/management/commands/load_unit_data.py +17 -19
  130. endoreg_db/management/commands/setup_endoreg_db.py +3 -3
  131. endoreg_db/management/commands/start_filewatcher.py +46 -37
  132. endoreg_db/management/commands/storage_management.py +271 -203
  133. endoreg_db/management/commands/validate_video_files.py +1 -5
  134. endoreg_db/migrations/0001_initial.py +297 -250
  135. endoreg_db/models/__init__.py +78 -123
  136. endoreg_db/models/administration/__init__.py +21 -42
  137. endoreg_db/models/administration/ai/active_model.py +2 -2
  138. endoreg_db/models/administration/ai/ai_model.py +7 -6
  139. endoreg_db/models/administration/case/__init__.py +1 -15
  140. endoreg_db/models/administration/case/case.py +3 -3
  141. endoreg_db/models/administration/case/case_template/__init__.py +2 -14
  142. endoreg_db/models/administration/case/case_template/case_template.py +2 -124
  143. endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
  144. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
  145. endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
  146. endoreg_db/models/administration/center/center.py +33 -19
  147. endoreg_db/models/administration/center/center_product.py +12 -9
  148. endoreg_db/models/administration/center/center_resource.py +25 -19
  149. endoreg_db/models/administration/center/center_shift.py +21 -17
  150. endoreg_db/models/administration/center/center_waste.py +16 -8
  151. endoreg_db/models/administration/person/__init__.py +2 -0
  152. endoreg_db/models/administration/person/employee/employee.py +10 -5
  153. endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
  154. endoreg_db/models/administration/person/employee/employee_type.py +12 -6
  155. endoreg_db/models/administration/person/examiner/examiner.py +13 -11
  156. endoreg_db/models/administration/person/patient/__init__.py +2 -0
  157. endoreg_db/models/administration/person/patient/patient.py +129 -100
  158. endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
  159. endoreg_db/models/administration/person/person.py +4 -0
  160. endoreg_db/models/administration/person/profession/__init__.py +8 -4
  161. endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
  162. endoreg_db/models/administration/product/product.py +20 -15
  163. endoreg_db/models/administration/product/product_material.py +17 -18
  164. endoreg_db/models/administration/product/product_weight.py +12 -8
  165. endoreg_db/models/administration/product/reference_product.py +23 -55
  166. endoreg_db/models/administration/qualification/qualification.py +7 -3
  167. endoreg_db/models/administration/qualification/qualification_type.py +7 -3
  168. endoreg_db/models/administration/shift/scheduled_days.py +8 -5
  169. endoreg_db/models/administration/shift/shift.py +16 -12
  170. endoreg_db/models/administration/shift/shift_type.py +23 -31
  171. endoreg_db/models/label/__init__.py +8 -9
  172. endoreg_db/models/label/annotation/image_classification.py +10 -9
  173. endoreg_db/models/label/annotation/video_segmentation_annotation.py +23 -28
  174. endoreg_db/models/label/label.py +15 -15
  175. endoreg_db/models/label/label_set.py +19 -6
  176. endoreg_db/models/label/label_type.py +1 -1
  177. endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
  178. endoreg_db/models/label/label_video_segment/label_video_segment.py +98 -102
  179. endoreg_db/models/label/video_segmentation_label.py +4 -0
  180. endoreg_db/models/label/video_segmentation_labelset.py +4 -3
  181. endoreg_db/models/media/frame/frame.py +22 -22
  182. endoreg_db/models/media/pdf/raw_pdf.py +194 -194
  183. endoreg_db/models/media/pdf/report_file.py +25 -29
  184. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +55 -47
  185. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
  186. endoreg_db/models/media/processing_history/__init__.py +5 -0
  187. endoreg_db/models/media/processing_history/processing_history.py +96 -0
  188. endoreg_db/models/media/video/__init__.py +1 -0
  189. endoreg_db/models/media/video/create_from_file.py +139 -77
  190. endoreg_db/models/media/video/pipe_2.py +8 -9
  191. endoreg_db/models/media/video/video_file.py +174 -112
  192. endoreg_db/models/media/video/video_file_ai.py +288 -74
  193. endoreg_db/models/media/video/video_file_anonymize.py +38 -38
  194. endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
  195. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
  196. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
  197. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
  198. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
  199. endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
  200. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
  201. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
  202. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
  203. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
  204. endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
  205. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
  206. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
  207. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
  208. endoreg_db/models/media/video/video_file_io.py +113 -61
  209. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
  210. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
  211. endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
  212. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
  213. endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
  214. endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
  215. endoreg_db/models/media/video/video_file_segments.py +24 -17
  216. endoreg_db/models/media/video/video_metadata.py +19 -35
  217. endoreg_db/models/media/video/video_processing.py +96 -95
  218. endoreg_db/models/medical/contraindication/README.md +1 -0
  219. endoreg_db/models/medical/contraindication/__init__.py +13 -3
  220. endoreg_db/models/medical/disease.py +22 -16
  221. endoreg_db/models/medical/event.py +31 -18
  222. endoreg_db/models/medical/examination/__init__.py +13 -6
  223. endoreg_db/models/medical/examination/examination.py +39 -20
  224. endoreg_db/models/medical/examination/examination_indication.py +30 -95
  225. endoreg_db/models/medical/examination/examination_time.py +23 -8
  226. endoreg_db/models/medical/examination/examination_time_type.py +9 -6
  227. endoreg_db/models/medical/examination/examination_type.py +3 -4
  228. endoreg_db/models/medical/finding/finding.py +32 -40
  229. endoreg_db/models/medical/finding/finding_classification.py +42 -72
  230. endoreg_db/models/medical/finding/finding_intervention.py +25 -22
  231. endoreg_db/models/medical/finding/finding_type.py +13 -12
  232. endoreg_db/models/medical/hardware/endoscope.py +26 -26
  233. endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
  234. endoreg_db/models/medical/laboratory/lab_value.py +62 -91
  235. endoreg_db/models/medical/medication/medication.py +22 -10
  236. endoreg_db/models/medical/medication/medication_indication.py +29 -3
  237. endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
  238. endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
  239. endoreg_db/models/medical/medication/medication_schedule.py +27 -16
  240. endoreg_db/models/medical/organ/__init__.py +15 -12
  241. endoreg_db/models/medical/patient/medication_examples.py +6 -6
  242. endoreg_db/models/medical/patient/patient_disease.py +20 -23
  243. endoreg_db/models/medical/patient/patient_event.py +19 -22
  244. endoreg_db/models/medical/patient/patient_examination.py +48 -54
  245. endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
  246. endoreg_db/models/medical/patient/patient_finding.py +122 -139
  247. endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
  248. endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
  249. endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
  250. endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
  251. endoreg_db/models/medical/patient/patient_medication.py +27 -38
  252. endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
  253. endoreg_db/models/medical/risk/risk.py +7 -6
  254. endoreg_db/models/medical/risk/risk_type.py +8 -5
  255. endoreg_db/models/metadata/model_meta.py +60 -29
  256. endoreg_db/models/metadata/model_meta_logic.py +125 -18
  257. endoreg_db/models/metadata/pdf_meta.py +31 -24
  258. endoreg_db/models/metadata/sensitive_meta.py +105 -85
  259. endoreg_db/models/metadata/sensitive_meta_logic.py +198 -103
  260. endoreg_db/models/metadata/video_meta.py +51 -31
  261. endoreg_db/models/metadata/video_prediction_logic.py +16 -23
  262. endoreg_db/models/metadata/video_prediction_meta.py +29 -33
  263. endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
  264. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
  265. endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
  266. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
  267. endoreg_db/models/other/emission/emission_factor.py +18 -8
  268. endoreg_db/models/other/gender.py +10 -5
  269. endoreg_db/models/other/information_source.py +50 -29
  270. endoreg_db/models/other/material.py +9 -5
  271. endoreg_db/models/other/resource.py +6 -4
  272. endoreg_db/models/other/tag.py +10 -5
  273. endoreg_db/models/other/transport_route.py +13 -8
  274. endoreg_db/models/other/unit.py +10 -6
  275. endoreg_db/models/other/waste.py +6 -5
  276. endoreg_db/models/report/report.py +6 -0
  277. endoreg_db/models/requirement/requirement.py +329 -361
  278. endoreg_db/models/requirement/requirement_error.py +85 -0
  279. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  280. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
  281. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
  282. endoreg_db/models/requirement/requirement_operator.py +103 -112
  283. endoreg_db/models/requirement/requirement_set.py +74 -57
  284. endoreg_db/models/state/__init__.py +4 -4
  285. endoreg_db/models/state/abstract.py +2 -2
  286. endoreg_db/models/state/anonymization.py +12 -0
  287. endoreg_db/models/state/audit_ledger.py +49 -51
  288. endoreg_db/models/state/label_video_segment.py +9 -0
  289. endoreg_db/models/state/raw_pdf.py +101 -68
  290. endoreg_db/models/state/sensitive_meta.py +6 -2
  291. endoreg_db/models/state/video.py +110 -90
  292. endoreg_db/models/upload_job.py +35 -34
  293. endoreg_db/models/utils.py +28 -25
  294. endoreg_db/queries/__init__.py +3 -1
  295. endoreg_db/root_urls.py +21 -2
  296. endoreg_db/schemas/examination_evaluation.py +1 -1
  297. endoreg_db/serializers/__init__.py +2 -10
  298. endoreg_db/serializers/anonymization.py +18 -10
  299. endoreg_db/serializers/label_video_segment/label_video_segment.py +2 -29
  300. endoreg_db/serializers/meta/__init__.py +1 -6
  301. endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
  302. endoreg_db/serializers/misc/file_overview.py +11 -99
  303. endoreg_db/serializers/misc/sensitive_patient_data.py +50 -26
  304. endoreg_db/serializers/patient_examination/patient_examination.py +3 -3
  305. endoreg_db/serializers/pdf/anony_text_validation.py +39 -23
  306. endoreg_db/serializers/requirements/requirement_sets.py +92 -22
  307. endoreg_db/serializers/video/segmentation.py +2 -1
  308. endoreg_db/serializers/video/video_file_list.py +65 -34
  309. endoreg_db/serializers/video/video_processing_history.py +20 -5
  310. endoreg_db/services/__old/pdf_import.py +1487 -0
  311. endoreg_db/services/__old/video_import.py +1306 -0
  312. endoreg_db/services/anonymization.py +128 -89
  313. endoreg_db/services/lookup_service.py +65 -52
  314. endoreg_db/services/lookup_store.py +2 -2
  315. endoreg_db/services/pdf_import.py +0 -1382
  316. endoreg_db/services/report_import.py +10 -0
  317. endoreg_db/services/video_import.py +6 -1255
  318. endoreg_db/tasks/upload_tasks.py +79 -70
  319. endoreg_db/tasks/video_ingest.py +8 -4
  320. endoreg_db/urls/__init__.py +5 -32
  321. endoreg_db/urls/ai.py +32 -0
  322. endoreg_db/urls/media.py +121 -83
  323. endoreg_db/urls/root_urls.py +29 -0
  324. endoreg_db/utils/__init__.py +15 -5
  325. endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
  326. endoreg_db/utils/case_generator/__init__.py +3 -0
  327. endoreg_db/utils/dataloader.py +142 -40
  328. endoreg_db/utils/defaults/set_default_center.py +32 -0
  329. endoreg_db/utils/names.py +22 -16
  330. endoreg_db/utils/paths.py +110 -46
  331. endoreg_db/utils/permissions.py +2 -1
  332. endoreg_db/utils/pipelines/Readme.md +1 -1
  333. endoreg_db/utils/pipelines/process_video_dir.py +1 -1
  334. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +655 -0
  335. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +97 -0
  336. endoreg_db/utils/setup_config.py +8 -5
  337. endoreg_db/utils/storage.py +115 -0
  338. endoreg_db/utils/validate_endo_roi.py +8 -2
  339. endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
  340. endoreg_db/views/__init__.py +85 -183
  341. endoreg_db/views/ai/__init__.py +8 -0
  342. endoreg_db/views/ai/label.py +155 -0
  343. endoreg_db/views/anonymization/media_management.py +202 -166
  344. endoreg_db/views/anonymization/overview.py +99 -67
  345. endoreg_db/views/anonymization/validate.py +182 -44
  346. endoreg_db/views/media/__init__.py +7 -20
  347. endoreg_db/views/media/pdf_media.py +197 -174
  348. endoreg_db/views/media/sensitive_metadata.py +193 -138
  349. endoreg_db/views/media/video_media.py +89 -82
  350. endoreg_db/views/meta/__init__.py +0 -8
  351. endoreg_db/views/misc/__init__.py +1 -7
  352. endoreg_db/views/misc/upload_views.py +94 -93
  353. endoreg_db/views/patient/patient.py +5 -4
  354. endoreg_db/views/report/__init__.py +5 -7
  355. endoreg_db/views/{pdf → report}/reimport.py +22 -22
  356. endoreg_db/views/{pdf/pdf_stream.py → report/report_stream.py} +46 -39
  357. endoreg_db/views/requirement/evaluate.py +188 -187
  358. endoreg_db/views/requirement/lookup.py +17 -3
  359. endoreg_db/views/requirement/lookup_store.py +22 -90
  360. endoreg_db/views/requirement/requirement_utils.py +89 -0
  361. endoreg_db/views/video/__init__.py +23 -24
  362. endoreg_db/views/video/correction.py +201 -172
  363. endoreg_db/views/video/reimport.py +1 -1
  364. endoreg_db/views/{media/video_segments.py → video/segments_crud.py} +77 -40
  365. endoreg_db/views/video/{video_meta.py → video_meta_stats.py} +2 -2
  366. endoreg_db/views/video/video_stream.py +7 -8
  367. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/METADATA +7 -3
  368. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/RECORD +391 -413
  369. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/WHEEL +1 -1
  370. endoreg_db/data/finding/anatomy_colon.yaml +0 -128
  371. endoreg_db/data/finding/colonoscopy.yaml +0 -40
  372. endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +0 -56
  373. endoreg_db/data/finding/complication.yaml +0 -16
  374. endoreg_db/data/finding/data.yaml +0 -105
  375. endoreg_db/data/finding/examination_setting.yaml +0 -16
  376. endoreg_db/data/finding/medication_related.yaml +0 -18
  377. endoreg_db/data/finding/outcome.yaml +0 -12
  378. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +0 -95
  379. endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +0 -22
  380. endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +0 -25
  381. endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
  382. endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
  383. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +0 -68
  384. endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
  385. endoreg_db/data/finding_classification/colonoscopy_location.yaml +0 -80
  386. endoreg_db/data/finding_classification/colonoscopy_lst.yaml +0 -21
  387. endoreg_db/data/finding_classification/colonoscopy_nice.yaml +0 -20
  388. endoreg_db/data/finding_classification/colonoscopy_paris.yaml +0 -26
  389. endoreg_db/data/finding_classification/colonoscopy_sano.yaml +0 -22
  390. endoreg_db/data/finding_classification/colonoscopy_summary.yaml +0 -53
  391. endoreg_db/data/finding_classification/complication_generic.yaml +0 -25
  392. endoreg_db/data/finding_classification/examination_setting_generic.yaml +0 -40
  393. endoreg_db/data/finding_classification/histology_colo.yaml +0 -51
  394. endoreg_db/data/finding_classification/intervention_required.yaml +0 -26
  395. endoreg_db/data/finding_classification/medication_related.yaml +0 -23
  396. endoreg_db/data/finding_classification/visualized.yaml +0 -33
  397. endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
  398. endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
  399. endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
  400. endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +0 -15
  401. endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +0 -17
  402. endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
  403. endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +0 -14
  404. endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
  405. endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +0 -82
  406. endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
  407. endoreg_db/data/finding_classification_choice/outcome.yaml +0 -19
  408. endoreg_db/data/finding_intervention/endoscopy.yaml +0 -43
  409. endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
  410. endoreg_db/data/finding_intervention/endoscopy_egd.yaml +0 -128
  411. endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +0 -32
  412. endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +0 -9
  413. endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +0 -36
  414. endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +0 -79
  415. endoreg_db/data/requirement/age.yaml +0 -26
  416. endoreg_db/data/requirement/gender.yaml +0 -25
  417. endoreg_db/management/commands/init_default_ai_model.py +0 -112
  418. endoreg_db/management/commands/reset_celery_schedule.py +0 -9
  419. endoreg_db/management/commands/validate_video.py +0 -204
  420. endoreg_db/migrations/0002_add_video_correction_models.py +0 -52
  421. endoreg_db/migrations/0003_add_center_display_name.py +0 -30
  422. endoreg_db/models/administration/permissions/__init__.py +0 -44
  423. endoreg_db/models/rule/__init__.py +0 -13
  424. endoreg_db/models/rule/rule.py +0 -27
  425. endoreg_db/models/rule/rule_applicator.py +0 -224
  426. endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
  427. endoreg_db/models/rule/rule_type.py +0 -20
  428. endoreg_db/models/rule/ruleset.py +0 -17
  429. endoreg_db/renames.yml +0 -8
  430. endoreg_db/serializers/_old/raw_pdf_meta_validation.py +0 -223
  431. endoreg_db/serializers/_old/raw_video_meta_validation.py +0 -179
  432. endoreg_db/serializers/_old/video.py +0 -71
  433. endoreg_db/serializers/meta/pdf_file_meta_extraction.py +0 -115
  434. endoreg_db/serializers/meta/report_meta.py +0 -53
  435. endoreg_db/serializers/report/__init__.py +0 -9
  436. endoreg_db/serializers/report/mixins.py +0 -45
  437. endoreg_db/serializers/report/report.py +0 -105
  438. endoreg_db/serializers/report/report_list.py +0 -22
  439. endoreg_db/serializers/report/secure_file_url.py +0 -26
  440. endoreg_db/serializers/video/video_metadata.py +0 -105
  441. endoreg_db/services/requirements_object.py +0 -147
  442. endoreg_db/services/storage_aware_video_processor.py +0 -344
  443. endoreg_db/urls/files.py +0 -6
  444. endoreg_db/urls/label_video_segment_validate.py +0 -33
  445. endoreg_db/urls/label_video_segments.py +0 -46
  446. endoreg_db/urls/report.py +0 -48
  447. endoreg_db/urls/video.py +0 -61
  448. endoreg_db/utils/case_generator/case_generator.py +0 -159
  449. endoreg_db/utils/case_generator/utils.py +0 -30
  450. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +0 -368
  451. endoreg_db/views/label/__init__.py +0 -5
  452. endoreg_db/views/label/label.py +0 -15
  453. endoreg_db/views/label_video_segment/__init__.py +0 -16
  454. endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +0 -44
  455. endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +0 -50
  456. endoreg_db/views/label_video_segment/label_video_segment.py +0 -77
  457. endoreg_db/views/label_video_segment/label_video_segment_by_label.py +0 -174
  458. endoreg_db/views/label_video_segment/label_video_segment_detail.py +0 -73
  459. endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +0 -46
  460. endoreg_db/views/label_video_segment/validate.py +0 -226
  461. endoreg_db/views/media/segments.py +0 -71
  462. endoreg_db/views/meta/available_files_list.py +0 -146
  463. endoreg_db/views/meta/report_meta.py +0 -53
  464. endoreg_db/views/meta/sensitive_meta_detail.py +0 -148
  465. endoreg_db/views/misc/secure_file_serving_view.py +0 -80
  466. endoreg_db/views/misc/secure_file_url_view.py +0 -84
  467. endoreg_db/views/misc/secure_url_validate.py +0 -79
  468. endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +0 -164
  469. endoreg_db/views/patient_finding_location/__init__.py +0 -5
  470. endoreg_db/views/patient_finding_location/pfl_create.py +0 -70
  471. endoreg_db/views/patient_finding_morphology/__init__.py +0 -5
  472. endoreg_db/views/patient_finding_morphology/pfm_create.py +0 -70
  473. endoreg_db/views/pdf/__init__.py +0 -8
  474. endoreg_db/views/report/report_list.py +0 -112
  475. endoreg_db/views/report/report_with_secure_url.py +0 -28
  476. endoreg_db/views/report/start_examination.py +0 -7
  477. endoreg_db/views/video/segmentation.py +0 -274
  478. endoreg_db/views/video/task_status.py +0 -49
  479. endoreg_db/views/video/timeline.py +0 -46
  480. endoreg_db/views/video/video_analyze.py +0 -52
  481. endoreg_db/views.py +0 -0
  482. /endoreg_db/data/requirement/{colonoscopy_baseline_austria.yaml → old/colonoscopy_baseline_austria.yaml} +0 -0
  483. /endoreg_db/data/requirement/{disease_cardiovascular.yaml → old/disease_cardiovascular.yaml} +0 -0
  484. /endoreg_db/data/requirement/{disease_classification_choice_cardiovascular.yaml → old/disease_classification_choice_cardiovascular.yaml} +0 -0
  485. /endoreg_db/data/requirement/{disease_hepatology.yaml → old/disease_hepatology.yaml} +0 -0
  486. /endoreg_db/data/requirement/{disease_misc.yaml → old/disease_misc.yaml} +0 -0
  487. /endoreg_db/data/requirement/{disease_renal.yaml → old/disease_renal.yaml} +0 -0
  488. /endoreg_db/data/requirement/{endoscopy_bleeding_risk.yaml → old/endoscopy_bleeding_risk.yaml} +0 -0
  489. /endoreg_db/data/requirement/{event_cardiology.yaml → old/event_cardiology.yaml} +0 -0
  490. /endoreg_db/data/requirement/{event_requirements.yaml → old/event_requirements.yaml} +0 -0
  491. /endoreg_db/data/requirement/{finding_colon_polyp.yaml → old/finding_colon_polyp.yaml} +0 -0
  492. /endoreg_db/{migrations/__init__.py → data/requirement/old/gender.yaml} +0 -0
  493. /endoreg_db/data/requirement/{lab_value.yaml → old/lab_value.yaml} +0 -0
  494. /endoreg_db/data/requirement/{medication.yaml → old/medication.yaml} +0 -0
  495. /endoreg_db/data/requirement_operator/{age.yaml → _old/age.yaml} +0 -0
  496. /endoreg_db/data/requirement_operator/{lab_operators.yaml → _old/lab_operators.yaml} +0 -0
  497. /endoreg_db/data/requirement_operator/{model_operators.yaml → _old/model_operators.yaml} +0 -0
  498. /endoreg_db/{models/media/video/refactor_plan.md → import_files/pseudonymization/__init__.py} +0 -0
  499. /endoreg_db/{models/media/video/video_file_frames.py → import_files/pseudonymization/pseudonymize.py} +0 -0
  500. /endoreg_db/models/{metadata/frame_ocr_result.py → report/__init__.py} +0 -0
  501. /endoreg_db/{urls/sensitive_meta.py → models/report/images.py} +0 -0
  502. /endoreg_db/utils/requirement_operator_logic/{lab_value_operators.py → _old/lab_value_operators.py} +0 -0
  503. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/licenses/LICENSE +0 -0
@@ -1,114 +1,121 @@
1
+ import logging
1
2
  import os
2
3
  import shutil
3
- import logging
4
- from pathlib import Path
5
4
  from datetime import datetime, timedelta
6
- from django.core.management.base import BaseCommand, CommandError
5
+ from pathlib import Path
6
+
7
7
  from django.conf import settings
8
+ from django.core.management.base import BaseCommand, CommandError
9
+
8
10
  from endoreg_db.models import VideoFile
9
11
 
10
12
  logger = logging.getLogger(__name__)
11
13
 
12
14
 
13
15
  class Command(BaseCommand):
14
- help = 'Automated storage management and cleanup to prevent disk space issues'
16
+ help = "Automated storage management and cleanup to prevent disk space issues"
15
17
 
16
18
  def add_arguments(self, parser):
17
19
  parser.add_argument(
18
- '--dry-run',
19
- action='store_true',
20
- help='Show what would be cleaned without actually deleting files',
20
+ "--dry-run",
21
+ action="store_true",
22
+ help="Show what would be cleaned without actually deleting files",
21
23
  )
22
24
  parser.add_argument(
23
- '--force',
24
- action='store_true',
25
- help='Force cleanup even if not critically low on space',
25
+ "--force",
26
+ action="store_true",
27
+ help="Force cleanup even if not critically low on space",
26
28
  )
27
29
  parser.add_argument(
28
- '--cleanup-frames',
29
- action='store_true',
30
- help='Clean up extracted video frames after processing',
30
+ "--cleanup-frames",
31
+ action="store_true",
32
+ help="Clean up extracted video frames after processing",
31
33
  )
32
34
  parser.add_argument(
33
- '--cleanup-old-videos',
34
- action='store_true',
35
- help='Clean up old processed videos (keep raw files)',
35
+ "--cleanup-old-videos",
36
+ action="store_true",
37
+ help="Clean up old processed videos (keep raw files)",
36
38
  )
37
39
  parser.add_argument(
38
- '--cleanup-uploads',
39
- action='store_true',
40
- help='Clean up old upload cache files',
40
+ "--cleanup-uploads",
41
+ action="store_true",
42
+ help="Clean up old upload cache files",
41
43
  )
42
44
  parser.add_argument(
43
- '--cleanup-logs',
44
- action='store_true',
45
- help='Clean up old log files',
45
+ "--cleanup-logs",
46
+ action="store_true",
47
+ help="Clean up old log files",
46
48
  )
47
49
  parser.add_argument(
48
- '--max-age-days',
50
+ "--max-age-days",
49
51
  type=int,
50
52
  default=30,
51
- help='Maximum age in days for files to keep (default: 30)',
53
+ help="Maximum age in days for files to keep (default: 30)",
52
54
  )
53
55
  parser.add_argument(
54
- '--emergency-threshold',
56
+ "--emergency-threshold",
55
57
  type=float,
56
58
  default=95.0,
57
- help='Storage usage percentage that triggers emergency cleanup (default: 95%%)',
59
+ help="Storage usage percentage that triggers emergency cleanup (default: 95%%)",
58
60
  )
59
61
 
60
62
  def handle(self, *args, **options):
61
63
  """Main command handler for storage management."""
62
- self.dry_run = options['dry_run']
63
- self.force = options['force']
64
- self.max_age_days = options['max_age_days']
65
- self.emergency_threshold = options['emergency_threshold']
64
+ self.dry_run = options["dry_run"]
65
+ self.force = options["force"]
66
+ self.max_age_days = options["max_age_days"]
67
+ self.emergency_threshold = options["emergency_threshold"]
66
68
 
67
69
  # Validate emergency_threshold range
68
70
  if not (0 <= self.emergency_threshold <= 100):
69
- raise CommandError("The --emergency-threshold value must be between 0 and 100 (inclusive).")
71
+ raise CommandError(
72
+ "The --emergency-threshold value must be between 0 and 100 (inclusive)."
73
+ )
70
74
 
71
75
  if self.dry_run:
72
- self.stdout.write(self.style.WARNING('DRY RUN MODE - No files will be deleted'))
73
-
76
+ self.stdout.write(
77
+ self.style.WARNING("DRY RUN MODE - No files will be deleted")
78
+ )
79
+
74
80
  try:
75
81
  # Check current storage status
76
82
  storage_info = self.get_storage_info()
77
83
  self.display_storage_status(storage_info)
78
-
84
+
79
85
  # Determine if emergency cleanup is needed
80
86
  needs_emergency_cleanup = (
81
- storage_info['usage_percent'] >= self.emergency_threshold
82
- or self.force
87
+ storage_info["usage_percent"] >= self.emergency_threshold or self.force
83
88
  )
84
-
89
+
85
90
  if needs_emergency_cleanup:
86
91
  self.stdout.write(
87
92
  self.style.ERROR(
88
- f'🚨 EMERGENCY CLEANUP TRIGGERED - Storage at {storage_info["usage_percent"]:.1f}%'
93
+ f"🚨 EMERGENCY CLEANUP TRIGGERED - Storage at {storage_info['usage_percent']:.1f}%"
89
94
  )
90
95
  )
91
96
  self.emergency_cleanup(options)
92
97
  else:
93
98
  self.stdout.write(
94
99
  self.style.SUCCESS(
95
- f'✅ Storage usage is acceptable at {storage_info["usage_percent"]:.1f}%'
100
+ f"✅ Storage usage is acceptable at {storage_info['usage_percent']:.1f}%"
96
101
  )
97
102
  )
98
-
103
+
99
104
  # Always run maintenance cleanup if requested
100
- if any([
101
- options['cleanup_frames'],
102
- options['cleanup_old_videos'],
103
- options['cleanup_uploads'],
104
- options['cleanup_logs']
105
- ]):
105
+ if any(
106
+ [
107
+ options["cleanup_frames"],
108
+ options["cleanup_old_videos"],
109
+ options["cleanup_uploads"],
110
+ options["cleanup_logs"],
111
+ ]
112
+ ):
106
113
  self.maintenance_cleanup(options)
107
-
114
+
108
115
  # Show final storage status
109
116
  final_storage = self.get_storage_info()
110
117
  self.display_cleanup_summary(storage_info, final_storage)
111
-
118
+
112
119
  except Exception as e:
113
120
  logger.error(f"Storage management failed: {e}")
114
121
  raise CommandError(f"Storage management failed: {e}")
@@ -117,21 +124,21 @@ class Command(BaseCommand):
117
124
  """Get current storage information."""
118
125
  try:
119
126
  # Get storage stats for the root filesystem
120
- total, used, free = shutil.disk_usage('/')
127
+ total, used, free = shutil.disk_usage("/")
121
128
  usage_percent = (used / total) * 100
122
-
129
+
123
130
  # Get project-specific storage info
124
131
  project_root = Path(settings.BASE_DIR).parent
125
132
  project_storage = self.get_directory_size(project_root)
126
-
133
+
127
134
  return {
128
- 'total_gb': total / (1024**3),
129
- 'used_gb': used / (1024**3),
130
- 'free_gb': free / (1024**3),
131
- 'usage_percent': usage_percent,
132
- 'project_storage_gb': project_storage / (1024**3),
133
- 'critical': usage_percent >= 95.0,
134
- 'warning': usage_percent >= 85.0,
135
+ "total_gb": total / (1024**3),
136
+ "used_gb": used / (1024**3),
137
+ "free_gb": free / (1024**3),
138
+ "usage_percent": usage_percent,
139
+ "project_storage_gb": project_storage / (1024**3),
140
+ "critical": usage_percent >= 95.0,
141
+ "warning": usage_percent >= 85.0,
135
142
  }
136
143
  except Exception as e:
137
144
  logger.error(f"Failed to get storage info: {e}")
@@ -155,350 +162,397 @@ class Command(BaseCommand):
155
162
  def display_storage_status(self, storage_info):
156
163
  """Display current storage status."""
157
164
  status_color = (
158
- self.style.ERROR if storage_info['critical'] else
159
- self.style.WARNING if storage_info['warning'] else
160
- self.style.SUCCESS
165
+ self.style.ERROR
166
+ if storage_info["critical"]
167
+ else self.style.WARNING
168
+ if storage_info["warning"]
169
+ else self.style.SUCCESS
161
170
  )
162
-
163
- self.stdout.write("\n" + "="*60)
171
+
172
+ self.stdout.write("\n" + "=" * 60)
164
173
  self.stdout.write(status_color("💾 STORAGE STATUS"))
165
- self.stdout.write("="*60)
174
+ self.stdout.write("=" * 60)
166
175
  self.stdout.write(f"Total Space: {storage_info['total_gb']:.1f} GB")
167
176
  self.stdout.write(f"Used Space: {storage_info['used_gb']:.1f} GB")
168
177
  self.stdout.write(f"Free Space: {storage_info['free_gb']:.1f} GB")
169
- self.stdout.write(status_color(f"Usage: {storage_info['usage_percent']:.1f}%"))
178
+ self.stdout.write(
179
+ status_color(f"Usage: {storage_info['usage_percent']:.1f}%")
180
+ )
170
181
  self.stdout.write(f"Project Size: {storage_info['project_storage_gb']:.1f} GB")
171
-
172
- if storage_info['critical']:
182
+
183
+ if storage_info["critical"]:
173
184
  self.stdout.write(self.style.ERROR("🚨 CRITICAL: Storage critically low!"))
174
- elif storage_info['warning']:
185
+ elif storage_info["warning"]:
175
186
  self.stdout.write(self.style.WARNING("⚠️ WARNING: Storage getting low"))
176
187
 
177
188
  def emergency_cleanup(self, options):
178
189
  """Perform emergency cleanup to free critical storage space."""
179
190
  self.stdout.write(self.style.ERROR("\n🚨 PERFORMING EMERGENCY CLEANUP"))
180
-
191
+
181
192
  total_freed = 0
182
-
193
+
183
194
  # 1. AGGRESSIVE: Clean up ALL extracted frames (usually largest space saver)
184
195
  frames_freed = self.cleanup_all_extracted_frames()
185
196
  total_freed += frames_freed
186
-
197
+
187
198
  # 2. AGGRESSIVE: Clean up ALL upload cache
188
199
  uploads_freed = self.cleanup_all_uploads()
189
200
  total_freed += uploads_freed
190
-
201
+
191
202
  # 3. Clean up old logs
192
203
  logs_freed = self.cleanup_old_logs()
193
204
  total_freed += logs_freed
194
-
205
+
195
206
  # 4. Clean up temporary files
196
207
  temp_freed = self.cleanup_temp_files()
197
208
  total_freed += temp_freed
198
-
209
+
199
210
  # 5. If still critical, clean up ALL processed videos (more aggressive)
200
211
  storage_info = self.get_storage_info()
201
- if storage_info['usage_percent'] >= 90.0:
212
+ if storage_info["usage_percent"] >= 90.0:
202
213
  # Use 0 days to clean up ALL processed videos
203
214
  videos_freed = self.cleanup_all_processed_videos()
204
215
  total_freed += videos_freed
205
-
216
+
206
217
  self.stdout.write(
207
- self.style.SUCCESS(f"✅ Emergency cleanup completed: {total_freed / (1024**3):.2f} GB freed")
218
+ self.style.SUCCESS(
219
+ f"✅ Emergency cleanup completed: {total_freed / (1024**3):.2f} GB freed"
220
+ )
208
221
  )
209
222
 
210
223
  def maintenance_cleanup(self, options):
211
224
  """Perform regular maintenance cleanup."""
212
225
  self.stdout.write(self.style.SUCCESS("\n🔧 PERFORMING MAINTENANCE CLEANUP"))
213
-
226
+
214
227
  total_freed = 0
215
-
216
- if options['cleanup_frames']:
228
+
229
+ if options["cleanup_frames"]:
217
230
  freed = self.cleanup_extracted_frames()
218
231
  total_freed += freed
219
-
220
- if options['cleanup_old_videos']:
232
+
233
+ if options["cleanup_old_videos"]:
221
234
  freed = self.cleanup_old_processed_videos(self.max_age_days)
222
235
  total_freed += freed
223
-
224
- if options['cleanup_uploads']:
236
+
237
+ if options["cleanup_uploads"]:
225
238
  freed = self.cleanup_upload_cache()
226
239
  total_freed += freed
227
-
228
- if options['cleanup_logs']:
240
+
241
+ if options["cleanup_logs"]:
229
242
  freed = self.cleanup_old_logs()
230
243
  total_freed += freed
231
-
244
+
232
245
  self.stdout.write(
233
- self.style.SUCCESS(f"✅ Maintenance cleanup completed: {total_freed / (1024**3):.2f} GB freed")
246
+ self.style.SUCCESS(
247
+ f"✅ Maintenance cleanup completed: {total_freed / (1024**3):.2f} GB freed"
248
+ )
234
249
  )
235
250
 
236
251
  def cleanup_extracted_frames(self):
237
252
  """Clean up extracted video frames that are no longer needed."""
238
253
  self.stdout.write("🖼️ Cleaning up extracted video frames...")
239
-
254
+
240
255
  total_freed = 0
241
- frames_dir = Path(settings.BASE_DIR).parent / 'storage' / 'frames'
242
-
256
+ frames_dir = Path(settings.BASE_DIR).parent / "storage" / "frames"
257
+
243
258
  if not frames_dir.exists():
244
259
  return 0
245
-
260
+
246
261
  # Find videos that have completed processing
247
262
  completed_videos = VideoFile.objects.filter(
248
- anonymization_tasks__status='done'
263
+ anonymization_tasks__status="done_processing_anonymization"
249
264
  ).distinct()
250
-
265
+
251
266
  for video in completed_videos:
252
267
  try:
253
268
  # Find frame directories for this video
254
269
  video_frame_dirs = list(frames_dir.glob(f"*{video.uuid}*"))
255
-
270
+
256
271
  for frame_dir in video_frame_dirs:
257
272
  if frame_dir.is_dir():
258
273
  dir_size = self.get_directory_size(frame_dir)
259
-
274
+
260
275
  if not self.dry_run:
261
276
  shutil.rmtree(frame_dir, ignore_errors=True)
262
-
277
+
263
278
  total_freed += dir_size
264
- self.stdout.write(f" Removed frames for {video.uuid}: {dir_size / (1024**2):.1f} MB")
265
-
279
+ self.stdout.write(
280
+ f" Removed frames for {video.uuid}: {dir_size / (1024**2):.1f} MB"
281
+ )
282
+
266
283
  except Exception as e:
267
284
  logger.warning(f"Failed to clean frames for video {video.uuid}: {e}")
268
285
  continue
269
-
286
+
270
287
  self.stdout.write(f"✅ Frames cleanup: {total_freed / (1024**3):.2f} GB freed")
271
288
  return total_freed
272
289
 
273
290
  def cleanup_upload_cache(self):
274
291
  """Clean up old upload cache files."""
275
292
  self.stdout.write("📤 Cleaning up upload cache...")
276
-
293
+
277
294
  total_freed = 0
278
- uploads_dir = Path(settings.BASE_DIR).parent / 'storage' / 'uploads'
279
-
295
+ uploads_dir = Path(settings.BASE_DIR).parent / "storage" / "uploads"
296
+
280
297
  if not uploads_dir.exists():
281
298
  return 0
282
-
299
+
283
300
  cutoff_date = datetime.now() - timedelta(days=self.max_age_days)
284
-
285
- for file_path in uploads_dir.rglob('*'):
301
+
302
+ for file_path in uploads_dir.rglob("*"):
286
303
  if file_path.is_file():
287
304
  try:
288
305
  file_mtime = datetime.fromtimestamp(file_path.stat().st_mtime)
289
-
306
+
290
307
  if file_mtime < cutoff_date:
291
308
  file_size = file_path.stat().st_size
292
-
309
+
293
310
  if not self.dry_run:
294
311
  file_path.unlink()
295
-
312
+
296
313
  total_freed += file_size
297
-
314
+
298
315
  except Exception as e:
299
316
  logger.warning(f"Failed to clean upload file {file_path}: {e}")
300
317
  continue
301
-
302
- self.stdout.write(f"✅ Upload cache cleanup: {total_freed / (1024**3):.2f} GB freed")
318
+
319
+ self.stdout.write(
320
+ f"✅ Upload cache cleanup: {total_freed / (1024**3):.2f} GB freed"
321
+ )
303
322
  return total_freed
304
323
 
305
324
  def cleanup_old_logs(self):
306
325
  """Clean up old log files."""
307
326
  self.stdout.write("📋 Cleaning up old log files...")
308
-
327
+
309
328
  total_freed = 0
310
329
  project_root = Path(settings.BASE_DIR).parent
311
-
330
+
312
331
  # Find and clean large log files
313
- for log_file in project_root.rglob('*.log'):
332
+ for log_file in project_root.rglob("*.log"):
314
333
  try:
315
334
  if log_file.stat().st_size > 10 * 1024 * 1024: # Files larger than 10MB
316
335
  file_size = log_file.stat().st_size
317
-
336
+
318
337
  if not self.dry_run:
319
338
  # Truncate instead of delete to preserve file handles
320
- with open(log_file, 'w') as f:
321
- f.write('')
322
-
339
+ with open(log_file, "w") as f:
340
+ f.write("")
341
+
323
342
  total_freed += file_size
324
- self.stdout.write(f" Truncated {log_file}: {file_size / (1024**2):.1f} MB")
325
-
343
+ self.stdout.write(
344
+ f" Truncated {log_file}: {file_size / (1024**2):.1f} MB"
345
+ )
346
+
326
347
  except Exception as e:
327
348
  logger.warning(f"Failed to clean log file {log_file}: {e}")
328
349
  continue
329
-
350
+
330
351
  self.stdout.write(f"✅ Log cleanup: {total_freed / (1024**3):.2f} GB freed")
331
352
  return total_freed
332
353
 
333
354
  def cleanup_temp_files(self):
334
355
  """Clean up temporary files."""
335
356
  self.stdout.write("🗂️ Cleaning up temporary files...")
336
-
357
+
337
358
  total_freed = 0
338
359
  temp_dirs = [
339
- '/tmp',
340
- Path(settings.BASE_DIR).parent / 'data' / 'tmp',
341
- Path(settings.BASE_DIR).parent / 'storage' / 'tmp',
360
+ "/tmp",
361
+ Path(settings.BASE_DIR).parent / "data" / "tmp",
362
+ Path(settings.BASE_DIR).parent / "storage" / "tmp",
342
363
  ]
343
-
364
+
344
365
  for temp_dir in temp_dirs:
345
366
  if not Path(temp_dir).exists():
346
367
  continue
347
-
368
+
348
369
  try:
349
- for item in Path(temp_dir).glob('*'):
350
- if item.is_file() and item.name.startswith(('tmp', 'temp')):
370
+ for item in Path(temp_dir).glob("*"):
371
+ if item.is_file() and item.name.startswith(("tmp", "temp")):
351
372
  file_size = item.stat().st_size
352
-
373
+
353
374
  if not self.dry_run:
354
375
  item.unlink()
355
-
376
+
356
377
  total_freed += file_size
357
-
378
+
358
379
  except Exception as e:
359
380
  logger.warning(f"Failed to clean temp dir {temp_dir}: {e}")
360
381
  continue
361
-
362
- self.stdout.write(f"✅ Temp files cleanup: {total_freed / (1024**3):.2f} GB freed")
382
+
383
+ self.stdout.write(
384
+ f"✅ Temp files cleanup: {total_freed / (1024**3):.2f} GB freed"
385
+ )
363
386
  return total_freed
364
387
 
365
388
  def cleanup_old_processed_videos(self, max_age_days):
366
389
  """Clean up old processed videos while keeping raw files."""
367
- self.stdout.write(f"🎥 Cleaning up processed videos older than {max_age_days} days...")
368
-
390
+ self.stdout.write(
391
+ f"🎥 Cleaning up processed videos older than {max_age_days} days..."
392
+ )
393
+
369
394
  total_freed = 0
370
395
  cutoff_date = datetime.now() - timedelta(days=max_age_days)
371
-
396
+
372
397
  # Find old processed videos - fix the date filtering
373
398
  try:
374
399
  old_videos = VideoFile.objects.filter(
375
400
  created_at__lt=cutoff_date, # Use created_at instead of date_created
376
- processed_file__isnull=False
401
+ processed_file__isnull=False,
377
402
  ).exclude(
378
- anonymization_tasks__status__in=['processing_anonymization', 'extracting_frames']
403
+ anonymization_tasks__status__in=[
404
+ "processing_anonymization",
405
+ "extracting_frames",
406
+ ]
407
+ )
408
+
409
+ self.stdout.write(
410
+ f"Found {old_videos.count()} processed videos older than {max_age_days} days"
379
411
  )
380
-
381
- self.stdout.write(f"Found {old_videos.count()} processed videos older than {max_age_days} days")
382
-
412
+
383
413
  except Exception as e:
384
414
  # Fallback: try different date field names
385
415
  try:
386
416
  old_videos = VideoFile.objects.filter(
387
417
  processed_file__isnull=False
388
418
  ).exclude(
389
- anonymization_tasks__status__in=['processing_anonymization', 'extracting_frames']
419
+ anonymization_tasks__status__in=[
420
+ "processing_anonymization",
421
+ "extracting_frames",
422
+ ]
423
+ )
424
+ self.stdout.write(
425
+ f"Using fallback filter, found {old_videos.count()} processed videos"
390
426
  )
391
- self.stdout.write(f"Using fallback filter, found {old_videos.count()} processed videos")
392
427
  except Exception as e2:
393
428
  logger.error(f"Failed to query videos: {e2}")
394
429
  return total_freed
395
-
430
+
396
431
  for video in old_videos:
397
432
  try:
398
- if video.processed_file and hasattr(video.processed_file, 'path'):
433
+ if video.processed_file and hasattr(video.processed_file, "path"):
399
434
  processed_path = Path(video.processed_file.path)
400
-
435
+
401
436
  if processed_path.exists():
402
437
  file_size = processed_path.stat().st_size
403
-
438
+
404
439
  if not self.dry_run:
405
440
  processed_path.unlink()
406
441
  video.processed_file = None
407
- video.save(update_fields=['processed_file'])
408
-
442
+ video.save(update_fields=["processed_file"])
443
+
409
444
  total_freed += file_size
410
- self.stdout.write(f" Removed processed video {video.uuid}: {file_size / (1024**2):.1f} MB")
411
-
445
+ self.stdout.write(
446
+ f" Removed processed video {video.uuid}: {file_size / (1024**2):.1f} MB"
447
+ )
448
+
412
449
  except Exception as e:
413
450
  logger.warning(f"Failed to clean processed video {video.uuid}: {e}")
414
451
  continue
415
-
416
- self.stdout.write(f"✅ Processed videos cleanup: {total_freed / (1024**3):.2f} GB freed")
452
+
453
+ self.stdout.write(
454
+ f"✅ Processed videos cleanup: {total_freed / (1024**3):.2f} GB freed"
455
+ )
417
456
  return total_freed
418
457
 
419
458
  def cleanup_all_extracted_frames(self):
420
459
  """More aggressive cleanup - remove ALL extracted frames regardless of status."""
421
460
  self.stdout.write("🖼️ AGGRESSIVE: Cleaning up ALL extracted video frames...")
422
-
461
+
423
462
  total_freed = 0
424
- frames_dir = Path(settings.BASE_DIR).parent / 'storage' / 'frames'
425
-
463
+ frames_dir = Path(settings.BASE_DIR).parent / "storage" / "frames"
464
+
426
465
  if not frames_dir.exists():
427
466
  self.stdout.write("No frames directory found")
428
467
  return 0
429
-
468
+
430
469
  try:
431
470
  # Get directory size before cleanup
432
471
  initial_size = self.get_directory_size(frames_dir)
433
- self.stdout.write(f"Found frames directory with {initial_size / (1024**3):.2f} GB")
434
-
472
+ self.stdout.write(
473
+ f"Found frames directory with {initial_size / (1024**3):.2f} GB"
474
+ )
475
+
435
476
  # Remove all frame directories and files
436
477
  for item in frames_dir.iterdir():
437
478
  if item.is_dir():
438
479
  dir_size = self.get_directory_size(item)
439
-
480
+
440
481
  if not self.dry_run:
441
482
  shutil.rmtree(item, ignore_errors=True)
442
-
483
+
443
484
  total_freed += dir_size
444
- self.stdout.write(f" Removed frame directory {item.name}: {dir_size / (1024**2):.1f} MB")
485
+ self.stdout.write(
486
+ f" Removed frame directory {item.name}: {dir_size / (1024**2):.1f} MB"
487
+ )
445
488
  elif item.is_file():
446
489
  file_size = item.stat().st_size
447
-
490
+
448
491
  if not self.dry_run:
449
492
  item.unlink()
450
-
493
+
451
494
  total_freed += file_size
452
- self.stdout.write(f" Removed frame file {item.name}: {file_size / (1024**2):.1f} MB")
453
-
495
+ self.stdout.write(
496
+ f" Removed frame file {item.name}: {file_size / (1024**2):.1f} MB"
497
+ )
498
+
454
499
  except Exception as e:
455
500
  logger.error(f"Failed to clean frames directory: {e}")
456
-
457
- self.stdout.write(f"✅ Aggressive frames cleanup: {total_freed / (1024**3):.2f} GB freed")
501
+
502
+ self.stdout.write(
503
+ f"✅ Aggressive frames cleanup: {total_freed / (1024**3):.2f} GB freed"
504
+ )
458
505
  return total_freed
459
506
 
460
507
  def cleanup_all_uploads(self):
461
508
  """More aggressive cleanup - remove ALL upload cache files."""
462
509
  self.stdout.write("📤 AGGRESSIVE: Cleaning up ALL upload cache...")
463
-
510
+
464
511
  total_freed = 0
465
- uploads_dir = Path(settings.BASE_DIR).parent / 'storage' / 'uploads'
466
-
512
+ uploads_dir = Path(settings.BASE_DIR).parent / "storage" / "uploads"
513
+
467
514
  if not uploads_dir.exists():
468
515
  return 0
469
-
516
+
470
517
  try:
471
- for item in uploads_dir.rglob('*'):
518
+ for item in uploads_dir.rglob("*"):
472
519
  if item.is_file():
473
520
  file_size = item.stat().st_size
474
-
521
+
475
522
  if not self.dry_run:
476
523
  item.unlink()
477
-
524
+
478
525
  total_freed += file_size
479
-
526
+
480
527
  except Exception as e:
481
528
  logger.error(f"Failed to clean uploads directory: {e}")
482
-
483
- self.stdout.write(f"✅ Aggressive upload cleanup: {total_freed / (1024**3):.2f} GB freed")
529
+
530
+ self.stdout.write(
531
+ f"✅ Aggressive upload cleanup: {total_freed / (1024**3):.2f} GB freed"
532
+ )
484
533
  return total_freed
485
534
 
486
535
  def cleanup_all_processed_videos(self):
487
536
  """AGGRESSIVE: Clean up ALL processed videos while keeping raw files."""
488
537
  self.stdout.write("🎥 AGGRESSIVE: Cleaning up ALL processed videos...")
489
-
538
+
490
539
  total_freed = 0
491
-
540
+
492
541
  try:
493
542
  # Find ALL processed videos
494
543
  processed_videos = VideoFile.objects.filter(
495
544
  processed_file__isnull=False
496
545
  ).exclude(
497
- anonymization_tasks__status__in=['processing_anonymization', 'extracting_frames']
546
+ anonymization_tasks__status__in=[
547
+ "processing_anonymization",
548
+ "extracting_frames",
549
+ ]
550
+ )
551
+
552
+ self.stdout.write(
553
+ f"Found {processed_videos.count()} processed videos to clean"
498
554
  )
499
-
500
- self.stdout.write(f"Found {processed_videos.count()} processed videos to clean")
501
-
555
+
502
556
  for video in processed_videos:
503
557
  try:
504
558
  freed = self._cleanup_processed_video_file(video)
@@ -506,43 +560,57 @@ class Command(BaseCommand):
506
560
  except Exception as e:
507
561
  logger.warning(f"Failed to clean processed video {video.uuid}: {e}")
508
562
  continue
509
-
563
+
510
564
  except Exception as e:
511
565
  logger.error(f"Failed to query processed videos: {e}")
512
-
513
- self.stdout.write(f"✅ Aggressive processed videos cleanup: {total_freed / (1024**3):.2f} GB freed")
566
+
567
+ self.stdout.write(
568
+ f"✅ Aggressive processed videos cleanup: {total_freed / (1024**3):.2f} GB freed"
569
+ )
514
570
  return total_freed
515
571
 
516
572
  def _cleanup_processed_video_file(self, video):
517
573
  """
518
574
  Helper to clean up a single processed video file, update DB, and return freed size in bytes.
519
575
  """
520
- if video.processed_file and hasattr(video.processed_file, 'path'):
576
+ if video.processed_file and hasattr(video.processed_file, "path"):
521
577
  processed_path = Path(video.processed_file.path)
522
578
  if processed_path.exists():
523
579
  file_size = processed_path.stat().st_size
524
580
  if not self.dry_run:
525
581
  processed_path.unlink()
526
582
  video.processed_file = None
527
- video.save(update_fields=['processed_file'])
528
- self.stdout.write(f" Removed processed video {video.uuid}: {file_size / (1024**2):.1f} MB")
583
+ video.save(update_fields=["processed_file"])
584
+ self.stdout.write(
585
+ f" Removed processed video {video.uuid}: {file_size / (1024**2):.1f} MB"
586
+ )
529
587
  return file_size
530
588
  return 0
531
589
 
532
590
  def display_cleanup_summary(self, before, after):
533
591
  """Display cleanup summary."""
534
- freed_gb = before['used_gb'] - after['used_gb']
535
-
536
- self.stdout.write("\n" + "="*60)
592
+ freed_gb = before["used_gb"] - after["used_gb"]
593
+
594
+ self.stdout.write("\n" + "=" * 60)
537
595
  self.stdout.write(self.style.SUCCESS("📊 CLEANUP SUMMARY"))
538
- self.stdout.write("="*60)
539
- self.stdout.write(f"Before: {before['used_gb']:.1f} GB used ({before['usage_percent']:.1f}%)")
540
- self.stdout.write(f"After: {after['used_gb']:.1f} GB used ({after['usage_percent']:.1f}%)")
596
+ self.stdout.write("=" * 60)
597
+ self.stdout.write(
598
+ f"Before: {before['used_gb']:.1f} GB used ({before['usage_percent']:.1f}%)"
599
+ )
600
+ self.stdout.write(
601
+ f"After: {after['used_gb']:.1f} GB used ({after['usage_percent']:.1f}%)"
602
+ )
541
603
  self.stdout.write(self.style.SUCCESS(f"Freed: {freed_gb:.2f} GB"))
542
-
543
- if after['usage_percent'] < 90.0:
604
+
605
+ if after["usage_percent"] < 90.0:
544
606
  self.stdout.write(self.style.SUCCESS("✅ Storage levels are now healthy!"))
545
- elif after['usage_percent'] < 95.0:
546
- self.stdout.write(self.style.WARNING("⚠️ Storage levels improved but still high"))
607
+ elif after["usage_percent"] < 95.0:
608
+ self.stdout.write(
609
+ self.style.WARNING("⚠️ Storage levels improved but still high")
610
+ )
547
611
  else:
548
- self.stdout.write(self.style.ERROR("🚨 Storage levels still critical - manual intervention needed"))
612
+ self.stdout.write(
613
+ self.style.ERROR(
614
+ "🚨 Storage levels still critical - manual intervention needed"
615
+ )
616
+ )