endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (372) hide show
  1. endoreg_db/authz/auth.py +74 -0
  2. endoreg_db/authz/backends.py +168 -0
  3. endoreg_db/authz/management/commands/list_routes.py +18 -0
  4. endoreg_db/authz/middleware.py +83 -0
  5. endoreg_db/authz/permissions.py +127 -0
  6. endoreg_db/authz/policy.py +218 -0
  7. endoreg_db/authz/views_auth.py +66 -0
  8. endoreg_db/config/env.py +13 -8
  9. endoreg_db/data/__init__.py +8 -31
  10. endoreg_db/data/_examples/disease.yaml +55 -0
  11. endoreg_db/data/_examples/disease_classification.yaml +13 -0
  12. endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
  13. endoreg_db/data/_examples/event.yaml +64 -0
  14. endoreg_db/data/_examples/examination.yaml +72 -0
  15. endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
  16. endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
  17. endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
  18. endoreg_db/data/_examples/finding/complication.yaml +16 -0
  19. endoreg_db/data/_examples/finding/data.yaml +105 -0
  20. endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
  21. endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
  22. endoreg_db/data/_examples/finding/outcome.yaml +12 -0
  23. endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
  24. endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
  25. endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
  26. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  27. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  28. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  29. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  30. endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
  31. endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
  32. endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
  33. endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
  34. endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
  35. endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
  36. endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
  37. endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
  38. endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
  39. endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
  40. endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
  41. endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
  42. endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
  43. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
  44. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  45. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  46. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  47. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
  48. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
  49. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
  50. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  51. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
  52. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
  53. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  54. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
  55. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  56. endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
  57. endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  58. endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
  59. endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
  60. endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
  61. endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
  62. endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
  63. endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
  64. endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
  65. endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
  66. endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
  67. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
  68. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
  69. endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
  70. endoreg_db/data/_examples/finding_type/data.yaml +43 -0
  71. endoreg_db/data/_examples/requirement/age.yaml +26 -0
  72. endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
  73. endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
  74. endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
  75. endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
  76. endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
  77. endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
  78. endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
  79. endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
  80. endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
  81. endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
  82. endoreg_db/data/_examples/requirement/gender.yaml +25 -0
  83. endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
  84. endoreg_db/data/_examples/requirement/medication.yaml +93 -0
  85. endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
  86. endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
  87. endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
  88. endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
  89. endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  90. endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
  91. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
  92. endoreg_db/data/event_classification/data.yaml +4 -0
  93. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  94. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
  95. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
  96. endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
  97. endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
  98. endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
  99. endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
  100. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
  101. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  102. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  103. endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
  104. endoreg_db/data/requirement_set/_old_ +109 -0
  105. endoreg_db/data/requirement_set_type/data.yaml +21 -0
  106. endoreg_db/data/setup_config.yaml +4 -4
  107. endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
  108. endoreg_db/exceptions.py +5 -2
  109. endoreg_db/helpers/data_loader.py +1 -1
  110. endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
  111. endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
  112. endoreg_db/management/commands/import_video.py +9 -10
  113. endoreg_db/management/commands/import_video_with_classification.py +1 -1
  114. endoreg_db/management/commands/init_default_ai_model.py +1 -1
  115. endoreg_db/management/commands/list_routes.py +18 -0
  116. endoreg_db/management/commands/load_ai_model_data.py +2 -1
  117. endoreg_db/management/commands/load_center_data.py +12 -12
  118. endoreg_db/management/commands/load_requirement_data.py +60 -31
  119. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  120. endoreg_db/management/commands/setup_endoreg_db.py +14 -10
  121. endoreg_db/management/commands/storage_management.py +271 -203
  122. endoreg_db/migrations/0001_initial.py +1799 -1300
  123. endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
  124. endoreg_db/migrations/_old/0001_initial.py +1857 -0
  125. endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
  126. endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
  127. endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
  128. endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
  129. endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
  130. endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
  131. endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
  132. endoreg_db/models/__init__.py +78 -123
  133. endoreg_db/models/administration/__init__.py +21 -42
  134. endoreg_db/models/administration/ai/active_model.py +2 -2
  135. endoreg_db/models/administration/ai/ai_model.py +7 -6
  136. endoreg_db/models/administration/case/__init__.py +1 -15
  137. endoreg_db/models/administration/case/case.py +3 -3
  138. endoreg_db/models/administration/case/case_template/__init__.py +2 -14
  139. endoreg_db/models/administration/case/case_template/case_template.py +2 -124
  140. endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
  141. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
  142. endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
  143. endoreg_db/models/administration/center/center.py +33 -19
  144. endoreg_db/models/administration/center/center_product.py +12 -9
  145. endoreg_db/models/administration/center/center_resource.py +25 -19
  146. endoreg_db/models/administration/center/center_shift.py +21 -17
  147. endoreg_db/models/administration/center/center_waste.py +16 -8
  148. endoreg_db/models/administration/person/__init__.py +2 -0
  149. endoreg_db/models/administration/person/employee/employee.py +10 -5
  150. endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
  151. endoreg_db/models/administration/person/employee/employee_type.py +12 -6
  152. endoreg_db/models/administration/person/examiner/examiner.py +13 -11
  153. endoreg_db/models/administration/person/patient/__init__.py +2 -0
  154. endoreg_db/models/administration/person/patient/patient.py +103 -100
  155. endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
  156. endoreg_db/models/administration/person/person.py +4 -0
  157. endoreg_db/models/administration/person/profession/__init__.py +8 -4
  158. endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
  159. endoreg_db/models/administration/product/product.py +20 -15
  160. endoreg_db/models/administration/product/product_material.py +17 -18
  161. endoreg_db/models/administration/product/product_weight.py +12 -8
  162. endoreg_db/models/administration/product/reference_product.py +23 -55
  163. endoreg_db/models/administration/qualification/qualification.py +7 -3
  164. endoreg_db/models/administration/qualification/qualification_type.py +7 -3
  165. endoreg_db/models/administration/shift/scheduled_days.py +8 -5
  166. endoreg_db/models/administration/shift/shift.py +16 -12
  167. endoreg_db/models/administration/shift/shift_type.py +23 -31
  168. endoreg_db/models/label/__init__.py +7 -8
  169. endoreg_db/models/label/annotation/image_classification.py +10 -9
  170. endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
  171. endoreg_db/models/label/label.py +15 -15
  172. endoreg_db/models/label/label_set.py +19 -6
  173. endoreg_db/models/label/label_type.py +1 -1
  174. endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
  175. endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
  176. endoreg_db/models/label/video_segmentation_label.py +4 -0
  177. endoreg_db/models/label/video_segmentation_labelset.py +4 -3
  178. endoreg_db/models/media/frame/frame.py +22 -22
  179. endoreg_db/models/media/pdf/raw_pdf.py +249 -177
  180. endoreg_db/models/media/pdf/report_file.py +25 -29
  181. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
  182. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
  183. endoreg_db/models/media/video/__init__.py +1 -0
  184. endoreg_db/models/media/video/create_from_file.py +48 -56
  185. endoreg_db/models/media/video/pipe_1.py +30 -33
  186. endoreg_db/models/media/video/pipe_2.py +8 -9
  187. endoreg_db/models/media/video/video_file.py +359 -204
  188. endoreg_db/models/media/video/video_file_ai.py +288 -74
  189. endoreg_db/models/media/video/video_file_anonymize.py +38 -38
  190. endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
  191. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
  192. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
  193. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
  194. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
  195. endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
  196. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
  197. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
  198. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
  199. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
  200. endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
  201. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
  202. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
  203. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
  204. endoreg_db/models/media/video/video_file_io.py +109 -62
  205. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
  206. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
  207. endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
  208. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
  209. endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
  210. endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
  211. endoreg_db/models/media/video/video_file_segments.py +24 -17
  212. endoreg_db/models/media/video/video_metadata.py +19 -35
  213. endoreg_db/models/media/video/video_processing.py +96 -95
  214. endoreg_db/models/medical/contraindication/__init__.py +13 -3
  215. endoreg_db/models/medical/disease.py +22 -16
  216. endoreg_db/models/medical/event.py +31 -18
  217. endoreg_db/models/medical/examination/__init__.py +13 -6
  218. endoreg_db/models/medical/examination/examination.py +17 -18
  219. endoreg_db/models/medical/examination/examination_indication.py +26 -25
  220. endoreg_db/models/medical/examination/examination_time.py +16 -6
  221. endoreg_db/models/medical/examination/examination_time_type.py +9 -6
  222. endoreg_db/models/medical/examination/examination_type.py +3 -4
  223. endoreg_db/models/medical/finding/finding.py +38 -39
  224. endoreg_db/models/medical/finding/finding_classification.py +37 -48
  225. endoreg_db/models/medical/finding/finding_intervention.py +27 -22
  226. endoreg_db/models/medical/finding/finding_type.py +13 -12
  227. endoreg_db/models/medical/hardware/endoscope.py +20 -26
  228. endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
  229. endoreg_db/models/medical/laboratory/lab_value.py +62 -91
  230. endoreg_db/models/medical/medication/medication.py +22 -10
  231. endoreg_db/models/medical/medication/medication_indication.py +29 -3
  232. endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
  233. endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
  234. endoreg_db/models/medical/medication/medication_schedule.py +27 -16
  235. endoreg_db/models/medical/organ/__init__.py +15 -12
  236. endoreg_db/models/medical/patient/medication_examples.py +1 -5
  237. endoreg_db/models/medical/patient/patient_disease.py +20 -23
  238. endoreg_db/models/medical/patient/patient_event.py +19 -22
  239. endoreg_db/models/medical/patient/patient_examination.py +48 -54
  240. endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
  241. endoreg_db/models/medical/patient/patient_finding.py +122 -139
  242. endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
  243. endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
  244. endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
  245. endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
  246. endoreg_db/models/medical/patient/patient_medication.py +27 -38
  247. endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
  248. endoreg_db/models/medical/risk/risk.py +7 -6
  249. endoreg_db/models/medical/risk/risk_type.py +8 -5
  250. endoreg_db/models/metadata/model_meta.py +60 -29
  251. endoreg_db/models/metadata/model_meta_logic.py +139 -18
  252. endoreg_db/models/metadata/pdf_meta.py +19 -24
  253. endoreg_db/models/metadata/sensitive_meta.py +102 -85
  254. endoreg_db/models/metadata/sensitive_meta_logic.py +383 -43
  255. endoreg_db/models/metadata/video_meta.py +51 -31
  256. endoreg_db/models/metadata/video_prediction_logic.py +16 -23
  257. endoreg_db/models/metadata/video_prediction_meta.py +29 -33
  258. endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
  259. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
  260. endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
  261. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
  262. endoreg_db/models/other/emission/emission_factor.py +18 -8
  263. endoreg_db/models/other/gender.py +10 -5
  264. endoreg_db/models/other/information_source.py +25 -25
  265. endoreg_db/models/other/material.py +9 -5
  266. endoreg_db/models/other/resource.py +6 -4
  267. endoreg_db/models/other/tag.py +10 -5
  268. endoreg_db/models/other/transport_route.py +13 -8
  269. endoreg_db/models/other/unit.py +10 -6
  270. endoreg_db/models/other/waste.py +6 -5
  271. endoreg_db/models/requirement/requirement.py +580 -272
  272. endoreg_db/models/requirement/requirement_error.py +85 -0
  273. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  274. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
  275. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
  276. endoreg_db/models/requirement/requirement_operator.py +36 -33
  277. endoreg_db/models/requirement/requirement_set.py +74 -57
  278. endoreg_db/models/state/__init__.py +4 -4
  279. endoreg_db/models/state/abstract.py +2 -2
  280. endoreg_db/models/state/anonymization.py +12 -0
  281. endoreg_db/models/state/audit_ledger.py +46 -47
  282. endoreg_db/models/state/label_video_segment.py +9 -0
  283. endoreg_db/models/state/raw_pdf.py +40 -46
  284. endoreg_db/models/state/sensitive_meta.py +6 -2
  285. endoreg_db/models/state/video.py +58 -53
  286. endoreg_db/models/upload_job.py +32 -55
  287. endoreg_db/models/utils.py +1 -2
  288. endoreg_db/root_urls.py +21 -2
  289. endoreg_db/serializers/__init__.py +26 -57
  290. endoreg_db/serializers/anonymization.py +18 -10
  291. endoreg_db/serializers/meta/report_meta.py +1 -1
  292. endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
  293. endoreg_db/serializers/misc/__init__.py +1 -1
  294. endoreg_db/serializers/misc/file_overview.py +33 -91
  295. endoreg_db/serializers/misc/{vop_patient_data.py → sensitive_patient_data.py} +1 -1
  296. endoreg_db/serializers/requirements/requirement_sets.py +92 -22
  297. endoreg_db/serializers/video/segmentation.py +2 -1
  298. endoreg_db/serializers/video/video_processing_history.py +20 -5
  299. endoreg_db/serializers/video_examination.py +198 -0
  300. endoreg_db/services/anonymization.py +75 -73
  301. endoreg_db/services/lookup_service.py +256 -73
  302. endoreg_db/services/lookup_store.py +174 -30
  303. endoreg_db/services/pdf_import.py +711 -310
  304. endoreg_db/services/storage_aware_video_processor.py +140 -114
  305. endoreg_db/services/video_import.py +266 -117
  306. endoreg_db/urls/__init__.py +27 -27
  307. endoreg_db/urls/label_video_segments.py +2 -0
  308. endoreg_db/urls/media.py +108 -66
  309. endoreg_db/urls/root_urls.py +29 -0
  310. endoreg_db/utils/__init__.py +15 -5
  311. endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
  312. endoreg_db/utils/case_generator/__init__.py +3 -0
  313. endoreg_db/utils/dataloader.py +88 -16
  314. endoreg_db/utils/defaults/set_default_center.py +32 -0
  315. endoreg_db/utils/names.py +22 -16
  316. endoreg_db/utils/permissions.py +2 -1
  317. endoreg_db/utils/pipelines/process_video_dir.py +1 -1
  318. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
  319. endoreg_db/utils/setup_config.py +8 -5
  320. endoreg_db/utils/storage.py +115 -0
  321. endoreg_db/utils/validate_endo_roi.py +8 -2
  322. endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
  323. endoreg_db/views/__init__.py +5 -12
  324. endoreg_db/views/anonymization/media_management.py +198 -163
  325. endoreg_db/views/anonymization/overview.py +4 -1
  326. endoreg_db/views/anonymization/validate.py +174 -40
  327. endoreg_db/views/media/__init__.py +2 -0
  328. endoreg_db/views/media/pdf_media.py +131 -150
  329. endoreg_db/views/media/sensitive_metadata.py +46 -6
  330. endoreg_db/views/media/video_media.py +89 -82
  331. endoreg_db/views/media/video_segments.py +187 -260
  332. endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
  333. endoreg_db/views/patient/patient.py +5 -4
  334. endoreg_db/views/pdf/__init__.py +5 -8
  335. endoreg_db/views/pdf/pdf_stream.py +186 -0
  336. endoreg_db/views/pdf/pdf_stream_views.py +0 -127
  337. endoreg_db/views/pdf/reimport.py +86 -91
  338. endoreg_db/views/requirement/evaluate.py +188 -187
  339. endoreg_db/views/requirement/lookup.py +186 -288
  340. endoreg_db/views/requirement/requirement_utils.py +89 -0
  341. endoreg_db/views/video/__init__.py +0 -4
  342. endoreg_db/views/video/correction.py +2 -2
  343. endoreg_db/views/video/video_examination_viewset.py +202 -289
  344. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
  345. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +350 -255
  346. endoreg_db/models/administration/permissions/__init__.py +0 -44
  347. endoreg_db/models/media/video/refactor_plan.md +0 -0
  348. endoreg_db/models/media/video/video_file_frames.py +0 -0
  349. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  350. endoreg_db/models/rule/__init__.py +0 -13
  351. endoreg_db/models/rule/rule.py +0 -27
  352. endoreg_db/models/rule/rule_applicator.py +0 -224
  353. endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
  354. endoreg_db/models/rule/rule_type.py +0 -20
  355. endoreg_db/models/rule/ruleset.py +0 -17
  356. endoreg_db/serializers/video/video_metadata.py +0 -105
  357. endoreg_db/urls/report.py +0 -48
  358. endoreg_db/urls/video.py +0 -61
  359. endoreg_db/utils/case_generator/case_generator.py +0 -159
  360. endoreg_db/utils/case_generator/utils.py +0 -30
  361. endoreg_db/views/pdf/pdf_media.py +0 -239
  362. endoreg_db/views/report/__init__.py +0 -9
  363. endoreg_db/views/report/report_list.py +0 -112
  364. endoreg_db/views/report/report_with_secure_url.py +0 -28
  365. endoreg_db/views/report/start_examination.py +0 -7
  366. endoreg_db/views/video/video_media.py +0 -158
  367. endoreg_db/views.py +0 -0
  368. /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
  369. /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
  370. /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
  371. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
  372. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,14 +1,14 @@
1
- from tqdm import tqdm
2
- from pathlib import Path
3
- from typing import List
4
- from typing import TYPE_CHECKING, Optional
5
1
  import logging
6
- from django.db import OperationalError
7
2
  import time
3
+ from pathlib import Path
4
+ from typing import TYPE_CHECKING, List, Optional
5
+
6
+ from django.db import OperationalError
7
+ from tqdm import tqdm
8
8
 
9
9
  if TYPE_CHECKING:
10
10
  from endoreg_db.models import VideoFile
11
-
11
+
12
12
  logger = logging.getLogger(__name__)
13
13
 
14
14
 
@@ -35,7 +35,6 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
35
35
  from endoreg_db.models.media.video.video_file_frames._bulk_create_frames import _bulk_create_frames
36
36
  from endoreg_db.models.media.video.video_file_frames._create_frame_object import _create_frame_object
37
37
 
38
-
39
38
  frames_to_create = []
40
39
  num_expected_or_provided = 0
41
40
  mark_as_extracted = False
@@ -46,11 +45,9 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
46
45
  num_expected_or_provided = len(frame_paths)
47
46
  for frame_path in tqdm(frame_paths, desc=f"Initializing Frames from Paths {video.uuid}", unit="frame"):
48
47
  try:
49
- frame_number = int(frame_path.stem.split('_')[-1])
48
+ frame_number = int(frame_path.stem.split("_")[-1])
50
49
  relative_path_str = frame_path.name
51
- frames_to_create.append(
52
- _create_frame_object(video, frame_number, relative_path_str, extracted=mark_as_extracted)
53
- )
50
+ frames_to_create.append(_create_frame_object(video, frame_number, relative_path_str, extracted=mark_as_extracted))
54
51
  except (ValueError, IndexError) as e:
55
52
  logger.warning("Could not parse frame number from %s: %s", frame_path.name, e)
56
53
  continue
@@ -63,7 +60,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
63
60
  if state.frames_initialized or state.frame_count is not None:
64
61
  state.frames_initialized = False
65
62
  state.frame_count = None
66
- state.save(update_fields=['frames_initialized', 'frame_count'])
63
+ state.save(update_fields=["frames_initialized", "frame_count"])
67
64
  except Exception as state_e:
68
65
  logger.error("Failed to reset state during empty initialization for video %s: %s", video.uuid, state_e, exc_info=True)
69
66
  return
@@ -73,13 +70,10 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
73
70
  num_expected_or_provided = expected_frame_count
74
71
  for frame_number in tqdm(range(expected_frame_count), desc=f"Initializing Expected Frames {video.uuid}", unit="frame"):
75
72
  relative_path_str = f"frame_{frame_number:07d}.jpg"
76
- frames_to_create.append(
77
- _create_frame_object(video, frame_number, relative_path_str, extracted=mark_as_extracted)
78
- )
79
-
73
+ frames_to_create.append(_create_frame_object(video, frame_number, relative_path_str, extracted=mark_as_extracted))
74
+
80
75
  if frames_to_create:
81
76
  for attempt in range(5):
82
-
83
77
  try:
84
78
  _bulk_create_frames(video, frames_to_create)
85
79
  num_created_or_ignored = len(frames_to_create)
@@ -88,11 +82,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
88
82
  if mark_as_extracted:
89
83
  frame_numbers_to_update = [f.frame_number for f in frames_to_create]
90
84
  if frame_numbers_to_update:
91
- update_count = Frame.objects.filter(
92
- video=video,
93
- frame_number__in=frame_numbers_to_update,
94
- is_extracted=False
95
- ).update(is_extracted=True)
85
+ update_count = Frame.objects.filter(video=video, frame_number__in=frame_numbers_to_update, is_extracted=False).update(is_extracted=True)
96
86
  if update_count > 0:
97
87
  logger.info("Marked %d existing Frame objects as is_extracted=True for video %s.", update_count, video.uuid)
98
88
 
@@ -100,7 +90,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
100
90
  state = video.get_or_create_state()
101
91
  state.frames_initialized = True
102
92
  state.frame_count = num_expected_or_provided
103
- state.save(update_fields=['frames_initialized', 'frame_count'])
93
+ state.save(update_fields=["frames_initialized", "frame_count"])
104
94
  logger.info("Set frames_initialized=True and frame_count=%d for video %s.", num_expected_or_provided, video.uuid)
105
95
  except Exception as state_e:
106
96
  logger.error("Failed to update state after frame initialization for video %s: %s", video.uuid, state_e, exc_info=True)
@@ -109,7 +99,7 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
109
99
  except OperationalError as e:
110
100
  if "database is locked" in str(e):
111
101
  logger.warning("Database is locked, retrying frame initialization for video %s (attempt %d/5).", video.uuid, attempt + 1)
112
- time.sleep(2 ** attempt)
102
+ time.sleep(2**attempt)
113
103
  if attempt < 4:
114
104
  continue
115
105
  else:
@@ -124,6 +114,6 @@ def _initialize_frames(video: "VideoFile", frame_paths: Optional[List[Path]] = N
124
114
  if state.frames_initialized or state.frame_count is not None:
125
115
  state.frames_initialized = False
126
116
  state.frame_count = None
127
- state.save(update_fields=['frames_initialized', 'frame_count'])
117
+ state.save(update_fields=["frames_initialized", "frame_count"])
128
118
  except Exception as state_e:
129
119
  logger.error("Failed to reset state during empty initialization for video %s: %s", video.uuid, state_e, exc_info=True)
@@ -1,18 +1,20 @@
1
1
  import logging
2
2
  import os
3
3
  from typing import TYPE_CHECKING
4
+
4
5
  from django.db import transaction
5
6
 
7
+ from endoreg_db.models.media.video.video_file_io import _get_frame_dir_path
8
+
6
9
  # Assuming ffmpeg_wrapper has or will have this function
7
10
  from endoreg_db.utils.video.ffmpeg_wrapper import extract_frame_range as ffmpeg_extract_frame_range
8
11
 
9
- from endoreg_db.models.media.video.video_file_io import _get_frame_dir_path
10
-
11
12
  if TYPE_CHECKING:
12
13
  from endoreg_db.models import VideoFile
13
14
 
14
15
  logger = logging.getLogger(__name__)
15
16
 
17
+
16
18
  def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
17
19
  """
18
20
  Deletes frame image files within the specified range [start_frame, end_frame)
@@ -20,19 +22,14 @@ def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
20
22
  """
21
23
 
22
24
  logger.info("Deleting frame files for video %s in range [%d, %d)", video.uuid, start_frame, end_frame)
23
- frames_to_delete = video.frames.filter(
24
- frame_number__gte=start_frame,
25
- frame_number__lt=end_frame,
26
- is_extracted=True
27
- )
25
+ frames_to_delete = video.frames.filter(frame_number__gte=start_frame, frame_number__lt=end_frame, is_extracted=True)
28
26
 
29
27
  deleted_count = 0
30
28
  paths_to_delete = [frame.file_path for frame in frames_to_delete] # Get paths before potential DB changes
31
29
 
32
30
  # Update DB first
33
31
  update_count = frames_to_delete.update(is_extracted=False)
34
- logger.info("Marked %d Frame objects as is_extracted=False for video %s range [%d, %d).",
35
- update_count, video.uuid, start_frame, end_frame)
32
+ logger.info("Marked %d Frame objects as is_extracted=False for video %s range [%d, %d).", update_count, video.uuid, start_frame, end_frame)
36
33
 
37
34
  # Then delete files
38
35
  for frame_path in paths_to_delete:
@@ -44,10 +41,14 @@ def _delete_frame_range(video: "VideoFile", start_frame: int, end_frame: int):
44
41
  # Log warning but continue; DB state is already updated.
45
42
  logger.warning("Could not delete frame file %s for video %s: %s", frame_path, video.uuid, e)
46
43
 
47
- logger.info("Attempted deletion of %d files for video %s range [%d, %d). Actual deleted: %d.",
48
- len(paths_to_delete), video.uuid, start_frame, end_frame, deleted_count)
49
-
50
-
44
+ logger.info(
45
+ "Attempted deletion of %d files for video %s range [%d, %d). Actual deleted: %d.",
46
+ len(paths_to_delete),
47
+ video.uuid,
48
+ start_frame,
49
+ end_frame,
50
+ deleted_count,
51
+ )
51
52
 
52
53
 
53
54
  @transaction.atomic
@@ -79,10 +80,7 @@ def _extract_frame_range(
79
80
  raise ValueError(f"Cannot determine frame directory path for video {video.uuid}.")
80
81
 
81
82
  # Check frames within the range that might already exist
82
- frames_in_range = video.frames.filter(
83
- frame_number__gte=start_frame,
84
- frame_number__lt=end_frame
85
- )
83
+ frames_in_range = video.frames.filter(frame_number__gte=start_frame, frame_number__lt=end_frame)
86
84
  existing_extracted_in_range = frames_in_range.filter(is_extracted=True)
87
85
 
88
86
  if existing_extracted_in_range.exists():
@@ -90,7 +88,12 @@ def _extract_frame_range(
90
88
  logger.info("Overwrite=True, deleting existing frame files in range [%d, %d) for video %s before extraction.", start_frame, end_frame, video.uuid)
91
89
  _delete_frame_range(video, start_frame, end_frame)
92
90
  else:
93
- logger.info("Frames already exist in range [%d, %d) for video %s and overwrite=False. Skipping extraction for this range.", start_frame, end_frame, video.uuid)
91
+ logger.info(
92
+ "Frames already exist in range [%d, %d) for video %s and overwrite=False. Skipping extraction for this range.",
93
+ start_frame,
94
+ end_frame,
95
+ video.uuid,
96
+ )
94
97
  updated_count = frames_in_range.filter(is_extracted=False).update(is_extracted=True)
95
98
  if updated_count > 0:
96
99
  logger.info("Marked %d existing Frame objects in range [%d, %d) as extracted for video %s.", updated_count, start_frame, end_frame, video.uuid)
@@ -101,11 +104,11 @@ def _extract_frame_range(
101
104
 
102
105
  try:
103
106
  logger.info("Starting frame range extraction [%d, %d) for video %s to %s", start_frame, end_frame, video.uuid, frame_dir)
104
- extracted_paths = ffmpeg_extract_frame_range(
105
- raw_file_path, frame_dir, start_frame, end_frame, quality=quality, ext=ext
106
- )
107
+ extracted_paths = ffmpeg_extract_frame_range(raw_file_path, frame_dir, start_frame, end_frame, quality=quality, ext=ext)
107
108
 
108
- logger.info("ffmpeg extraction process completed for video %s range [%d, %d). Found %d files.", video.uuid, start_frame, end_frame, len(extracted_paths))
109
+ logger.info(
110
+ "ffmpeg extraction process completed for video %s range [%d, %d). Found %d files.", video.uuid, start_frame, end_frame, len(extracted_paths)
111
+ )
109
112
 
110
113
  update_count = frames_in_range.update(is_extracted=True)
111
114
  logger.info("Marked %d Frame objects in range [%d, %d) as is_extracted=True for video %s.", update_count, start_frame, end_frame, video.uuid)
@@ -127,7 +130,7 @@ def _extract_frame_range(
127
130
  logger.error("Frame range extraction [%d, %d) or DB update failed for video %s: %s", start_frame, end_frame, video.uuid, e, exc_info=True)
128
131
 
129
132
  logger.warning("Attempting file cleanup in range [%d, %d) for video %s due to extraction error.", start_frame, end_frame, video.uuid)
130
- files_to_check = extracted_paths if 'extracted_paths' in locals() and extracted_paths else []
133
+ files_to_check = extracted_paths if "extracted_paths" in locals() and extracted_paths else []
131
134
  if not files_to_check:
132
135
  files_to_check = [frame_dir / f"frame_{i:07d}.{ext}" for i in range(start_frame, end_frame)]
133
136
 
@@ -1,5 +1,6 @@
1
1
  import logging
2
2
  from typing import TYPE_CHECKING, Set
3
+
3
4
  from django.db import transaction
4
5
 
5
6
  if TYPE_CHECKING:
@@ -7,43 +8,49 @@ if TYPE_CHECKING:
7
8
 
8
9
  logger = logging.getLogger(__name__)
9
10
 
11
+
10
12
  @transaction.atomic
11
13
  def _mark_frames_extracted_status(video: "VideoFile", extracted_frame_numbers: Set[int], status: bool):
12
14
  """
13
15
  Bulk updates the is_extracted status for a set of frame numbers.
14
16
  """
15
17
  from endoreg_db.models.media.frame import Frame
18
+
16
19
  if not extracted_frame_numbers:
17
20
  logger.warning("No frame numbers provided to update status for video %s.", video.uuid)
18
21
  return 0
19
22
 
20
23
  # --- Enhanced Logging ---
21
- min_frame = min(extracted_frame_numbers) if extracted_frame_numbers else 'N/A'
22
- max_frame = max(extracted_frame_numbers) if extracted_frame_numbers else 'N/A'
24
+ min_frame = min(extracted_frame_numbers) if extracted_frame_numbers else "N/A"
25
+ max_frame = max(extracted_frame_numbers) if extracted_frame_numbers else "N/A"
23
26
  contains_zero = 0 in extracted_frame_numbers
24
27
  logger.info(
25
28
  "Attempting to mark %d Frame objects as is_extracted=%s for video %s. Frame numbers range: [%s-%s]. Contains frame 0: %s",
26
- len(extracted_frame_numbers), status, video.uuid, min_frame, max_frame, contains_zero
29
+ len(extracted_frame_numbers),
30
+ status,
31
+ video.uuid,
32
+ min_frame,
33
+ max_frame,
34
+ contains_zero,
27
35
  )
28
36
  # --- End Enhanced Logging ---
29
37
 
30
38
  try:
31
39
  # Update Frame objects based on frame_number
32
40
  # Convert set to list for potentially better compatibility with some DB backends
33
- updated_count = Frame.objects.filter(
34
- video=video,
35
- frame_number__in=list(extracted_frame_numbers)
36
- ).update(is_extracted=status)
41
+ updated_count = Frame.objects.filter(video=video, frame_number__in=list(extracted_frame_numbers)).update(is_extracted=status)
37
42
 
38
43
  logger.info("Database reported updating %d Frame objects to is_extracted=%s for video %s.", updated_count, status, video.uuid)
39
44
 
40
45
  # Verification step
41
46
  if updated_count != len(extracted_frame_numbers):
42
47
  logger.warning(
43
- "Mismatch during status update for video %s. Expected to update %d frames, but DB reported updating %d.",
44
- video.uuid, len(extracted_frame_numbers), updated_count
45
- )
46
- # --- Add detailed check for frame 0 if status is True and it should have been updated ---
48
+ "Mismatch during status update for video %s. Expected to update %d frames, but DB reported updating %d.",
49
+ video.uuid,
50
+ len(extracted_frame_numbers),
51
+ updated_count,
52
+ )
53
+ # --- Add detailed check for frame 0 if status is True and it should have been updated ---
47
54
  if status is True and contains_zero and updated_count < len(extracted_frame_numbers):
48
55
  try:
49
56
  # Check the status of frame 0 directly after the update attempt
@@ -52,14 +59,16 @@ def _mark_frames_extracted_status(video: "VideoFile", extracted_frame_numbers: S
52
59
  logger.error("Verification check: Frame 0 (PK: %s) was NOT updated to is_extracted=True for video %s.", frame_zero.pk, video.uuid)
53
60
  else:
54
61
  # This case should ideally not happen if updated_count < expected count, but log just in case
55
- logger.info("Verification check: Frame 0 (PK: %s) IS is_extracted=True for video %s, despite count mismatch.", frame_zero.pk, video.uuid)
62
+ logger.info(
63
+ "Verification check: Frame 0 (PK: %s) IS is_extracted=True for video %s, despite count mismatch.", frame_zero.pk, video.uuid
64
+ )
56
65
  except Frame.DoesNotExist:
57
66
  logger.error("Verification check: Frame 0 does not exist for video %s during status check.", video.uuid)
58
67
  except Exception as verify_e:
59
68
  logger.error("Verification check: Error checking frame 0 status for video %s: %s", video.uuid, verify_e)
60
- # --- End detailed check ---
69
+ # --- End detailed check ---
61
70
 
62
71
  return updated_count
63
72
  except Exception as e:
64
73
  logger.error("Failed to bulk update is_extracted status for video %s: %s", video.uuid, e, exc_info=True)
65
- raise # Re-raise to ensure transaction rollback if needed
74
+ raise # Re-raise to ensure transaction rollback if needed
@@ -1,75 +1,110 @@
1
1
  import logging
2
+ from contextlib import contextmanager
2
3
  from pathlib import Path
3
- from typing import TYPE_CHECKING, Optional
4
+ from typing import TYPE_CHECKING, Iterator, Optional
5
+
4
6
  from django.db import transaction
5
7
 
6
- from ...utils import data_paths, ANONYM_VIDEO_DIR, VIDEO_DIR # Import VIDEO_DIR for correct path resolution
8
+ from ....utils import delete_field_file, ensure_local_file, storage_file_exists
9
+ from ...utils import (
10
+ ANONYM_VIDEO_DIR,
11
+ VIDEO_DIR,
12
+ data_paths,
13
+ )
7
14
 
8
15
  if TYPE_CHECKING:
9
16
  from .video_file import VideoFile
10
17
 
11
18
  logger = logging.getLogger("video_file")
12
19
 
20
+
13
21
  def _get_raw_file_path(video: "VideoFile") -> Optional[Path]:
14
- """
15
- Resolves and returns the absolute path to the raw video file if available.
16
- The FileField stores a path relative to the storage root. We need to join
17
- that relative path onto the actual video directory under STORAGE_DIR.
18
- """
19
- try:
20
- if video.has_raw and video.raw_file.name:
21
- # raw_file.name is a relative storage path like 'videos/<filename>'
22
- raw_rel = Path(video.raw_file.name)
23
-
24
- # If it already contains the video directory name, keep the tail
25
- rel_name = raw_rel.name if raw_rel.parent.name == VIDEO_DIR.name else raw_rel
26
- full_path = data_paths["video"] / rel_name
27
-
28
- # If primary path doesn't exist, check alternative locations
29
- if not full_path.exists():
30
- # Check if file is in sensitive subdirectory
31
- sensitive_path = data_paths["video"] / "sensitive" / rel_name
32
- if sensitive_path.exists():
33
- return sensitive_path.resolve()
34
-
35
- # Check direct raw_file.path if available
36
- # Check direct raw_file.path if available
37
- try:
38
- direct_path = Path(video.raw_file.path)
39
- if direct_path.exists():
40
- return direct_path.resolve()
41
- except Exception as e:
42
- logger.debug("Could not access direct raw_file.path for video %s: %s", video.uuid, e)
43
- # Fallback to checking alternative paths
44
-
45
- # Check common alternative paths
46
- alternative_paths = [
47
- Path("/home/admin/dev/lx-annotate/libs/data/videos") / rel_name,
48
- Path("/home/admin/dev/lx-annotate/libs/data/videos/sensitive") / rel_name,
49
- data_paths["video"].parent / "libs" / "data" / "videos" / rel_name,
50
- ]
51
-
52
- for alt_path in alternative_paths:
53
- if alt_path.exists():
54
- return alt_path.resolve()
55
-
56
- return full_path.resolve()
57
- else:
58
- return None
59
- except Exception as e:
60
- logger.warning("Could not get path for raw file of VideoFile %s: %s", video.uuid, e)
22
+ """Return the best-effort absolute path to the raw video on disk."""
23
+ if not (video.has_raw and video.raw_file.name):
61
24
  return None
62
25
 
26
+ # raw_file.name is a relative storage path like 'videos/<filename>'
27
+ raw_rel = Path(video.raw_file.name)
28
+
29
+ # If it already contains the video directory name, keep the tail
30
+ rel_name = raw_rel.name if raw_rel.parent.name == VIDEO_DIR.name else raw_rel
31
+ full_path = data_paths["video"] / rel_name
32
+
33
+ if full_path.exists():
34
+ return full_path.resolve()
35
+
36
+ sensitive_path = data_paths["video"] / "sensitive" / rel_name
37
+ if sensitive_path.exists():
38
+ return sensitive_path.resolve()
39
+
40
+ if storage_file_exists(video.raw_file):
41
+ try:
42
+ direct_path = Path(video.raw_file.path)
43
+ if direct_path.exists():
44
+ return direct_path.resolve()
45
+ except Exception as exc:
46
+ logger.debug(
47
+ "Could not access direct raw_file.path for video %s: %s",
48
+ video.uuid,
49
+ exc,
50
+ )
51
+
52
+ logger.warning(
53
+ "Raw video file '%s' not found under %s or via stored FileField path for video %s.",
54
+ rel_name,
55
+ data_paths["video"],
56
+ video.uuid,
57
+ )
58
+ return None
59
+
60
+
61
+ @contextmanager
62
+ def _ensure_local_raw_file(video: "VideoFile") -> Iterator[Path]:
63
+ """Yield a local filesystem path for the raw file, downloading if required."""
64
+ if not video.has_raw:
65
+ raise ValueError(f"Video {video.uuid} has no raw file")
66
+
67
+ with ensure_local_file(video.raw_file) as local_path:
68
+ yield local_path
69
+
70
+
63
71
  def _get_processed_file_path(video: "VideoFile") -> Optional[Path]:
64
72
  """Returns the absolute Path object for the processed file, if it exists."""
65
- try:
66
- if video.is_processed and video.processed_file.name:
67
- return Path(video.processed_file.path)
68
- else:
69
- return None
70
- except Exception as e:
71
- logger.warning("Could not get path for processed file of VideoFile %s: %s", video.uuid, e)
73
+ processed_field = getattr(video, "processed_file", None)
74
+ if not (video.is_processed and processed_field and processed_field.name):
72
75
  return None
76
+ try:
77
+ direct_path = Path(processed_field.path)
78
+ if direct_path.exists():
79
+ return direct_path.resolve()
80
+ except Exception as exc:
81
+ logger.debug(
82
+ "Could not access direct processed_file.path for video %s: %s",
83
+ video.uuid,
84
+ exc,
85
+ )
86
+ direct_path = None
87
+
88
+ if processed_field and storage_file_exists(processed_field):
89
+ logger.debug("Processed file for %s available only via storage backend", video.uuid)
90
+ else:
91
+ logger.warning(
92
+ "Could not get path for processed file of VideoFile %s: %s",
93
+ video.uuid,
94
+ "path unavailable",
95
+ )
96
+ return None
97
+
98
+
99
+ @contextmanager
100
+ def _ensure_local_processed_file(video: "VideoFile") -> Iterator[Path]:
101
+ """Yield a local path to the processed file, downloading if necessary."""
102
+ if not video.is_processed:
103
+ raise ValueError(f"Video {video.uuid} has no processed file")
104
+
105
+ with ensure_local_file(video.processed_file) as local_path:
106
+ yield local_path
107
+
73
108
 
74
109
  @transaction.atomic
75
110
  def _delete_with_file(video: "VideoFile", *args, **kwargs):
@@ -96,6 +131,11 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
96
131
  except Exception as e:
97
132
  # Log error but continue
98
133
  logger.error("Error deleting raw video file %s for video %s: %s", raw_file_path, video.uuid, e, exc_info=True)
134
+ else:
135
+ if delete_field_file(getattr(video, "raw_file", None), save=False):
136
+ logger.info("Deleted raw file from storage for video %s", video.uuid)
137
+ else:
138
+ logger.warning("Raw video file not found during deletion for video %s.", video.uuid)
99
139
 
100
140
  # 3. Delete Processed File
101
141
  processed_file_path = _get_processed_file_path(video)
@@ -109,6 +149,11 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
109
149
  except Exception as e:
110
150
  # Log error but continue
111
151
  logger.error("Error deleting processed video file %s for video %s: %s", processed_file_path, video.uuid, e, exc_info=True)
152
+ else:
153
+ if delete_field_file(getattr(video, "processed_file", None), save=False):
154
+ logger.info("Deleted processed file from storage for video %s", video.uuid)
155
+ else:
156
+ logger.warning("Processed file missing in storage for video %s", video.uuid)
112
157
 
113
158
  # 4. Delete Database Record
114
159
  try:
@@ -119,7 +164,8 @@ def _delete_with_file(video: "VideoFile", *args, **kwargs):
119
164
  return f"Successfully deleted VideoFile {video.uuid} and attempted file cleanup."
120
165
  except Exception as e:
121
166
  logger.error("Error deleting VideoFile database record PK %s (UUID: %s): %s", video.pk, video.uuid, e, exc_info=True)
122
- raise # Re-raise the exception for DB deletion failure
167
+ raise # Re-raise the exception for DB deletion failure
168
+
123
169
 
124
170
  def _get_base_frame_dir(video: "VideoFile") -> Path:
125
171
  """Gets the base directory path for storing extracted frames."""
@@ -130,22 +176,23 @@ def _get_base_frame_dir(video: "VideoFile") -> Path:
130
176
  def _set_frame_dir(video: "VideoFile", force_update: bool = False):
131
177
  """Sets the frame_dir field based on the video's UUID."""
132
178
  target_dir = _get_base_frame_dir(video)
133
- target_path_str = target_dir.as_posix() # Store as POSIX path string
179
+ target_path_str = target_dir.as_posix() # Store as POSIX path string
134
180
 
135
181
  if not video.frame_dir or video.frame_dir != target_path_str or force_update:
136
182
  video.frame_dir = target_path_str
137
183
  logger.info("Set frame_dir for video %s to %s", video.uuid, video.frame_dir)
138
184
  # Avoid saving if called from within the save method itself
139
- if not getattr(video, '_saving', False):
140
- video.save(update_fields=['frame_dir'])
185
+ if not getattr(video, "_saving", False):
186
+ video.save(update_fields=["frame_dir"])
141
187
 
142
188
 
143
189
  def _get_frame_dir_path(video: "VideoFile") -> Optional[Path]:
144
190
  """Returns the Path object for the frame directory, if set."""
145
191
  if not video.frame_dir:
146
192
  _set_frame_dir(video)
147
-
148
- return Path(video.frame_dir)
193
+
194
+ return Path(video.frame_dir)
195
+
149
196
 
150
197
  def _get_temp_anonymized_frame_dir(video: "VideoFile") -> Path:
151
198
  """Gets the path for the temporary directory used during anonymization frame creation."""
@@ -1,5 +1,6 @@
1
1
  import logging
2
- from typing import TYPE_CHECKING, Optional, List, Dict
2
+ from typing import TYPE_CHECKING, List, Optional
3
+
3
4
  from .get_endo_roi import _get_endo_roi
4
5
 
5
6
  if TYPE_CHECKING:
@@ -10,7 +11,7 @@ logger = logging.getLogger(__name__)
10
11
 
11
12
  def _get_crop_template(video: "VideoFile") -> Optional[List[int]]:
12
13
  """Generates a crop template [y1, y2, x1, x2] from the endo ROI."""
13
- endo_roi = _get_endo_roi(video) # Use the helper function
14
+ endo_roi = _get_endo_roi(video) # Use the helper function
14
15
  if not endo_roi:
15
16
  logger.warning("Cannot generate crop template for video %s: Endo ROI not available.", video.uuid)
16
17
  return None
@@ -40,6 +41,5 @@ def _get_crop_template(video: "VideoFile") -> Optional[List[int]]:
40
41
  # Proceed without boundary check if image dimensions unknown
41
42
  crop_template = [y, y + height, x, x + width]
42
43
 
43
-
44
44
  logger.debug("Generated crop template for video %s: %s", video.uuid, crop_template)
45
45
  return crop_template
@@ -1,11 +1,12 @@
1
1
  import logging
2
- from typing import TYPE_CHECKING, Optional, Dict
2
+ from typing import TYPE_CHECKING, Dict, Optional
3
3
 
4
4
  if TYPE_CHECKING:
5
5
  from ..video_file import VideoFile
6
6
 
7
7
  logger = logging.getLogger(__name__)
8
8
 
9
+
9
10
  def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
10
11
  """
11
12
  Gets the endoscope region of interest (ROI) dictionary from the linked VideoMeta.
@@ -26,8 +27,9 @@ def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
26
27
  and all(k in endo_roi for k in ("x", "y", "width", "height"))
27
28
  and all(isinstance(v, int) and not isinstance(v, bool) for v in endo_roi.values())
28
29
  ):
29
- logger.debug("Retrieved endo ROI for video %s: %s", video.uuid, endo_roi)
30
- return endo_roi
30
+ cleaned_roi = {k: int(endo_roi[k] or 0) for k in ("x", "y", "width", "height")}
31
+ logger.debug("Retrieved endo ROI for video %s: %s", video.uuid, cleaned_roi)
32
+ return cleaned_roi
31
33
  else:
32
34
  logger.warning("Endo ROI not fully defined or invalid in VideoMeta for video %s. ROI: %s", video.uuid, endo_roi)
33
35
  return None