endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (372) hide show
  1. endoreg_db/authz/auth.py +74 -0
  2. endoreg_db/authz/backends.py +168 -0
  3. endoreg_db/authz/management/commands/list_routes.py +18 -0
  4. endoreg_db/authz/middleware.py +83 -0
  5. endoreg_db/authz/permissions.py +127 -0
  6. endoreg_db/authz/policy.py +218 -0
  7. endoreg_db/authz/views_auth.py +66 -0
  8. endoreg_db/config/env.py +13 -8
  9. endoreg_db/data/__init__.py +8 -31
  10. endoreg_db/data/_examples/disease.yaml +55 -0
  11. endoreg_db/data/_examples/disease_classification.yaml +13 -0
  12. endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
  13. endoreg_db/data/_examples/event.yaml +64 -0
  14. endoreg_db/data/_examples/examination.yaml +72 -0
  15. endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
  16. endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
  17. endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
  18. endoreg_db/data/_examples/finding/complication.yaml +16 -0
  19. endoreg_db/data/_examples/finding/data.yaml +105 -0
  20. endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
  21. endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
  22. endoreg_db/data/_examples/finding/outcome.yaml +12 -0
  23. endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
  24. endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
  25. endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
  26. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  27. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  28. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  29. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  30. endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
  31. endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
  32. endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
  33. endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
  34. endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
  35. endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
  36. endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
  37. endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
  38. endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
  39. endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
  40. endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
  41. endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
  42. endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
  43. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
  44. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  45. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  46. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  47. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
  48. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
  49. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
  50. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  51. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
  52. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
  53. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  54. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
  55. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  56. endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
  57. endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  58. endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
  59. endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
  60. endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
  61. endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
  62. endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
  63. endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
  64. endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
  65. endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
  66. endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
  67. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
  68. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
  69. endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
  70. endoreg_db/data/_examples/finding_type/data.yaml +43 -0
  71. endoreg_db/data/_examples/requirement/age.yaml +26 -0
  72. endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
  73. endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
  74. endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
  75. endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
  76. endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
  77. endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
  78. endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
  79. endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
  80. endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
  81. endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
  82. endoreg_db/data/_examples/requirement/gender.yaml +25 -0
  83. endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
  84. endoreg_db/data/_examples/requirement/medication.yaml +93 -0
  85. endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
  86. endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
  87. endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
  88. endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
  89. endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  90. endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
  91. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
  92. endoreg_db/data/event_classification/data.yaml +4 -0
  93. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  94. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
  95. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
  96. endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
  97. endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
  98. endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
  99. endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
  100. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
  101. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  102. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  103. endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
  104. endoreg_db/data/requirement_set/_old_ +109 -0
  105. endoreg_db/data/requirement_set_type/data.yaml +21 -0
  106. endoreg_db/data/setup_config.yaml +4 -4
  107. endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
  108. endoreg_db/exceptions.py +5 -2
  109. endoreg_db/helpers/data_loader.py +1 -1
  110. endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
  111. endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
  112. endoreg_db/management/commands/import_video.py +9 -10
  113. endoreg_db/management/commands/import_video_with_classification.py +1 -1
  114. endoreg_db/management/commands/init_default_ai_model.py +1 -1
  115. endoreg_db/management/commands/list_routes.py +18 -0
  116. endoreg_db/management/commands/load_ai_model_data.py +2 -1
  117. endoreg_db/management/commands/load_center_data.py +12 -12
  118. endoreg_db/management/commands/load_requirement_data.py +60 -31
  119. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  120. endoreg_db/management/commands/setup_endoreg_db.py +14 -10
  121. endoreg_db/management/commands/storage_management.py +271 -203
  122. endoreg_db/migrations/0001_initial.py +1799 -1300
  123. endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
  124. endoreg_db/migrations/_old/0001_initial.py +1857 -0
  125. endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
  126. endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
  127. endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
  128. endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
  129. endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
  130. endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
  131. endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
  132. endoreg_db/models/__init__.py +78 -123
  133. endoreg_db/models/administration/__init__.py +21 -42
  134. endoreg_db/models/administration/ai/active_model.py +2 -2
  135. endoreg_db/models/administration/ai/ai_model.py +7 -6
  136. endoreg_db/models/administration/case/__init__.py +1 -15
  137. endoreg_db/models/administration/case/case.py +3 -3
  138. endoreg_db/models/administration/case/case_template/__init__.py +2 -14
  139. endoreg_db/models/administration/case/case_template/case_template.py +2 -124
  140. endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
  141. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
  142. endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
  143. endoreg_db/models/administration/center/center.py +33 -19
  144. endoreg_db/models/administration/center/center_product.py +12 -9
  145. endoreg_db/models/administration/center/center_resource.py +25 -19
  146. endoreg_db/models/administration/center/center_shift.py +21 -17
  147. endoreg_db/models/administration/center/center_waste.py +16 -8
  148. endoreg_db/models/administration/person/__init__.py +2 -0
  149. endoreg_db/models/administration/person/employee/employee.py +10 -5
  150. endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
  151. endoreg_db/models/administration/person/employee/employee_type.py +12 -6
  152. endoreg_db/models/administration/person/examiner/examiner.py +13 -11
  153. endoreg_db/models/administration/person/patient/__init__.py +2 -0
  154. endoreg_db/models/administration/person/patient/patient.py +103 -100
  155. endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
  156. endoreg_db/models/administration/person/person.py +4 -0
  157. endoreg_db/models/administration/person/profession/__init__.py +8 -4
  158. endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
  159. endoreg_db/models/administration/product/product.py +20 -15
  160. endoreg_db/models/administration/product/product_material.py +17 -18
  161. endoreg_db/models/administration/product/product_weight.py +12 -8
  162. endoreg_db/models/administration/product/reference_product.py +23 -55
  163. endoreg_db/models/administration/qualification/qualification.py +7 -3
  164. endoreg_db/models/administration/qualification/qualification_type.py +7 -3
  165. endoreg_db/models/administration/shift/scheduled_days.py +8 -5
  166. endoreg_db/models/administration/shift/shift.py +16 -12
  167. endoreg_db/models/administration/shift/shift_type.py +23 -31
  168. endoreg_db/models/label/__init__.py +7 -8
  169. endoreg_db/models/label/annotation/image_classification.py +10 -9
  170. endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
  171. endoreg_db/models/label/label.py +15 -15
  172. endoreg_db/models/label/label_set.py +19 -6
  173. endoreg_db/models/label/label_type.py +1 -1
  174. endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
  175. endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
  176. endoreg_db/models/label/video_segmentation_label.py +4 -0
  177. endoreg_db/models/label/video_segmentation_labelset.py +4 -3
  178. endoreg_db/models/media/frame/frame.py +22 -22
  179. endoreg_db/models/media/pdf/raw_pdf.py +249 -177
  180. endoreg_db/models/media/pdf/report_file.py +25 -29
  181. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
  182. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
  183. endoreg_db/models/media/video/__init__.py +1 -0
  184. endoreg_db/models/media/video/create_from_file.py +48 -56
  185. endoreg_db/models/media/video/pipe_1.py +30 -33
  186. endoreg_db/models/media/video/pipe_2.py +8 -9
  187. endoreg_db/models/media/video/video_file.py +359 -204
  188. endoreg_db/models/media/video/video_file_ai.py +288 -74
  189. endoreg_db/models/media/video/video_file_anonymize.py +38 -38
  190. endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
  191. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
  192. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
  193. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
  194. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
  195. endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
  196. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
  197. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
  198. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
  199. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
  200. endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
  201. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
  202. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
  203. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
  204. endoreg_db/models/media/video/video_file_io.py +109 -62
  205. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
  206. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
  207. endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
  208. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
  209. endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
  210. endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
  211. endoreg_db/models/media/video/video_file_segments.py +24 -17
  212. endoreg_db/models/media/video/video_metadata.py +19 -35
  213. endoreg_db/models/media/video/video_processing.py +96 -95
  214. endoreg_db/models/medical/contraindication/__init__.py +13 -3
  215. endoreg_db/models/medical/disease.py +22 -16
  216. endoreg_db/models/medical/event.py +31 -18
  217. endoreg_db/models/medical/examination/__init__.py +13 -6
  218. endoreg_db/models/medical/examination/examination.py +17 -18
  219. endoreg_db/models/medical/examination/examination_indication.py +26 -25
  220. endoreg_db/models/medical/examination/examination_time.py +16 -6
  221. endoreg_db/models/medical/examination/examination_time_type.py +9 -6
  222. endoreg_db/models/medical/examination/examination_type.py +3 -4
  223. endoreg_db/models/medical/finding/finding.py +38 -39
  224. endoreg_db/models/medical/finding/finding_classification.py +37 -48
  225. endoreg_db/models/medical/finding/finding_intervention.py +27 -22
  226. endoreg_db/models/medical/finding/finding_type.py +13 -12
  227. endoreg_db/models/medical/hardware/endoscope.py +20 -26
  228. endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
  229. endoreg_db/models/medical/laboratory/lab_value.py +62 -91
  230. endoreg_db/models/medical/medication/medication.py +22 -10
  231. endoreg_db/models/medical/medication/medication_indication.py +29 -3
  232. endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
  233. endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
  234. endoreg_db/models/medical/medication/medication_schedule.py +27 -16
  235. endoreg_db/models/medical/organ/__init__.py +15 -12
  236. endoreg_db/models/medical/patient/medication_examples.py +1 -5
  237. endoreg_db/models/medical/patient/patient_disease.py +20 -23
  238. endoreg_db/models/medical/patient/patient_event.py +19 -22
  239. endoreg_db/models/medical/patient/patient_examination.py +48 -54
  240. endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
  241. endoreg_db/models/medical/patient/patient_finding.py +122 -139
  242. endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
  243. endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
  244. endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
  245. endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
  246. endoreg_db/models/medical/patient/patient_medication.py +27 -38
  247. endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
  248. endoreg_db/models/medical/risk/risk.py +7 -6
  249. endoreg_db/models/medical/risk/risk_type.py +8 -5
  250. endoreg_db/models/metadata/model_meta.py +60 -29
  251. endoreg_db/models/metadata/model_meta_logic.py +139 -18
  252. endoreg_db/models/metadata/pdf_meta.py +19 -24
  253. endoreg_db/models/metadata/sensitive_meta.py +102 -85
  254. endoreg_db/models/metadata/sensitive_meta_logic.py +383 -43
  255. endoreg_db/models/metadata/video_meta.py +51 -31
  256. endoreg_db/models/metadata/video_prediction_logic.py +16 -23
  257. endoreg_db/models/metadata/video_prediction_meta.py +29 -33
  258. endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
  259. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
  260. endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
  261. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
  262. endoreg_db/models/other/emission/emission_factor.py +18 -8
  263. endoreg_db/models/other/gender.py +10 -5
  264. endoreg_db/models/other/information_source.py +25 -25
  265. endoreg_db/models/other/material.py +9 -5
  266. endoreg_db/models/other/resource.py +6 -4
  267. endoreg_db/models/other/tag.py +10 -5
  268. endoreg_db/models/other/transport_route.py +13 -8
  269. endoreg_db/models/other/unit.py +10 -6
  270. endoreg_db/models/other/waste.py +6 -5
  271. endoreg_db/models/requirement/requirement.py +580 -272
  272. endoreg_db/models/requirement/requirement_error.py +85 -0
  273. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  274. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
  275. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
  276. endoreg_db/models/requirement/requirement_operator.py +36 -33
  277. endoreg_db/models/requirement/requirement_set.py +74 -57
  278. endoreg_db/models/state/__init__.py +4 -4
  279. endoreg_db/models/state/abstract.py +2 -2
  280. endoreg_db/models/state/anonymization.py +12 -0
  281. endoreg_db/models/state/audit_ledger.py +46 -47
  282. endoreg_db/models/state/label_video_segment.py +9 -0
  283. endoreg_db/models/state/raw_pdf.py +40 -46
  284. endoreg_db/models/state/sensitive_meta.py +6 -2
  285. endoreg_db/models/state/video.py +58 -53
  286. endoreg_db/models/upload_job.py +32 -55
  287. endoreg_db/models/utils.py +1 -2
  288. endoreg_db/root_urls.py +21 -2
  289. endoreg_db/serializers/__init__.py +26 -57
  290. endoreg_db/serializers/anonymization.py +18 -10
  291. endoreg_db/serializers/meta/report_meta.py +1 -1
  292. endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
  293. endoreg_db/serializers/misc/__init__.py +1 -1
  294. endoreg_db/serializers/misc/file_overview.py +33 -91
  295. endoreg_db/serializers/misc/{vop_patient_data.py → sensitive_patient_data.py} +1 -1
  296. endoreg_db/serializers/requirements/requirement_sets.py +92 -22
  297. endoreg_db/serializers/video/segmentation.py +2 -1
  298. endoreg_db/serializers/video/video_processing_history.py +20 -5
  299. endoreg_db/serializers/video_examination.py +198 -0
  300. endoreg_db/services/anonymization.py +75 -73
  301. endoreg_db/services/lookup_service.py +256 -73
  302. endoreg_db/services/lookup_store.py +174 -30
  303. endoreg_db/services/pdf_import.py +711 -310
  304. endoreg_db/services/storage_aware_video_processor.py +140 -114
  305. endoreg_db/services/video_import.py +266 -117
  306. endoreg_db/urls/__init__.py +27 -27
  307. endoreg_db/urls/label_video_segments.py +2 -0
  308. endoreg_db/urls/media.py +108 -66
  309. endoreg_db/urls/root_urls.py +29 -0
  310. endoreg_db/utils/__init__.py +15 -5
  311. endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
  312. endoreg_db/utils/case_generator/__init__.py +3 -0
  313. endoreg_db/utils/dataloader.py +88 -16
  314. endoreg_db/utils/defaults/set_default_center.py +32 -0
  315. endoreg_db/utils/names.py +22 -16
  316. endoreg_db/utils/permissions.py +2 -1
  317. endoreg_db/utils/pipelines/process_video_dir.py +1 -1
  318. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
  319. endoreg_db/utils/setup_config.py +8 -5
  320. endoreg_db/utils/storage.py +115 -0
  321. endoreg_db/utils/validate_endo_roi.py +8 -2
  322. endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
  323. endoreg_db/views/__init__.py +5 -12
  324. endoreg_db/views/anonymization/media_management.py +198 -163
  325. endoreg_db/views/anonymization/overview.py +4 -1
  326. endoreg_db/views/anonymization/validate.py +174 -40
  327. endoreg_db/views/media/__init__.py +2 -0
  328. endoreg_db/views/media/pdf_media.py +131 -150
  329. endoreg_db/views/media/sensitive_metadata.py +46 -6
  330. endoreg_db/views/media/video_media.py +89 -82
  331. endoreg_db/views/media/video_segments.py +187 -260
  332. endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
  333. endoreg_db/views/patient/patient.py +5 -4
  334. endoreg_db/views/pdf/__init__.py +5 -8
  335. endoreg_db/views/pdf/pdf_stream.py +186 -0
  336. endoreg_db/views/pdf/pdf_stream_views.py +0 -127
  337. endoreg_db/views/pdf/reimport.py +86 -91
  338. endoreg_db/views/requirement/evaluate.py +188 -187
  339. endoreg_db/views/requirement/lookup.py +186 -288
  340. endoreg_db/views/requirement/requirement_utils.py +89 -0
  341. endoreg_db/views/video/__init__.py +0 -4
  342. endoreg_db/views/video/correction.py +2 -2
  343. endoreg_db/views/video/video_examination_viewset.py +202 -289
  344. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
  345. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +350 -255
  346. endoreg_db/models/administration/permissions/__init__.py +0 -44
  347. endoreg_db/models/media/video/refactor_plan.md +0 -0
  348. endoreg_db/models/media/video/video_file_frames.py +0 -0
  349. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  350. endoreg_db/models/rule/__init__.py +0 -13
  351. endoreg_db/models/rule/rule.py +0 -27
  352. endoreg_db/models/rule/rule_applicator.py +0 -224
  353. endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
  354. endoreg_db/models/rule/rule_type.py +0 -20
  355. endoreg_db/models/rule/ruleset.py +0 -17
  356. endoreg_db/serializers/video/video_metadata.py +0 -105
  357. endoreg_db/urls/report.py +0 -48
  358. endoreg_db/urls/video.py +0 -61
  359. endoreg_db/utils/case_generator/case_generator.py +0 -159
  360. endoreg_db/utils/case_generator/utils.py +0 -30
  361. endoreg_db/views/pdf/pdf_media.py +0 -239
  362. endoreg_db/views/report/__init__.py +0 -9
  363. endoreg_db/views/report/report_list.py +0 -112
  364. endoreg_db/views/report/report_with_secure_url.py +0 -28
  365. endoreg_db/views/report/start_examination.py +0 -7
  366. endoreg_db/views/video/video_media.py +0 -158
  367. endoreg_db/views.py +0 -0
  368. /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
  369. /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
  370. /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
  371. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
  372. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,96 +1,102 @@
1
1
  """Concrete model for video files, handling both raw and processed states."""
2
2
 
3
3
  import logging
4
- from pathlib import Path
4
+ import os
5
5
  import uuid
6
- from typing import TYPE_CHECKING, Optional, Union, cast
6
+ from pathlib import Path
7
+ from typing import TYPE_CHECKING, Optional, Self, Union, cast
7
8
 
8
- from django.db import models
9
9
  from django.core.files import File
10
- from django.db.models.fields.files import FieldFile
11
10
  from django.core.validators import FileExtensionValidator
11
+ from django.db import models
12
12
  from django.db.models import F
13
+ from django.db.models.fields.files import FieldFile
14
+ from librosa import frames_to_samples
15
+ from pandas.core import frame
16
+
13
17
  from endoreg_db.utils.calc_duration_seconds import _calc_duration_vf
18
+ from endoreg_db.utils.video.ffmpeg_wrapper import assemble_video_from_frames
19
+
20
+ from ...label import Label, LabelVideoSegment
21
+ from ...state import VideoState
22
+ from ...utils import ANONYM_VIDEO_DIR, VIDEO_DIR
14
23
 
15
24
  # --- Import model-specific function modules ---
16
25
  from .create_from_file import _create_from_file
17
- from .video_file_anonymize import (
18
- _anonymize,
19
- _create_anonymized_frame_files,
20
- _cleanup_raw_assets,
21
- )
22
- from .video_file_meta import (
23
- _update_text_metadata,
24
- _update_video_meta,
25
- _get_fps,
26
- _get_endo_roi,
27
- _get_crop_template,
28
- _initialize_video_specs,
29
- )
26
+ from .pipe_1 import _pipe_1, _test_after_pipe_1
27
+ from .pipe_2 import _pipe_2
28
+ from .video_file_ai import _extract_text_from_video_frames, _predict_video_pipeline
29
+ from .video_file_anonymize import _anonymize, _censor_outside_frames, _cleanup_raw_assets, _create_anonymized_frame_files
30
30
  from .video_file_frames import (
31
- _extract_frames,
32
- _initialize_frames,
31
+ _bulk_create_frames,
32
+ _create_frame_object,
33
33
  _delete_frames,
34
+ _extract_frames,
35
+ _get_frame,
36
+ _get_frame_number,
34
37
  _get_frame_path,
35
38
  _get_frame_paths,
36
- _get_frame_number,
37
- _get_frames,
38
- _get_frame,
39
39
  _get_frame_range,
40
- _create_frame_object,
41
- _bulk_create_frames,
40
+ _get_frames,
41
+ _initialize_frames,
42
42
  )
43
+
43
44
  # Update import aliases for clarity and to use as helpers
44
- from .video_file_frames._manage_frame_range import _extract_frame_range as _extract_frame_range_helper
45
- from .video_file_frames._manage_frame_range import _delete_frame_range as _delete_frame_range_helper
45
+ from .video_file_frames._manage_frame_range import (
46
+ _delete_frame_range as _delete_frame_range_helper,
47
+ )
48
+ from .video_file_frames._manage_frame_range import (
49
+ _extract_frame_range as _extract_frame_range_helper,
50
+ )
46
51
  from .video_file_io import (
47
52
  _delete_with_file,
48
53
  _get_base_frame_dir,
49
- _set_frame_dir,
50
54
  _get_frame_dir_path,
51
- _get_temp_anonymized_frame_dir,
52
- _get_target_anonymized_video_path,
53
- _get_raw_file_path,
54
55
  _get_processed_file_path,
56
+ _get_raw_file_path,
57
+ _get_target_anonymized_video_path,
58
+ _get_temp_anonymized_frame_dir,
59
+ _set_frame_dir,
55
60
  )
56
- from .video_file_ai import (
57
- _predict_video_pipeline,
58
- _extract_text_from_video_frames,
61
+ from .video_file_meta import (
62
+ _get_crop_template,
63
+ _get_endo_roi,
64
+ _get_fps,
65
+ _initialize_video_specs,
66
+ _update_text_metadata,
67
+ _update_video_meta,
59
68
  )
60
69
 
61
- from .pipe_1 import _pipe_1, _test_after_pipe_1
62
- from .pipe_2 import _pipe_2
63
-
64
- from ...utils import VIDEO_DIR, ANONYM_VIDEO_DIR
65
- from ...state import VideoState
66
- from ...label import LabelVideoSegment, Label
67
-
68
-
69
70
  # Configure logging
70
71
  logger = logging.getLogger(__name__) # Changed from "video_file"
71
72
 
72
73
  if TYPE_CHECKING:
74
+ from django.db.models.fields.files import FieldFile
75
+
73
76
  from endoreg_db.models import (
74
77
  Center,
78
+ EndoscopyProcessor,
79
+ FFMpegMeta,
75
80
  Frame,
81
+ ModelMeta,
82
+ Patient,
83
+ PatientExamination,
76
84
  SensitiveMeta,
77
- EndoscopyProcessor,
85
+ VideoImportMeta,
78
86
  VideoMeta,
79
- PatientExamination,
80
- Patient,
81
87
  VideoState,
82
- ModelMeta,
83
- VideoImportMeta,
84
- FFMpegMeta,
85
- )
88
+ SensitiveMeta
89
+ )
90
+
91
+
86
92
  class VideoQuerySet(models.QuerySet):
87
93
  def next_after(self, last_id=None):
88
94
  """
89
95
  Return the next VideoFile instance with a primary key greater than the given last_id.
90
-
96
+
91
97
  Parameters:
92
98
  last_id (int or None): The primary key to start after. If None or invalid, returns the first instance.
93
-
99
+
94
100
  Returns:
95
101
  VideoFile or None: The next VideoFile instance, or None if not found.
96
102
  """
@@ -102,9 +108,10 @@ class VideoQuerySet(models.QuerySet):
102
108
  q = self if last_id is None else self.filter(pk__gt=last_id)
103
109
  return q.order_by("pk").first()
104
110
 
111
+
105
112
  class VideoFile(models.Model):
106
113
  uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
107
-
114
+
108
115
  objects = VideoQuerySet.as_manager()
109
116
 
110
117
  raw_file = models.FileField(
@@ -122,81 +129,107 @@ class VideoFile(models.Model):
122
129
 
123
130
  video_hash = models.CharField(max_length=255, unique=True, help_text="Hash of the raw video file.")
124
131
  processed_video_hash = models.CharField(
125
- max_length=255, unique=True, null=True, blank=True, help_text="Hash of the processed video file, unique if not null."
132
+ max_length=255,
133
+ unique=True,
134
+ null=True,
135
+ blank=True,
136
+ help_text="Hash of the processed video file, unique if not null.",
126
137
  )
127
138
 
128
139
  sensitive_meta = models.OneToOneField(
129
- "SensitiveMeta", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
130
- ) # type: ignore
131
- center = models.ForeignKey("Center", on_delete=models.PROTECT) # type: ignore
132
- processor = models.ForeignKey(
133
- "EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True
134
- ) # type: ignore
140
+ "SensitiveMeta",
141
+ on_delete=models.SET_NULL,
142
+ null=True,
143
+ blank=True,
144
+ related_name="video_file",
145
+ )
146
+ center = models.ForeignKey("Center", on_delete=models.PROTECT)
147
+ processor = models.ForeignKey("EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True)
135
148
  video_meta = models.OneToOneField(
136
- "VideoMeta", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
137
- ) # type: ignore
149
+ "VideoMeta",
150
+ on_delete=models.SET_NULL,
151
+ null=True,
152
+ blank=True,
153
+ related_name="video_file",
154
+ )
138
155
  examination = models.ForeignKey(
139
156
  "PatientExamination",
140
157
  on_delete=models.SET_NULL,
141
158
  blank=True,
142
159
  null=True,
143
160
  related_name="video_files",
144
- ) # type: ignore
161
+ )
145
162
  patient = models.ForeignKey(
146
163
  "Patient",
147
164
  on_delete=models.SET_NULL,
148
165
  blank=True,
149
166
  null=True,
150
167
  related_name="video_files",
151
- ) # type: ignore
152
- ai_model_meta = models.ForeignKey(
153
- "ModelMeta", on_delete=models.SET_NULL, blank=True, null=True
154
- ) # type: ignore
168
+ )
169
+ ai_model_meta = models.ForeignKey("ModelMeta", on_delete=models.SET_NULL, blank=True, null=True)
155
170
  state = models.OneToOneField(
156
- "VideoState", on_delete=models.SET_NULL, null=True, blank=True, related_name="video_file"
157
- ) # type: ignore
158
- import_meta = models.OneToOneField(
159
- "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
160
- ) # type: ignore
171
+ "VideoState",
172
+ on_delete=models.SET_NULL,
173
+ null=True,
174
+ blank=True,
175
+ related_name="video_file",
176
+ )
177
+ import_meta = models.OneToOneField("VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True)
161
178
 
162
179
  original_file_name = models.CharField(max_length=255, blank=True, null=True)
163
180
  uploaded_at = models.DateTimeField(auto_now_add=True)
164
- frame_dir = models.CharField(max_length=512, blank=True, help_text="Path to frames extracted from the raw video.")
181
+ frame_dir = models.CharField(
182
+ max_length=512,
183
+ blank=True,
184
+ help_text="Path to frames extracted from the raw video.",
185
+ )
165
186
  fps = models.FloatField(blank=True, null=True)
166
187
  duration = models.FloatField(blank=True, null=True)
167
188
  frame_count = models.IntegerField(blank=True, null=True)
168
189
  width = models.IntegerField(blank=True, null=True)
169
190
  height = models.IntegerField(blank=True, null=True)
170
191
  suffix = models.CharField(max_length=10, blank=True, null=True)
171
- sequences = models.JSONField(default=dict, blank=True, help_text="AI prediction sequences based on raw frames.")
192
+ sequences = models.JSONField(
193
+ default=dict,
194
+ blank=True,
195
+ help_text="AI prediction sequences based on raw frames.",
196
+ )
172
197
  date = models.DateField(blank=True, null=True)
173
198
  meta = models.JSONField(blank=True, null=True)
174
199
  date_created = models.DateTimeField(auto_now_add=True)
175
200
  date_modified = models.DateTimeField(auto_now=True)
176
201
 
177
202
  if TYPE_CHECKING:
178
- label_video_segments: "models.QuerySet[LabelVideoSegment]"
179
- frames: "models.QuerySet[Frame]"
180
- center: "Center"
181
- processor: "EndoscopyProcessor"
182
- video_meta: "VideoMeta"
183
- examination: "PatientExamination"
184
- patient: "Patient"
185
- sensitive_meta: "SensitiveMeta"
186
- state: "VideoState"
187
- ai_model_meta: "ModelMeta"
188
- import_meta: "VideoImportMeta"
189
-
203
+ from django.db.models.manager import RelatedManager
204
+
205
+ @property
206
+ def label_video_segments(self) -> RelatedManager[LabelVideoSegment]: ...
207
+
208
+ @property
209
+ def frames(self) -> RelatedManager[Frame]: ...
210
+
211
+ center: models.ForeignKey["Center"]
212
+ processor: models.ForeignKey["EndoscopyProcessor | None"]
213
+ video_meta: models.OneToOneField["VideoMeta | None"]
214
+ examination: models.ForeignKey["PatientExamination | None"]
215
+ patient: models.ForeignKey["Patient | None"]
216
+ sensitive_meta: models.OneToOneField["SensitiveMeta | None"]
217
+ state: models.OneToOneField["VideoState | None"]
218
+ ai_model_meta: models.ForeignKey["ModelMeta | None"]
219
+ import_meta: models.OneToOneField["VideoImportMeta | None"]
220
+ raw_file = cast(FieldFile, raw_file)
221
+ processed_file = cast(FieldFile, processed_file)
190
222
 
191
223
  @property
192
224
  def ffmpeg_meta(self) -> "FFMpegMeta":
193
225
  """
194
226
  Return the associated FFMpegMeta instance for this video, initializing video specs if necessary.
195
-
227
+
196
228
  Returns:
197
229
  FFMpegMeta: The FFMpegMeta object containing metadata for this video.
198
230
  """
199
231
  from endoreg_db.models import FFMpegMeta
232
+
200
233
  if self.video_meta is not None:
201
234
  if self.video_meta.ffmpeg_meta is not None:
202
235
  return self.video_meta.ffmpeg_meta
@@ -207,47 +240,29 @@ class VideoFile(models.Model):
207
240
  assert isinstance(ffmpeg_meta, FFMpegMeta), "Expected FFMpegMeta instance."
208
241
  return ffmpeg_meta
209
242
 
243
+ # Exception message constants
244
+
245
+ NO_ACTIVE_FILE = "Has no raw file"
246
+ NO_FILE_ASSOCIATED = "Active file has no associated file."
210
247
 
211
248
  @property
212
- def active_file_url(self) -> str:
213
- """
214
- Return the URL of the active processed file.
215
-
216
- Returns:
217
- str: The URL of the active video file.
218
-
219
- Raises:
220
- Value Error if no active VideoFile is available.
221
- """
222
- active = self.active_file
223
- if not isinstance(active, FieldFile):
224
- raise ValueError("Active file is not a stored FieldFile instance.")
225
- if not active.name:
226
- raise ValueError("Active file has no associated name.")
227
- return active.url
228
-
229
- @property
230
- def active_raw_file(self) -> FieldFile:
231
- raw = self.raw_file
232
- if isinstance(raw, FieldFile) and raw.name:
233
- return raw
234
- raise ValueError("No raw file available for this video")
235
-
236
- @property
237
- def active_raw_file_url(self)-> str:
238
- """
239
- Return the path of the URL of the active raw file for name reading.
249
+ def active_raw_file(self) -> File:
250
+ """Return the raw file if available, otherwise raise ValueError."""
251
+ if self.has_raw:
252
+ return self.raw_file
253
+ raise ValueError(self.NO_ACTIVE_FILE)
240
254
 
241
- Raises:
242
- ValueError("Active file has no associated file")
243
-
244
- Returns:
245
- """
246
- raw = self.active_raw_file
247
- if not raw.name:
248
- raise ValueError("Active raw file has no associated name.")
249
- return raw.url
250
-
255
+ @property
256
+ def active_raw_file_url(self) -> str:
257
+ """Return the URL of the active raw file, or raise ValueError if unavailable."""
258
+ _file = self.active_raw_file
259
+ assert _file is not None, self.NO_ACTIVE_FILE
260
+ if not _file or not _file.name:
261
+ raise ValueError(self.NO_FILE_ASSOCIATED)
262
+ url = getattr(_file, "url", None)
263
+ if not url:
264
+ raise ValueError("Active raw file URL could not be resolved.")
265
+ return str(url)
251
266
 
252
267
  # Pipeline Functions
253
268
  pipe_1 = _pipe_1
@@ -275,32 +290,30 @@ class VideoFile(models.Model):
275
290
  create_frame_object = _create_frame_object
276
291
  bulk_create_frames = _bulk_create_frames
277
292
 
278
-
279
-
280
293
  # Define new methods that call the helper functions
281
294
  def extract_specific_frame_range(self, start_frame: int, end_frame: int, overwrite: bool = False, **kwargs) -> bool:
282
295
  """
283
296
  Extract frames from the video within the specified frame range.
284
-
297
+
285
298
  Parameters:
286
299
  start_frame (int): The starting frame number (inclusive).
287
300
  end_frame (int): The ending frame number (exclusive).
288
301
  overwrite (bool): Whether to overwrite existing frames in the range.
289
-
302
+
290
303
  Returns:
291
304
  bool: True if frame extraction was successful, False otherwise.
292
-
305
+
293
306
  Additional keyword arguments:
294
307
  quality (int, optional): Quality setting for extracted frames.
295
308
  ext (str, optional): File extension for extracted frames.
296
309
  verbose (bool, optional): Whether to enable verbose output.
297
310
  """
298
- quality = kwargs.get('quality', 2)
299
- ext = kwargs.get('ext', "jpg")
300
- verbose = kwargs.get('verbose', False)
311
+ quality = kwargs.get("quality", 2)
312
+ ext = kwargs.get("ext", "jpg")
313
+ verbose = kwargs.get("verbose", False)
301
314
 
302
315
  # Log if unexpected kwargs are passed, beyond those used by the helper
303
- expected_helper_kwargs = {'quality', 'ext', 'verbose'}
316
+ expected_helper_kwargs = {"quality", "ext", "verbose"}
304
317
  unexpected_kwargs = {k: v for k, v in kwargs.items() if k not in expected_helper_kwargs}
305
318
  if unexpected_kwargs:
306
319
  logger.warning(f"Unexpected keyword arguments for extract_specific_frame_range, will be ignored by helper: {unexpected_kwargs}")
@@ -312,18 +325,14 @@ class VideoFile(models.Model):
312
325
  quality=quality,
313
326
  overwrite=overwrite,
314
327
  ext=ext,
315
- verbose=verbose
328
+ verbose=verbose,
316
329
  )
317
330
 
318
331
  def delete_specific_frame_range(self, start_frame: int, end_frame: int) -> None:
319
332
  """
320
333
  Deletes frame files for a specific range [start_frame, end_frame).
321
334
  """
322
- _delete_frame_range_helper(
323
- video=self,
324
- start_frame=start_frame,
325
- end_frame=end_frame
326
- )
335
+ _delete_frame_range_helper(video=self, start_frame=start_frame, end_frame=end_frame)
327
336
 
328
337
  delete_with_file = _delete_with_file
329
338
  get_base_frame_dir = _get_base_frame_dir
@@ -340,8 +349,6 @@ class VideoFile(models.Model):
340
349
 
341
350
  predict_video = _predict_video_pipeline
342
351
  extract_text_from_frames = _extract_text_from_video_frames
343
-
344
-
345
352
 
346
353
  @classmethod
347
354
  def check_hash_exists(cls, video_hash: str) -> bool:
@@ -360,16 +367,15 @@ class VideoFile(models.Model):
360
367
  Return True if a raw video file is associated with this instance.
361
368
  """
362
369
  return bool(self.raw_file and self.raw_file.name)
363
-
364
370
 
365
371
  @property
366
372
  def active_file(self) -> FieldFile:
367
373
  """
368
374
  Return the active video file, preferring the processed file if available.
369
-
375
+
370
376
  Returns:
371
377
  File: The processed file if present; otherwise, the raw file.
372
-
378
+
373
379
  Raises:
374
380
  ValueError: If neither a processed nor a raw file is available.
375
381
  """
@@ -382,16 +388,15 @@ class VideoFile(models.Model):
382
388
  return raw
383
389
 
384
390
  raise ValueError("No active file available. VideoFile has neither raw nor processed file.")
385
-
386
391
 
387
392
  @property
388
393
  def active_file_path(self) -> Path:
389
394
  """
390
395
  Return the filesystem path of the active video file.
391
-
396
+
392
397
  Returns:
393
398
  Path: The path to the processed file if available, otherwise the raw file.
394
-
399
+
395
400
  Raises:
396
401
  ValueError: If neither a processed nor raw file is present.
397
402
  """
@@ -404,9 +409,29 @@ class VideoFile(models.Model):
404
409
  raise ValueError("No active file path available. VideoFile has neither raw nor processed file.")
405
410
 
406
411
  if path is None:
407
- raise ValueError("Active file path could not be resolved.")
412
+ raise ValueError("Active file path could not be resolved. VideoFile raw file is missing.")
408
413
  return path
409
414
 
415
+ @property
416
+ def active_file_url(self) -> str:
417
+ """Return the URL of the active video file, if available."""
418
+ file_obj = self.active_file
419
+ if not isinstance(file_obj, FieldFile):
420
+ raise ValueError("Active file is not a valid Django FieldFile instance.")
421
+ try:
422
+ url = getattr(file_obj, "url", None)
423
+ except Exception as exc: # storage backends may raise when missing
424
+ logger.warning(
425
+ "Active file URL unavailable for video %s: %s",
426
+ self.uuid,
427
+ exc,
428
+ )
429
+ raise ValueError("Active file URL could not be resolved for this VideoFile.") from exc
430
+
431
+ if not url:
432
+ raise ValueError("Active file URL is empty for this VideoFile.")
433
+
434
+ return str(url)
410
435
 
411
436
  @classmethod
412
437
  def create_from_file(cls, file_path: Union[str, Path], center_name: str, **kwargs) -> Optional["VideoFile"]:
@@ -414,16 +439,22 @@ class VideoFile(models.Model):
414
439
  if isinstance(file_path, str):
415
440
  file_path = Path(file_path)
416
441
  # Pass center_name and other kwargs to the helper function
442
+ if not center_name:
443
+ try:
444
+ center_name = os.environ["CENTER_NAME"]
445
+ except KeyError:
446
+ logger.error("Center name must be provided to create VideoFile from file. You can set CENTER_NAME in environment variables.")
447
+ return None
417
448
  return _create_from_file(cls, file_path, center_name=center_name, **kwargs)
418
449
 
419
450
  @classmethod
420
451
  def create_from_file_initialized(
421
452
  cls,
422
453
  file_path: Union[str, Path],
423
- center_name:str,
454
+ center_name: str,
424
455
  processor_name: Optional[str] = None,
425
- delete_source:bool = False,
426
- save_video_file:bool = True, # Add this line
456
+ delete_source: bool = False,
457
+ save_video_file: bool = True, # Add this line
427
458
  ):
428
459
  """
429
460
  Creates a VideoFile instance from a given video file path.
@@ -441,38 +472,43 @@ class VideoFile(models.Model):
441
472
  center_name=center_name,
442
473
  processor_name=processor_name,
443
474
  delete_source=delete_source,
444
- save=save_video_file, # Add this line
475
+ save=save_video_file, # Add this line
445
476
  )
446
477
 
447
478
  video_file = video_file.initialize()
448
479
  return video_file
449
-
480
+
450
481
  def delete(self, using=None, keep_parents=False) -> tuple[int, dict[str, int]]:
451
482
  """
452
483
  Delete the VideoFile instance, including associated files and frames.
453
-
484
+
454
485
  Overrides the default delete method to ensure proper cleanup of related resources.
455
486
  """
456
487
  # Ensure frames are deleted before the main instance
457
488
  _delete_frames(self)
458
489
 
459
490
  # Call the original delete method to remove the instance from the database
460
- active_path = self.active_file_path
461
- logger.info(f"Deleting VideoFile: {self.uuid} - {active_path}")
491
+ try:
492
+ active_path = self.active_file_path
493
+ logger.info(f"Deleting VideoFile: {self.uuid} - {active_path}")
494
+
495
+ except ValueError:
496
+ logger.info(f"Deleting VideoFile: {self.uuid} - No active file path found.")
497
+ active_path = None
462
498
 
463
499
  # Delete associated files if they exist
464
- if active_path.exists():
500
+ if active_path and active_path.exists():
465
501
  active_path.unlink(missing_ok=True)
466
-
502
+
467
503
  # Delete file storage
468
504
  if self.raw_file and self.raw_file.storage.exists(self.raw_file.name):
469
505
  self.raw_file.storage.delete(self.raw_file.name)
470
506
  if self.processed_file and self.processed_file.storage.exists(self.processed_file.name):
471
507
  self.processed_file.storage.delete(self.processed_file.name)
472
-
508
+
473
509
  # Use proper database connection
474
510
  if using is None:
475
- using = 'default'
511
+ using = "default"
476
512
 
477
513
  raw_file_path = self.get_raw_file_path()
478
514
  if raw_file_path:
@@ -484,7 +520,7 @@ class VideoFile(models.Model):
484
520
  logger.info(f"Removed processing lock: {lock_path}")
485
521
  except Exception as e:
486
522
  logger.warning(f"Could not remove processing lock {lock_path}: {e}")
487
-
523
+
488
524
  try:
489
525
  # Call parent delete with proper parameters
490
526
  result = super().delete(using=using, keep_parents=keep_parents)
@@ -497,24 +533,47 @@ class VideoFile(models.Model):
497
533
  def validate_metadata_annotation(self, extracted_data_dict: Optional[dict] = None) -> bool:
498
534
  """
499
535
  Validate the metadata of the VideoFile instance.
500
-
501
- Called after annotation in the frontend, this method deletes the associated active file, updates the sensitive meta data with the user annotated data.
502
- It also ensures the video file is properly saved after the metadata update.
536
+
537
+ Called after annotation in the frontend, this method:
538
+ 1. Updates sensitive metadata with user-annotated data
539
+ 2. Deletes the RAW video file (keeping only the anonymized version)
540
+ 3. Marks the video as validated
541
+
542
+ **IMPORTANT:** Only the raw video is deleted. The processed (anonymized)
543
+ video is preserved as the final validated output.
503
544
  """
504
- from endoreg_db.models import SensitiveMeta
545
+
505
546
  if not self.sensitive_meta:
506
- self.sensitive_meta = SensitiveMeta.objects.create(center=self.center)
507
-
508
- # Delete the active file to ensure it is reprocessed with the new metadata
509
- if self.active_file_path.exists():
510
- self.active_file_path.unlink(missing_ok=True)
511
-
512
- # Update sensitive metadata with user annotations
513
- sensitive_meta = _update_text_metadata(self, extracted_data_dict, overwrite=True)
514
-
515
- if sensitive_meta:
547
+ # Ensure a SensitiveMeta exists so validation can proceed.
548
+ self.sensitive_meta = self.get_or_create_sensitive_meta()
549
+ # CRITICAL FIX: Delete RAW video file, not the processed (anonymized) one
550
+ # CRITICAL: Update metadata BEFORE deleting raw video
551
+ if extracted_data_dict:
552
+ self.sensitive_meta.update_from_dict(extracted_data_dict)
553
+ else:
554
+ return False
555
+
556
+ # After validation and metadata update, only the anonymized video should remain
557
+ from .video_file_io import _get_raw_file_path
558
+
559
+ raw_path = _get_raw_file_path(self)
560
+
561
+ if raw_path and raw_path.exists():
562
+ logger.info(f"Deleting raw video file after validation: {raw_path}")
563
+ raw_path.unlink(missing_ok=True)
564
+ # Clear the raw_file field in database (use delete() to avoid save issues)
565
+ if self.raw_file:
566
+ self.raw_file.delete(save=False)
567
+ logger.info(f"Raw video deleted for {self.uuid}. Anonymized video preserved.")
568
+ else:
569
+ logger.warning(
570
+ "Raw video file not found for deletion during validation %s.",
571
+ self.uuid,
572
+ )
573
+
574
+ if self.sensitive_meta:
516
575
  # Mark as processed after validation
517
- self.get_or_create_state().mark_sensitive_meta_processed(save=True)
576
+ self.get_or_create_state().mark_anonymization_validated(save=True)
518
577
  # Save the VideoFile instance to persist changes
519
578
  self.save()
520
579
  logger.info(f"Metadata annotation validated and saved for video {self.uuid}.")
@@ -522,13 +581,11 @@ class VideoFile(models.Model):
522
581
  else:
523
582
  logger.error(f"Failed to validate metadata annotation for video {self.uuid}.")
524
583
  return False
525
-
526
-
527
-
584
+
528
585
  def initialize(self):
529
586
  """
530
587
  Initialize the VideoFile instance by updating metadata, setting up video specs, assigning frame directory, ensuring related state and sensitive metadata exist, saving the instance, and initializing frames.
531
-
588
+
532
589
  Returns:
533
590
  VideoFile: The initialized VideoFile instance.
534
591
  """
@@ -548,7 +605,6 @@ class VideoFile(models.Model):
548
605
  # Initialize frames based on the video specs
549
606
  self.initialize_frames()
550
607
 
551
-
552
608
  return self
553
609
 
554
610
  def __str__(self):
@@ -586,7 +642,7 @@ class VideoFile(models.Model):
586
642
  # Now call the original save method
587
643
  """
588
644
  Saves the VideoFile instance to the database.
589
-
645
+
590
646
  Overrides the default save method to persist changes to the VideoFile model.
591
647
  """
592
648
  super().save(*args, **kwargs)
@@ -618,23 +674,69 @@ class VideoFile(models.Model):
618
674
  def get_or_create_sensitive_meta(self) -> "SensitiveMeta":
619
675
  """
620
676
  Retrieve the associated SensitiveMeta instance for this video, creating and assigning one if it does not exist.
621
-
677
+
678
+ **Two-Phase Patient Data Pattern:**
679
+ This method implements a two-phase approach to handle incomplete patient data:
680
+
681
+ **Phase 1: Initial Creation (with defaults)**
682
+ - Creates SensitiveMeta with default patient data to prevent hash calculation errors
683
+ - Default values: patient_first_name="Patient", patient_last_name="Unknown", patient_dob=1990-01-01
684
+ - Allows video import to proceed even without extracted patient data
685
+ - Temporary hash and pseudo-entities are created
686
+
687
+ **Phase 2: Update (with extracted data)**
688
+ - Real patient data is extracted later (e.g., from video OCR via lx_anonymizer)
689
+ - update_from_dict() is called with actual patient information
690
+ - Hash is recalculated automatically using real data
691
+ - Correct pseudo-entities are created/linked based on new hash
692
+
693
+ **Example workflow:**
694
+ ```python
695
+ # Phase 1: Video creation
696
+ video = VideoFile.create_from_file_initialized(...)
697
+ video.initialize() # Calls this method
698
+ # → SensitiveMeta created with defaults
699
+ # → Hash: sha256("Patient Unknown 1990-01-01...")
700
+
701
+ # Phase 2: Frame cleaning extracts real data
702
+ extracted = {"patient_first_name": "Max", "patient_last_name": "Mustermann", ...}
703
+ video.sensitive_meta.update_from_dict(extracted)
704
+ # → Hash: sha256("Max Mustermann 1985-03-15...") (RECALCULATED)
705
+ ```
706
+
622
707
  Returns:
623
708
  SensitiveMeta: The related SensitiveMeta instance.
709
+
710
+ See Also:
711
+ - sensitive_meta_logic.perform_save_logic() for hash calculation details
712
+ - sensitive_meta_logic.update_sensitive_meta_from_dict() for update mechanism
624
713
  """
714
+ from datetime import date as dt_date
715
+
625
716
  from endoreg_db.models import SensitiveMeta
717
+
626
718
  if self.sensitive_meta is None:
627
- self.sensitive_meta = SensitiveMeta.objects.create(center = self.center)
628
- # Do not mark processed here; it will be set after extraction/validation steps
719
+ # Use create_from_dict with default patient data
720
+ # to prevent "First name is required to calculate patient hash" error
721
+ default_data = {
722
+ "patient_first_name": "Patient",
723
+ "patient_last_name": "Unknown",
724
+ "patient_dob": dt_date(1990, 1, 1),
725
+ "examination_date": dt_date.today(),
726
+ "center": self.center,
727
+ }
728
+ self.sensitive_meta = SensitiveMeta.create_from_dict(default_data)
729
+ self.save(update_fields=["sensitive_meta"])
730
+ # Do not mark state as processed here; it will be set after extraction/validation steps
629
731
  return self.sensitive_meta
630
732
 
631
733
  def get_outside_segments(self, only_validated: bool = False) -> models.QuerySet["LabelVideoSegment"]:
632
734
  """
633
735
  Return all video segments labeled as "outside" for this video.
634
-
736
+
635
737
  Parameters:
636
738
  only_validated (bool): If True, only segments with a validated state are included.
637
-
739
+
638
740
  Returns:
639
741
  QuerySet: A queryset of LabelVideoSegment instances labeled as "outside". Returns an empty queryset if the label does not exist or an error occurs.
640
742
  """
@@ -651,43 +753,96 @@ class VideoFile(models.Model):
651
753
  logger.warning("Outside label not found in the database.")
652
754
  return self.label_video_segments.none()
653
755
  except Exception as e:
654
- logger.error("Error getting outside segments for video %s: %s", self.uuid, e, exc_info=True)
756
+ logger.error(
757
+ "Error getting outside segments for video %s: %s",
758
+ self.uuid,
759
+ e,
760
+ exc_info=True,
761
+ )
655
762
  return self.label_video_segments.none()
656
-
763
+
764
+ @classmethod
765
+ def create_video_without_outside_frames(cls, instance: "VideoFile", only_validated: bool = False) -> bool:
766
+ """
767
+ Creates a new video by excluding frames that belong to 'outside' segments.
768
+
769
+ Parameters:
770
+ only_validated (bool): If True, only validated segments are considered for frame exclusion.
771
+
772
+ Returns:
773
+ VideoFile: A new VideoFile instance with the frames excluding those labeled as 'outside'.
774
+ """
775
+ video = instance
776
+
777
+ if not video:
778
+ logger.warning("No processed video file available for VideoFile %s.", cls.uuid)
779
+ return False
780
+ try:
781
+ extracted = video.extract_frames(quality=2, overwrite=False, ext="jpg", verbose=False, from_processed=True)
782
+ assert extracted is True
783
+ except AssertionError:
784
+ # Use default anonymization here
785
+ video.anonymize
786
+ extracted = video.extract_frames(quality=2, overwrite=False, ext="jpg", verbose=False, from_processed=True)
787
+ assert extracted is True
788
+ try:
789
+ # Step 1: Get the "outside" labeled frames
790
+ censored = _censor_outside_frames(video)
791
+ frames = [instance.get_frame_dir_path()]
792
+ assert len(frames) != 0
793
+ fps = video.fps if video.fps else 120.0 # Default to 30 FPS if fps is not set
794
+ assert fps is not None
795
+ assert video.width is not None
796
+ assert video.height is not None
797
+
798
+ # Step 2: Reassemble the video with frames excluding the 'outside' labeled frames
799
+ output_video_path = Path(f"/path/to/output/{cls.uuid}_filtered.mp4")
800
+ fps = cls.fps if cls.fps else 30.0 # Default to 30 FPS if fps is not set
801
+ new_video_file = assemble_video_from_frames(frames, output_video_path, fps, width=video.width, height=video.height)
802
+ video.processed_file = new_video_file
803
+ return True
804
+ except AssertionError as ae:
805
+ logger.error(f"Assertion error while creating video without 'outside' frames for VideoFile {cls.uuid}: {ae}", exc_info=True)
806
+ return False
807
+ except Label.DoesNotExist:
808
+ logger.warning("Outside label not found in the database.")
809
+ return False
810
+ except Exception as e:
811
+ logger.error(f"Error creating video without 'outside' frames for VideoFile {cls.uuid}: {e}", exc_info=True)
812
+ return False
813
+
657
814
  @classmethod
658
815
  def get_all_videos(cls) -> models.QuerySet["VideoFile"]:
659
816
  """
660
817
  Returns a queryset containing all VideoFile records.
661
-
818
+
662
819
  This class method retrieves every VideoFile instance in the database without filtering.
663
820
  """
664
821
  return cast(models.QuerySet["VideoFile"], cls.objects.all())
665
-
822
+
666
823
  def count_unmodified_others(self) -> int:
667
824
  """
668
825
  Count the number of other VideoFile instances that have not been modified since creation.
669
-
826
+
670
827
  Returns:
671
828
  int: The count of VideoFile records, excluding this instance, where the modification timestamp matches the creation timestamp.
672
829
  """
673
830
  return (
674
- VideoFile.objects
675
- .filter(date_modified=F('date_created')) # compare the two fields in SQL
676
- .exclude(pk=self.pk) # exclude this instance
677
- .count() # run a fast COUNT(*) on the filtered set
831
+ VideoFile.objects.filter(date_modified=F("date_created")) # compare the two fields in SQL
832
+ .exclude(pk=self.pk) # exclude this instance
833
+ .count() # run a fast COUNT(*) on the filtered set
678
834
  )
679
835
 
680
-
681
836
  def frame_number_to_s(self, frame_number: int) -> float:
682
837
  """
683
838
  Convert a frame number to its corresponding time in seconds based on the video's frames per second (FPS).
684
-
839
+
685
840
  Parameters:
686
841
  frame_number (int): The frame number to convert.
687
-
842
+
688
843
  Returns:
689
844
  float: The time in seconds corresponding to the given frame number.
690
-
845
+
691
846
  Raises:
692
847
  ValueError: If the video's FPS is not set or is less than or equal to zero.
693
848
  """
@@ -695,18 +850,18 @@ class VideoFile(models.Model):
695
850
  if fps is None or fps <= 0:
696
851
  raise ValueError("FPS must be set and greater than zero.")
697
852
  return frame_number / fps
698
-
853
+
699
854
  def get_video_by_id(self, video_id: int) -> "VideoFile":
700
855
  """
701
856
  Retrieve a VideoFile instance by its primary key (ID).
702
-
857
+
703
858
  Parameters:
704
859
  video_id (int): The primary key of the VideoFile to retrieve.
705
-
860
+
706
861
  Returns:
707
862
  VideoFile: The VideoFile instance with the specified ID.
708
-
863
+
709
864
  Raises:
710
865
  VideoFile.DoesNotExist: If no VideoFile with the given ID exists.
711
866
  """
712
- return self.objects.get(pk=video_id)
867
+ return self.objects.get(pk=video_id)