endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (372) hide show
  1. endoreg_db/authz/auth.py +74 -0
  2. endoreg_db/authz/backends.py +168 -0
  3. endoreg_db/authz/management/commands/list_routes.py +18 -0
  4. endoreg_db/authz/middleware.py +83 -0
  5. endoreg_db/authz/permissions.py +127 -0
  6. endoreg_db/authz/policy.py +218 -0
  7. endoreg_db/authz/views_auth.py +66 -0
  8. endoreg_db/config/env.py +13 -8
  9. endoreg_db/data/__init__.py +8 -31
  10. endoreg_db/data/_examples/disease.yaml +55 -0
  11. endoreg_db/data/_examples/disease_classification.yaml +13 -0
  12. endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
  13. endoreg_db/data/_examples/event.yaml +64 -0
  14. endoreg_db/data/_examples/examination.yaml +72 -0
  15. endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
  16. endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
  17. endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
  18. endoreg_db/data/_examples/finding/complication.yaml +16 -0
  19. endoreg_db/data/_examples/finding/data.yaml +105 -0
  20. endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
  21. endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
  22. endoreg_db/data/_examples/finding/outcome.yaml +12 -0
  23. endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
  24. endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
  25. endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
  26. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  27. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  28. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  29. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  30. endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
  31. endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
  32. endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
  33. endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
  34. endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
  35. endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
  36. endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
  37. endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
  38. endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
  39. endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
  40. endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
  41. endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
  42. endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
  43. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
  44. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  45. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  46. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  47. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
  48. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
  49. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
  50. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  51. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
  52. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
  53. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  54. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
  55. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  56. endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
  57. endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  58. endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
  59. endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
  60. endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
  61. endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
  62. endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
  63. endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
  64. endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
  65. endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
  66. endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
  67. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
  68. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
  69. endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
  70. endoreg_db/data/_examples/finding_type/data.yaml +43 -0
  71. endoreg_db/data/_examples/requirement/age.yaml +26 -0
  72. endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
  73. endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
  74. endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
  75. endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
  76. endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
  77. endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
  78. endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
  79. endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
  80. endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
  81. endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
  82. endoreg_db/data/_examples/requirement/gender.yaml +25 -0
  83. endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
  84. endoreg_db/data/_examples/requirement/medication.yaml +93 -0
  85. endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
  86. endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
  87. endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
  88. endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
  89. endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  90. endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
  91. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
  92. endoreg_db/data/event_classification/data.yaml +4 -0
  93. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  94. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
  95. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
  96. endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
  97. endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
  98. endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
  99. endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
  100. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
  101. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  102. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  103. endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
  104. endoreg_db/data/requirement_set/_old_ +109 -0
  105. endoreg_db/data/requirement_set_type/data.yaml +21 -0
  106. endoreg_db/data/setup_config.yaml +4 -4
  107. endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
  108. endoreg_db/exceptions.py +5 -2
  109. endoreg_db/helpers/data_loader.py +1 -1
  110. endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
  111. endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
  112. endoreg_db/management/commands/import_video.py +9 -10
  113. endoreg_db/management/commands/import_video_with_classification.py +1 -1
  114. endoreg_db/management/commands/init_default_ai_model.py +1 -1
  115. endoreg_db/management/commands/list_routes.py +18 -0
  116. endoreg_db/management/commands/load_ai_model_data.py +2 -1
  117. endoreg_db/management/commands/load_center_data.py +12 -12
  118. endoreg_db/management/commands/load_requirement_data.py +60 -31
  119. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  120. endoreg_db/management/commands/setup_endoreg_db.py +14 -10
  121. endoreg_db/management/commands/storage_management.py +271 -203
  122. endoreg_db/migrations/0001_initial.py +1799 -1300
  123. endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
  124. endoreg_db/migrations/_old/0001_initial.py +1857 -0
  125. endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
  126. endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
  127. endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
  128. endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
  129. endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
  130. endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
  131. endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
  132. endoreg_db/models/__init__.py +78 -123
  133. endoreg_db/models/administration/__init__.py +21 -42
  134. endoreg_db/models/administration/ai/active_model.py +2 -2
  135. endoreg_db/models/administration/ai/ai_model.py +7 -6
  136. endoreg_db/models/administration/case/__init__.py +1 -15
  137. endoreg_db/models/administration/case/case.py +3 -3
  138. endoreg_db/models/administration/case/case_template/__init__.py +2 -14
  139. endoreg_db/models/administration/case/case_template/case_template.py +2 -124
  140. endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
  141. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
  142. endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
  143. endoreg_db/models/administration/center/center.py +33 -19
  144. endoreg_db/models/administration/center/center_product.py +12 -9
  145. endoreg_db/models/administration/center/center_resource.py +25 -19
  146. endoreg_db/models/administration/center/center_shift.py +21 -17
  147. endoreg_db/models/administration/center/center_waste.py +16 -8
  148. endoreg_db/models/administration/person/__init__.py +2 -0
  149. endoreg_db/models/administration/person/employee/employee.py +10 -5
  150. endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
  151. endoreg_db/models/administration/person/employee/employee_type.py +12 -6
  152. endoreg_db/models/administration/person/examiner/examiner.py +13 -11
  153. endoreg_db/models/administration/person/patient/__init__.py +2 -0
  154. endoreg_db/models/administration/person/patient/patient.py +103 -100
  155. endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
  156. endoreg_db/models/administration/person/person.py +4 -0
  157. endoreg_db/models/administration/person/profession/__init__.py +8 -4
  158. endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
  159. endoreg_db/models/administration/product/product.py +20 -15
  160. endoreg_db/models/administration/product/product_material.py +17 -18
  161. endoreg_db/models/administration/product/product_weight.py +12 -8
  162. endoreg_db/models/administration/product/reference_product.py +23 -55
  163. endoreg_db/models/administration/qualification/qualification.py +7 -3
  164. endoreg_db/models/administration/qualification/qualification_type.py +7 -3
  165. endoreg_db/models/administration/shift/scheduled_days.py +8 -5
  166. endoreg_db/models/administration/shift/shift.py +16 -12
  167. endoreg_db/models/administration/shift/shift_type.py +23 -31
  168. endoreg_db/models/label/__init__.py +7 -8
  169. endoreg_db/models/label/annotation/image_classification.py +10 -9
  170. endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
  171. endoreg_db/models/label/label.py +15 -15
  172. endoreg_db/models/label/label_set.py +19 -6
  173. endoreg_db/models/label/label_type.py +1 -1
  174. endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
  175. endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
  176. endoreg_db/models/label/video_segmentation_label.py +4 -0
  177. endoreg_db/models/label/video_segmentation_labelset.py +4 -3
  178. endoreg_db/models/media/frame/frame.py +22 -22
  179. endoreg_db/models/media/pdf/raw_pdf.py +249 -177
  180. endoreg_db/models/media/pdf/report_file.py +25 -29
  181. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
  182. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
  183. endoreg_db/models/media/video/__init__.py +1 -0
  184. endoreg_db/models/media/video/create_from_file.py +48 -56
  185. endoreg_db/models/media/video/pipe_1.py +30 -33
  186. endoreg_db/models/media/video/pipe_2.py +8 -9
  187. endoreg_db/models/media/video/video_file.py +359 -204
  188. endoreg_db/models/media/video/video_file_ai.py +288 -74
  189. endoreg_db/models/media/video/video_file_anonymize.py +38 -38
  190. endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
  191. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
  192. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
  193. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
  194. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
  195. endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
  196. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
  197. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
  198. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
  199. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
  200. endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
  201. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
  202. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
  203. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
  204. endoreg_db/models/media/video/video_file_io.py +109 -62
  205. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
  206. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
  207. endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
  208. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
  209. endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
  210. endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
  211. endoreg_db/models/media/video/video_file_segments.py +24 -17
  212. endoreg_db/models/media/video/video_metadata.py +19 -35
  213. endoreg_db/models/media/video/video_processing.py +96 -95
  214. endoreg_db/models/medical/contraindication/__init__.py +13 -3
  215. endoreg_db/models/medical/disease.py +22 -16
  216. endoreg_db/models/medical/event.py +31 -18
  217. endoreg_db/models/medical/examination/__init__.py +13 -6
  218. endoreg_db/models/medical/examination/examination.py +17 -18
  219. endoreg_db/models/medical/examination/examination_indication.py +26 -25
  220. endoreg_db/models/medical/examination/examination_time.py +16 -6
  221. endoreg_db/models/medical/examination/examination_time_type.py +9 -6
  222. endoreg_db/models/medical/examination/examination_type.py +3 -4
  223. endoreg_db/models/medical/finding/finding.py +38 -39
  224. endoreg_db/models/medical/finding/finding_classification.py +37 -48
  225. endoreg_db/models/medical/finding/finding_intervention.py +27 -22
  226. endoreg_db/models/medical/finding/finding_type.py +13 -12
  227. endoreg_db/models/medical/hardware/endoscope.py +20 -26
  228. endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
  229. endoreg_db/models/medical/laboratory/lab_value.py +62 -91
  230. endoreg_db/models/medical/medication/medication.py +22 -10
  231. endoreg_db/models/medical/medication/medication_indication.py +29 -3
  232. endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
  233. endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
  234. endoreg_db/models/medical/medication/medication_schedule.py +27 -16
  235. endoreg_db/models/medical/organ/__init__.py +15 -12
  236. endoreg_db/models/medical/patient/medication_examples.py +1 -5
  237. endoreg_db/models/medical/patient/patient_disease.py +20 -23
  238. endoreg_db/models/medical/patient/patient_event.py +19 -22
  239. endoreg_db/models/medical/patient/patient_examination.py +48 -54
  240. endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
  241. endoreg_db/models/medical/patient/patient_finding.py +122 -139
  242. endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
  243. endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
  244. endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
  245. endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
  246. endoreg_db/models/medical/patient/patient_medication.py +27 -38
  247. endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
  248. endoreg_db/models/medical/risk/risk.py +7 -6
  249. endoreg_db/models/medical/risk/risk_type.py +8 -5
  250. endoreg_db/models/metadata/model_meta.py +60 -29
  251. endoreg_db/models/metadata/model_meta_logic.py +139 -18
  252. endoreg_db/models/metadata/pdf_meta.py +19 -24
  253. endoreg_db/models/metadata/sensitive_meta.py +102 -85
  254. endoreg_db/models/metadata/sensitive_meta_logic.py +383 -43
  255. endoreg_db/models/metadata/video_meta.py +51 -31
  256. endoreg_db/models/metadata/video_prediction_logic.py +16 -23
  257. endoreg_db/models/metadata/video_prediction_meta.py +29 -33
  258. endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
  259. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
  260. endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
  261. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
  262. endoreg_db/models/other/emission/emission_factor.py +18 -8
  263. endoreg_db/models/other/gender.py +10 -5
  264. endoreg_db/models/other/information_source.py +25 -25
  265. endoreg_db/models/other/material.py +9 -5
  266. endoreg_db/models/other/resource.py +6 -4
  267. endoreg_db/models/other/tag.py +10 -5
  268. endoreg_db/models/other/transport_route.py +13 -8
  269. endoreg_db/models/other/unit.py +10 -6
  270. endoreg_db/models/other/waste.py +6 -5
  271. endoreg_db/models/requirement/requirement.py +580 -272
  272. endoreg_db/models/requirement/requirement_error.py +85 -0
  273. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  274. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
  275. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
  276. endoreg_db/models/requirement/requirement_operator.py +36 -33
  277. endoreg_db/models/requirement/requirement_set.py +74 -57
  278. endoreg_db/models/state/__init__.py +4 -4
  279. endoreg_db/models/state/abstract.py +2 -2
  280. endoreg_db/models/state/anonymization.py +12 -0
  281. endoreg_db/models/state/audit_ledger.py +46 -47
  282. endoreg_db/models/state/label_video_segment.py +9 -0
  283. endoreg_db/models/state/raw_pdf.py +40 -46
  284. endoreg_db/models/state/sensitive_meta.py +6 -2
  285. endoreg_db/models/state/video.py +58 -53
  286. endoreg_db/models/upload_job.py +32 -55
  287. endoreg_db/models/utils.py +1 -2
  288. endoreg_db/root_urls.py +21 -2
  289. endoreg_db/serializers/__init__.py +26 -57
  290. endoreg_db/serializers/anonymization.py +18 -10
  291. endoreg_db/serializers/meta/report_meta.py +1 -1
  292. endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
  293. endoreg_db/serializers/misc/__init__.py +1 -1
  294. endoreg_db/serializers/misc/file_overview.py +33 -91
  295. endoreg_db/serializers/misc/{vop_patient_data.py → sensitive_patient_data.py} +1 -1
  296. endoreg_db/serializers/requirements/requirement_sets.py +92 -22
  297. endoreg_db/serializers/video/segmentation.py +2 -1
  298. endoreg_db/serializers/video/video_processing_history.py +20 -5
  299. endoreg_db/serializers/video_examination.py +198 -0
  300. endoreg_db/services/anonymization.py +75 -73
  301. endoreg_db/services/lookup_service.py +256 -73
  302. endoreg_db/services/lookup_store.py +174 -30
  303. endoreg_db/services/pdf_import.py +711 -310
  304. endoreg_db/services/storage_aware_video_processor.py +140 -114
  305. endoreg_db/services/video_import.py +266 -117
  306. endoreg_db/urls/__init__.py +27 -27
  307. endoreg_db/urls/label_video_segments.py +2 -0
  308. endoreg_db/urls/media.py +108 -66
  309. endoreg_db/urls/root_urls.py +29 -0
  310. endoreg_db/utils/__init__.py +15 -5
  311. endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
  312. endoreg_db/utils/case_generator/__init__.py +3 -0
  313. endoreg_db/utils/dataloader.py +88 -16
  314. endoreg_db/utils/defaults/set_default_center.py +32 -0
  315. endoreg_db/utils/names.py +22 -16
  316. endoreg_db/utils/permissions.py +2 -1
  317. endoreg_db/utils/pipelines/process_video_dir.py +1 -1
  318. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
  319. endoreg_db/utils/setup_config.py +8 -5
  320. endoreg_db/utils/storage.py +115 -0
  321. endoreg_db/utils/validate_endo_roi.py +8 -2
  322. endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
  323. endoreg_db/views/__init__.py +5 -12
  324. endoreg_db/views/anonymization/media_management.py +198 -163
  325. endoreg_db/views/anonymization/overview.py +4 -1
  326. endoreg_db/views/anonymization/validate.py +174 -40
  327. endoreg_db/views/media/__init__.py +2 -0
  328. endoreg_db/views/media/pdf_media.py +131 -150
  329. endoreg_db/views/media/sensitive_metadata.py +46 -6
  330. endoreg_db/views/media/video_media.py +89 -82
  331. endoreg_db/views/media/video_segments.py +187 -260
  332. endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
  333. endoreg_db/views/patient/patient.py +5 -4
  334. endoreg_db/views/pdf/__init__.py +5 -8
  335. endoreg_db/views/pdf/pdf_stream.py +186 -0
  336. endoreg_db/views/pdf/pdf_stream_views.py +0 -127
  337. endoreg_db/views/pdf/reimport.py +86 -91
  338. endoreg_db/views/requirement/evaluate.py +188 -187
  339. endoreg_db/views/requirement/lookup.py +186 -288
  340. endoreg_db/views/requirement/requirement_utils.py +89 -0
  341. endoreg_db/views/video/__init__.py +0 -4
  342. endoreg_db/views/video/correction.py +2 -2
  343. endoreg_db/views/video/video_examination_viewset.py +202 -289
  344. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
  345. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +350 -255
  346. endoreg_db/models/administration/permissions/__init__.py +0 -44
  347. endoreg_db/models/media/video/refactor_plan.md +0 -0
  348. endoreg_db/models/media/video/video_file_frames.py +0 -0
  349. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  350. endoreg_db/models/rule/__init__.py +0 -13
  351. endoreg_db/models/rule/rule.py +0 -27
  352. endoreg_db/models/rule/rule_applicator.py +0 -224
  353. endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
  354. endoreg_db/models/rule/rule_type.py +0 -20
  355. endoreg_db/models/rule/ruleset.py +0 -17
  356. endoreg_db/serializers/video/video_metadata.py +0 -105
  357. endoreg_db/urls/report.py +0 -48
  358. endoreg_db/urls/video.py +0 -61
  359. endoreg_db/utils/case_generator/case_generator.py +0 -159
  360. endoreg_db/utils/case_generator/utils.py +0 -30
  361. endoreg_db/views/pdf/pdf_media.py +0 -239
  362. endoreg_db/views/report/__init__.py +0 -9
  363. endoreg_db/views/report/report_list.py +0 -112
  364. endoreg_db/views/report/report_with_secure_url.py +0 -28
  365. endoreg_db/views/report/start_examination.py +0 -7
  366. endoreg_db/views/video/video_media.py +0 -158
  367. endoreg_db/views.py +0 -0
  368. /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
  369. /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
  370. /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
  371. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
  372. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
@@ -9,25 +9,24 @@ Changelog:
9
9
  during concurrent video imports (matches PDF import pattern)
10
10
  """
11
11
 
12
- from datetime import date
13
12
  import logging
14
- import sys
15
13
  import os
16
14
  import shutil
17
15
  import time
18
16
  from contextlib import contextmanager
17
+ from datetime import date
19
18
  from pathlib import Path
20
- from typing import Union, Dict, Any, Optional, List, Tuple
19
+ from typing import Any, Dict, List, Optional, Tuple, Union
20
+ import subprocess
21
21
  from django.db import transaction
22
- from lx_anonymizer import FrameCleaner
23
- from moviepy import video
24
- from endoreg_db.models import VideoFile, SensitiveMeta
25
- from endoreg_db.utils.paths import STORAGE_DIR, VIDEO_DIR, ANONYM_VIDEO_DIR
26
- import random
27
- from endoreg_db.utils.hashs import get_video_hash
28
- from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
29
22
  from django.db.models.fields.files import FieldFile
30
- from endoreg_db.models import EndoscopyProcessor
23
+
24
+ from endoreg_db.models import EndoscopyProcessor, SensitiveMeta, VideoFile
25
+ from endoreg_db.models.media.video.video_file_anonymize import _cleanup_raw_assets
26
+ from endoreg_db.utils import ensure_local_file, storage_file_exists
27
+ from endoreg_db.utils.hashs import get_video_hash
28
+ from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, STORAGE_DIR, VIDEO_DIR
29
+ from endoreg_db.models.state import VideoState
31
30
 
32
31
  # File lock configuration (matches PDF import)
33
32
  STALE_LOCK_SECONDS = 6000 # 100 minutes - reclaim locks older than this
@@ -74,8 +73,11 @@ class VideoImportService:
74
73
  self.processing_context: Dict[str, Any] = {}
75
74
 
76
75
  self.delete_source = True
76
+ self.original_file_path = None
77
77
 
78
78
  self.logger = logging.getLogger(__name__)
79
+
80
+ self.current_video_id = Optional[int]
79
81
 
80
82
  self.cleaner = None # This gets instantiated in the perform_frame_cleaning method
81
83
 
@@ -116,7 +118,11 @@ class VideoImportService:
116
118
 
117
119
  if age is not None and age > STALE_LOCK_SECONDS:
118
120
  try:
119
- logger.warning("Stale lock detected for %s (age %.0fs). Reclaiming lock...", path, age)
121
+ logger.warning(
122
+ "Stale lock detected for %s (age %.0fs). Reclaiming lock...",
123
+ path,
124
+ age,
125
+ )
120
126
  lock_path.unlink()
121
127
  except Exception as e:
122
128
  logger.warning("Failed to remove stale lock %s: %s", lock_path, e)
@@ -206,7 +212,14 @@ class VideoImportService:
206
212
  finally:
207
213
  self._cleanup_processing_context()
208
214
 
209
- def _initialize_processing_context(self, file_path: Union[Path, str], center_name: str, processor_name: str, save_video: bool, delete_source: bool):
215
+ def _initialize_processing_context(
216
+ self,
217
+ file_path: Union[Path, str],
218
+ center_name: str,
219
+ processor_name: str,
220
+ save_video: bool,
221
+ delete_source: bool,
222
+ ):
210
223
  """Initialize the processing context for the current video import."""
211
224
  self.processing_context = {
212
225
  "file_path": Path(file_path),
@@ -219,6 +232,7 @@ class VideoImportService:
219
232
  "anonymization_completed": False,
220
233
  "error_reason": None,
221
234
  }
235
+ self.original_file_path = str(file_path)
222
236
 
223
237
  self.logger.info(f"Initialized processing context for: {file_path}")
224
238
 
@@ -270,6 +284,7 @@ class VideoImportService:
270
284
  delete_source=self.processing_context["delete_source"],
271
285
  save_video_file=self.processing_context["save_video"],
272
286
  )
287
+ self.current_video_id = self.current_video.pk
273
288
 
274
289
  if not self.current_video:
275
290
  raise RuntimeError("Failed to create VideoFile instance")
@@ -319,12 +334,21 @@ class VideoImportService:
319
334
  except Exception:
320
335
  stored_raw_path = None
321
336
 
322
- # Fallback: derive from UUID + suffix
337
+ # Fallback: derive from UUID + suffix - ALWAYS use UUID for consistency
323
338
  if not stored_raw_path:
324
339
  suffix = source_path.suffix or ".mp4"
325
340
  uuid_str = getattr(_current_video, "uuid", None)
326
- filename = f"{uuid_str}{suffix}" if uuid_str else source_path.name
341
+ if uuid_str:
342
+ filename = f"{uuid_str}{suffix}"
343
+ else:
344
+ # Emergency fallback with timestamp to avoid conflicts
345
+ import time
346
+
347
+ timestamp = int(time.time())
348
+ filename = f"video_{timestamp}{suffix}"
349
+ self.logger.warning("No UUID available, using timestamp-based filename: %s", filename)
327
350
  stored_raw_path = videos_dir / filename
351
+ self.logger.debug("Using UUID-based raw filename: %s", filename)
328
352
 
329
353
  delete_source = bool(self.processing_context.get("delete_source", True))
330
354
  stored_raw_path.parent.mkdir(parents=True, exist_ok=True)
@@ -369,9 +393,6 @@ class VideoImportService:
369
393
  # Initialize video specifications
370
394
  video.initialize_video_specs()
371
395
 
372
- # Initialize frame objects in database
373
- video.initialize_frames()
374
-
375
396
  # Extract frames BEFORE processing to prevent pipeline 1 conflicts
376
397
  self.logger.info("Pre-extracting frames to avoid pipeline conflicts...")
377
398
  try:
@@ -379,6 +400,8 @@ class VideoImportService:
379
400
  if frames_extracted:
380
401
  self.processing_context["frames_extracted"] = True
381
402
  self.logger.info("Frame extraction completed successfully")
403
+ # Initialize frame objects in database
404
+ video.initialize_frames(video.get_frame_paths())
382
405
 
383
406
  # CRITICAL: Immediately save the frames_extracted state to database
384
407
  # to prevent refresh_from_db() in pipeline 1 from overriding it
@@ -420,20 +443,23 @@ class VideoImportService:
420
443
  endoscope_data_roi_nested, endoscope_image_roi = self._get_processor_roi_info()
421
444
 
422
445
  # Perform frame cleaning with timeout to prevent blocking
423
- from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError
446
+ from concurrent.futures import ThreadPoolExecutor
447
+ from concurrent.futures import TimeoutError as FutureTimeoutError
424
448
 
425
449
  with ThreadPoolExecutor(max_workers=1) as executor:
426
- future = executor.submit(self._perform_frame_cleaning, endoscope_data_roi_nested, endoscope_image_roi)
450
+ future = executor.submit(
451
+ self._perform_frame_cleaning,
452
+ endoscope_data_roi_nested,
453
+ endoscope_image_roi,
454
+ )
427
455
  try:
428
456
  # Increased timeout to better accommodate ffmpeg + OCR
429
- future.result(timeout=50000)
457
+ future.result(timeout=5000)
430
458
  self.processing_context["anonymization_completed"] = True
431
459
  self.logger.info("Frame cleaning completed successfully within timeout")
432
460
  except FutureTimeoutError:
433
461
  self.logger.warning("Frame cleaning timed out; entering grace period check for cleaned output")
434
462
  # Grace period: detect if cleaned file appears shortly after timeout
435
- raw_video_path = self.processing_context.get("raw_video_path")
436
- video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name if raw_video_path else "video.mp4")
437
463
  grace_seconds = 60
438
464
  expected_cleaned_path: Optional[Path] = None
439
465
  processed_field = video.processed_file
@@ -448,7 +474,10 @@ class VideoImportService:
448
474
  if expected_cleaned_path.exists():
449
475
  self.processing_context["cleaned_video_path"] = expected_cleaned_path
450
476
  self.processing_context["anonymization_completed"] = True
451
- self.logger.info("Detected cleaned video during grace period: %s", expected_cleaned_path)
477
+ self.logger.info(
478
+ "Detected cleaned video during grace period: %s",
479
+ expected_cleaned_path,
480
+ )
452
481
  found = True
453
482
  break
454
483
  time.sleep(1)
@@ -494,21 +523,31 @@ class VideoImportService:
494
523
  original_raw_file_path_to_delete = video.get_raw_file_path()
495
524
  original_raw_frame_dir_to_delete = video.get_frame_dir_path()
496
525
 
497
- video.raw_file.name = None # type: ignore[assignment]
526
+ video.raw_file.name = ""
498
527
 
499
528
  update_fields.extend(["raw_file", "video_hash"])
500
529
 
501
530
  transaction.on_commit(
502
531
  lambda: _cleanup_raw_assets(
503
- video_uuid=video.uuid, raw_file_path=original_raw_file_path_to_delete, raw_frame_dir=original_raw_frame_dir_to_delete
532
+ video_uuid=video.uuid,
533
+ raw_file_path=original_raw_file_path_to_delete,
534
+ raw_frame_dir=original_raw_frame_dir_to_delete,
504
535
  )
505
536
  )
506
537
 
507
538
  video.save(update_fields=update_fields)
508
- video.state.mark_anonymized(save=True)
509
- video.refresh_from_db()
510
- self.current_video = video
511
- return True
539
+ if not isinstance(video.state, VideoState):
540
+ try:
541
+ video.get_or_create_state()
542
+ except ValueError as e:
543
+ raise RuntimeError(f"Video state not found for video {video.uuid}. Error {e}")
544
+
545
+ else:
546
+ video.state.mark_anonymized(save=True)
547
+ video.refresh_from_db()
548
+ self.current_video = video
549
+
550
+ return True
512
551
 
513
552
  def _fallback_anonymize_video(self):
514
553
  """
@@ -539,7 +578,11 @@ class VideoImportService:
539
578
  try:
540
579
  video.refresh_from_db()
541
580
  except Exception as refresh_error:
542
- self.logger.warning("Could not refresh VideoFile %s from DB: %s", video.uuid, refresh_error)
581
+ self.logger.warning(
582
+ "Could not refresh VideoFile %s from DB: %s",
583
+ video.uuid,
584
+ refresh_error,
585
+ )
543
586
 
544
587
  state = video.get_or_create_state()
545
588
 
@@ -587,8 +630,9 @@ class VideoImportService:
587
630
  else:
588
631
  raw_video_path = self.processing_context.get("raw_video_path")
589
632
  if raw_video_path and Path(raw_video_path).exists():
590
- video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name)
591
- processed_filename = f"processed_{video_filename}"
633
+ # Use UUID-based naming to avoid conflicts
634
+ suffix = Path(raw_video_path).suffix or ".mp4"
635
+ processed_filename = f"processed_{video.uuid}{suffix}"
592
636
  processed_video_path = Path(raw_video_path).parent / processed_filename
593
637
  try:
594
638
  shutil.copy2(str(raw_video_path), str(processed_video_path))
@@ -624,7 +668,10 @@ class VideoImportService:
624
668
 
625
669
  self.processing_context["anonymization_completed"] = True
626
670
  else:
627
- self.logger.warning("Processed video file not found after move: %s", anonym_target_path)
671
+ self.logger.warning(
672
+ "Processed video file not found after move: %s",
673
+ anonym_target_path,
674
+ )
628
675
  except Exception as exc:
629
676
  self.logger.error("Failed to move processed video to anonym_videos: %s", exc)
630
677
  else:
@@ -646,7 +693,7 @@ class VideoImportService:
646
693
  except Exception as exc:
647
694
  self.logger.warning("Failed to remove source file %s: %s", source_path, exc)
648
695
 
649
- if not video.processed_file or not Path(video.processed_file.path).exists():
696
+ if not video.processed_file or not storage_file_exists(video.processed_file):
650
697
  self.logger.warning("No processed_file found after cleanup - video will be unprocessed")
651
698
  try:
652
699
  video.anonymize(delete_original_raw=self.delete_source)
@@ -663,6 +710,12 @@ class VideoImportService:
663
710
  with transaction.atomic():
664
711
  video.refresh_from_db()
665
712
  if hasattr(video, "state") and self.processing_context.get("anonymization_completed"):
713
+ if not isinstance(video.state, VideoState):
714
+ try:
715
+ video.get_or_create_state()
716
+ except:
717
+ raise RuntimeError(f"Video state not found for video {video.uuid}")
718
+
666
719
  video.state.mark_sensitive_meta_processed(save=True)
667
720
 
668
721
  self.logger.info("Import and anonymization completed for VideoFile UUID: %s", video.uuid)
@@ -677,48 +730,82 @@ class VideoImportService:
677
730
  """Create or move a sensitive copy of the raw video file inside storage."""
678
731
 
679
732
  video = video_instance or self._require_current_video()
680
-
681
733
  raw_field: FieldFile | None = getattr(video, "raw_file", None)
682
- source_path: Path | None = None
683
- try:
684
- if raw_field and raw_field.path:
685
- source_path = Path(raw_field.path)
686
- except Exception:
687
- source_path = None
688
734
 
689
- if source_path is None and file_path is not None:
690
- source_path = Path(file_path)
735
+ def copy_into_sensitive(source: Path) -> Path:
736
+ target_dir = VIDEO_DIR / "sensitive"
737
+ if not target_dir.exists():
738
+ self.logger.info("Creating sensitive file directory: %s", target_dir)
739
+ os.makedirs(target_dir, exist_ok=True)
691
740
 
692
- if source_path is None:
693
- raise ValueError("No file path available for creating sensitive file")
694
- if not raw_field:
695
- raise ValueError("VideoFile must have a raw_file to create a sensitive file")
741
+ target_name = source.name or "raw_video"
742
+ target_file_path = target_dir / target_name
743
+
744
+ if source != target_file_path:
745
+ try:
746
+ shutil.copy2(source, target_file_path)
747
+ self.logger.info("Copied raw file to sensitive directory: %s", target_file_path)
748
+ except Exception as exc:
749
+ self.logger.warning("Failed to copy raw file to sensitive dir: %s", exc)
750
+ shutil.copy(source, target_file_path)
751
+ self.logger.info(
752
+ "Fallback copy succeeded for sensitive directory: %s",
753
+ target_file_path,
754
+ )
755
+ else:
756
+ self.logger.debug(
757
+ "Source path already in sensitive directory: %s",
758
+ target_file_path,
759
+ )
696
760
 
697
- target_dir = VIDEO_DIR / "sensitive"
698
- if not target_dir.exists():
699
- self.logger.info("Creating sensitive file directory: %s", target_dir)
700
- os.makedirs(target_dir, exist_ok=True)
761
+ return target_file_path
701
762
 
702
- target_file_path = target_dir / source_path.name
703
- try:
704
- shutil.move(str(source_path), str(target_file_path))
705
- self.logger.info("Moved raw file to sensitive directory: %s", target_file_path)
706
- except Exception as exc:
707
- self.logger.warning("Failed to move raw file to sensitive dir, copying instead: %s", exc)
708
- shutil.copy(str(source_path), str(target_file_path))
763
+ target_file_path: Path | None = None
764
+
765
+ # Prefer an on-disk path from the FieldFile when available
766
+ if raw_field:
709
767
  try:
710
- os.remove(source_path)
711
- except FileNotFoundError:
712
- pass
768
+ local_candidate = Path(raw_field.path)
769
+ if local_candidate.exists():
770
+ target_file_path = copy_into_sensitive(local_candidate)
771
+ except Exception:
772
+ target_file_path = None
773
+
774
+ if target_file_path is None and storage_file_exists(raw_field):
775
+ try:
776
+ with ensure_local_file(raw_field) as temp_source:
777
+ target_file_path = copy_into_sensitive(Path(temp_source))
778
+ except Exception as exc:
779
+ self.logger.warning("Failed to download raw_field for sensitive copy: %s", exc)
780
+
781
+ if target_file_path is None and file_path is not None:
782
+ file_candidate = Path(file_path)
783
+ if file_candidate.exists():
784
+ target_file_path = copy_into_sensitive(file_candidate)
785
+
786
+ if target_file_path is None:
787
+ context_path = self.processing_context.get("raw_video_path")
788
+ if context_path:
789
+ context_candidate = Path(context_path)
790
+ if context_candidate.exists():
791
+ target_file_path = copy_into_sensitive(context_candidate)
792
+
793
+ if target_file_path is None:
794
+ raise ValueError("No file path available for creating sensitive file")
795
+ if not raw_field:
796
+ raise ValueError("VideoFile must have a raw_file to create a sensitive file")
713
797
 
714
798
  try:
715
799
  from endoreg_db.utils import data_paths
716
800
 
717
801
  storage_root = data_paths["storage"]
718
802
  relative_path = target_file_path.relative_to(storage_root)
719
- video.raw_file.name = str(relative_path)
803
+ video.raw_file.name = relative_path.as_posix()
720
804
  video.save(update_fields=["raw_file"])
721
- self.logger.info("Updated video.raw_file to point to sensitive location: %s", relative_path)
805
+ self.logger.info(
806
+ "Updated video.raw_file to point to sensitive location: %s",
807
+ relative_path,
808
+ )
722
809
  except Exception as exc:
723
810
  self.logger.warning("Failed to set relative path, using fallback: %s", exc)
724
811
  video.raw_file.name = f"videos/sensitive/{target_file_path.name}"
@@ -734,7 +821,9 @@ class VideoImportService:
734
821
  self.logger.info("Created sensitive file for %s at %s", video.uuid, target_file_path)
735
822
  return target_file_path
736
823
 
737
- def _get_processor_roi_info(self) -> Tuple[Optional[List[List[Dict[str, Any]]]], Optional[Dict[str, Any]]]:
824
+ def _get_processor_roi_info(
825
+ self,
826
+ ) -> Tuple[Optional[List[List[Dict[str, Any]]]], Optional[Dict[str, Any]]]:
738
827
  """Get processor ROI information for masking."""
739
828
  endoscope_data_roi_nested = None
740
829
  endoscope_image_roi = None
@@ -748,7 +837,10 @@ class VideoImportService:
748
837
  assert isinstance(processor, EndoscopyProcessor), "Processor is not of type EndoscopyProcessor"
749
838
  endoscope_image_roi = processor.get_roi_endoscope_image()
750
839
  endoscope_data_roi_nested = processor.get_sensitive_rois()
751
- self.logger.info("Retrieved processor ROI information: endoscope_image_roi=%s", endoscope_image_roi)
840
+ self.logger.info(
841
+ "Retrieved processor ROI information: endoscope_image_roi=%s",
842
+ endoscope_image_roi,
843
+ )
752
844
  else:
753
845
  self.logger.warning(
754
846
  "No processor found for video %s, proceeding without ROI masking",
@@ -791,31 +883,16 @@ class VideoImportService:
791
883
  video.save(update_fields=["sensitive_meta"])
792
884
  self.logger.info("Created default SensitiveMeta for video %s", video.uuid)
793
885
  except Exception as exc:
794
- self.logger.error("Failed to create default SensitiveMeta for video %s: %s", video.uuid, exc)
886
+ self.logger.error(
887
+ "Failed to create default SensitiveMeta for video %s: %s",
888
+ video.uuid,
889
+ exc,
890
+ )
795
891
  return
796
892
  else:
797
- update_data: Dict[str, Any] = {}
798
- if not sensitive_meta.patient_first_name:
799
- update_data["patient_first_name"] = "Patient"
800
- if not sensitive_meta.patient_last_name:
801
- update_data["patient_last_name"] = "Unknown"
802
- if not sensitive_meta.patient_dob:
803
- update_data["patient_dob"] = date(1990, 1, 1)
804
- if not sensitive_meta.examination_date:
805
- update_data["examination_date"] = date.today()
806
-
807
- if update_data:
808
- try:
809
- sensitive_meta.update_from_dict(update_data)
810
- state = video.get_or_create_state()
811
- state.mark_sensitive_meta_processed(save=True)
812
- self.logger.info(
813
- "Updated missing SensitiveMeta fields for video %s: %s",
814
- video.uuid,
815
- list(update_data.keys()),
816
- )
817
- except Exception as exc:
818
- self.logger.error("Failed to update SensitiveMeta for video %s: %s", video.uuid, exc)
893
+ state = video.get_or_create_state()
894
+ state.mark_sensitive_meta_processed(save=True)
895
+
819
896
 
820
897
  def _ensure_frame_cleaning_available(self):
821
898
  """
@@ -825,23 +902,24 @@ class VideoImportService:
825
902
  Tuple of (availability_flag, FrameCleaner_class, ReportReader_class)
826
903
  """
827
904
  try:
828
- # Check if we can find lx-anonymizer
829
- from lx_anonymizer import FrameCleaner # type: ignore[import]
830
-
831
- if FrameCleaner:
832
- return True, FrameCleaner()
833
-
905
+ from lx_anonymizer import FrameCleaner
834
906
  except Exception as e:
835
907
  self.logger.warning(f"Frame cleaning not available: {e} Please install or update lx_anonymizer.")
908
+ _available = False
909
+ FrameCleaner = None
836
910
 
837
- return False, None
911
+ assert FrameCleaner is not None
912
+ frame_cleaner = FrameCleaner()
913
+ _available = True
914
+
915
+ return _available, frame_cleaner
838
916
 
839
917
  def _perform_frame_cleaning(self, endoscope_data_roi_nested, endoscope_image_roi):
840
918
  """Perform frame cleaning and anonymization."""
841
919
  # Instantiate frame cleaner
842
920
  is_available, frame_cleaner = self._ensure_frame_cleaning_available()
843
921
 
844
- if not is_available:
922
+ if not is_available or frame_cleaner is None:
845
923
  raise RuntimeError("Frame cleaning not available")
846
924
 
847
925
  # Prepare parameters for frame cleaning
@@ -854,12 +932,15 @@ class VideoImportService:
854
932
  except Exception:
855
933
  raise RuntimeError(f"Raw video path not found: {raw_video_path}")
856
934
 
857
- # Create temporary output path for cleaned video
858
- video_filename = self.processing_context.get("video_filename", Path(raw_video_path).name)
859
- cleaned_filename = f"cleaned_{video_filename}"
935
+ # Create temporary output path for cleaned video using UUID to avoid naming conflicts
936
+ video = self._require_current_video()
937
+ # Ensure raw_video_path is not None
860
938
  if not raw_video_path:
861
- raise RuntimeError("raw_video_path is None after fallback, cannot construct cleaned_video_path")
939
+ raise RuntimeError("raw_video_path is None, cannot construct cleaned_video_path")
940
+ suffix = Path(raw_video_path).suffix or ".mp4"
941
+ cleaned_filename = f"cleaned_{video.uuid}{suffix}"
862
942
  cleaned_video_path = Path(raw_video_path).parent / cleaned_filename
943
+ self.logger.debug("Using UUID-based cleaned filename: %s", cleaned_filename)
863
944
 
864
945
  # Clean video with ROI masking (heavy I/O operation)
865
946
  actual_cleaned_path, extracted_metadata = frame_cleaner.clean_video(
@@ -896,21 +977,58 @@ class VideoImportService:
896
977
  sm = sensitive_meta
897
978
  updated_fields = []
898
979
 
980
+ # Ensure center is set from video.center if not in extracted_metadata
981
+ metadata_to_update = extracted_metadata.copy()
982
+
983
+ # FIX: Set center object instead of center_name string
984
+ if not hasattr(sm, "center") or not sm.center:
985
+ if video.center:
986
+ metadata_to_update["center"] = video.center
987
+ self.logger.debug(
988
+ "Added center object '%s' to metadata for SensitiveMeta update",
989
+ video.center.name,
990
+ )
991
+ else:
992
+ center_name = metadata_to_update.get("center_name")
993
+ if center_name:
994
+ try:
995
+ from ..models.administration import Center
996
+
997
+ center_obj = Center.objects.get(name=center_name)
998
+ metadata_to_update["center"] = center_obj
999
+ self.logger.debug("Loaded center object '%s' from center_name", center_name)
1000
+ metadata_to_update.pop("center_name", None)
1001
+ except Center.DoesNotExist:
1002
+ self.logger.error("Center '%s' not found in database", center_name)
1003
+ return
1004
+
899
1005
  try:
900
- sm.update_from_dict(extracted_metadata)
901
- updated_fields = list(extracted_metadata.keys())
1006
+ sm.update_from_dict(metadata_to_update)
1007
+ updated_fields = list(extracted_metadata.keys()) # Only log originally extracted fields
902
1008
  except KeyError as e:
903
1009
  self.logger.warning(f"Failed to update SensitiveMeta field {e}")
1010
+ return
904
1011
 
905
1012
  if updated_fields:
906
- sm.save(update_fields=updated_fields)
907
- self.logger.info("Updated SensitiveMeta fields for video %s: %s", video.uuid, updated_fields)
1013
+ try:
1014
+ sm.save() # Remove update_fields to allow all necessary fields to be saved
1015
+ self.logger.info(
1016
+ "Updated SensitiveMeta fields for video %s: %s",
1017
+ video.uuid,
1018
+ updated_fields,
1019
+ )
908
1020
 
909
- state = video.get_or_create_state()
910
- state.mark_sensitive_meta_processed(save=True)
911
- self.logger.info("Marked sensitive metadata as processed for video %s", video.uuid)
1021
+ state = video.get_or_create_state()
1022
+ state.mark_sensitive_meta_processed(save=True)
1023
+ self.logger.info("Marked sensitive metadata as processed for video %s", video.uuid)
1024
+ except Exception as e:
1025
+ self.logger.error(f"Failed to save SensitiveMeta: {e}")
1026
+ raise # Re-raise to trigger fallback in calling method
912
1027
  else:
913
- self.logger.info("No SensitiveMeta fields updated for video %s - all existing values preserved", video.uuid)
1028
+ self.logger.info(
1029
+ "No SensitiveMeta fields updated for video %s - all existing values preserved",
1030
+ video.uuid,
1031
+ )
914
1032
 
915
1033
  def _signal_completion(self):
916
1034
  """Signal completion to the tracking system."""
@@ -918,21 +1036,23 @@ class VideoImportService:
918
1036
  video = self._require_current_video()
919
1037
 
920
1038
  raw_field: FieldFile | None = getattr(video, "raw_file", None)
921
- raw_exists = False
922
- if raw_field and getattr(raw_field, "path", None):
923
- try:
924
- raw_exists = Path(raw_field.path).exists()
925
- except (ValueError, OSError):
926
- raw_exists = False
1039
+ raw_exists = storage_file_exists(raw_field)
927
1040
 
928
1041
  video_processing_complete = video.sensitive_meta is not None and video.video_meta is not None and raw_exists
929
1042
 
930
1043
  if video_processing_complete:
931
- self.logger.info("Video %s processing completed successfully - ready for validation", video.uuid)
1044
+ self.logger.info(
1045
+ "Video %s processing completed successfully - ready for validation",
1046
+ video.uuid,
1047
+ )
932
1048
 
933
1049
  # Update completion flags if they exist
934
1050
  completion_fields = []
935
- for field_name in ["import_completed", "processing_complete", "ready_for_validation"]:
1051
+ for field_name in [
1052
+ "import_completed",
1053
+ "processing_complete",
1054
+ "ready_for_validation",
1055
+ ]:
936
1056
  if hasattr(video, field_name):
937
1057
  setattr(video, field_name, True)
938
1058
  completion_fields.append(field_name)
@@ -952,15 +1072,44 @@ class VideoImportService:
952
1072
  def _cleanup_on_error(self):
953
1073
  """Cleanup processing context on error."""
954
1074
  if self.current_video and hasattr(self.current_video, "state"):
1075
+ if self.current_video.state is None:
1076
+ try:
1077
+ self.current_video.get_or_create_state()
1078
+ except Exception as e:
1079
+ self.logger.warning(f"Video state not found for video {self.current_video.uuid} during error cleanup {e}")
1080
+ return
1081
+ self.current_video.state = self.current_video.get_or_create_state()
1082
+ try:
1083
+ if self.original_file_path is not None:
1084
+ assert Path(self.original_file_path).exists()
1085
+ else:
1086
+ self.logger.warning("Original file path is None")
1087
+ self.logger.info("Marked video import as failed in state")
1088
+ raw_file_path = getattr(self.current_video.raw_file, "path", None)
1089
+ original_file_path = self.original_file_path
1090
+ if raw_file_path and original_file_path:
1091
+ shutil.copy2(str(raw_file_path), str(original_file_path))
1092
+ else:
1093
+ self.logger.warning("Cannot restore original raw file: path is None")
1094
+ except AssertionError:
1095
+ self.logger.warning("Original file path does not exist")
955
1096
  try:
1097
+
1098
+ if not isinstance(self.current_video.state, VideoState):
1099
+ logger.error("Current video is none after Assertion for Video File")
1100
+ raise AssertionError
1101
+
1102
+
956
1103
  if self.processing_context.get("processing_started"):
957
1104
  self.current_video.state.frames_extracted = False
958
1105
  self.current_video.state.frames_initialized = False
959
1106
  self.current_video.state.video_meta_extracted = False
960
1107
  self.current_video.state.text_meta_extracted = False
961
1108
  self.current_video.state.save()
1109
+
962
1110
  except Exception as e:
963
1111
  self.logger.warning(f"Error during cleanup: {e}")
1112
+
964
1113
 
965
1114
  def _cleanup_processing_context(self):
966
1115
  """