endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (450) hide show
  1. endoreg_db/admin.py +10 -5
  2. endoreg_db/apps.py +4 -7
  3. endoreg_db/authz/auth.py +1 -0
  4. endoreg_db/authz/backends.py +1 -1
  5. endoreg_db/authz/management/commands/list_routes.py +2 -0
  6. endoreg_db/authz/middleware.py +8 -7
  7. endoreg_db/authz/permissions.py +21 -10
  8. endoreg_db/authz/policy.py +14 -19
  9. endoreg_db/authz/views_auth.py +14 -10
  10. endoreg_db/codemods/rename_datetime_fields.py +8 -1
  11. endoreg_db/exceptions.py +5 -2
  12. endoreg_db/forms/__init__.py +0 -1
  13. endoreg_db/forms/examination_form.py +4 -3
  14. endoreg_db/forms/patient_finding_intervention_form.py +30 -8
  15. endoreg_db/forms/patient_form.py +9 -13
  16. endoreg_db/forms/questionnaires/__init__.py +1 -1
  17. endoreg_db/forms/settings/__init__.py +4 -1
  18. endoreg_db/forms/unit.py +2 -1
  19. endoreg_db/helpers/count_db.py +17 -14
  20. endoreg_db/helpers/default_objects.py +2 -1
  21. endoreg_db/helpers/download_segmentation_model.py +4 -3
  22. endoreg_db/helpers/interact.py +0 -5
  23. endoreg_db/helpers/test_video_helper.py +33 -25
  24. endoreg_db/import_files/__init__.py +1 -1
  25. endoreg_db/import_files/context/__init__.py +1 -1
  26. endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
  27. endoreg_db/import_files/context/ensure_center.py +4 -4
  28. endoreg_db/import_files/context/file_lock.py +3 -3
  29. endoreg_db/import_files/context/import_context.py +11 -12
  30. endoreg_db/import_files/context/validate_directories.py +1 -0
  31. endoreg_db/import_files/file_storage/create_report_file.py +57 -34
  32. endoreg_db/import_files/file_storage/create_video_file.py +64 -35
  33. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
  34. endoreg_db/import_files/file_storage/state_management.py +89 -122
  35. endoreg_db/import_files/file_storage/storage.py +5 -1
  36. endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
  37. endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
  38. endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
  39. endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
  40. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
  41. endoreg_db/import_files/report_import_service.py +36 -30
  42. endoreg_db/import_files/video_import_service.py +27 -23
  43. endoreg_db/logger_conf.py +56 -40
  44. endoreg_db/management/__init__.py +1 -1
  45. endoreg_db/management/commands/__init__.py +1 -1
  46. endoreg_db/management/commands/check_auth.py +45 -38
  47. endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
  48. endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
  49. endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
  50. endoreg_db/management/commands/fix_video_paths.py +75 -54
  51. endoreg_db/management/commands/import_report.py +1 -3
  52. endoreg_db/management/commands/list_routes.py +2 -0
  53. endoreg_db/management/commands/load_ai_model_data.py +8 -2
  54. endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
  55. endoreg_db/management/commands/load_center_data.py +3 -3
  56. endoreg_db/management/commands/load_distribution_data.py +35 -38
  57. endoreg_db/management/commands/load_endoscope_data.py +0 -3
  58. endoreg_db/management/commands/load_examination_data.py +20 -4
  59. endoreg_db/management/commands/load_finding_data.py +18 -3
  60. endoreg_db/management/commands/load_gender_data.py +17 -24
  61. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
  62. endoreg_db/management/commands/load_information_source.py +0 -3
  63. endoreg_db/management/commands/load_lab_value_data.py +14 -3
  64. endoreg_db/management/commands/load_legacy_data.py +303 -0
  65. endoreg_db/management/commands/load_name_data.py +1 -2
  66. endoreg_db/management/commands/load_pdf_type_data.py +4 -8
  67. endoreg_db/management/commands/load_profession_data.py +0 -1
  68. endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
  69. endoreg_db/management/commands/load_requirement_data.py +6 -2
  70. endoreg_db/management/commands/load_unit_data.py +0 -4
  71. endoreg_db/management/commands/load_user_groups.py +5 -7
  72. endoreg_db/management/commands/model_input.py +169 -0
  73. endoreg_db/management/commands/register_ai_model.py +22 -16
  74. endoreg_db/management/commands/setup_endoreg_db.py +110 -32
  75. endoreg_db/management/commands/storage_management.py +14 -8
  76. endoreg_db/management/commands/summarize_db_content.py +154 -63
  77. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  78. endoreg_db/management/commands/validate_video_files.py +82 -50
  79. endoreg_db/management/commands/video_validation.py +4 -6
  80. endoreg_db/migrations/0001_initial.py +112 -63
  81. endoreg_db/models/__init__.py +8 -0
  82. endoreg_db/models/administration/ai/active_model.py +5 -5
  83. endoreg_db/models/administration/ai/ai_model.py +41 -18
  84. endoreg_db/models/administration/ai/model_type.py +1 -0
  85. endoreg_db/models/administration/case/case.py +22 -22
  86. endoreg_db/models/administration/center/__init__.py +5 -5
  87. endoreg_db/models/administration/center/center.py +6 -2
  88. endoreg_db/models/administration/center/center_resource.py +18 -4
  89. endoreg_db/models/administration/center/center_shift.py +3 -1
  90. endoreg_db/models/administration/center/center_waste.py +6 -2
  91. endoreg_db/models/administration/person/__init__.py +1 -1
  92. endoreg_db/models/administration/person/employee/__init__.py +1 -1
  93. endoreg_db/models/administration/person/employee/employee_type.py +3 -1
  94. endoreg_db/models/administration/person/examiner/__init__.py +1 -1
  95. endoreg_db/models/administration/person/examiner/examiner.py +10 -2
  96. endoreg_db/models/administration/person/names/first_name.py +6 -4
  97. endoreg_db/models/administration/person/names/last_name.py +4 -3
  98. endoreg_db/models/administration/person/patient/__init__.py +1 -1
  99. endoreg_db/models/administration/person/patient/patient.py +0 -1
  100. endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
  101. endoreg_db/models/administration/person/person.py +1 -1
  102. endoreg_db/models/administration/product/__init__.py +7 -6
  103. endoreg_db/models/administration/product/product.py +6 -2
  104. endoreg_db/models/administration/product/product_group.py +9 -7
  105. endoreg_db/models/administration/product/product_material.py +9 -2
  106. endoreg_db/models/administration/product/reference_product.py +64 -15
  107. endoreg_db/models/administration/qualification/qualification.py +3 -1
  108. endoreg_db/models/administration/shift/shift.py +3 -1
  109. endoreg_db/models/administration/shift/shift_type.py +12 -4
  110. endoreg_db/models/aidataset/__init__.py +5 -0
  111. endoreg_db/models/aidataset/aidataset.py +193 -0
  112. endoreg_db/models/label/__init__.py +1 -1
  113. endoreg_db/models/label/label.py +10 -2
  114. endoreg_db/models/label/label_set.py +3 -1
  115. endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
  116. endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
  117. endoreg_db/models/media/__init__.py +12 -5
  118. endoreg_db/models/media/frame/__init__.py +1 -1
  119. endoreg_db/models/media/frame/frame.py +34 -8
  120. endoreg_db/models/media/pdf/__init__.py +2 -1
  121. endoreg_db/models/media/pdf/raw_pdf.py +11 -4
  122. endoreg_db/models/media/pdf/report_file.py +6 -2
  123. endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
  124. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
  125. endoreg_db/models/media/video/create_from_file.py +20 -41
  126. endoreg_db/models/media/video/pipe_1.py +75 -30
  127. endoreg_db/models/media/video/pipe_2.py +37 -12
  128. endoreg_db/models/media/video/video_file.py +36 -24
  129. endoreg_db/models/media/video/video_file_ai.py +235 -70
  130. endoreg_db/models/media/video/video_file_anonymize.py +240 -65
  131. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
  132. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
  133. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
  134. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
  135. endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
  136. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
  137. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
  138. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
  139. endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
  140. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
  141. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
  142. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
  143. endoreg_db/models/media/video/video_file_io.py +85 -33
  144. endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
  145. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
  146. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
  147. endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
  148. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
  149. endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
  150. endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
  151. endoreg_db/models/media/video/video_file_segments.py +118 -27
  152. endoreg_db/models/media/video/video_metadata.py +25 -6
  153. endoreg_db/models/media/video/video_processing.py +54 -15
  154. endoreg_db/models/medical/__init__.py +3 -13
  155. endoreg_db/models/medical/contraindication/__init__.py +3 -1
  156. endoreg_db/models/medical/disease.py +18 -6
  157. endoreg_db/models/medical/event.py +6 -2
  158. endoreg_db/models/medical/examination/__init__.py +5 -1
  159. endoreg_db/models/medical/examination/examination.py +22 -6
  160. endoreg_db/models/medical/examination/examination_indication.py +23 -7
  161. endoreg_db/models/medical/examination/examination_time.py +6 -2
  162. endoreg_db/models/medical/finding/__init__.py +3 -1
  163. endoreg_db/models/medical/finding/finding.py +37 -12
  164. endoreg_db/models/medical/finding/finding_classification.py +27 -8
  165. endoreg_db/models/medical/finding/finding_intervention.py +19 -6
  166. endoreg_db/models/medical/finding/finding_type.py +3 -1
  167. endoreg_db/models/medical/hardware/__init__.py +1 -1
  168. endoreg_db/models/medical/hardware/endoscope.py +14 -2
  169. endoreg_db/models/medical/laboratory/__init__.py +1 -1
  170. endoreg_db/models/medical/laboratory/lab_value.py +139 -39
  171. endoreg_db/models/medical/medication/__init__.py +7 -3
  172. endoreg_db/models/medical/medication/medication.py +3 -1
  173. endoreg_db/models/medical/medication/medication_indication.py +3 -1
  174. endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
  175. endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
  176. endoreg_db/models/medical/medication/medication_schedule.py +3 -1
  177. endoreg_db/models/medical/patient/__init__.py +2 -10
  178. endoreg_db/models/medical/patient/medication_examples.py +3 -14
  179. endoreg_db/models/medical/patient/patient_disease.py +17 -5
  180. endoreg_db/models/medical/patient/patient_event.py +12 -4
  181. endoreg_db/models/medical/patient/patient_examination.py +52 -15
  182. endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
  183. endoreg_db/models/medical/patient/patient_finding.py +105 -29
  184. endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
  185. endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
  186. endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
  187. endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
  188. endoreg_db/models/medical/patient/patient_medication.py +25 -7
  189. endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
  190. endoreg_db/models/metadata/model_meta.py +40 -12
  191. endoreg_db/models/metadata/model_meta_logic.py +51 -16
  192. endoreg_db/models/metadata/sensitive_meta.py +65 -28
  193. endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
  194. endoreg_db/models/metadata/video_meta.py +146 -39
  195. endoreg_db/models/metadata/video_prediction_logic.py +70 -21
  196. endoreg_db/models/metadata/video_prediction_meta.py +80 -27
  197. endoreg_db/models/operation_log.py +63 -0
  198. endoreg_db/models/other/__init__.py +10 -10
  199. endoreg_db/models/other/distribution/__init__.py +9 -7
  200. endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
  201. endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
  202. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
  203. endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
  204. endoreg_db/models/other/emission/__init__.py +1 -1
  205. endoreg_db/models/other/emission/emission_factor.py +9 -3
  206. endoreg_db/models/other/information_source.py +15 -5
  207. endoreg_db/models/other/material.py +3 -1
  208. endoreg_db/models/other/transport_route.py +3 -1
  209. endoreg_db/models/other/unit.py +6 -2
  210. endoreg_db/models/report/report.py +0 -1
  211. endoreg_db/models/requirement/requirement.py +84 -27
  212. endoreg_db/models/requirement/requirement_error.py +5 -6
  213. endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
  214. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
  215. endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
  216. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
  217. endoreg_db/models/requirement/requirement_operator.py +28 -8
  218. endoreg_db/models/requirement/requirement_set.py +34 -11
  219. endoreg_db/models/state/__init__.py +1 -0
  220. endoreg_db/models/state/audit_ledger.py +9 -2
  221. endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
  222. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  223. endoreg_db/models/state/raw_pdf.py +0 -1
  224. endoreg_db/models/state/video.py +2 -4
  225. endoreg_db/models/utils.py +4 -2
  226. endoreg_db/queries/__init__.py +2 -6
  227. endoreg_db/queries/annotations/__init__.py +1 -3
  228. endoreg_db/queries/annotations/legacy.py +37 -26
  229. endoreg_db/root_urls.py +3 -4
  230. endoreg_db/schemas/examination_evaluation.py +3 -0
  231. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
  232. endoreg_db/serializers/__init__.py +2 -8
  233. endoreg_db/serializers/administration/__init__.py +1 -2
  234. endoreg_db/serializers/administration/ai/__init__.py +0 -1
  235. endoreg_db/serializers/administration/ai/active_model.py +3 -1
  236. endoreg_db/serializers/administration/ai/ai_model.py +5 -3
  237. endoreg_db/serializers/administration/ai/model_type.py +3 -1
  238. endoreg_db/serializers/administration/center.py +7 -2
  239. endoreg_db/serializers/administration/gender.py +4 -2
  240. endoreg_db/serializers/anonymization.py +13 -13
  241. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
  242. endoreg_db/serializers/examination/__init__.py +1 -1
  243. endoreg_db/serializers/examination/base.py +12 -13
  244. endoreg_db/serializers/examination/dropdown.py +6 -7
  245. endoreg_db/serializers/examination_serializer.py +3 -6
  246. endoreg_db/serializers/finding/__init__.py +1 -1
  247. endoreg_db/serializers/finding/finding.py +14 -7
  248. endoreg_db/serializers/finding_classification/__init__.py +3 -3
  249. endoreg_db/serializers/finding_classification/choice.py +3 -3
  250. endoreg_db/serializers/finding_classification/classification.py +2 -4
  251. endoreg_db/serializers/label_video_segment/__init__.py +5 -3
  252. endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
  253. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  254. endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
  255. endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
  256. endoreg_db/serializers/meta/__init__.py +1 -2
  257. endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
  258. endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
  259. endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
  260. endoreg_db/serializers/misc/__init__.py +2 -2
  261. endoreg_db/serializers/misc/file_overview.py +11 -7
  262. endoreg_db/serializers/misc/stats.py +10 -8
  263. endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
  264. endoreg_db/serializers/misc/upload_job.py +32 -29
  265. endoreg_db/serializers/patient/__init__.py +2 -1
  266. endoreg_db/serializers/patient/patient.py +32 -15
  267. endoreg_db/serializers/patient/patient_dropdown.py +11 -3
  268. endoreg_db/serializers/patient_examination/__init__.py +1 -1
  269. endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
  270. endoreg_db/serializers/patient_finding/__init__.py +1 -1
  271. endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
  272. endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
  273. endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
  274. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
  275. endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
  276. endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
  277. endoreg_db/serializers/pdf/__init__.py +1 -3
  278. endoreg_db/serializers/requirements/requirement_schema.py +1 -6
  279. endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
  280. endoreg_db/serializers/video/__init__.py +2 -2
  281. endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
  282. endoreg_db/serializers/video/video_file_brief.py +6 -2
  283. endoreg_db/serializers/video/video_file_detail.py +36 -23
  284. endoreg_db/serializers/video/video_file_list.py +4 -2
  285. endoreg_db/serializers/video/video_processing_history.py +54 -50
  286. endoreg_db/services/__init__.py +1 -1
  287. endoreg_db/services/anonymization.py +2 -2
  288. endoreg_db/services/examination_evaluation.py +40 -17
  289. endoreg_db/services/model_meta_from_hf.py +76 -0
  290. endoreg_db/services/polling_coordinator.py +101 -70
  291. endoreg_db/services/pseudonym_service.py +27 -22
  292. endoreg_db/services/report_import.py +6 -3
  293. endoreg_db/services/segment_sync.py +75 -59
  294. endoreg_db/services/video_import.py +6 -7
  295. endoreg_db/urls/__init__.py +2 -2
  296. endoreg_db/urls/ai.py +7 -25
  297. endoreg_db/urls/anonymization.py +61 -15
  298. endoreg_db/urls/auth.py +4 -4
  299. endoreg_db/urls/classification.py +4 -9
  300. endoreg_db/urls/examination.py +27 -18
  301. endoreg_db/urls/media.py +27 -34
  302. endoreg_db/urls/patient.py +11 -7
  303. endoreg_db/urls/requirements.py +3 -1
  304. endoreg_db/urls/root_urls.py +2 -3
  305. endoreg_db/urls/stats.py +24 -16
  306. endoreg_db/urls/upload.py +3 -11
  307. endoreg_db/utils/__init__.py +14 -15
  308. endoreg_db/utils/ai/__init__.py +1 -1
  309. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  310. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  311. endoreg_db/utils/ai/get.py +2 -1
  312. endoreg_db/utils/ai/inference_dataset.py +14 -15
  313. endoreg_db/utils/ai/model_training/config.py +117 -0
  314. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  315. endoreg_db/utils/ai/model_training/losses.py +68 -0
  316. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  317. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  318. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  319. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  320. endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
  321. endoreg_db/utils/ai/predict.py +4 -4
  322. endoreg_db/utils/ai/preprocess.py +19 -11
  323. endoreg_db/utils/calc_duration_seconds.py +4 -4
  324. endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
  325. endoreg_db/utils/check_video_files.py +74 -47
  326. endoreg_db/utils/cropping.py +10 -9
  327. endoreg_db/utils/dataloader.py +11 -3
  328. endoreg_db/utils/dates.py +3 -4
  329. endoreg_db/utils/defaults/set_default_center.py +7 -6
  330. endoreg_db/utils/env.py +6 -2
  331. endoreg_db/utils/extract_specific_frames.py +24 -9
  332. endoreg_db/utils/file_operations.py +30 -18
  333. endoreg_db/utils/fix_video_path_direct.py +57 -41
  334. endoreg_db/utils/frame_anonymization_utils.py +157 -157
  335. endoreg_db/utils/hashs.py +3 -18
  336. endoreg_db/utils/links/requirement_link.py +96 -52
  337. endoreg_db/utils/ocr.py +30 -25
  338. endoreg_db/utils/operation_log.py +61 -0
  339. endoreg_db/utils/parse_and_generate_yaml.py +12 -13
  340. endoreg_db/utils/paths.py +6 -6
  341. endoreg_db/utils/permissions.py +40 -24
  342. endoreg_db/utils/pipelines/process_video_dir.py +50 -26
  343. endoreg_db/utils/product/sum_emissions.py +5 -3
  344. endoreg_db/utils/product/sum_weights.py +4 -2
  345. endoreg_db/utils/pydantic_models/__init__.py +3 -4
  346. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
  347. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
  348. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
  349. endoreg_db/utils/setup_config.py +21 -5
  350. endoreg_db/utils/storage.py +3 -1
  351. endoreg_db/utils/translation.py +19 -15
  352. endoreg_db/utils/uuid.py +1 -0
  353. endoreg_db/utils/validate_endo_roi.py +12 -4
  354. endoreg_db/utils/validate_subcategory_dict.py +26 -24
  355. endoreg_db/utils/validate_video_detailed.py +207 -149
  356. endoreg_db/utils/video/__init__.py +7 -3
  357. endoreg_db/utils/video/extract_frames.py +30 -18
  358. endoreg_db/utils/video/names.py +11 -6
  359. endoreg_db/utils/video/streaming_processor.py +175 -101
  360. endoreg_db/utils/video/video_splitter.py +30 -19
  361. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
  362. endoreg_db/views/__init__.py +0 -20
  363. endoreg_db/views/anonymization/__init__.py +6 -2
  364. endoreg_db/views/anonymization/media_management.py +2 -6
  365. endoreg_db/views/anonymization/overview.py +34 -1
  366. endoreg_db/views/anonymization/validate.py +79 -18
  367. endoreg_db/views/auth/__init__.py +1 -1
  368. endoreg_db/views/auth/keycloak.py +16 -14
  369. endoreg_db/views/examination/__init__.py +12 -15
  370. endoreg_db/views/examination/examination.py +5 -5
  371. endoreg_db/views/examination/examination_manifest_cache.py +5 -5
  372. endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
  373. endoreg_db/views/examination/get_finding_classifications.py +9 -7
  374. endoreg_db/views/examination/get_findings.py +8 -10
  375. endoreg_db/views/examination/get_instruments.py +3 -2
  376. endoreg_db/views/examination/get_interventions.py +1 -1
  377. endoreg_db/views/finding/__init__.py +2 -2
  378. endoreg_db/views/finding/finding.py +58 -54
  379. endoreg_db/views/finding/get_classifications.py +1 -1
  380. endoreg_db/views/finding/get_interventions.py +1 -1
  381. endoreg_db/views/finding_classification/__init__.py +5 -5
  382. endoreg_db/views/finding_classification/finding_classification.py +5 -6
  383. endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
  384. endoreg_db/views/media/__init__.py +13 -13
  385. endoreg_db/views/media/pdf_media.py +9 -9
  386. endoreg_db/views/media/sensitive_metadata.py +10 -7
  387. endoreg_db/views/media/video_media.py +4 -4
  388. endoreg_db/views/meta/__init__.py +1 -1
  389. endoreg_db/views/meta/sensitive_meta_list.py +20 -22
  390. endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
  391. endoreg_db/views/misc/__init__.py +6 -34
  392. endoreg_db/views/misc/center.py +2 -1
  393. endoreg_db/views/misc/csrf.py +2 -1
  394. endoreg_db/views/misc/gender.py +2 -1
  395. endoreg_db/views/misc/stats.py +141 -106
  396. endoreg_db/views/patient/__init__.py +1 -3
  397. endoreg_db/views/patient/patient.py +141 -99
  398. endoreg_db/views/patient_examination/__init__.py +5 -5
  399. endoreg_db/views/patient_examination/patient_examination.py +43 -42
  400. endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
  401. endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
  402. endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
  403. endoreg_db/views/patient_examination/video.py +114 -80
  404. endoreg_db/views/patient_finding/__init__.py +1 -1
  405. endoreg_db/views/patient_finding/patient_finding.py +17 -10
  406. endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
  407. endoreg_db/views/patient_finding_classification/__init__.py +1 -1
  408. endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
  409. endoreg_db/views/report/reimport.py +1 -1
  410. endoreg_db/views/report/report_stream.py +5 -8
  411. endoreg_db/views/requirement/__init__.py +2 -1
  412. endoreg_db/views/requirement/evaluate.py +7 -9
  413. endoreg_db/views/requirement/lookup.py +2 -3
  414. endoreg_db/views/requirement/lookup_store.py +0 -1
  415. endoreg_db/views/requirement/requirement_utils.py +2 -4
  416. endoreg_db/views/stats/__init__.py +4 -4
  417. endoreg_db/views/stats/stats_views.py +152 -115
  418. endoreg_db/views/video/__init__.py +18 -27
  419. endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
  420. endoreg_db/views/{ai → video/ai}/label.py +20 -16
  421. endoreg_db/views/video/correction.py +5 -6
  422. endoreg_db/views/video/reimport.py +134 -99
  423. endoreg_db/views/video/segments_crud.py +134 -44
  424. endoreg_db/views/video/video_apply_mask.py +13 -12
  425. endoreg_db/views/video/video_correction.py +2 -1
  426. endoreg_db/views/video/video_download_processed.py +15 -15
  427. endoreg_db/views/video/video_meta_stats.py +7 -6
  428. endoreg_db/views/video/video_processing_history.py +3 -2
  429. endoreg_db/views/video/video_remove_frames.py +13 -12
  430. endoreg_db/views/video/video_stream.py +110 -82
  431. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
  432. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
  433. endoreg_db/management/commands/import_fallback_video.py +0 -203
  434. endoreg_db/management/commands/import_video.py +0 -422
  435. endoreg_db/management/commands/import_video_with_classification.py +0 -367
  436. endoreg_db/models/media/processing_history/processing_history.py +0 -96
  437. endoreg_db/serializers/label/__init__.py +0 -7
  438. endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
  439. endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
  440. endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
  441. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
  442. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
  443. endoreg_db/services/__old/pdf_import.py +0 -1487
  444. endoreg_db/services/__old/video_import.py +0 -1306
  445. endoreg_db/tasks/upload_tasks.py +0 -216
  446. endoreg_db/tasks/video_ingest.py +0 -161
  447. endoreg_db/tasks/video_processing_tasks.py +0 -327
  448. endoreg_db/views/misc/translation.py +0 -182
  449. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
  450. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
@@ -1,6 +1,5 @@
1
1
  import logging
2
2
  import shutil
3
- import uuid
4
3
  from pathlib import Path
5
4
  from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
6
5
 
@@ -53,48 +52,92 @@ def _create_anonymized_frame_files(
53
52
  generated_paths = []
54
53
  frame_iterator = frames.filter(is_extracted=True).iterator()
55
54
  total_frames = frames.filter(is_extracted=True).count()
56
- progress_bar = tqdm(frame_iterator, total=total_frames, desc=f"Anonymizing frames for {video.uuid}")
55
+ progress_bar = tqdm(
56
+ frame_iterator,
57
+ total=total_frames,
58
+ desc=f"Anonymizing frames for {video.video_hash}",
59
+ )
57
60
 
58
61
  for frame_obj in progress_bar:
59
62
  try:
60
63
  _frame_number = frame_obj.frame_number
61
- target_path = anonymized_frame_dir / f"frame_{frame_obj.frame_number:07d}.jpg"
64
+ target_path = (
65
+ anonymized_frame_dir / f"frame_{frame_obj.frame_number:07d}.jpg"
66
+ )
62
67
  make_all_black = frame_obj.frame_number in outside_frame_numbers
63
68
 
64
69
  try:
65
70
  source_path = frame_obj.file_path
66
71
  if not isinstance(source_path, Path):
67
- raise TypeError(f"Frame.file_path did not return a Path object for frame {frame_obj.frame_number}")
72
+ raise TypeError(
73
+ f"Frame.file_path did not return a Path object for frame {frame_obj.frame_number}"
74
+ )
68
75
  except (AttributeError, TypeError, Exception) as path_err:
69
76
  logger.error(
70
- "Could not determine source path for Frame %d (PK: %s) using frame_obj.file_path: %s", frame_obj.frame_number, frame_obj.pk, path_err
77
+ "Could not determine source path for Frame %d (PK: %s) using frame_obj.file_path: %s",
78
+ frame_obj.frame_number,
79
+ frame_obj.pk,
80
+ path_err,
71
81
  )
72
- raise RuntimeError(f"Failed to get source path for frame {frame_obj.frame_number}") from path_err
82
+ raise RuntimeError(
83
+ f"Failed to get source path for frame {frame_obj.frame_number}"
84
+ ) from path_err
73
85
 
74
86
  if not source_path.exists():
75
87
  error_msg = (
76
88
  f"CRITICAL INCONSISTENCY: Source frame file missing for frame {frame_obj.frame_number} "
77
- f"(PK: {frame_obj.pk}, Path: {source_path}) despite is_extracted=True for video {video.uuid}. "
89
+ f"(PK: {frame_obj.pk}, Path: {source_path}) despite is_extracted=True for video {video.video_hash}. "
78
90
  f"Halting anonymization."
79
91
  )
80
92
  logger.error(error_msg)
81
93
  raise FileNotFoundError(error_msg)
82
94
 
83
- anonymize_frame(raw_frame_path=source_path, target_frame_path=target_path, endo_roi=endo_roi, all_black=make_all_black, censor_color=censor_color)
95
+ anonymize_frame(
96
+ raw_frame_path=source_path,
97
+ target_frame_path=target_path,
98
+ endo_roi=endo_roi,
99
+ all_black=make_all_black,
100
+ censor_color=censor_color,
101
+ )
84
102
 
85
103
  generated_paths.append(target_path)
86
- except (FileNotFoundError, IOError, ValueError, AttributeError, TypeError, Exception) as e:
87
- logger.error("Error anonymizing frame %d (PK: %s): %s", frame_obj.frame_number, frame_obj.pk, e, exc_info=True)
88
- raise RuntimeError(f"Failed to anonymize frame {frame_obj.frame_number}") from e
104
+ except (
105
+ FileNotFoundError,
106
+ IOError,
107
+ ValueError,
108
+ AttributeError,
109
+ TypeError,
110
+ Exception,
111
+ ) as e:
112
+ logger.error(
113
+ "Error anonymizing frame %d (PK: %s): %s",
114
+ frame_obj.frame_number,
115
+ frame_obj.pk,
116
+ e,
117
+ exc_info=True,
118
+ )
119
+ raise RuntimeError(
120
+ f"Failed to anonymize frame {frame_obj.frame_number}"
121
+ ) from e
89
122
 
90
123
  if len(generated_paths) != total_frames:
91
- logger.error("Mismatch in generated frames count. Expected %d, got %d.", total_frames, len(generated_paths))
92
- raise RuntimeError("Anonymized frame generation resulted in incorrect number of files.")
124
+ logger.error(
125
+ "Mismatch in generated frames count. Expected %d, got %d.",
126
+ total_frames,
127
+ len(generated_paths),
128
+ )
129
+ raise RuntimeError(
130
+ "Anonymized frame generation resulted in incorrect number of files."
131
+ )
93
132
 
94
133
  return generated_paths
95
134
 
96
135
 
97
- def _censor_outside_frames(video: "VideoFile", outside_label_name: str = "outside", censor_color: Tuple[int, int, int] = (0, 0, 0)) -> bool:
136
+ def _censor_outside_frames(
137
+ video: "VideoFile",
138
+ outside_label_name: str = "outside",
139
+ censor_color: Tuple[int, int, int] = (0, 0, 0),
140
+ ) -> bool:
98
141
  """
99
142
  Overwrites frame files marked as 'outside' with a censored version (e.g., black).
100
143
  This modifies the original raw frames directly. Use with caution. Requires frames to be extracted.
@@ -104,14 +147,21 @@ def _censor_outside_frames(video: "VideoFile", outside_label_name: str = "outsid
104
147
  - Pre-condition: Requires state.frames_extracted=True.
105
148
  - Post-condition: No state changes.
106
149
  """
107
- logger.warning("Starting direct censoring of 'outside' frames for video %s. This modifies raw frame files.", video.uuid)
150
+ logger.warning(
151
+ "Starting direct censoring of 'outside' frames for video %s. This modifies raw frame files.",
152
+ video.video_hash,
153
+ )
108
154
  state = video.get_or_create_state()
109
155
  if not state.frames_extracted:
110
- raise ValueError(f"Frames not extracted for video {video.uuid}. Cannot censor.")
156
+ raise ValueError(
157
+ f"Frames not extracted for video {video.video_hash}. Cannot censor."
158
+ )
111
159
 
112
160
  outside_frames = _get_outside_frames(video, outside_label_name)
113
161
  if not outside_frames:
114
- logger.info("No 'outside' frames found to censor for video %s.", video.uuid)
162
+ logger.info(
163
+ "No 'outside' frames found to censor for video %s.", video.video_hash
164
+ )
115
165
  return True
116
166
 
117
167
  censored_count = 0
@@ -120,17 +170,24 @@ def _censor_outside_frames(video: "VideoFile", outside_label_name: str = "outsid
120
170
  try:
121
171
  frame_path = frame_obj.file_path
122
172
  if not frame_path:
123
- logger.warning("Could not get file path for frame %d. Skipping censoring.", frame_obj.frame_number)
173
+ logger.warning(
174
+ "Could not get file path for frame %d. Skipping censoring.",
175
+ frame_obj.frame_number,
176
+ )
124
177
  error_count += 1
125
178
  continue
126
179
 
127
180
  if not frame_path.exists():
128
- logger.warning("Frame file %s not found for censoring. Skipping.", frame_path)
181
+ logger.warning(
182
+ "Frame file %s not found for censoring. Skipping.", frame_path
183
+ )
129
184
  continue
130
185
 
131
186
  img = cv2.imread(str(frame_path))
132
187
  if img is None:
133
- logger.warning("Could not read frame %s for censoring. Skipping.", frame_path)
188
+ logger.warning(
189
+ "Could not read frame %s for censoring. Skipping.", frame_path
190
+ )
134
191
  continue
135
192
 
136
193
  img[:] = censor_color
@@ -142,14 +199,27 @@ def _censor_outside_frames(video: "VideoFile", outside_label_name: str = "outsid
142
199
  error_count += 1
143
200
 
144
201
  except Exception as e:
145
- logger.error("Error censoring frame %d (%s): %s", frame_obj.frame_number, getattr(frame_obj, "relative_path", "N/A"), e, exc_info=True)
202
+ logger.error(
203
+ "Error censoring frame %d (%s): %s",
204
+ frame_obj.frame_number,
205
+ getattr(frame_obj, "relative_path", "N/A"),
206
+ e,
207
+ exc_info=True,
208
+ )
146
209
  error_count += 1
147
210
 
148
- logger.info("Finished censoring for video %s. Censored: %d, Errors: %d", video.uuid, censored_count, error_count)
211
+ logger.info(
212
+ "Finished censoring for video %s. Censored: %d, Errors: %d",
213
+ video.video_hash,
214
+ censored_count,
215
+ error_count,
216
+ )
149
217
  return error_count == 0
150
218
 
151
219
 
152
- def _make_temporary_anonymized_frames(video: "VideoFile", roi_processing = True) -> Tuple[Path, List[Path]]:
220
+ def _make_temporary_anonymized_frames(
221
+ video: "VideoFile", roi_processing=True
222
+ ) -> Tuple[Path, List[Path]]:
153
223
  """
154
224
  Creates temporary anonymized frames in a separate directory.
155
225
  Requires raw file and extracted frames. Raises ValueError or RuntimeError on failure.
@@ -159,43 +229,73 @@ def _make_temporary_anonymized_frames(video: "VideoFile", roi_processing = True)
159
229
  - Post-condition: No state changes directly by this function.
160
230
  """
161
231
  if video.is_processed:
162
- raise ValueError(f"Cannot create temporary anonymized frames for video {video.uuid}: already processed.")
232
+ raise ValueError(
233
+ f"Cannot create temporary anonymized frames for video {video.video_hash}: already processed."
234
+ )
163
235
  if not video.has_raw:
164
- raise ValueError(f"Cannot create temporary anonymized frames for video {video.uuid}: Raw file is missing.")
236
+ raise ValueError(
237
+ f"Cannot create temporary anonymized frames for video {video.video_hash}: Raw file is missing."
238
+ )
165
239
 
166
240
  temp_anonym_frame_dir = video.get_temp_anonymized_frame_dir()
167
241
  temp_anonym_frame_dir.mkdir(parents=True, exist_ok=True)
168
- logger.info("Creating temporary anonymized frames for video %s in %s", video.uuid, temp_anonym_frame_dir)
242
+ logger.info(
243
+ "Creating temporary anonymized frames for video %s in %s",
244
+ video.video_hash,
245
+ temp_anonym_frame_dir,
246
+ )
169
247
  if roi_processing:
170
248
  endo_roi = video.get_endo_roi()
171
249
  if not validate_endo_roi(endo_roi_dict=endo_roi):
172
- raise ValueError(f"Endoscope ROI is not valid for video {video.uuid}")
250
+ raise ValueError(f"Endoscope ROI is not valid for video {video.video_hash}")
173
251
  else:
174
- endo_roi = {"x": 0, "y": 0, "width": 0, "height": 0} # Dummy ROI to skip processing
252
+ endo_roi = {
253
+ "x": 0,
254
+ "y": 0,
255
+ "width": 0,
256
+ "height": 0,
257
+ } # Dummy ROI to skip processing
175
258
  assert endo_roi is not None # For type checker
176
-
177
-
178
259
 
179
260
  state = video.get_or_create_state()
180
261
  if not state.frames_extracted:
181
- logger.info("Raw frames not extracted for %s, extracting now.", video.uuid)
262
+ logger.info(
263
+ "Raw frames not extracted for %s, extracting now.", video.video_hash
264
+ )
182
265
  try:
183
266
  if not video.extract_frames(overwrite=False):
184
- raise RuntimeError(f"Frame extraction method returned False unexpectedly for video {video.uuid}.")
267
+ raise RuntimeError(
268
+ f"Frame extraction method returned False unexpectedly for video {video.video_hash}."
269
+ )
185
270
  state.refresh_from_db()
186
271
  if not state.frames_extracted:
187
- raise RuntimeError(f"Frame extraction did not update state for video {video.uuid}, cannot create anonymized frames.")
272
+ raise RuntimeError(
273
+ f"Frame extraction did not update state for video {video.video_hash}, cannot create anonymized frames."
274
+ )
188
275
  except Exception as extract_e:
189
- logger.error("Frame extraction failed during anonymization prep for video %s: %s", video.uuid, extract_e, exc_info=True)
190
- raise RuntimeError(f"Frame extraction failed for video {video.uuid}, cannot create anonymized frames.") from extract_e
276
+ logger.error(
277
+ "Frame extraction failed during anonymization prep for video %s: %s",
278
+ video.video_hash,
279
+ extract_e,
280
+ exc_info=True,
281
+ )
282
+ raise RuntimeError(
283
+ f"Frame extraction failed for video {video.video_hash}, cannot create anonymized frames."
284
+ ) from extract_e
191
285
 
192
286
  all_frames = video.get_frames()
193
287
  if not all_frames.exists():
194
- raise FileNotFoundError(f"No frame objects found for video {video.uuid} after extraction attempt.")
288
+ raise FileNotFoundError(
289
+ f"No frame objects found for video {video.video_hash} after extraction attempt."
290
+ )
195
291
 
196
292
  outside_frame_numbers = _get_outside_frame_numbers(video)
197
293
 
198
- logger.info("Generating %d temporary anonymized frame files for video %s...", all_frames.filter(is_extracted=True).count(), video.uuid)
294
+ logger.info(
295
+ "Generating %d temporary anonymized frame files for video %s...",
296
+ all_frames.filter(is_extracted=True).count(),
297
+ video.video_hash,
298
+ )
199
299
  generated_frame_paths = _create_anonymized_frame_files(
200
300
  video=video,
201
301
  anonymized_frame_dir=temp_anonym_frame_dir,
@@ -203,7 +303,11 @@ def _make_temporary_anonymized_frames(video: "VideoFile", roi_processing = True)
203
303
  frames=all_frames,
204
304
  outside_frame_numbers=outside_frame_numbers,
205
305
  )
206
- logger.info("Generated %d temporary anonymized frame files for video %s.", len(generated_frame_paths), video.uuid)
306
+ logger.info(
307
+ "Generated %d temporary anonymized frame files for video %s.",
308
+ len(generated_frame_paths),
309
+ video.video_hash,
310
+ )
207
311
  return temp_anonym_frame_dir, generated_frame_paths
208
312
 
209
313
 
@@ -223,25 +327,38 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
223
327
  state = video.get_or_create_state()
224
328
 
225
329
  if state.anonymized:
226
- logger.info("Video %s is already marked as anonymized in state. Skipping.", video.uuid)
330
+ logger.info(
331
+ "Video %s is already marked as anonymized in state. Skipping.",
332
+ video.video_hash,
333
+ )
227
334
  return True
228
335
  if not video.has_raw:
229
- raise FileNotFoundError(f"Raw file is missing for video {video.uuid}, cannot anonymize.")
336
+ raise FileNotFoundError(
337
+ f"Raw file is missing for video {video.video_hash}, cannot anonymize."
338
+ )
230
339
  if not state.frames_extracted:
231
- raise ValueError(f"Frames not extracted for video {video.uuid}, cannot anonymize.")
340
+ raise ValueError(
341
+ f"Frames not extracted for video {video.video_hash}, cannot anonymize."
342
+ )
232
343
  if not video.sensitive_meta or not video.sensitive_meta.is_verified:
233
- raise ValueError(f"Sensitive metadata for video {video.uuid} is not validated. Cannot anonymize.")
344
+ raise ValueError(
345
+ f"Sensitive metadata for video {video.video_hash} is not validated. Cannot anonymize."
346
+ )
234
347
  # outside_segments = video.get_outside_segments(only_validated=False)
235
348
  # unvalidated_outside = outside_segments.filter(state__is_validated=False)
236
349
 
237
- logger.info("Starting anonymization process for video %s", video.uuid)
350
+ logger.info("Starting anonymization process for video %s", video.video_hash)
238
351
 
239
352
  temp_anonym_frame_dir = None
240
353
  anonymized_video_path = None
241
354
  try:
242
- temp_anonym_frame_dir, generated_frame_paths = _make_temporary_anonymized_frames(video)
355
+ temp_anonym_frame_dir, generated_frame_paths = (
356
+ _make_temporary_anonymized_frames(video)
357
+ )
243
358
  if not generated_frame_paths:
244
- raise RuntimeError(f"Failed to generate temporary anonymized frames for video {video.uuid}.")
359
+ raise RuntimeError(
360
+ f"Failed to generate temporary anonymized frames for video {video.video_hash}."
361
+ )
245
362
 
246
363
  anonymized_video_path = video.get_target_anonymized_video_path()
247
364
  anonymized_video_path.parent.mkdir(parents=True, exist_ok=True)
@@ -250,9 +367,15 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
250
367
 
251
368
  fps = video.get_fps()
252
369
  if fps is None:
253
- raise ValueError(f"FPS could not be determined for {video.uuid}, cannot assemble video.")
370
+ raise ValueError(
371
+ f"FPS could not be determined for {video.video_hash}, cannot assemble video."
372
+ )
254
373
 
255
- logger.info("Assembling anonymized video for %s at %s", video.uuid, anonymized_video_path)
374
+ logger.info(
375
+ "Assembling anonymized video for %s at %s",
376
+ video.video_hash,
377
+ anonymized_video_path,
378
+ )
256
379
  assemble_video_from_frames(
257
380
  frame_paths=generated_frame_paths,
258
381
  output_path=anonymized_video_path,
@@ -260,14 +383,25 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
260
383
  )
261
384
 
262
385
  if not anonymized_video_path.exists():
263
- raise RuntimeError(f"Processed video file not found after assembly for {video.uuid}: {anonymized_video_path}")
386
+ raise RuntimeError(
387
+ f"Processed video file not found after assembly for {video.video_hash}: {anonymized_video_path}"
388
+ )
264
389
 
265
390
  new_processed_hash = get_video_hash(anonymized_video_path)
266
- if type(video).objects.filter(processed_video_hash=new_processed_hash).exclude(pk=video.pk).exists():
267
- raise ValueError(f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.uuid}).")
391
+ if (
392
+ type(video)
393
+ .objects.filter(processed_video_hash=new_processed_hash)
394
+ .exclude(pk=video.pk)
395
+ .exists()
396
+ ):
397
+ raise ValueError(
398
+ f"Processed video hash {new_processed_hash} already exists for another video (Video: {video.video_hash})."
399
+ )
268
400
 
269
401
  video.processed_video_hash = new_processed_hash
270
- video.processed_file.name = video.get_target_anonymized_video_path().relative_to(STORAGE_DIR).as_posix()
402
+ video.processed_file.name = (
403
+ video.get_target_anonymized_video_path().relative_to(STORAGE_DIR).as_posix()
404
+ )
271
405
 
272
406
  update_fields = [
273
407
  "processed_video_hash",
@@ -285,7 +419,9 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
285
419
 
286
420
  transaction.on_commit(
287
421
  lambda: _cleanup_raw_assets(
288
- video_uuid=video.uuid, raw_file_path=original_raw_file_path_to_delete, raw_frame_dir=original_raw_frame_dir_to_delete
422
+ video_hash=video.video_hash,
423
+ raw_file_path=original_raw_file_path_to_delete,
424
+ raw_frame_dir=original_raw_frame_dir_to_delete,
289
425
  )
290
426
  )
291
427
 
@@ -296,19 +432,33 @@ def _anonymize(video: "VideoFile", delete_original_raw: bool = True) -> bool:
296
432
  return True
297
433
 
298
434
  except Exception as e:
299
- logger.error("Anonymization failed for video %s: %s", video.uuid, e, exc_info=True)
435
+ logger.error(
436
+ "Anonymization failed for video %s: %s", video.video_hash, e, exc_info=True
437
+ )
300
438
  if anonymized_video_path and anonymized_video_path.exists():
301
- logger.warning("Cleaning up potentially orphaned processed file for video %s due to error: %s", video.uuid, anonymized_video_path)
439
+ logger.warning(
440
+ "Cleaning up potentially orphaned processed file for video %s due to error: %s",
441
+ video.video_hash,
442
+ anonymized_video_path,
443
+ )
302
444
  anonymized_video_path.unlink(missing_ok=True)
303
- raise RuntimeError(f"Anonymization failed for video {video.uuid}") from e
445
+ raise RuntimeError(f"Anonymization failed for video {video.video_hash}") from e
304
446
 
305
447
  finally:
306
448
  if temp_anonym_frame_dir and temp_anonym_frame_dir.exists():
307
- logger.info("Cleaning up temporary anonymized frame directory for video %s: %s", video.uuid, temp_anonym_frame_dir)
449
+ logger.info(
450
+ "Cleaning up temporary anonymized frame directory for video %s: %s",
451
+ video.video_hash,
452
+ temp_anonym_frame_dir,
453
+ )
308
454
  shutil.rmtree(temp_anonym_frame_dir)
309
455
 
310
456
 
311
- def _cleanup_raw_assets(video_uuid: uuid.UUID, raw_file_path: Optional[Path] = None, raw_frame_dir: Optional[Path] = None):
457
+ def _cleanup_raw_assets(
458
+ video_hash: "str",
459
+ raw_file_path: Optional[Path] = None,
460
+ raw_frame_dir: Optional[Path] = None,
461
+ ):
312
462
  """
313
463
  Deletes the original raw video file and its extracted frames directory.
314
464
  Called via transaction.on_commit after successful anonymization.
@@ -318,32 +468,57 @@ def _cleanup_raw_assets(video_uuid: uuid.UUID, raw_file_path: Optional[Path] = N
318
468
  """
319
469
  from endoreg_db.models import VideoFile, VideoState
320
470
 
321
- logger.info("Performing post-commit cleanup of raw assets for video %s.", video_uuid)
471
+ logger.info(
472
+ "Performing post-commit cleanup of raw assets for video %s.", video_hash
473
+ )
322
474
  try:
323
- video_file = VideoFile.objects.select_related("state").filter(uuid=video_uuid).first()
475
+ video_file = (
476
+ VideoFile.objects.select_related("state")
477
+ .filter(content_hash=video_hash)
478
+ .first()
479
+ )
324
480
  if not video_file:
325
- logger.error("VideoFile %s not found during post-commit cleanup.", video_uuid)
481
+ logger.error(
482
+ "VideoFile %s not found during post-commit cleanup.", video_hash
483
+ )
326
484
  return
327
485
  if not video_file.state:
328
- logger.error("VideoState not found for VideoFile %s during post-commit cleanup.", video_uuid)
486
+ logger.error(
487
+ "VideoState not found for VideoFile %s during post-commit cleanup.",
488
+ video_hash,
489
+ )
329
490
  video_file.state = VideoState.objects.create(video_file=video_file)
330
491
 
331
492
  if raw_file_path and raw_file_path.exists():
332
493
  logger.info("Deleting original raw video file: %s", raw_file_path)
333
494
  raw_file_path.unlink()
334
495
  elif raw_file_path:
335
- logger.warning("Original raw video file %s not found for post-commit deletion.", raw_file_path)
496
+ logger.warning(
497
+ "Original raw video file %s not found for post-commit deletion.",
498
+ raw_file_path,
499
+ )
336
500
 
337
501
  if raw_frame_dir and raw_frame_dir.exists():
338
502
  logger.info("Deleting original raw frame directory: %s", raw_frame_dir)
339
503
  shutil.rmtree(raw_frame_dir, ignore_errors=True)
340
504
  elif raw_frame_dir:
341
- logger.warning("Original raw frame directory %s not found for post-commit deletion.", raw_frame_dir)
505
+ logger.warning(
506
+ "Original raw frame directory %s not found for post-commit deletion.",
507
+ raw_frame_dir,
508
+ )
342
509
 
343
510
  if video_file.state.frames_extracted:
344
511
  video_file.state.frames_extracted = False
345
512
  video_file.state.save(update_fields=["frames_extracted"])
346
- logger.info("Set state.frames_extracted=False for video %s after raw asset cleanup.", video_uuid)
513
+ logger.info(
514
+ "Set state.frames_extracted=False for video %s after raw asset cleanup.",
515
+ video_hash,
516
+ )
347
517
 
348
518
  except Exception as e:
349
- logger.error("Error during post-commit cleanup of raw assets for video %s: %s", video_uuid, e, exc_info=True)
519
+ logger.error(
520
+ "Error during post-commit cleanup of raw assets for video %s: %s",
521
+ video_hash,
522
+ e,
523
+ exc_info=True,
524
+ )
@@ -16,5 +16,10 @@ def _bulk_create_frames(video: "VideoFile", frames_to_create: List["Frame"]):
16
16
  try:
17
17
  Frame.objects.bulk_create(frames_to_create, ignore_conflicts=True)
18
18
  except Exception as e:
19
- logger.error("Error during bulk creation of frames for video %s: %s", video.uuid, e, exc_info=True)
19
+ logger.error(
20
+ "Error during bulk creation of frames for video %s: %s",
21
+ video.video_hash,
22
+ e,
23
+ exc_info=True,
24
+ )
20
25
  raise
@@ -9,7 +9,9 @@ logger = logging.getLogger(__name__)
9
9
  __all__ = ["_create_frame_object"]
10
10
 
11
11
 
12
- def _create_frame_object(video: "VideoFile", frame_number: int, relative_path: str, extracted: bool = False) -> "Frame":
12
+ def _create_frame_object(
13
+ video: "VideoFile", frame_number: int, relative_path: str, extracted: bool = False
14
+ ) -> "Frame":
13
15
  """Instantiates a Frame object (does not save it)."""
14
16
  from endoreg_db.models import Frame
15
17
 
@@ -4,7 +4,10 @@ from typing import TYPE_CHECKING
4
4
 
5
5
  from django.db import transaction
6
6
 
7
- from endoreg_db.models.media.video.video_file_io import _get_frame_dir_path, _get_temp_anonymized_frame_dir
7
+ from endoreg_db.models.media.video.video_file_io import (
8
+ _get_frame_dir_path,
9
+ _get_temp_anonymized_frame_dir,
10
+ )
8
11
 
9
12
  if TYPE_CHECKING:
10
13
  from endoreg_db.models import VideoFile, VideoState
@@ -44,7 +47,7 @@ def _delete_frames(video: "VideoFile") -> str:
44
47
  msg = f"Frame directory not found, skipping deletion: {frame_dir}"
45
48
  logger.debug(msg)
46
49
  else:
47
- msg = f"Frame directory path not set for video {video.uuid}, cannot delete standard frames."
50
+ msg = f"Frame directory path not set for video {video.video_hash}, cannot delete standard frames."
48
51
  logger.warning(msg)
49
52
 
50
53
  temp_anonym_frame_dir = None
@@ -52,7 +55,9 @@ def _delete_frames(video: "VideoFile") -> str:
52
55
  temp_anonym_frame_dir = _get_temp_anonymized_frame_dir(video)
53
56
  if temp_anonym_frame_dir and temp_anonym_frame_dir.exists():
54
57
  shutil.rmtree(temp_anonym_frame_dir)
55
- msg = f"Deleted temporary anonymized frame directory: {temp_anonym_frame_dir}"
58
+ msg = (
59
+ f"Deleted temporary anonymized frame directory: {temp_anonym_frame_dir}"
60
+ )
56
61
  logger.info(msg)
57
62
  deleted_messages.append(msg)
58
63
  except Exception as e:
@@ -69,16 +74,28 @@ def _delete_frames(video: "VideoFile") -> str:
69
74
 
70
75
  if update_fields_state:
71
76
  state.save(update_fields=update_fields_state)
72
- logger.info("Reset frame state flags (%s) for video %s.", ", ".join(update_fields_state), video.uuid)
77
+ logger.info(
78
+ "Reset frame state flags (%s) for video %s.",
79
+ ", ".join(update_fields_state),
80
+ video.video_hash,
81
+ )
73
82
  state_updated = True
74
83
  else:
75
- logger.info("Frame state flags already False for video %s.", video.uuid)
84
+ logger.info(
85
+ "Frame state flags already False for video %s.", video.video_hash
86
+ )
76
87
  state_updated = True
77
88
 
78
89
  try:
79
- update_count = Frame.objects.filter(video=video, is_extracted=True).update(is_extracted=False)
90
+ update_count = Frame.objects.filter(video=video, is_extracted=True).update(
91
+ is_extracted=False
92
+ )
80
93
  if update_count > 0:
81
- logger.info("Marked %d Frame objects as is_extracted=False for video %s.", update_count, video.uuid)
94
+ logger.info(
95
+ "Marked %d Frame objects as is_extracted=False for video %s.",
96
+ update_count,
97
+ video.video_hash,
98
+ )
82
99
  db_updated = True
83
100
  except Exception as db_err:
84
101
  msg = f"Failed to update is_extracted flag for Frame objects for video %s: {db_err}"
@@ -87,10 +104,14 @@ def _delete_frames(video: "VideoFile") -> str:
87
104
  db_updated = False
88
105
 
89
106
  except Exception as state_e:
90
- msg = f"Failed to update state after deleting frame files for video %s: {state_e}"
107
+ msg = (
108
+ f"Failed to update state after deleting frame files for video %s: {state_e}"
109
+ )
91
110
  logger.error(msg, exc_info=True)
92
111
  error_messages.append(msg)
93
- raise RuntimeError(f"Failed to update state during frame file deletion for video {video.uuid}") from state_e
112
+ raise RuntimeError(
113
+ f"Failed to update state during frame file deletion for video {video.video_hash}"
114
+ ) from state_e
94
115
 
95
116
  final_message = "; ".join(deleted_messages)
96
117
  if error_messages: