endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (450) hide show
  1. endoreg_db/admin.py +10 -5
  2. endoreg_db/apps.py +4 -7
  3. endoreg_db/authz/auth.py +1 -0
  4. endoreg_db/authz/backends.py +1 -1
  5. endoreg_db/authz/management/commands/list_routes.py +2 -0
  6. endoreg_db/authz/middleware.py +8 -7
  7. endoreg_db/authz/permissions.py +21 -10
  8. endoreg_db/authz/policy.py +14 -19
  9. endoreg_db/authz/views_auth.py +14 -10
  10. endoreg_db/codemods/rename_datetime_fields.py +8 -1
  11. endoreg_db/exceptions.py +5 -2
  12. endoreg_db/forms/__init__.py +0 -1
  13. endoreg_db/forms/examination_form.py +4 -3
  14. endoreg_db/forms/patient_finding_intervention_form.py +30 -8
  15. endoreg_db/forms/patient_form.py +9 -13
  16. endoreg_db/forms/questionnaires/__init__.py +1 -1
  17. endoreg_db/forms/settings/__init__.py +4 -1
  18. endoreg_db/forms/unit.py +2 -1
  19. endoreg_db/helpers/count_db.py +17 -14
  20. endoreg_db/helpers/default_objects.py +2 -1
  21. endoreg_db/helpers/download_segmentation_model.py +4 -3
  22. endoreg_db/helpers/interact.py +0 -5
  23. endoreg_db/helpers/test_video_helper.py +33 -25
  24. endoreg_db/import_files/__init__.py +1 -1
  25. endoreg_db/import_files/context/__init__.py +1 -1
  26. endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
  27. endoreg_db/import_files/context/ensure_center.py +4 -4
  28. endoreg_db/import_files/context/file_lock.py +3 -3
  29. endoreg_db/import_files/context/import_context.py +11 -12
  30. endoreg_db/import_files/context/validate_directories.py +1 -0
  31. endoreg_db/import_files/file_storage/create_report_file.py +57 -34
  32. endoreg_db/import_files/file_storage/create_video_file.py +64 -35
  33. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
  34. endoreg_db/import_files/file_storage/state_management.py +89 -122
  35. endoreg_db/import_files/file_storage/storage.py +5 -1
  36. endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
  37. endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
  38. endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
  39. endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
  40. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
  41. endoreg_db/import_files/report_import_service.py +36 -30
  42. endoreg_db/import_files/video_import_service.py +27 -23
  43. endoreg_db/logger_conf.py +56 -40
  44. endoreg_db/management/__init__.py +1 -1
  45. endoreg_db/management/commands/__init__.py +1 -1
  46. endoreg_db/management/commands/check_auth.py +45 -38
  47. endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
  48. endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
  49. endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
  50. endoreg_db/management/commands/fix_video_paths.py +75 -54
  51. endoreg_db/management/commands/import_report.py +1 -3
  52. endoreg_db/management/commands/list_routes.py +2 -0
  53. endoreg_db/management/commands/load_ai_model_data.py +8 -2
  54. endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
  55. endoreg_db/management/commands/load_center_data.py +3 -3
  56. endoreg_db/management/commands/load_distribution_data.py +35 -38
  57. endoreg_db/management/commands/load_endoscope_data.py +0 -3
  58. endoreg_db/management/commands/load_examination_data.py +20 -4
  59. endoreg_db/management/commands/load_finding_data.py +18 -3
  60. endoreg_db/management/commands/load_gender_data.py +17 -24
  61. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
  62. endoreg_db/management/commands/load_information_source.py +0 -3
  63. endoreg_db/management/commands/load_lab_value_data.py +14 -3
  64. endoreg_db/management/commands/load_legacy_data.py +303 -0
  65. endoreg_db/management/commands/load_name_data.py +1 -2
  66. endoreg_db/management/commands/load_pdf_type_data.py +4 -8
  67. endoreg_db/management/commands/load_profession_data.py +0 -1
  68. endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
  69. endoreg_db/management/commands/load_requirement_data.py +6 -2
  70. endoreg_db/management/commands/load_unit_data.py +0 -4
  71. endoreg_db/management/commands/load_user_groups.py +5 -7
  72. endoreg_db/management/commands/model_input.py +169 -0
  73. endoreg_db/management/commands/register_ai_model.py +22 -16
  74. endoreg_db/management/commands/setup_endoreg_db.py +110 -32
  75. endoreg_db/management/commands/storage_management.py +14 -8
  76. endoreg_db/management/commands/summarize_db_content.py +154 -63
  77. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  78. endoreg_db/management/commands/validate_video_files.py +82 -50
  79. endoreg_db/management/commands/video_validation.py +4 -6
  80. endoreg_db/migrations/0001_initial.py +112 -63
  81. endoreg_db/models/__init__.py +8 -0
  82. endoreg_db/models/administration/ai/active_model.py +5 -5
  83. endoreg_db/models/administration/ai/ai_model.py +41 -18
  84. endoreg_db/models/administration/ai/model_type.py +1 -0
  85. endoreg_db/models/administration/case/case.py +22 -22
  86. endoreg_db/models/administration/center/__init__.py +5 -5
  87. endoreg_db/models/administration/center/center.py +6 -2
  88. endoreg_db/models/administration/center/center_resource.py +18 -4
  89. endoreg_db/models/administration/center/center_shift.py +3 -1
  90. endoreg_db/models/administration/center/center_waste.py +6 -2
  91. endoreg_db/models/administration/person/__init__.py +1 -1
  92. endoreg_db/models/administration/person/employee/__init__.py +1 -1
  93. endoreg_db/models/administration/person/employee/employee_type.py +3 -1
  94. endoreg_db/models/administration/person/examiner/__init__.py +1 -1
  95. endoreg_db/models/administration/person/examiner/examiner.py +10 -2
  96. endoreg_db/models/administration/person/names/first_name.py +6 -4
  97. endoreg_db/models/administration/person/names/last_name.py +4 -3
  98. endoreg_db/models/administration/person/patient/__init__.py +1 -1
  99. endoreg_db/models/administration/person/patient/patient.py +0 -1
  100. endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
  101. endoreg_db/models/administration/person/person.py +1 -1
  102. endoreg_db/models/administration/product/__init__.py +7 -6
  103. endoreg_db/models/administration/product/product.py +6 -2
  104. endoreg_db/models/administration/product/product_group.py +9 -7
  105. endoreg_db/models/administration/product/product_material.py +9 -2
  106. endoreg_db/models/administration/product/reference_product.py +64 -15
  107. endoreg_db/models/administration/qualification/qualification.py +3 -1
  108. endoreg_db/models/administration/shift/shift.py +3 -1
  109. endoreg_db/models/administration/shift/shift_type.py +12 -4
  110. endoreg_db/models/aidataset/__init__.py +5 -0
  111. endoreg_db/models/aidataset/aidataset.py +193 -0
  112. endoreg_db/models/label/__init__.py +1 -1
  113. endoreg_db/models/label/label.py +10 -2
  114. endoreg_db/models/label/label_set.py +3 -1
  115. endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
  116. endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
  117. endoreg_db/models/media/__init__.py +12 -5
  118. endoreg_db/models/media/frame/__init__.py +1 -1
  119. endoreg_db/models/media/frame/frame.py +34 -8
  120. endoreg_db/models/media/pdf/__init__.py +2 -1
  121. endoreg_db/models/media/pdf/raw_pdf.py +11 -4
  122. endoreg_db/models/media/pdf/report_file.py +6 -2
  123. endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
  124. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
  125. endoreg_db/models/media/video/create_from_file.py +20 -41
  126. endoreg_db/models/media/video/pipe_1.py +75 -30
  127. endoreg_db/models/media/video/pipe_2.py +37 -12
  128. endoreg_db/models/media/video/video_file.py +36 -24
  129. endoreg_db/models/media/video/video_file_ai.py +235 -70
  130. endoreg_db/models/media/video/video_file_anonymize.py +240 -65
  131. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
  132. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
  133. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
  134. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
  135. endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
  136. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
  137. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
  138. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
  139. endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
  140. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
  141. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
  142. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
  143. endoreg_db/models/media/video/video_file_io.py +85 -33
  144. endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
  145. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
  146. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
  147. endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
  148. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
  149. endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
  150. endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
  151. endoreg_db/models/media/video/video_file_segments.py +118 -27
  152. endoreg_db/models/media/video/video_metadata.py +25 -6
  153. endoreg_db/models/media/video/video_processing.py +54 -15
  154. endoreg_db/models/medical/__init__.py +3 -13
  155. endoreg_db/models/medical/contraindication/__init__.py +3 -1
  156. endoreg_db/models/medical/disease.py +18 -6
  157. endoreg_db/models/medical/event.py +6 -2
  158. endoreg_db/models/medical/examination/__init__.py +5 -1
  159. endoreg_db/models/medical/examination/examination.py +22 -6
  160. endoreg_db/models/medical/examination/examination_indication.py +23 -7
  161. endoreg_db/models/medical/examination/examination_time.py +6 -2
  162. endoreg_db/models/medical/finding/__init__.py +3 -1
  163. endoreg_db/models/medical/finding/finding.py +37 -12
  164. endoreg_db/models/medical/finding/finding_classification.py +27 -8
  165. endoreg_db/models/medical/finding/finding_intervention.py +19 -6
  166. endoreg_db/models/medical/finding/finding_type.py +3 -1
  167. endoreg_db/models/medical/hardware/__init__.py +1 -1
  168. endoreg_db/models/medical/hardware/endoscope.py +14 -2
  169. endoreg_db/models/medical/laboratory/__init__.py +1 -1
  170. endoreg_db/models/medical/laboratory/lab_value.py +139 -39
  171. endoreg_db/models/medical/medication/__init__.py +7 -3
  172. endoreg_db/models/medical/medication/medication.py +3 -1
  173. endoreg_db/models/medical/medication/medication_indication.py +3 -1
  174. endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
  175. endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
  176. endoreg_db/models/medical/medication/medication_schedule.py +3 -1
  177. endoreg_db/models/medical/patient/__init__.py +2 -10
  178. endoreg_db/models/medical/patient/medication_examples.py +3 -14
  179. endoreg_db/models/medical/patient/patient_disease.py +17 -5
  180. endoreg_db/models/medical/patient/patient_event.py +12 -4
  181. endoreg_db/models/medical/patient/patient_examination.py +52 -15
  182. endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
  183. endoreg_db/models/medical/patient/patient_finding.py +105 -29
  184. endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
  185. endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
  186. endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
  187. endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
  188. endoreg_db/models/medical/patient/patient_medication.py +25 -7
  189. endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
  190. endoreg_db/models/metadata/model_meta.py +40 -12
  191. endoreg_db/models/metadata/model_meta_logic.py +51 -16
  192. endoreg_db/models/metadata/sensitive_meta.py +65 -28
  193. endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
  194. endoreg_db/models/metadata/video_meta.py +146 -39
  195. endoreg_db/models/metadata/video_prediction_logic.py +70 -21
  196. endoreg_db/models/metadata/video_prediction_meta.py +80 -27
  197. endoreg_db/models/operation_log.py +63 -0
  198. endoreg_db/models/other/__init__.py +10 -10
  199. endoreg_db/models/other/distribution/__init__.py +9 -7
  200. endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
  201. endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
  202. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
  203. endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
  204. endoreg_db/models/other/emission/__init__.py +1 -1
  205. endoreg_db/models/other/emission/emission_factor.py +9 -3
  206. endoreg_db/models/other/information_source.py +15 -5
  207. endoreg_db/models/other/material.py +3 -1
  208. endoreg_db/models/other/transport_route.py +3 -1
  209. endoreg_db/models/other/unit.py +6 -2
  210. endoreg_db/models/report/report.py +0 -1
  211. endoreg_db/models/requirement/requirement.py +84 -27
  212. endoreg_db/models/requirement/requirement_error.py +5 -6
  213. endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
  214. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
  215. endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
  216. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
  217. endoreg_db/models/requirement/requirement_operator.py +28 -8
  218. endoreg_db/models/requirement/requirement_set.py +34 -11
  219. endoreg_db/models/state/__init__.py +1 -0
  220. endoreg_db/models/state/audit_ledger.py +9 -2
  221. endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
  222. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  223. endoreg_db/models/state/raw_pdf.py +0 -1
  224. endoreg_db/models/state/video.py +2 -4
  225. endoreg_db/models/utils.py +4 -2
  226. endoreg_db/queries/__init__.py +2 -6
  227. endoreg_db/queries/annotations/__init__.py +1 -3
  228. endoreg_db/queries/annotations/legacy.py +37 -26
  229. endoreg_db/root_urls.py +3 -4
  230. endoreg_db/schemas/examination_evaluation.py +3 -0
  231. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
  232. endoreg_db/serializers/__init__.py +2 -8
  233. endoreg_db/serializers/administration/__init__.py +1 -2
  234. endoreg_db/serializers/administration/ai/__init__.py +0 -1
  235. endoreg_db/serializers/administration/ai/active_model.py +3 -1
  236. endoreg_db/serializers/administration/ai/ai_model.py +5 -3
  237. endoreg_db/serializers/administration/ai/model_type.py +3 -1
  238. endoreg_db/serializers/administration/center.py +7 -2
  239. endoreg_db/serializers/administration/gender.py +4 -2
  240. endoreg_db/serializers/anonymization.py +13 -13
  241. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
  242. endoreg_db/serializers/examination/__init__.py +1 -1
  243. endoreg_db/serializers/examination/base.py +12 -13
  244. endoreg_db/serializers/examination/dropdown.py +6 -7
  245. endoreg_db/serializers/examination_serializer.py +3 -6
  246. endoreg_db/serializers/finding/__init__.py +1 -1
  247. endoreg_db/serializers/finding/finding.py +14 -7
  248. endoreg_db/serializers/finding_classification/__init__.py +3 -3
  249. endoreg_db/serializers/finding_classification/choice.py +3 -3
  250. endoreg_db/serializers/finding_classification/classification.py +2 -4
  251. endoreg_db/serializers/label_video_segment/__init__.py +5 -3
  252. endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
  253. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  254. endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
  255. endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
  256. endoreg_db/serializers/meta/__init__.py +1 -2
  257. endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
  258. endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
  259. endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
  260. endoreg_db/serializers/misc/__init__.py +2 -2
  261. endoreg_db/serializers/misc/file_overview.py +11 -7
  262. endoreg_db/serializers/misc/stats.py +10 -8
  263. endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
  264. endoreg_db/serializers/misc/upload_job.py +32 -29
  265. endoreg_db/serializers/patient/__init__.py +2 -1
  266. endoreg_db/serializers/patient/patient.py +32 -15
  267. endoreg_db/serializers/patient/patient_dropdown.py +11 -3
  268. endoreg_db/serializers/patient_examination/__init__.py +1 -1
  269. endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
  270. endoreg_db/serializers/patient_finding/__init__.py +1 -1
  271. endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
  272. endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
  273. endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
  274. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
  275. endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
  276. endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
  277. endoreg_db/serializers/pdf/__init__.py +1 -3
  278. endoreg_db/serializers/requirements/requirement_schema.py +1 -6
  279. endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
  280. endoreg_db/serializers/video/__init__.py +2 -2
  281. endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
  282. endoreg_db/serializers/video/video_file_brief.py +6 -2
  283. endoreg_db/serializers/video/video_file_detail.py +36 -23
  284. endoreg_db/serializers/video/video_file_list.py +4 -2
  285. endoreg_db/serializers/video/video_processing_history.py +54 -50
  286. endoreg_db/services/__init__.py +1 -1
  287. endoreg_db/services/anonymization.py +2 -2
  288. endoreg_db/services/examination_evaluation.py +40 -17
  289. endoreg_db/services/model_meta_from_hf.py +76 -0
  290. endoreg_db/services/polling_coordinator.py +101 -70
  291. endoreg_db/services/pseudonym_service.py +27 -22
  292. endoreg_db/services/report_import.py +6 -3
  293. endoreg_db/services/segment_sync.py +75 -59
  294. endoreg_db/services/video_import.py +6 -7
  295. endoreg_db/urls/__init__.py +2 -2
  296. endoreg_db/urls/ai.py +7 -25
  297. endoreg_db/urls/anonymization.py +61 -15
  298. endoreg_db/urls/auth.py +4 -4
  299. endoreg_db/urls/classification.py +4 -9
  300. endoreg_db/urls/examination.py +27 -18
  301. endoreg_db/urls/media.py +27 -34
  302. endoreg_db/urls/patient.py +11 -7
  303. endoreg_db/urls/requirements.py +3 -1
  304. endoreg_db/urls/root_urls.py +2 -3
  305. endoreg_db/urls/stats.py +24 -16
  306. endoreg_db/urls/upload.py +3 -11
  307. endoreg_db/utils/__init__.py +14 -15
  308. endoreg_db/utils/ai/__init__.py +1 -1
  309. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  310. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  311. endoreg_db/utils/ai/get.py +2 -1
  312. endoreg_db/utils/ai/inference_dataset.py +14 -15
  313. endoreg_db/utils/ai/model_training/config.py +117 -0
  314. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  315. endoreg_db/utils/ai/model_training/losses.py +68 -0
  316. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  317. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  318. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  319. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  320. endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
  321. endoreg_db/utils/ai/predict.py +4 -4
  322. endoreg_db/utils/ai/preprocess.py +19 -11
  323. endoreg_db/utils/calc_duration_seconds.py +4 -4
  324. endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
  325. endoreg_db/utils/check_video_files.py +74 -47
  326. endoreg_db/utils/cropping.py +10 -9
  327. endoreg_db/utils/dataloader.py +11 -3
  328. endoreg_db/utils/dates.py +3 -4
  329. endoreg_db/utils/defaults/set_default_center.py +7 -6
  330. endoreg_db/utils/env.py +6 -2
  331. endoreg_db/utils/extract_specific_frames.py +24 -9
  332. endoreg_db/utils/file_operations.py +30 -18
  333. endoreg_db/utils/fix_video_path_direct.py +57 -41
  334. endoreg_db/utils/frame_anonymization_utils.py +157 -157
  335. endoreg_db/utils/hashs.py +3 -18
  336. endoreg_db/utils/links/requirement_link.py +96 -52
  337. endoreg_db/utils/ocr.py +30 -25
  338. endoreg_db/utils/operation_log.py +61 -0
  339. endoreg_db/utils/parse_and_generate_yaml.py +12 -13
  340. endoreg_db/utils/paths.py +6 -6
  341. endoreg_db/utils/permissions.py +40 -24
  342. endoreg_db/utils/pipelines/process_video_dir.py +50 -26
  343. endoreg_db/utils/product/sum_emissions.py +5 -3
  344. endoreg_db/utils/product/sum_weights.py +4 -2
  345. endoreg_db/utils/pydantic_models/__init__.py +3 -4
  346. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
  347. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
  348. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
  349. endoreg_db/utils/setup_config.py +21 -5
  350. endoreg_db/utils/storage.py +3 -1
  351. endoreg_db/utils/translation.py +19 -15
  352. endoreg_db/utils/uuid.py +1 -0
  353. endoreg_db/utils/validate_endo_roi.py +12 -4
  354. endoreg_db/utils/validate_subcategory_dict.py +26 -24
  355. endoreg_db/utils/validate_video_detailed.py +207 -149
  356. endoreg_db/utils/video/__init__.py +7 -3
  357. endoreg_db/utils/video/extract_frames.py +30 -18
  358. endoreg_db/utils/video/names.py +11 -6
  359. endoreg_db/utils/video/streaming_processor.py +175 -101
  360. endoreg_db/utils/video/video_splitter.py +30 -19
  361. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
  362. endoreg_db/views/__init__.py +0 -20
  363. endoreg_db/views/anonymization/__init__.py +6 -2
  364. endoreg_db/views/anonymization/media_management.py +2 -6
  365. endoreg_db/views/anonymization/overview.py +34 -1
  366. endoreg_db/views/anonymization/validate.py +79 -18
  367. endoreg_db/views/auth/__init__.py +1 -1
  368. endoreg_db/views/auth/keycloak.py +16 -14
  369. endoreg_db/views/examination/__init__.py +12 -15
  370. endoreg_db/views/examination/examination.py +5 -5
  371. endoreg_db/views/examination/examination_manifest_cache.py +5 -5
  372. endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
  373. endoreg_db/views/examination/get_finding_classifications.py +9 -7
  374. endoreg_db/views/examination/get_findings.py +8 -10
  375. endoreg_db/views/examination/get_instruments.py +3 -2
  376. endoreg_db/views/examination/get_interventions.py +1 -1
  377. endoreg_db/views/finding/__init__.py +2 -2
  378. endoreg_db/views/finding/finding.py +58 -54
  379. endoreg_db/views/finding/get_classifications.py +1 -1
  380. endoreg_db/views/finding/get_interventions.py +1 -1
  381. endoreg_db/views/finding_classification/__init__.py +5 -5
  382. endoreg_db/views/finding_classification/finding_classification.py +5 -6
  383. endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
  384. endoreg_db/views/media/__init__.py +13 -13
  385. endoreg_db/views/media/pdf_media.py +9 -9
  386. endoreg_db/views/media/sensitive_metadata.py +10 -7
  387. endoreg_db/views/media/video_media.py +4 -4
  388. endoreg_db/views/meta/__init__.py +1 -1
  389. endoreg_db/views/meta/sensitive_meta_list.py +20 -22
  390. endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
  391. endoreg_db/views/misc/__init__.py +6 -34
  392. endoreg_db/views/misc/center.py +2 -1
  393. endoreg_db/views/misc/csrf.py +2 -1
  394. endoreg_db/views/misc/gender.py +2 -1
  395. endoreg_db/views/misc/stats.py +141 -106
  396. endoreg_db/views/patient/__init__.py +1 -3
  397. endoreg_db/views/patient/patient.py +141 -99
  398. endoreg_db/views/patient_examination/__init__.py +5 -5
  399. endoreg_db/views/patient_examination/patient_examination.py +43 -42
  400. endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
  401. endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
  402. endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
  403. endoreg_db/views/patient_examination/video.py +114 -80
  404. endoreg_db/views/patient_finding/__init__.py +1 -1
  405. endoreg_db/views/patient_finding/patient_finding.py +17 -10
  406. endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
  407. endoreg_db/views/patient_finding_classification/__init__.py +1 -1
  408. endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
  409. endoreg_db/views/report/reimport.py +1 -1
  410. endoreg_db/views/report/report_stream.py +5 -8
  411. endoreg_db/views/requirement/__init__.py +2 -1
  412. endoreg_db/views/requirement/evaluate.py +7 -9
  413. endoreg_db/views/requirement/lookup.py +2 -3
  414. endoreg_db/views/requirement/lookup_store.py +0 -1
  415. endoreg_db/views/requirement/requirement_utils.py +2 -4
  416. endoreg_db/views/stats/__init__.py +4 -4
  417. endoreg_db/views/stats/stats_views.py +152 -115
  418. endoreg_db/views/video/__init__.py +18 -27
  419. endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
  420. endoreg_db/views/{ai → video/ai}/label.py +20 -16
  421. endoreg_db/views/video/correction.py +5 -6
  422. endoreg_db/views/video/reimport.py +134 -99
  423. endoreg_db/views/video/segments_crud.py +134 -44
  424. endoreg_db/views/video/video_apply_mask.py +13 -12
  425. endoreg_db/views/video/video_correction.py +2 -1
  426. endoreg_db/views/video/video_download_processed.py +15 -15
  427. endoreg_db/views/video/video_meta_stats.py +7 -6
  428. endoreg_db/views/video/video_processing_history.py +3 -2
  429. endoreg_db/views/video/video_remove_frames.py +13 -12
  430. endoreg_db/views/video/video_stream.py +110 -82
  431. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
  432. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
  433. endoreg_db/management/commands/import_fallback_video.py +0 -203
  434. endoreg_db/management/commands/import_video.py +0 -422
  435. endoreg_db/management/commands/import_video_with_classification.py +0 -367
  436. endoreg_db/models/media/processing_history/processing_history.py +0 -96
  437. endoreg_db/serializers/label/__init__.py +0 -7
  438. endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
  439. endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
  440. endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
  441. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
  442. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
  443. endoreg_db/services/__old/pdf_import.py +0 -1487
  444. endoreg_db/services/__old/video_import.py +0 -1306
  445. endoreg_db/tasks/upload_tasks.py +0 -216
  446. endoreg_db/tasks/video_ingest.py +0 -161
  447. endoreg_db/tasks/video_processing_tasks.py +0 -327
  448. endoreg_db/views/misc/translation.py +0 -182
  449. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
  450. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
@@ -15,7 +15,9 @@ def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
15
15
  Returns None if VideoMeta is not linked or ROI is not properly defined.
16
16
  """
17
17
  if not video.video_meta:
18
- logger.warning("VideoMeta not linked for video %s. Cannot get endo ROI.", video.uuid)
18
+ logger.warning(
19
+ "VideoMeta not linked for video %s. Cannot get endo ROI.", video.video_hash
20
+ )
19
21
  return None
20
22
 
21
23
  try:
@@ -25,17 +27,36 @@ def _get_endo_roi(video: "VideoFile") -> Optional[Dict[str, int]]:
25
27
  if (
26
28
  isinstance(endo_roi, dict)
27
29
  and all(k in endo_roi for k in ("x", "y", "width", "height"))
28
- and all(isinstance(v, int) and not isinstance(v, bool) for v in endo_roi.values())
30
+ and all(
31
+ isinstance(v, int) and not isinstance(v, bool)
32
+ for v in endo_roi.values()
33
+ )
29
34
  ):
30
- cleaned_roi = {k: int(endo_roi[k] or 0) for k in ("x", "y", "width", "height")}
31
- logger.debug("Retrieved endo ROI for video %s: %s", video.uuid, cleaned_roi)
35
+ cleaned_roi = {
36
+ k: int(endo_roi[k] or 0) for k in ("x", "y", "width", "height")
37
+ }
38
+ logger.debug(
39
+ "Retrieved endo ROI for video %s: %s", video.video_hash, cleaned_roi
40
+ )
32
41
  return cleaned_roi
33
42
  else:
34
- logger.warning("Endo ROI not fully defined or invalid in VideoMeta for video %s. ROI: %s", video.uuid, endo_roi)
43
+ logger.warning(
44
+ "Endo ROI not fully defined or invalid in VideoMeta for video %s. ROI: %s",
45
+ video.video_hash,
46
+ endo_roi,
47
+ )
35
48
  return None
36
49
  except AttributeError:
37
- logger.error("VideoMeta object for video %s does not have a 'get_endo_roi' method.", video.uuid)
50
+ logger.error(
51
+ "VideoMeta object for video %s does not have a 'get_endo_roi' method.",
52
+ video.video_hash,
53
+ )
38
54
  return None
39
55
  except Exception as e:
40
- logger.error("Error getting endo ROI from VideoMeta for video %s: %s", video.uuid, e, exc_info=True)
56
+ logger.error(
57
+ "Error getting endo ROI from VideoMeta for video %s: %s",
58
+ video.video_hash,
59
+ e,
60
+ exc_info=True,
61
+ )
41
62
  return None
@@ -45,10 +45,10 @@ def _get_fps(video: "VideoFile") -> float:
45
45
  if video.fps is not None:
46
46
  return video.fps
47
47
 
48
- logger.debug("FPS not set on instance %s, checking VideoMeta.", video.uuid)
48
+ logger.debug("FPS not set on instance %s, checking VideoMeta.", video.video_hash)
49
49
 
50
50
  if not video.video_meta:
51
- logger.info("VideoMeta not linked for %s, attempting update.", video.uuid)
51
+ logger.info("VideoMeta not linked for %s, attempting update.", video.video_hash)
52
52
 
53
53
  _update_video_meta(video, save_instance=True) # Call the helper function
54
54
 
@@ -56,20 +56,33 @@ def _get_fps(video: "VideoFile") -> float:
56
56
  if video.fps is not None:
57
57
  return video.fps
58
58
  elif video.video_meta and video.video_meta.fps is not None:
59
- logger.info("Retrieved FPS %.2f from VideoMeta for %s.", video.video_meta.fps, video.uuid)
59
+ logger.info(
60
+ "Retrieved FPS %.2f from VideoMeta for %s.",
61
+ video.video_meta.fps,
62
+ video.video_hash,
63
+ )
60
64
  _fps = video.video_meta.fps
61
65
  try:
62
66
  _fps = float(_fps)
63
67
  except (TypeError, ValueError):
64
- logger.warning("Invalid FPS value %.2f in VideoMeta for video %s.", video.video_meta.fps, video.uuid)
65
- raise ValueError(f"Could not determine FPS for video {video.uuid} due to invalid VideoMeta FPS value.")
68
+ logger.warning(
69
+ "Invalid FPS value %.2f in VideoMeta for video %s.",
70
+ video.video_meta.fps,
71
+ video.video_hash,
72
+ )
73
+ raise ValueError(
74
+ f"Could not determine FPS for video {video.video_hash} due to invalid VideoMeta FPS value."
75
+ )
66
76
  video.fps = _fps
67
77
  # Avoid saving if called from within the save method itself
68
78
  if not getattr(video, "_saving", False):
69
79
  video.save(update_fields=["fps"])
70
80
  return _fps
71
81
  else:
72
- logger.warning("Could not determine FPS from VideoMeta for video %s. Trying direct raw file access.", video.uuid)
82
+ logger.warning(
83
+ "Could not determine FPS from VideoMeta for video %s. Trying direct raw file access.",
84
+ video.video_hash,
85
+ )
73
86
  try:
74
87
  if video.has_raw:
75
88
  video_path = video.get_raw_file_path() # Use helper
@@ -88,23 +101,39 @@ def _get_fps(video: "VideoFile") -> float:
88
101
  cap.release()
89
102
  if fps and fps > 0:
90
103
  video.fps = fps
91
- logger.info("Determined FPS %.2f directly from file for %s.", video.fps, video.uuid)
104
+ logger.info(
105
+ "Determined FPS %.2f directly from file for %s.",
106
+ video.fps,
107
+ video.video_hash,
108
+ )
92
109
  if not getattr(video, "_saving", False):
93
110
  video.save(update_fields=["fps"])
94
111
  return fps
95
112
  else:
96
- logger.warning("Could not determine a valid FPS for video file %s.", video_path)
113
+ logger.warning(
114
+ "Could not determine a valid FPS for video file %s.",
115
+ video_path,
116
+ )
97
117
  elif video_path:
98
- logger.warning("Raw file path %s does not exist for direct FPS check.", video_path)
118
+ logger.warning(
119
+ "Raw file path %s does not exist for direct FPS check.",
120
+ video_path,
121
+ )
99
122
  else:
100
123
  logger.warning("Raw file path is None for direct FPS check.")
101
124
  else:
102
125
  logger.warning("Raw file not available for direct FPS check.")
103
126
 
104
127
  except Exception as e:
105
- logger.error("Error getting FPS directly from file %s: %s", video.raw_file.name if video.has_raw else "N/A", e)
128
+ logger.error(
129
+ "Error getting FPS directly from file %s: %s",
130
+ video.raw_file.name if video.has_raw else "N/A",
131
+ e,
132
+ )
106
133
 
107
- raise ValueError(f"Could not determine FPS for video {video.uuid}. Ensure the video file is valid and accessible.")
134
+ raise ValueError(
135
+ f"Could not determine FPS for video {video.video_hash}. Ensure the video file is valid and accessible."
136
+ )
108
137
 
109
138
 
110
139
  # TODO Refactor to utils / check if similar function exists in utils
@@ -132,7 +161,9 @@ def _calculate_fps_manually(cap, video_path: Path) -> float:
132
161
  Returns:
133
162
  float: The estimated FPS, or 0.0 if the duration is zero or calculation fails.
134
163
  """
135
- logger.warning(f"Could not get a valid FPS for {video_path}. Trying to calculate manually.")
164
+ logger.warning(
165
+ f"Could not get a valid FPS for {video_path}. Trying to calculate manually."
166
+ )
136
167
  # This is less accurate and slower
137
168
  num_frames = 0
138
169
  start_time = cv2.getTickCount()
@@ -146,5 +177,7 @@ def _calculate_fps_manually(cap, video_path: Path) -> float:
146
177
  if seconds > 0:
147
178
  return num_frames / seconds
148
179
 
149
- logger.error(f"Manual FPS calculation failed for {video_path} due to zero duration.")
180
+ logger.error(
181
+ f"Manual FPS calculation failed for {video_path} due to zero duration."
182
+ )
150
183
  return 0.0
@@ -31,24 +31,38 @@ def _initialize_video_specs(video: "VideoFile", use_raw: bool = True) -> bool:
31
31
  video_path = video.active_file_path # Use property relying on IO helpers
32
32
  target_file_name = video.active_file.name
33
33
  else:
34
- logger.error("No suitable video file found for spec initialization for %s.", video.uuid)
34
+ logger.error(
35
+ "No suitable video file found for spec initialization for %s.",
36
+ video.video_hash,
37
+ )
35
38
  return False
36
39
 
37
40
  if not video_path:
38
41
  # Raise exception
39
- raise FileNotFoundError(f"Could not determine video file path for spec initialization for {video.uuid}.")
40
-
41
- logger.info("Initializing video specs directly from file %s (%s) for %s", target_file_name, video_path, video.uuid)
42
+ raise FileNotFoundError(
43
+ f"Could not determine video file path for spec initialization for {video.video_hash}."
44
+ )
45
+
46
+ logger.info(
47
+ "Initializing video specs directly from file %s (%s) for %s",
48
+ target_file_name,
49
+ video_path,
50
+ video.video_hash,
51
+ )
42
52
  try:
43
53
  if not video_path.exists():
44
54
  # Raise exception
45
- raise FileNotFoundError(f"Video file not found at {video_path} for spec initialization (Video: {video.uuid}).")
55
+ raise FileNotFoundError(
56
+ f"Video file not found at {video_path} for spec initialization (Video: {video.video_hash})."
57
+ )
46
58
 
47
59
  video_cap = cv2.VideoCapture(video_path.as_posix())
48
60
  if not video_cap.isOpened():
49
61
  # Raise exception
50
62
  video_cap.release() # Ensure release
51
- raise RuntimeError(f"Could not open video file {video_path} with OpenCV for spec initialization (Video: {video.uuid}).")
63
+ raise RuntimeError(
64
+ f"Could not open video file {video_path} with OpenCV for spec initialization (Video: {video.video_hash})."
65
+ )
52
66
 
53
67
  updated = False
54
68
  fields_to_update = []
@@ -67,8 +81,16 @@ def _initialize_video_specs(video: "VideoFile", use_raw: bool = True) -> bool:
67
81
  file_height = int(video_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
68
82
  file_frame_count = int(video_cap.get(cv2.CAP_PROP_FRAME_COUNT))
69
83
  except Exception as cv_err:
70
- logger.error("Error getting properties from OpenCV for %s (Video: %s): %s", video_path, video.uuid, cv_err, exc_info=True)
71
- raise RuntimeError(f"OpenCV failed to get properties for {video_path}") from cv_err
84
+ logger.error(
85
+ "Error getting properties from OpenCV for %s (Video: %s): %s",
86
+ video_path,
87
+ video.video_hash,
88
+ cv_err,
89
+ exc_info=True,
90
+ )
91
+ raise RuntimeError(
92
+ f"OpenCV failed to get properties for {video_path}"
93
+ ) from cv_err
72
94
  finally:
73
95
  video_cap.release() # Ensure release after getting props
74
96
 
@@ -96,42 +118,81 @@ def _initialize_video_specs(video: "VideoFile", use_raw: bool = True) -> bool:
96
118
  video.frame_count = file_frame_count
97
119
  fields_to_update.append("frame_count")
98
120
  updated = True
99
- elif file_frame_count is None or file_frame_count <= 0: # Log if not updated due to invalid file_frame_count
100
- logger.warning("Invalid frame count (value: %s) obtained from OpenCV for %s. Video frame_count not updated.", file_frame_count, video_path)
121
+ elif (
122
+ file_frame_count is None or file_frame_count <= 0
123
+ ): # Log if not updated due to invalid file_frame_count
124
+ logger.warning(
125
+ "Invalid frame count (value: %s) obtained from OpenCV for %s. Video frame_count not updated.",
126
+ file_frame_count,
127
+ video_path,
128
+ )
101
129
 
102
130
  # --- Update Duration ---
103
131
  if current_duration is None: # Only if duration isn't already set
104
132
  # Use the potentially updated video.frame_count and current_fps (which reflects video.fps or file_fps)
105
133
  final_frame_count_for_duration = video.frame_count
106
- final_fps_for_duration = current_fps # This is video.fps after potential update from file_fps
107
-
108
- if final_frame_count_for_duration and final_frame_count_for_duration > 0 and final_fps_for_duration and final_fps_for_duration > 0:
134
+ final_fps_for_duration = (
135
+ current_fps # This is video.fps after potential update from file_fps
136
+ )
137
+
138
+ if (
139
+ final_frame_count_for_duration
140
+ and final_frame_count_for_duration > 0
141
+ and final_fps_for_duration
142
+ and final_fps_for_duration > 0
143
+ ):
109
144
  video.duration = final_frame_count_for_duration / final_fps_for_duration
110
145
  fields_to_update.append("duration")
111
146
  updated = True
112
147
  else:
113
148
  # Log if duration could not be calculated, indicating which component was missing/invalid
114
- if not (final_frame_count_for_duration and final_frame_count_for_duration > 0):
149
+ if not (
150
+ final_frame_count_for_duration
151
+ and final_frame_count_for_duration > 0
152
+ ):
115
153
  logger.warning(
116
- "Duration not calculated for %s: frame count is unavailable or invalid (value: %s).", video_path, final_frame_count_for_duration
154
+ "Duration not calculated for %s: frame count is unavailable or invalid (value: %s).",
155
+ video_path,
156
+ final_frame_count_for_duration,
117
157
  )
118
158
  if not (final_fps_for_duration and final_fps_for_duration > 0):
119
- logger.warning("Duration not calculated for %s: FPS is unavailable or invalid (value: %s).", video_path, final_fps_for_duration)
159
+ logger.warning(
160
+ "Duration not calculated for %s: FPS is unavailable or invalid (value: %s).",
161
+ video_path,
162
+ final_fps_for_duration,
163
+ )
120
164
 
121
165
  # --- Save if updated ---
122
166
  if updated:
123
- logger.info("Updated video specs for %s from file %s: %s", video.uuid, target_file_name, ", ".join(fields_to_update))
167
+ logger.info(
168
+ "Updated video specs for %s from file %s: %s",
169
+ video.video_hash,
170
+ target_file_name,
171
+ ", ".join(fields_to_update),
172
+ )
124
173
  video.save(update_fields=fields_to_update)
125
174
  return True
126
175
  else:
127
- logger.info("No video specs needed updating for %s from file %s.", video.uuid, target_file_name)
176
+ logger.info(
177
+ "No video specs needed updating for %s from file %s.",
178
+ video.video_hash,
179
+ target_file_name,
180
+ )
128
181
  return True
129
182
 
130
183
  except Exception as e:
131
184
  # Log and re-raise exception
132
- logger.error("Error initializing video specs for %s from file %s: %s", video.uuid, video_path, e, exc_info=True)
185
+ logger.error(
186
+ "Error initializing video specs for %s from file %s: %s",
187
+ video.video_hash,
188
+ video_path,
189
+ e,
190
+ exc_info=True,
191
+ )
133
192
  # Ensure capture is released in case of unexpected error
134
193
  if "video_cap" in locals() and video_cap.isOpened():
135
194
  video_cap.release()
136
195
  # Re-raise as RuntimeError
137
- raise RuntimeError(f"Failed to initialize video specs for {video.uuid} from {video_path}") from e
196
+ raise RuntimeError(
197
+ f"Failed to initialize video specs for {video.video_hash} from {video_path}"
198
+ ) from e
@@ -10,6 +10,7 @@ from ....metadata.sensitive_meta_logic import update_or_create_sensitive_meta_fr
10
10
 
11
11
  if TYPE_CHECKING:
12
12
  from ..video_file import VideoFile
13
+
13
14
  # SensitiveMeta is already imported above
14
15
 
15
16
  logger = logging.getLogger(__name__)
@@ -31,35 +32,46 @@ def _update_text_metadata(
31
32
  - Pre-condition: Requires state.frames_extracted=True.
32
33
  - Post-condition: Sets state.text_meta_extracted=True (even if no text found).
33
34
  """
34
- logger.debug(f"Updating text metadata for video {video.uuid}")
35
+ logger.debug(f"Updating text metadata for video {video.video_hash}")
35
36
  state = video.get_or_create_state()
36
37
 
37
38
  # --- Pre-condition Checks ---
38
39
  if not state.frames_extracted:
39
40
  # Attempt to extract frames automatically if they're not available
40
- logger.warning(f"Frames not extracted for video {video.uuid}. Attempting automatic frame extraction...")
41
+ logger.warning(
42
+ f"Frames not extracted for video {video.video_hash}. Attempting automatic frame extraction..."
43
+ )
41
44
  try:
42
45
  success = video.extract_frames(overwrite=False)
43
46
  except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc:
44
47
  logger.error(
45
48
  "Failed to extract frames for video %s: %s",
46
- video.uuid,
49
+ video.video_hash,
47
50
  exc,
48
51
  exc_info=True,
49
52
  )
50
- raise ValueError(f"Cannot update text metadata for video {video.uuid}: Frames not extracted and automatic extraction failed") from exc
53
+ raise ValueError(
54
+ f"Cannot update text metadata for video {video.video_hash}: Frames not extracted and automatic extraction failed"
55
+ ) from exc
51
56
 
52
57
  if not success:
53
- raise ValueError(f"Cannot update text metadata for video {video.uuid}: Frame extraction returned False")
58
+ raise ValueError(
59
+ f"Cannot update text metadata for video {video.video_hash}: Frame extraction returned False"
60
+ )
54
61
 
55
62
  state.refresh_from_db()
56
63
  if not state.frames_extracted:
57
- raise ValueError(f"Cannot update text metadata for video {video.uuid}: Frame extraction completed but state was not updated")
64
+ raise ValueError(
65
+ f"Cannot update text metadata for video {video.video_hash}: Frame extraction completed but state was not updated"
66
+ )
58
67
 
59
- logger.info(f"Successfully extracted frames for video {video.uuid}")
68
+ logger.info(f"Successfully extracted frames for video {video.video_hash}")
60
69
 
61
70
  if state.text_meta_extracted and not overwrite:
62
- logger.info("Text already extracted for video %s and overwrite=False. Skipping.", video.uuid) # Changed to info
71
+ logger.info(
72
+ "Text already extracted for video %s and overwrite=False. Skipping.",
73
+ video.video_hash,
74
+ ) # Changed to info
63
75
  return video.sensitive_meta # Return existing meta if available
64
76
  # --- End Pre-condition Checks ---
65
77
 
@@ -67,10 +79,19 @@ def _update_text_metadata(
67
79
  # _extract_text_from_video_frames raises ValueError on pre-condition failure
68
80
  try:
69
81
  if not extracted_data_dict:
70
- extracted_data_dict = video.extract_text_from_frames(frame_fraction=ocr_frame_fraction, cap=cap)
82
+ extracted_data_dict = video.extract_text_from_frames(
83
+ frame_fraction=ocr_frame_fraction, cap=cap
84
+ )
71
85
  except Exception as text_extract_e:
72
- logger.error("Failed during text extraction step for video %s: %s", video.uuid, text_extract_e, exc_info=True)
73
- raise RuntimeError(f"Text extraction failed for video {video.uuid}") from text_extract_e
86
+ logger.error(
87
+ "Failed during text extraction step for video %s: %s",
88
+ video.video_hash,
89
+ text_extract_e,
90
+ exc_info=True,
91
+ )
92
+ raise RuntimeError(
93
+ f"Text extraction failed for video {video.video_hash}"
94
+ ) from text_extract_e
74
95
 
75
96
  # --- Atomic Update Block ---
76
97
  try:
@@ -80,7 +101,10 @@ def _update_text_metadata(
80
101
  sensitive_meta_instance = video.sensitive_meta # Get current instance
81
102
 
82
103
  if not extracted_data_dict:
83
- logger.warning("No text extracted for video %s; skipping SensitiveMeta update.", video.uuid)
104
+ logger.warning(
105
+ "No text extracted for video %s; skipping SensitiveMeta update.",
106
+ video.video_hash,
107
+ )
84
108
  # Mark state as retrieved even if no data found, to avoid re-running unless overwrite=True
85
109
  if not state.text_meta_extracted:
86
110
  state.text_meta_extracted = True
@@ -90,7 +114,11 @@ def _update_text_metadata(
90
114
  # Add center info if not already present in extracted data
91
115
  if "center_name" not in extracted_data_dict and video.center:
92
116
  extracted_data_dict["center_name"] = video.center.name
93
- logger.debug("Data for SensitiveMeta update for video %s: %s", video.uuid, extracted_data_dict)
117
+ logger.debug(
118
+ "Data for SensitiveMeta update for video %s: %s",
119
+ video.video_hash,
120
+ extracted_data_dict,
121
+ )
94
122
 
95
123
  # Pass the Class, the data dict, and the current instance (or None)
96
124
  # This function might raise exceptions if data is invalid
@@ -102,7 +130,9 @@ def _update_text_metadata(
102
130
 
103
131
  # Update VideoFile fields if necessary
104
132
  update_fields_video = []
105
- if created or sensitive_meta != sensitive_meta_instance: # Check if relation needs update
133
+ if (
134
+ created or sensitive_meta != sensitive_meta_instance
135
+ ): # Check if relation needs update
106
136
  video.sensitive_meta = sensitive_meta
107
137
  update_fields_video.append("sensitive_meta")
108
138
 
@@ -111,7 +141,6 @@ def _update_text_metadata(
111
141
  if extracted_date: # Ensure date is not None or empty
112
142
  video.date = extracted_date
113
143
  update_fields_video.append("date")
114
-
115
144
 
116
145
  # Save VideoFile if fields changed
117
146
  if update_fields_video:
@@ -125,13 +154,25 @@ def _update_text_metadata(
125
154
  # Mark sensitive meta as processed when updated via text metadata
126
155
  if sensitive_meta:
127
156
  state.mark_sensitive_meta_processed(save=True)
128
- logger.info(f"Marked sensitive_meta_processed=True for video {video.uuid} after text metadata update")
129
-
130
- logger.info("Successfully updated/created SensitiveMeta and state for video %s.", video.uuid) # Changed to info
157
+ logger.info(
158
+ f"Marked sensitive_meta_processed=True for video {video.video_hash} after text metadata update"
159
+ )
160
+
161
+ logger.info(
162
+ "Successfully updated/created SensitiveMeta and state for video %s.",
163
+ video.video_hash,
164
+ ) # Changed to info
131
165
  return sensitive_meta
132
166
 
133
167
  except Exception as e:
134
- logger.error("Failed to update/create SensitiveMeta or state for video %s: %s", video.uuid, e, exc_info=True)
168
+ logger.error(
169
+ "Failed to update/create SensitiveMeta or state for video %s: %s",
170
+ video.video_hash,
171
+ e,
172
+ exc_info=True,
173
+ )
135
174
  # Re-raise exception for the pipeline to catch
136
- raise RuntimeError(f"Failed to update/create SensitiveMeta or state for video {video.uuid}") from e
175
+ raise RuntimeError(
176
+ f"Failed to update/create SensitiveMeta or state for video {video.video_hash}"
177
+ ) from e
137
178
  # --- End Atomic Update Block ---
@@ -14,12 +14,14 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
14
14
  """
15
15
  from ....metadata import VideoMeta # Local import
16
16
 
17
- logger.debug("Updating technical VideoMeta for video %s (from raw file).", video.uuid)
17
+ logger.debug(
18
+ "Updating technical VideoMeta for video %s (from raw file).", video.video_hash
19
+ )
18
20
 
19
21
  if not video.has_raw:
20
22
  # DEFENSIVE: Log warning and skip instead of crashing
21
23
  logger.warning(
22
- f"Raw video file path not available for {video.uuid}. Skipping VideoMeta update - this may indicate the video was processed and raw file moved."
24
+ f"Raw video file path not available for {video.video_hash}. Skipping VideoMeta update - this may indicate the video was processed and raw file moved."
23
25
  )
24
26
  return # Graceful skip instead of FileNotFoundError
25
27
 
@@ -27,22 +29,30 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
27
29
  if not raw_video_path or not raw_video_path.exists():
28
30
  # DEFENSIVE: Log warning and skip instead of crashing production pipeline
29
31
  logger.warning(
30
- f"Raw video file path {raw_video_path} does not exist for video {video.uuid}. Skipping VideoMeta update - this typically happens after video processing when raw files are moved to processed location."
32
+ f"Raw video file path {raw_video_path} does not exist for video {video.video_hash}. Skipping VideoMeta update - this typically happens after video processing when raw files are moved to processed location."
31
33
  )
32
34
  return # Graceful skip instead of FileNotFoundError that crashes production
33
35
 
34
36
  try:
35
37
  vm = video.video_meta
36
38
  if vm:
37
- logger.info("Updating existing VideoMeta (PK: %s) for video %s.", vm.pk, video.uuid)
38
- vm.update_meta(raw_video_path) # Assuming this method exists and raises on error
39
+ logger.info(
40
+ "Updating existing VideoMeta (PK: %s) for video %s.",
41
+ vm.pk,
42
+ video.video_hash,
43
+ )
44
+ vm.update_meta(
45
+ raw_video_path
46
+ ) # Assuming this method exists and raises on error
39
47
  vm.save()
40
48
  else:
41
49
  if not video.center or not video.processor:
42
50
  # Raise exception
43
- raise ValueError(f"Cannot create VideoMeta for {video.uuid}: Center or Processor is missing.")
51
+ raise ValueError(
52
+ f"Cannot create VideoMeta for {video.video_hash}: Center or Processor is missing."
53
+ )
44
54
 
45
- logger.info("Creating new VideoMeta for video %s.", video.uuid)
55
+ logger.info("Creating new VideoMeta for video %s.", video.video_hash)
46
56
  # Assuming create_from_file exists and raises on error
47
57
  video.video_meta = VideoMeta.create_from_file(
48
58
  video_path=raw_video_path,
@@ -52,7 +62,11 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
52
62
  )
53
63
  vm = video.video_meta
54
64
  assert vm is not None # For type checker
55
- logger.info("Created and linked VideoMeta (PK: %s) for video %s.", vm.pk, video.uuid)
65
+ logger.info(
66
+ "Created and linked VideoMeta (PK: %s) for video %s.",
67
+ vm.pk,
68
+ video.video_hash,
69
+ )
56
70
 
57
71
  # Save the VideoFile instance itself if requested and if video_meta was linked/updated
58
72
  if save_instance:
@@ -62,16 +76,30 @@ def _update_video_meta(video: "VideoFile", save_instance: bool = True):
62
76
  meta_fields = ["fps", "duration", "frame_count", "width", "height"]
63
77
  for field in meta_fields:
64
78
  # Check if field is None on video but has value on meta
65
- if getattr(video, field) is None and getattr(video.video_meta, field, None) is not None:
79
+ if (
80
+ getattr(video, field) is None
81
+ and getattr(video.video_meta, field, None) is not None
82
+ ):
66
83
  # No need to set attribute here, save method handles it
67
84
  update_fields.append(field)
68
85
  # Ensure update_fields has unique values before saving
69
86
  unique_update_fields = list(set(update_fields))
70
87
  if unique_update_fields:
71
88
  video.save(update_fields=unique_update_fields)
72
- logger.info("Saved video %s after VideoMeta update (Fields: %s).", video.uuid, unique_update_fields)
89
+ logger.info(
90
+ "Saved video %s after VideoMeta update (Fields: %s).",
91
+ video.video_hash,
92
+ unique_update_fields,
93
+ )
73
94
 
74
95
  except Exception as e:
75
- logger.error("Failed to update/create VideoMeta for video %s: %s", video.uuid, e, exc_info=True)
96
+ logger.error(
97
+ "Failed to update/create VideoMeta for video %s: %s",
98
+ video.video_hash,
99
+ e,
100
+ exc_info=True,
101
+ )
76
102
  # Re-raise exception
77
- raise RuntimeError(f"Failed to update/create VideoMeta for video {video.uuid}") from e
103
+ raise RuntimeError(
104
+ f"Failed to update/create VideoMeta for video {video.video_hash}"
105
+ ) from e