endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (450) hide show
  1. endoreg_db/admin.py +10 -5
  2. endoreg_db/apps.py +4 -7
  3. endoreg_db/authz/auth.py +1 -0
  4. endoreg_db/authz/backends.py +1 -1
  5. endoreg_db/authz/management/commands/list_routes.py +2 -0
  6. endoreg_db/authz/middleware.py +8 -7
  7. endoreg_db/authz/permissions.py +21 -10
  8. endoreg_db/authz/policy.py +14 -19
  9. endoreg_db/authz/views_auth.py +14 -10
  10. endoreg_db/codemods/rename_datetime_fields.py +8 -1
  11. endoreg_db/exceptions.py +5 -2
  12. endoreg_db/forms/__init__.py +0 -1
  13. endoreg_db/forms/examination_form.py +4 -3
  14. endoreg_db/forms/patient_finding_intervention_form.py +30 -8
  15. endoreg_db/forms/patient_form.py +9 -13
  16. endoreg_db/forms/questionnaires/__init__.py +1 -1
  17. endoreg_db/forms/settings/__init__.py +4 -1
  18. endoreg_db/forms/unit.py +2 -1
  19. endoreg_db/helpers/count_db.py +17 -14
  20. endoreg_db/helpers/default_objects.py +2 -1
  21. endoreg_db/helpers/download_segmentation_model.py +4 -3
  22. endoreg_db/helpers/interact.py +0 -5
  23. endoreg_db/helpers/test_video_helper.py +33 -25
  24. endoreg_db/import_files/__init__.py +1 -1
  25. endoreg_db/import_files/context/__init__.py +1 -1
  26. endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
  27. endoreg_db/import_files/context/ensure_center.py +4 -4
  28. endoreg_db/import_files/context/file_lock.py +3 -3
  29. endoreg_db/import_files/context/import_context.py +11 -12
  30. endoreg_db/import_files/context/validate_directories.py +1 -0
  31. endoreg_db/import_files/file_storage/create_report_file.py +57 -34
  32. endoreg_db/import_files/file_storage/create_video_file.py +64 -35
  33. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
  34. endoreg_db/import_files/file_storage/state_management.py +89 -122
  35. endoreg_db/import_files/file_storage/storage.py +5 -1
  36. endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
  37. endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
  38. endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
  39. endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
  40. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
  41. endoreg_db/import_files/report_import_service.py +36 -30
  42. endoreg_db/import_files/video_import_service.py +27 -23
  43. endoreg_db/logger_conf.py +56 -40
  44. endoreg_db/management/__init__.py +1 -1
  45. endoreg_db/management/commands/__init__.py +1 -1
  46. endoreg_db/management/commands/check_auth.py +45 -38
  47. endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
  48. endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
  49. endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
  50. endoreg_db/management/commands/fix_video_paths.py +75 -54
  51. endoreg_db/management/commands/import_report.py +1 -3
  52. endoreg_db/management/commands/list_routes.py +2 -0
  53. endoreg_db/management/commands/load_ai_model_data.py +8 -2
  54. endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
  55. endoreg_db/management/commands/load_center_data.py +3 -3
  56. endoreg_db/management/commands/load_distribution_data.py +35 -38
  57. endoreg_db/management/commands/load_endoscope_data.py +0 -3
  58. endoreg_db/management/commands/load_examination_data.py +20 -4
  59. endoreg_db/management/commands/load_finding_data.py +18 -3
  60. endoreg_db/management/commands/load_gender_data.py +17 -24
  61. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
  62. endoreg_db/management/commands/load_information_source.py +0 -3
  63. endoreg_db/management/commands/load_lab_value_data.py +14 -3
  64. endoreg_db/management/commands/load_legacy_data.py +303 -0
  65. endoreg_db/management/commands/load_name_data.py +1 -2
  66. endoreg_db/management/commands/load_pdf_type_data.py +4 -8
  67. endoreg_db/management/commands/load_profession_data.py +0 -1
  68. endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
  69. endoreg_db/management/commands/load_requirement_data.py +6 -2
  70. endoreg_db/management/commands/load_unit_data.py +0 -4
  71. endoreg_db/management/commands/load_user_groups.py +5 -7
  72. endoreg_db/management/commands/model_input.py +169 -0
  73. endoreg_db/management/commands/register_ai_model.py +22 -16
  74. endoreg_db/management/commands/setup_endoreg_db.py +110 -32
  75. endoreg_db/management/commands/storage_management.py +14 -8
  76. endoreg_db/management/commands/summarize_db_content.py +154 -63
  77. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  78. endoreg_db/management/commands/validate_video_files.py +82 -50
  79. endoreg_db/management/commands/video_validation.py +4 -6
  80. endoreg_db/migrations/0001_initial.py +112 -63
  81. endoreg_db/models/__init__.py +8 -0
  82. endoreg_db/models/administration/ai/active_model.py +5 -5
  83. endoreg_db/models/administration/ai/ai_model.py +41 -18
  84. endoreg_db/models/administration/ai/model_type.py +1 -0
  85. endoreg_db/models/administration/case/case.py +22 -22
  86. endoreg_db/models/administration/center/__init__.py +5 -5
  87. endoreg_db/models/administration/center/center.py +6 -2
  88. endoreg_db/models/administration/center/center_resource.py +18 -4
  89. endoreg_db/models/administration/center/center_shift.py +3 -1
  90. endoreg_db/models/administration/center/center_waste.py +6 -2
  91. endoreg_db/models/administration/person/__init__.py +1 -1
  92. endoreg_db/models/administration/person/employee/__init__.py +1 -1
  93. endoreg_db/models/administration/person/employee/employee_type.py +3 -1
  94. endoreg_db/models/administration/person/examiner/__init__.py +1 -1
  95. endoreg_db/models/administration/person/examiner/examiner.py +10 -2
  96. endoreg_db/models/administration/person/names/first_name.py +6 -4
  97. endoreg_db/models/administration/person/names/last_name.py +4 -3
  98. endoreg_db/models/administration/person/patient/__init__.py +1 -1
  99. endoreg_db/models/administration/person/patient/patient.py +0 -1
  100. endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
  101. endoreg_db/models/administration/person/person.py +1 -1
  102. endoreg_db/models/administration/product/__init__.py +7 -6
  103. endoreg_db/models/administration/product/product.py +6 -2
  104. endoreg_db/models/administration/product/product_group.py +9 -7
  105. endoreg_db/models/administration/product/product_material.py +9 -2
  106. endoreg_db/models/administration/product/reference_product.py +64 -15
  107. endoreg_db/models/administration/qualification/qualification.py +3 -1
  108. endoreg_db/models/administration/shift/shift.py +3 -1
  109. endoreg_db/models/administration/shift/shift_type.py +12 -4
  110. endoreg_db/models/aidataset/__init__.py +5 -0
  111. endoreg_db/models/aidataset/aidataset.py +193 -0
  112. endoreg_db/models/label/__init__.py +1 -1
  113. endoreg_db/models/label/label.py +10 -2
  114. endoreg_db/models/label/label_set.py +3 -1
  115. endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
  116. endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
  117. endoreg_db/models/media/__init__.py +12 -5
  118. endoreg_db/models/media/frame/__init__.py +1 -1
  119. endoreg_db/models/media/frame/frame.py +34 -8
  120. endoreg_db/models/media/pdf/__init__.py +2 -1
  121. endoreg_db/models/media/pdf/raw_pdf.py +11 -4
  122. endoreg_db/models/media/pdf/report_file.py +6 -2
  123. endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
  124. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
  125. endoreg_db/models/media/video/create_from_file.py +20 -41
  126. endoreg_db/models/media/video/pipe_1.py +75 -30
  127. endoreg_db/models/media/video/pipe_2.py +37 -12
  128. endoreg_db/models/media/video/video_file.py +36 -24
  129. endoreg_db/models/media/video/video_file_ai.py +235 -70
  130. endoreg_db/models/media/video/video_file_anonymize.py +240 -65
  131. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
  132. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
  133. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
  134. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
  135. endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
  136. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
  137. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
  138. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
  139. endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
  140. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
  141. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
  142. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
  143. endoreg_db/models/media/video/video_file_io.py +85 -33
  144. endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
  145. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
  146. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
  147. endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
  148. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
  149. endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
  150. endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
  151. endoreg_db/models/media/video/video_file_segments.py +118 -27
  152. endoreg_db/models/media/video/video_metadata.py +25 -6
  153. endoreg_db/models/media/video/video_processing.py +54 -15
  154. endoreg_db/models/medical/__init__.py +3 -13
  155. endoreg_db/models/medical/contraindication/__init__.py +3 -1
  156. endoreg_db/models/medical/disease.py +18 -6
  157. endoreg_db/models/medical/event.py +6 -2
  158. endoreg_db/models/medical/examination/__init__.py +5 -1
  159. endoreg_db/models/medical/examination/examination.py +22 -6
  160. endoreg_db/models/medical/examination/examination_indication.py +23 -7
  161. endoreg_db/models/medical/examination/examination_time.py +6 -2
  162. endoreg_db/models/medical/finding/__init__.py +3 -1
  163. endoreg_db/models/medical/finding/finding.py +37 -12
  164. endoreg_db/models/medical/finding/finding_classification.py +27 -8
  165. endoreg_db/models/medical/finding/finding_intervention.py +19 -6
  166. endoreg_db/models/medical/finding/finding_type.py +3 -1
  167. endoreg_db/models/medical/hardware/__init__.py +1 -1
  168. endoreg_db/models/medical/hardware/endoscope.py +14 -2
  169. endoreg_db/models/medical/laboratory/__init__.py +1 -1
  170. endoreg_db/models/medical/laboratory/lab_value.py +139 -39
  171. endoreg_db/models/medical/medication/__init__.py +7 -3
  172. endoreg_db/models/medical/medication/medication.py +3 -1
  173. endoreg_db/models/medical/medication/medication_indication.py +3 -1
  174. endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
  175. endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
  176. endoreg_db/models/medical/medication/medication_schedule.py +3 -1
  177. endoreg_db/models/medical/patient/__init__.py +2 -10
  178. endoreg_db/models/medical/patient/medication_examples.py +3 -14
  179. endoreg_db/models/medical/patient/patient_disease.py +17 -5
  180. endoreg_db/models/medical/patient/patient_event.py +12 -4
  181. endoreg_db/models/medical/patient/patient_examination.py +52 -15
  182. endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
  183. endoreg_db/models/medical/patient/patient_finding.py +105 -29
  184. endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
  185. endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
  186. endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
  187. endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
  188. endoreg_db/models/medical/patient/patient_medication.py +25 -7
  189. endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
  190. endoreg_db/models/metadata/model_meta.py +40 -12
  191. endoreg_db/models/metadata/model_meta_logic.py +51 -16
  192. endoreg_db/models/metadata/sensitive_meta.py +65 -28
  193. endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
  194. endoreg_db/models/metadata/video_meta.py +146 -39
  195. endoreg_db/models/metadata/video_prediction_logic.py +70 -21
  196. endoreg_db/models/metadata/video_prediction_meta.py +80 -27
  197. endoreg_db/models/operation_log.py +63 -0
  198. endoreg_db/models/other/__init__.py +10 -10
  199. endoreg_db/models/other/distribution/__init__.py +9 -7
  200. endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
  201. endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
  202. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
  203. endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
  204. endoreg_db/models/other/emission/__init__.py +1 -1
  205. endoreg_db/models/other/emission/emission_factor.py +9 -3
  206. endoreg_db/models/other/information_source.py +15 -5
  207. endoreg_db/models/other/material.py +3 -1
  208. endoreg_db/models/other/transport_route.py +3 -1
  209. endoreg_db/models/other/unit.py +6 -2
  210. endoreg_db/models/report/report.py +0 -1
  211. endoreg_db/models/requirement/requirement.py +84 -27
  212. endoreg_db/models/requirement/requirement_error.py +5 -6
  213. endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
  214. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
  215. endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
  216. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
  217. endoreg_db/models/requirement/requirement_operator.py +28 -8
  218. endoreg_db/models/requirement/requirement_set.py +34 -11
  219. endoreg_db/models/state/__init__.py +1 -0
  220. endoreg_db/models/state/audit_ledger.py +9 -2
  221. endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
  222. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  223. endoreg_db/models/state/raw_pdf.py +0 -1
  224. endoreg_db/models/state/video.py +2 -4
  225. endoreg_db/models/utils.py +4 -2
  226. endoreg_db/queries/__init__.py +2 -6
  227. endoreg_db/queries/annotations/__init__.py +1 -3
  228. endoreg_db/queries/annotations/legacy.py +37 -26
  229. endoreg_db/root_urls.py +3 -4
  230. endoreg_db/schemas/examination_evaluation.py +3 -0
  231. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
  232. endoreg_db/serializers/__init__.py +2 -8
  233. endoreg_db/serializers/administration/__init__.py +1 -2
  234. endoreg_db/serializers/administration/ai/__init__.py +0 -1
  235. endoreg_db/serializers/administration/ai/active_model.py +3 -1
  236. endoreg_db/serializers/administration/ai/ai_model.py +5 -3
  237. endoreg_db/serializers/administration/ai/model_type.py +3 -1
  238. endoreg_db/serializers/administration/center.py +7 -2
  239. endoreg_db/serializers/administration/gender.py +4 -2
  240. endoreg_db/serializers/anonymization.py +13 -13
  241. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
  242. endoreg_db/serializers/examination/__init__.py +1 -1
  243. endoreg_db/serializers/examination/base.py +12 -13
  244. endoreg_db/serializers/examination/dropdown.py +6 -7
  245. endoreg_db/serializers/examination_serializer.py +3 -6
  246. endoreg_db/serializers/finding/__init__.py +1 -1
  247. endoreg_db/serializers/finding/finding.py +14 -7
  248. endoreg_db/serializers/finding_classification/__init__.py +3 -3
  249. endoreg_db/serializers/finding_classification/choice.py +3 -3
  250. endoreg_db/serializers/finding_classification/classification.py +2 -4
  251. endoreg_db/serializers/label_video_segment/__init__.py +5 -3
  252. endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
  253. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  254. endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
  255. endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
  256. endoreg_db/serializers/meta/__init__.py +1 -2
  257. endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
  258. endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
  259. endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
  260. endoreg_db/serializers/misc/__init__.py +2 -2
  261. endoreg_db/serializers/misc/file_overview.py +11 -7
  262. endoreg_db/serializers/misc/stats.py +10 -8
  263. endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
  264. endoreg_db/serializers/misc/upload_job.py +32 -29
  265. endoreg_db/serializers/patient/__init__.py +2 -1
  266. endoreg_db/serializers/patient/patient.py +32 -15
  267. endoreg_db/serializers/patient/patient_dropdown.py +11 -3
  268. endoreg_db/serializers/patient_examination/__init__.py +1 -1
  269. endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
  270. endoreg_db/serializers/patient_finding/__init__.py +1 -1
  271. endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
  272. endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
  273. endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
  274. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
  275. endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
  276. endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
  277. endoreg_db/serializers/pdf/__init__.py +1 -3
  278. endoreg_db/serializers/requirements/requirement_schema.py +1 -6
  279. endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
  280. endoreg_db/serializers/video/__init__.py +2 -2
  281. endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
  282. endoreg_db/serializers/video/video_file_brief.py +6 -2
  283. endoreg_db/serializers/video/video_file_detail.py +36 -23
  284. endoreg_db/serializers/video/video_file_list.py +4 -2
  285. endoreg_db/serializers/video/video_processing_history.py +54 -50
  286. endoreg_db/services/__init__.py +1 -1
  287. endoreg_db/services/anonymization.py +2 -2
  288. endoreg_db/services/examination_evaluation.py +40 -17
  289. endoreg_db/services/model_meta_from_hf.py +76 -0
  290. endoreg_db/services/polling_coordinator.py +101 -70
  291. endoreg_db/services/pseudonym_service.py +27 -22
  292. endoreg_db/services/report_import.py +6 -3
  293. endoreg_db/services/segment_sync.py +75 -59
  294. endoreg_db/services/video_import.py +6 -7
  295. endoreg_db/urls/__init__.py +2 -2
  296. endoreg_db/urls/ai.py +7 -25
  297. endoreg_db/urls/anonymization.py +61 -15
  298. endoreg_db/urls/auth.py +4 -4
  299. endoreg_db/urls/classification.py +4 -9
  300. endoreg_db/urls/examination.py +27 -18
  301. endoreg_db/urls/media.py +27 -34
  302. endoreg_db/urls/patient.py +11 -7
  303. endoreg_db/urls/requirements.py +3 -1
  304. endoreg_db/urls/root_urls.py +2 -3
  305. endoreg_db/urls/stats.py +24 -16
  306. endoreg_db/urls/upload.py +3 -11
  307. endoreg_db/utils/__init__.py +14 -15
  308. endoreg_db/utils/ai/__init__.py +1 -1
  309. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  310. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  311. endoreg_db/utils/ai/get.py +2 -1
  312. endoreg_db/utils/ai/inference_dataset.py +14 -15
  313. endoreg_db/utils/ai/model_training/config.py +117 -0
  314. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  315. endoreg_db/utils/ai/model_training/losses.py +68 -0
  316. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  317. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  318. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  319. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  320. endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
  321. endoreg_db/utils/ai/predict.py +4 -4
  322. endoreg_db/utils/ai/preprocess.py +19 -11
  323. endoreg_db/utils/calc_duration_seconds.py +4 -4
  324. endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
  325. endoreg_db/utils/check_video_files.py +74 -47
  326. endoreg_db/utils/cropping.py +10 -9
  327. endoreg_db/utils/dataloader.py +11 -3
  328. endoreg_db/utils/dates.py +3 -4
  329. endoreg_db/utils/defaults/set_default_center.py +7 -6
  330. endoreg_db/utils/env.py +6 -2
  331. endoreg_db/utils/extract_specific_frames.py +24 -9
  332. endoreg_db/utils/file_operations.py +30 -18
  333. endoreg_db/utils/fix_video_path_direct.py +57 -41
  334. endoreg_db/utils/frame_anonymization_utils.py +157 -157
  335. endoreg_db/utils/hashs.py +3 -18
  336. endoreg_db/utils/links/requirement_link.py +96 -52
  337. endoreg_db/utils/ocr.py +30 -25
  338. endoreg_db/utils/operation_log.py +61 -0
  339. endoreg_db/utils/parse_and_generate_yaml.py +12 -13
  340. endoreg_db/utils/paths.py +6 -6
  341. endoreg_db/utils/permissions.py +40 -24
  342. endoreg_db/utils/pipelines/process_video_dir.py +50 -26
  343. endoreg_db/utils/product/sum_emissions.py +5 -3
  344. endoreg_db/utils/product/sum_weights.py +4 -2
  345. endoreg_db/utils/pydantic_models/__init__.py +3 -4
  346. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
  347. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
  348. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
  349. endoreg_db/utils/setup_config.py +21 -5
  350. endoreg_db/utils/storage.py +3 -1
  351. endoreg_db/utils/translation.py +19 -15
  352. endoreg_db/utils/uuid.py +1 -0
  353. endoreg_db/utils/validate_endo_roi.py +12 -4
  354. endoreg_db/utils/validate_subcategory_dict.py +26 -24
  355. endoreg_db/utils/validate_video_detailed.py +207 -149
  356. endoreg_db/utils/video/__init__.py +7 -3
  357. endoreg_db/utils/video/extract_frames.py +30 -18
  358. endoreg_db/utils/video/names.py +11 -6
  359. endoreg_db/utils/video/streaming_processor.py +175 -101
  360. endoreg_db/utils/video/video_splitter.py +30 -19
  361. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
  362. endoreg_db/views/__init__.py +0 -20
  363. endoreg_db/views/anonymization/__init__.py +6 -2
  364. endoreg_db/views/anonymization/media_management.py +2 -6
  365. endoreg_db/views/anonymization/overview.py +34 -1
  366. endoreg_db/views/anonymization/validate.py +79 -18
  367. endoreg_db/views/auth/__init__.py +1 -1
  368. endoreg_db/views/auth/keycloak.py +16 -14
  369. endoreg_db/views/examination/__init__.py +12 -15
  370. endoreg_db/views/examination/examination.py +5 -5
  371. endoreg_db/views/examination/examination_manifest_cache.py +5 -5
  372. endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
  373. endoreg_db/views/examination/get_finding_classifications.py +9 -7
  374. endoreg_db/views/examination/get_findings.py +8 -10
  375. endoreg_db/views/examination/get_instruments.py +3 -2
  376. endoreg_db/views/examination/get_interventions.py +1 -1
  377. endoreg_db/views/finding/__init__.py +2 -2
  378. endoreg_db/views/finding/finding.py +58 -54
  379. endoreg_db/views/finding/get_classifications.py +1 -1
  380. endoreg_db/views/finding/get_interventions.py +1 -1
  381. endoreg_db/views/finding_classification/__init__.py +5 -5
  382. endoreg_db/views/finding_classification/finding_classification.py +5 -6
  383. endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
  384. endoreg_db/views/media/__init__.py +13 -13
  385. endoreg_db/views/media/pdf_media.py +9 -9
  386. endoreg_db/views/media/sensitive_metadata.py +10 -7
  387. endoreg_db/views/media/video_media.py +4 -4
  388. endoreg_db/views/meta/__init__.py +1 -1
  389. endoreg_db/views/meta/sensitive_meta_list.py +20 -22
  390. endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
  391. endoreg_db/views/misc/__init__.py +6 -34
  392. endoreg_db/views/misc/center.py +2 -1
  393. endoreg_db/views/misc/csrf.py +2 -1
  394. endoreg_db/views/misc/gender.py +2 -1
  395. endoreg_db/views/misc/stats.py +141 -106
  396. endoreg_db/views/patient/__init__.py +1 -3
  397. endoreg_db/views/patient/patient.py +141 -99
  398. endoreg_db/views/patient_examination/__init__.py +5 -5
  399. endoreg_db/views/patient_examination/patient_examination.py +43 -42
  400. endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
  401. endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
  402. endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
  403. endoreg_db/views/patient_examination/video.py +114 -80
  404. endoreg_db/views/patient_finding/__init__.py +1 -1
  405. endoreg_db/views/patient_finding/patient_finding.py +17 -10
  406. endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
  407. endoreg_db/views/patient_finding_classification/__init__.py +1 -1
  408. endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
  409. endoreg_db/views/report/reimport.py +1 -1
  410. endoreg_db/views/report/report_stream.py +5 -8
  411. endoreg_db/views/requirement/__init__.py +2 -1
  412. endoreg_db/views/requirement/evaluate.py +7 -9
  413. endoreg_db/views/requirement/lookup.py +2 -3
  414. endoreg_db/views/requirement/lookup_store.py +0 -1
  415. endoreg_db/views/requirement/requirement_utils.py +2 -4
  416. endoreg_db/views/stats/__init__.py +4 -4
  417. endoreg_db/views/stats/stats_views.py +152 -115
  418. endoreg_db/views/video/__init__.py +18 -27
  419. endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
  420. endoreg_db/views/{ai → video/ai}/label.py +20 -16
  421. endoreg_db/views/video/correction.py +5 -6
  422. endoreg_db/views/video/reimport.py +134 -99
  423. endoreg_db/views/video/segments_crud.py +134 -44
  424. endoreg_db/views/video/video_apply_mask.py +13 -12
  425. endoreg_db/views/video/video_correction.py +2 -1
  426. endoreg_db/views/video/video_download_processed.py +15 -15
  427. endoreg_db/views/video/video_meta_stats.py +7 -6
  428. endoreg_db/views/video/video_processing_history.py +3 -2
  429. endoreg_db/views/video/video_remove_frames.py +13 -12
  430. endoreg_db/views/video/video_stream.py +110 -82
  431. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
  432. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
  433. endoreg_db/management/commands/import_fallback_video.py +0 -203
  434. endoreg_db/management/commands/import_video.py +0 -422
  435. endoreg_db/management/commands/import_video_with_classification.py +0 -367
  436. endoreg_db/models/media/processing_history/processing_history.py +0 -96
  437. endoreg_db/serializers/label/__init__.py +0 -7
  438. endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
  439. endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
  440. endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
  441. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
  442. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
  443. endoreg_db/services/__old/pdf_import.py +0 -1487
  444. endoreg_db/services/__old/video_import.py +0 -1306
  445. endoreg_db/tasks/upload_tasks.py +0 -216
  446. endoreg_db/tasks/video_ingest.py +0 -161
  447. endoreg_db/tasks/video_processing_tasks.py +0 -327
  448. endoreg_db/views/misc/translation.py +0 -182
  449. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
  450. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
@@ -7,14 +7,17 @@ from typing import TYPE_CHECKING, Optional, Type
7
7
 
8
8
  # Import the new exceptions from the correct path
9
9
  from endoreg_db.exceptions import InsufficientStorageError, TranscodingError
10
- from endoreg_db.utils.paths import IMPORT_VIDEO_DIR, SENSITIVE_VIDEO_DIR, TRANSCODING_DIR
10
+ from endoreg_db.utils.paths import (
11
+ IMPORT_VIDEO_DIR,
12
+ SENSITIVE_VIDEO_DIR,
13
+ TRANSCODING_DIR,
14
+ )
11
15
 
12
16
  if TYPE_CHECKING:
13
17
  from endoreg_db.models import VideoFile
14
18
 
15
19
  import endoreg_db.utils.paths as path_utils
16
- from ....utils.file_operations import get_uuid_filename
17
- from ....utils.hashs import get_video_hash
20
+
18
21
  from ....utils.video.ffmpeg_wrapper import transcode_videofile_if_required
19
22
 
20
23
  logger = logging.getLogger(__name__)
@@ -57,7 +60,9 @@ def check_storage_capacity(
57
60
  # Don't fail the operation, just log the warning
58
61
 
59
62
 
60
- def atomic_copy_with_fallback(src_path: Path = IMPORT_VIDEO_DIR, dst_path: Path = SENSITIVE_VIDEO_DIR) -> bool:
63
+ def atomic_copy_with_fallback(
64
+ src_path: Path = IMPORT_VIDEO_DIR, dst_path: Path = SENSITIVE_VIDEO_DIR
65
+ ) -> bool:
61
66
  """
62
67
  Atomically copy file from src to dst, preserving the source file.
63
68
 
@@ -192,7 +197,8 @@ def _create_from_file(
192
197
  cls_model: Type["VideoFile"],
193
198
  file_path: Path,
194
199
  center_name: str,
195
- processor_name: Optional[str] = None,
200
+ processor_name: Optional[str],
201
+ video_hash: str,
196
202
  video_dir: Path = IMPORT_VIDEO_DIR,
197
203
  save: bool = True,
198
204
  delete_source: bool = False,
@@ -218,7 +224,7 @@ def _create_from_file(
218
224
  try:
219
225
  # Ensure we operate under the canonical video path root
220
226
  data_paths = _get_data_paths()
221
- resolved_video_dir = _get_path(data_paths, "video", video_dir)
227
+ resolved_video_dir = _get_path(data_paths, "sensitive_video", video_dir)
222
228
  video_dir = Path(resolved_video_dir)
223
229
  storage_root_default = Path(video_dir).parent
224
230
  resolved_storage_root = _get_path(data_paths, "storage", storage_root_default)
@@ -251,23 +257,13 @@ def _create_from_file(
251
257
 
252
258
  logger.debug("Using file for hashing: %s", transcoded_file_path)
253
259
 
254
- # 2. Calculate hash (this will be the raw_video_hash)
255
- video_hash = get_video_hash(transcoded_file_path)
256
- if not video_hash:
257
- raise ValueError(
258
- f"Could not calculate video hash for {transcoded_file_path}"
259
- )
260
- logger.info(
261
- "Calculated raw video hash: %s for %s", video_hash, original_file_name
262
- )
263
-
264
260
  # 3. Check if hash already exists
265
261
  if cls_model.check_hash_exists(video_hash=video_hash):
266
262
  existing_video = cls_model.objects.get(video_hash=video_hash)
267
263
  logger.warning(
268
264
  "Video with hash %s already exists (UUID: %s)",
269
265
  video_hash,
270
- existing_video.uuid,
266
+ existing_video.video_hash,
271
267
  )
272
268
 
273
269
  # Check if the existing video has a valid file
@@ -292,9 +288,7 @@ def _create_from_file(
292
288
  )
293
289
  existing_video.delete()
294
290
 
295
- # 4. Generate UUID and final storage path
296
- new_file_name, uuid_val = get_uuid_filename(transcoded_file_path)
297
- final_storage_path = video_dir / new_file_name
291
+ final_storage_path = video_dir / video_hash / transcoded_file_path.suffix
298
292
  final_storage_path.parent.mkdir(parents=True, exist_ok=True)
299
293
 
300
294
  # 5. Move or Copy the file to final storage using improved method
@@ -327,19 +321,6 @@ def _create_from_file(
327
321
  except Exception as e:
328
322
  raise RuntimeError(f"Failed to move file to final storage: {e}") from e
329
323
 
330
- # 6. Verify hash after move/copy
331
- final_hash = get_video_hash(final_storage_path)
332
- if final_hash != video_hash:
333
- logger.error(
334
- "Hash mismatch after file operation! Expected %s, got %s",
335
- video_hash,
336
- final_hash,
337
- )
338
- final_storage_path.unlink(missing_ok=True)
339
- raise RuntimeError(
340
- f"Hash mismatch after file operation for {final_storage_path}"
341
- )
342
-
343
324
  # 7. Get related objects
344
325
  try:
345
326
  center = Center.objects.get(name=center_name)
@@ -365,14 +346,11 @@ def _create_from_file(
365
346
  raise ValueError(f"Processor '{processor_name}' not found.") from e
366
347
 
367
348
  # 8. Create the VideoFile instance
368
- logger.info("Creating new VideoFile instance with UUID: %s", uuid_val)
349
+ logger.info("Creating new VideoFile instance with hash: %s", video_hash)
369
350
  # Store FileField path relative to storage root including the videos prefix
370
- # TODO Review removal, since this is now newly handled by path_utils
371
- #storage_base = Path(_get_path(data_paths, "storage", final_storage_path.parent))
372
-
373
- relative_name = path_utils.to_storage_relative(final_storage_path)
351
+
352
+ relative_name = path_utils.to_storage_relative(final_storage_path)
374
353
  video = cls_model(
375
- uuid=uuid_val,
376
354
  raw_file=relative_name,
377
355
  processed_file=None,
378
356
  center=center,
@@ -385,10 +363,11 @@ def _create_from_file(
385
363
 
386
364
  # 9. Save the instance if requested
387
365
  if save:
388
- logger.info("Saving new VideoFile instance (UUID: %s)", uuid_val)
366
+ logger.info("Saving new VideoFile instance (Hash:%s", video_hash)
389
367
  video.save()
390
368
  logger.info(
391
- "Successfully created VideoFile PK %s (UUID: %s)", video.pk, video.uuid
369
+ "Successfully created VideoFile PK %s ",
370
+ video.pk,
392
371
  )
393
372
 
394
373
  return video
@@ -40,18 +40,24 @@ def _pipe_1(
40
40
  video_file.refresh_from_db()
41
41
  video_file.update_video_meta()
42
42
 
43
- logger.info(f"Starting Pipe 1 for video {video_file.uuid}")
43
+ logger.info(f"Starting Pipe 1 for video {video_file.video_hash}")
44
44
  try:
45
45
  # 1. Heavy I/O operations outside the transaction block
46
46
  logger.info("Pipe 1: Extracting frames...")
47
- video_file.extract_frames(overwrite=False) # Avoid overwriting if already extracted
47
+ video_file.extract_frames(
48
+ overwrite=False
49
+ ) # Avoid overwriting if already extracted
48
50
 
49
51
  logger.info("Pipe 1: Extracting text metadata...")
50
- video_file.update_text_metadata(ocr_frame_fraction=ocr_frame_fraction, cap=ocr_cap, overwrite=False)
52
+ video_file.update_text_metadata(
53
+ ocr_frame_fraction=ocr_frame_fraction, cap=ocr_cap, overwrite=False
54
+ )
51
55
  with transaction.atomic():
52
56
  state = video_file.get_or_create_state()
53
57
  if not state.frames_extracted:
54
- logger.error("Pipe 1 failed: Frame extraction did not complete successfully.")
58
+ logger.error(
59
+ "Pipe 1 failed: Frame extraction did not complete successfully."
60
+ )
55
61
  return False
56
62
 
57
63
  # 3. Perform Initial Prediction
@@ -59,7 +65,9 @@ def _pipe_1(
59
65
  try:
60
66
  ai_model_obj = AiModel.objects.get(name=model_name)
61
67
  if model_meta_version is not None:
62
- model_meta = ai_model_obj.metadata_versions.get(version=model_meta_version)
68
+ model_meta = ai_model_obj.metadata_versions.get(
69
+ version=model_meta_version
70
+ )
63
71
  else:
64
72
  model_meta = ai_model_obj.get_latest_version()
65
73
  except AiModel.DoesNotExist:
@@ -68,7 +76,9 @@ def _pipe_1(
68
76
  model_name = download_segmentation_model()
69
77
  ai_model_obj = AiModel.objects.get(name=model_name)
70
78
  if model_meta_version is not None:
71
- model_meta = ai_model_obj.metadata_versions.get(version=model_meta_version)
79
+ model_meta = ai_model_obj.metadata_versions.get(
80
+ version=model_meta_version
81
+ )
72
82
  else:
73
83
  model_meta = ai_model_obj.get_latest_version()
74
84
  except AiModel.DoesNotExist:
@@ -80,19 +90,25 @@ def _pipe_1(
80
90
  model_name = download_segmentation_model()
81
91
  ai_model_obj = AiModel.objects.get(name=model_name)
82
92
  if model_meta_version is not None:
83
- model_meta = ai_model_obj.metadata_versions.get(version=model_meta_version)
93
+ model_meta = ai_model_obj.metadata_versions.get(
94
+ version=model_meta_version
95
+ )
84
96
  else:
85
97
  model_meta = ai_model_obj.get_latest_version()
86
98
  except ModelMeta.DoesNotExist:
87
- logger.error(f"Pipe 1 failed: ModelMeta version {model_meta_version} for model '{model_name}' not found.")
99
+ logger.error(
100
+ f"Pipe 1 failed: ModelMeta version {model_meta_version} for model '{model_name}' not found."
101
+ )
88
102
  return False
89
103
  try:
90
- sequences: Optional[Dict[str, List[Tuple[int, int]]]] = video_file.predict_video(
91
- model_meta=model_meta,
92
- smooth_window_size_s=smooth_window_size_s,
93
- binarize_threshold=binarize_threshold,
94
- test_run=test_run,
95
- n_test_frames=n_test_frames,
104
+ sequences: Optional[Dict[str, List[Tuple[int, int]]]] = (
105
+ video_file.predict_video(
106
+ model_meta=model_meta,
107
+ smooth_window_size_s=smooth_window_size_s,
108
+ binarize_threshold=binarize_threshold,
109
+ test_run=test_run,
110
+ n_test_frames=n_test_frames,
111
+ )
96
112
  )
97
113
  except Exception as e:
98
114
  logger.error(f"Pipe 1 failed during prediction: {e}", exc_info=True)
@@ -110,13 +126,19 @@ def _pipe_1(
110
126
 
111
127
  logger.info(f"Pipe 1: Sequences returned from prediction: {sequences}")
112
128
  if not sequences:
113
- logger.warning("Pipe 1: Prediction returned empty sequences dictionary. No LabelVideoSegments will be created.")
129
+ logger.warning(
130
+ "Pipe 1: Prediction returned empty sequences dictionary. No LabelVideoSegments will be created."
131
+ )
114
132
 
115
133
  # 4. Create LabelVideoSegments
116
134
  logger.info("Pipe 1: Creating LabelVideoSegments from predictions...")
117
135
  try:
118
- video_prediction_meta = VideoPredictionMeta.objects.get(video_file=video_file, model_meta=model_meta)
119
- logger.info(f"Pipe 1: Calling _convert_sequences_to_db_segments for video {video_file.uuid} with prediction meta {video_prediction_meta.pk}")
136
+ video_prediction_meta = VideoPredictionMeta.objects.get(
137
+ video_file=video_file, model_meta=model_meta
138
+ )
139
+ logger.info(
140
+ f"Pipe 1: Calling _convert_sequences_to_db_segments for video {video_file.video_hash} with prediction meta {video_prediction_meta.pk}"
141
+ )
120
142
  _convert_sequences_to_db_segments(
121
143
  video=video_file,
122
144
  sequences=sequences,
@@ -128,18 +150,26 @@ def _pipe_1(
128
150
  state.save(update_fields=["lvs_created"])
129
151
  logger.info("Pipe 1: Set lvs_created state to True.")
130
152
  logger.info("Pipe 1: LabelVideoSegment creation complete.")
131
- lvs_count_after = LabelVideoSegment.objects.filter(video_file=video_file).count()
132
- logger.info(f"Pipe 1: Found {lvs_count_after} LabelVideoSegments after conversion attempt.")
153
+ lvs_count_after = LabelVideoSegment.objects.filter(
154
+ video_file=video_file
155
+ ).count()
156
+ logger.info(
157
+ f"Pipe 1: Found {lvs_count_after} LabelVideoSegments after conversion attempt."
158
+ )
133
159
  except VideoPredictionMeta.DoesNotExist:
134
- logger.error("Pipe 1 failed: Could not find VideoPredictionMeta after prediction.")
160
+ logger.error(
161
+ "Pipe 1 failed: Could not find VideoPredictionMeta after prediction."
162
+ )
135
163
  raise
136
164
 
137
- logger.info(f"Pipe 1 completed successfully for video {video_file.uuid}")
165
+ logger.info(f"Pipe 1 completed successfully for video {video_file.video_hash}")
138
166
  success = True # Set success flag
139
167
  return True
140
168
 
141
169
  except Exception as e:
142
- logger.error(f"Pipe 1 failed for video {video_file.uuid}: {e}", exc_info=True)
170
+ logger.error(
171
+ f"Pipe 1 failed for video {video_file.video_hash}: {e}", exc_info=True
172
+ )
143
173
  return False
144
174
  finally:
145
175
  # 5. Optionally delete frames
@@ -155,19 +185,23 @@ def _pipe_1(
155
185
 
156
186
 
157
187
  # --- Test after Pipe 1 ---
158
- def _test_after_pipe_1(video_file: "VideoFile", start_frame: int = 0, end_frame: int = 100) -> bool:
188
+ def _test_after_pipe_1(
189
+ video_file: "VideoFile", start_frame: int = 0, end_frame: int = 100
190
+ ) -> bool:
159
191
  """
160
192
  Simulates human annotation validation after Pipe 1.
161
193
  Creates 'outside' segments and marks sensitive meta as verified.
162
194
  """
163
195
  from ...label import Label, LabelVideoSegment
164
196
 
165
- logger.info(f"Starting _test_after_pipe_1 for video {video_file.uuid}")
197
+ logger.info(f"Starting _test_after_pipe_1 for video {video_file.video_hash}")
166
198
  try:
167
199
  # 1. Create 'outside' LabelVideoSegments
168
200
  try:
169
201
  outside_label = Label.objects.get(name__iexact="outside")
170
- logger.info(f"Creating 'outside' annotation segment [{start_frame}-{end_frame}]")
202
+ logger.info(
203
+ f"Creating 'outside' annotation segment [{start_frame}-{end_frame}]"
204
+ )
171
205
  # Create a segment - assuming custom_create handles saving
172
206
  outside_segment = LabelVideoSegment.objects.create( # Assign to variable
173
207
  video_file=video_file,
@@ -177,16 +211,22 @@ def _test_after_pipe_1(video_file: "VideoFile", start_frame: int = 0, end_frame:
177
211
  prediction_meta=None,
178
212
  )
179
213
  # Ensure the segment has a state and mark it as validated
180
- segment_state, created = outside_segment.get_or_create_state() # Unpack the tuple
214
+ segment_state, created = (
215
+ outside_segment.get_or_create_state()
216
+ ) # Unpack the tuple
181
217
  segment_state.is_validated = True
182
218
  segment_state.save()
183
- logger.info(f"Marked 'outside' segment {outside_segment.pk} as validated. Created: {created}")
219
+ logger.info(
220
+ f"Marked 'outside' segment {outside_segment.pk} as validated. Created: {created}"
221
+ )
184
222
 
185
223
  except Label.DoesNotExist:
186
224
  logger.error("_test_after_pipe_1 failed: 'outside' Label not found.")
187
225
  return False
188
226
  except Exception as e:
189
- logger.error(f"_test_after_pipe_1 failed during segment creation: {e}", exc_info=True)
227
+ logger.error(
228
+ f"_test_after_pipe_1 failed during segment creation: {e}", exc_info=True
229
+ )
190
230
  return False
191
231
 
192
232
  # 2. Set Sensitive Metadata state to verified
@@ -205,9 +245,14 @@ def _test_after_pipe_1(video_file: "VideoFile", start_frame: int = 0, end_frame:
205
245
  else:
206
246
  logger.warning("_test_after_pipe_1: No sensitive meta found to verify.")
207
247
 
208
- logger.info(f"_test_after_pipe_1 completed successfully for video {video_file.uuid}")
248
+ logger.info(
249
+ f"_test_after_pipe_1 completed successfully for video {video_file.video_hash}"
250
+ )
209
251
  return True
210
252
 
211
253
  except Exception as e:
212
- logger.error(f"_test_after_pipe_1 failed for video {video_file.uuid}: {e}", exc_info=True)
254
+ logger.error(
255
+ f"_test_after_pipe_1 failed for video {video_file.video_hash}: {e}",
256
+ exc_info=True,
257
+ )
213
258
  return False
@@ -22,7 +22,7 @@ def _pipe_2(video_file: "VideoFile") -> bool:
22
22
  Returns:
23
23
  bool: True if all operations complete successfully; otherwise, False.
24
24
  """
25
- logger.info("Starting Pipe 2 for video %s", video_file.uuid)
25
+ logger.info("Starting Pipe 2 for video %s", video_file.video_hash)
26
26
  try:
27
27
  # --- Part 1: Frame Extraction ---
28
28
  # Determine if frames are needed (short transaction for state read)
@@ -31,7 +31,9 @@ def _pipe_2(video_file: "VideoFile") -> bool:
31
31
  frames_needed = not state.frames_extracted
32
32
 
33
33
  if frames_needed:
34
- logger.info("Pipe 2: Frames not extracted. Extracting outside main DB transaction...")
34
+ logger.info(
35
+ "Pipe 2: Frames not extracted. Extracting outside main DB transaction..."
36
+ )
35
37
  if not video_file.extract_frames(overwrite=False): # Heavy I/O work
36
38
  logger.error("Pipe 2 failed: Frame extraction method returned False.")
37
39
  return False
@@ -40,7 +42,9 @@ def _pipe_2(video_file: "VideoFile") -> bool:
40
42
  with transaction.atomic():
41
43
  video_file.refresh_from_db()
42
44
  if not video_file.state or not video_file.state.frames_extracted:
43
- logger.error("Pipe 2 failed: Frame extraction did not update state successfully.")
45
+ logger.error(
46
+ "Pipe 2 failed: Frame extraction did not update state successfully."
47
+ )
44
48
  return False
45
49
  logger.info("Pipe 2: Frame extraction complete.")
46
50
  else:
@@ -55,17 +59,25 @@ def _pipe_2(video_file: "VideoFile") -> bool:
55
59
  state.sensitive_meta_processed = False
56
60
 
57
61
  if anonymization_needed:
58
- logger.info("Pipe 2: Video not anonymized. Anonymizing outside main DB transaction...")
59
- anonymize_success = video_file.anonymize(delete_original_raw=True) # Heavy I/O work
62
+ logger.info(
63
+ "Pipe 2: Video not anonymized. Anonymizing outside main DB transaction..."
64
+ )
65
+ anonymize_success = video_file.anonymize(
66
+ delete_original_raw=True
67
+ ) # Heavy I/O work
60
68
  if not anonymize_success:
61
- logger.error("Pipe 2 failed: Anonymization process failed (returned False).")
69
+ logger.error(
70
+ "Pipe 2 failed: Anonymization process failed (returned False)."
71
+ )
62
72
  return False
63
73
 
64
74
  # Verify anonymization and update state (short transaction)
65
75
  with transaction.atomic():
66
76
  video_file.refresh_from_db()
67
77
  if not video_file.state or not video_file.state.anonymized:
68
- logger.error("Pipe 2 Error: State.anonymized is False even after anonymize() call.")
78
+ logger.error(
79
+ "Pipe 2 Error: State.anonymized is False even after anonymize() call."
80
+ )
69
81
  return False
70
82
  logger.info("Pipe 2: Anonymization complete.")
71
83
  else:
@@ -86,19 +98,32 @@ def _pipe_2(video_file: "VideoFile") -> bool:
86
98
  sm_pk = video_file.sensitive_meta.pk
87
99
  video_file.sensitive_meta.delete()
88
100
  video_file.sensitive_meta = None # Important after SET_NULL
89
- video_file.save(update_fields=["sensitive_meta"]) # Persist the null relation
90
- logger.info("Pipe 2: Deleted sensitive meta object (PK: %s).", sm_pk)
101
+ video_file.save(
102
+ update_fields=["sensitive_meta"]
103
+ ) # Persist the null relation
104
+ logger.info(
105
+ "Pipe 2: Deleted sensitive meta object (PK: %s).", sm_pk
106
+ )
91
107
  except Exception as e:
92
- logger.error("Pipe 2: Failed to delete sensitive meta object: %s", e, exc_info=True)
108
+ logger.error(
109
+ "Pipe 2: Failed to delete sensitive meta object: %s",
110
+ e,
111
+ exc_info=True,
112
+ )
93
113
  raise # Reraise to ensure this transaction rolls back
94
114
  else:
95
115
  logger.info("Pipe 2: No sensitive meta object found to delete.")
96
116
 
97
- logger.info(f"Pipe 2 completed successfully for video {video_file.uuid}")
117
+ logger.info(
118
+ f"Pipe 2 completed successfully for video {video_file.video_hash}"
119
+ )
98
120
  return True
99
121
 
100
122
  except Exception as e:
101
123
  # This will catch exceptions from I/O operations if they raise,
102
124
  # or from the final transaction block, or any other unhandled error.
103
- logger.error(f"Pipe 2 failed for video {video_file.uuid} with unhandled exception: {e}", exc_info=True)
125
+ logger.error(
126
+ f"Pipe 2 failed for video {video_file.video_hash} with unhandled exception: {e}",
127
+ exc_info=True,
128
+ )
104
129
  return False
@@ -2,9 +2,8 @@
2
2
 
3
3
  import logging
4
4
  import os
5
- import uuid
6
5
  from pathlib import Path
7
- from typing import TYPE_CHECKING, Optional, Self, Union, cast
6
+ from typing import TYPE_CHECKING, Optional, Union, cast
8
7
 
9
8
  from django.core.files import File
10
9
  from django.core.validators import FileExtensionValidator
@@ -13,11 +12,11 @@ from django.db.models import F
13
12
  from django.db.models.fields.files import FieldFile
14
13
 
15
14
  from endoreg_db.utils.calc_duration_seconds import _calc_duration_vf
15
+ from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, SENSITIVE_VIDEO_DIR
16
16
  from endoreg_db.utils.video.ffmpeg_wrapper import assemble_video_from_frames
17
17
 
18
18
  from ...label import Label, LabelVideoSegment
19
19
  from ...state import VideoState
20
- from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, SENSITIVE_VIDEO_DIR
21
20
 
22
21
  # --- Import model-specific function modules ---
23
22
  from .create_from_file import _create_from_file
@@ -112,8 +111,6 @@ class VideoQuerySet(models.QuerySet):
112
111
 
113
112
 
114
113
  class VideoFile(models.Model):
115
- uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
116
-
117
114
  objects = VideoQuerySet.as_manager()
118
115
 
119
116
  raw_file = models.FileField(
@@ -447,7 +444,7 @@ class VideoFile(models.Model):
447
444
  except Exception as exc: # storage backends may raise when missing
448
445
  logger.warning(
449
446
  "Active file URL unavailable for video %s: %s",
450
- self.uuid,
447
+ self.video_hash,
451
448
  exc,
452
449
  )
453
450
  raise ValueError(
@@ -482,9 +479,10 @@ class VideoFile(models.Model):
482
479
  cls,
483
480
  file_path: Union[str, Path],
484
481
  center_name: str,
485
- processor_name: Optional[str] = None,
482
+ processor_name: Optional[str],
483
+ video_hash: str,
486
484
  delete_source: bool = False,
487
- save_video_file: bool = True, # Add this line
485
+ save_video_file: bool = True,
488
486
  ):
489
487
  """
490
488
  Creates a VideoFile instance from a given video file path.
@@ -501,6 +499,7 @@ class VideoFile(models.Model):
501
499
  file_path=file_path,
502
500
  center_name=center_name,
503
501
  processor_name=processor_name,
502
+ video_hash=video_hash,
504
503
  delete_source=delete_source,
505
504
  save=save_video_file, # Add this line
506
505
  )
@@ -520,10 +519,12 @@ class VideoFile(models.Model):
520
519
  # Call the original delete method to remove the instance from the database
521
520
  try:
522
521
  active_path = self.active_file_path
523
- logger.info(f"Deleting VideoFile: {self.uuid} - {active_path}")
522
+ logger.info(f"Deleting VideoFile: {self.video_hash} - {active_path}")
524
523
 
525
524
  except ValueError:
526
- logger.info(f"Deleting VideoFile: {self.uuid} - No active file path found.")
525
+ logger.info(
526
+ f"Deleting VideoFile: {self.video_hash} - No active file path found."
527
+ )
527
528
  active_path = None
528
529
 
529
530
  # Delete associated files if they exist
@@ -556,10 +557,10 @@ class VideoFile(models.Model):
556
557
  try:
557
558
  # Call parent delete with proper parameters
558
559
  result = super().delete(using=using, keep_parents=keep_parents)
559
- logger.info(f"VideoFile {self.uuid} deleted successfully.")
560
+ logger.info(f"VideoFile {self.video_hash} deleted successfully.")
560
561
  return result
561
562
  except Exception as e:
562
- logger.error(f"Error deleting VideoFile {self.uuid}: {e}")
563
+ logger.error(f"Error deleting VideoFile {self.video_hash}: {e}")
563
564
  raise
564
565
 
565
566
  def validate_metadata_annotation(
@@ -596,12 +597,12 @@ class VideoFile(models.Model):
596
597
  if self.raw_file:
597
598
  self.raw_file.delete(save=False)
598
599
  logger.info(
599
- f"Raw video deleted for {self.uuid}. Anonymized video preserved."
600
+ f"Raw video deleted for {self.video_hash}. Anonymized video preserved."
600
601
  )
601
602
  else:
602
603
  logger.warning(
603
604
  "Raw video file not found for deletion during validation %s.",
604
- self.uuid,
605
+ self.video_hash,
605
606
  )
606
607
 
607
608
  if self.sensitive_meta:
@@ -610,12 +611,12 @@ class VideoFile(models.Model):
610
611
  # Save the VideoFile instance to persist changes
611
612
  self.save()
612
613
  logger.info(
613
- f"Metadata annotation validated and saved for video {self.uuid}."
614
+ f"Metadata annotation validated and saved for video {self.video_hash}."
614
615
  )
615
616
  return True
616
617
  else:
617
618
  logger.error(
618
- f"Failed to validate metadata annotation for video {self.uuid}."
619
+ f"Failed to validate metadata annotation for video {self.video_hash}."
619
620
  )
620
621
  return False
621
622
 
@@ -652,7 +653,7 @@ class VideoFile(models.Model):
652
653
  state = (
653
654
  "Processed" if self.is_processed else ("Raw" if self.has_raw else "No File")
654
655
  )
655
- return f"VideoFile ({state}): {file_name} (UUID: {self.uuid})"
656
+ return f"VideoFile ({state}): {file_name} (UUID: {self.video_hash})"
656
657
 
657
658
  # --- Convenience state/meta helpers used in tests and admin workflows ---
658
659
  def mark_sensitive_meta_processed(self, *, save: bool = True) -> "VideoFile":
@@ -746,7 +747,7 @@ class VideoFile(models.Model):
746
747
  except Exception as e:
747
748
  logger.error(
748
749
  "Error getting outside segments for video %s: %s",
749
- self.uuid,
750
+ self.video_hash,
750
751
  e,
751
752
  exc_info=True,
752
753
  )
@@ -769,7 +770,8 @@ class VideoFile(models.Model):
769
770
 
770
771
  if not video:
771
772
  logger.warning(
772
- "No processed video file available for VideoFile %s.", cls.uuid
773
+ "No processed video file available for VideoFile %s.",
774
+ instance.video_hash,
773
775
  )
774
776
  return False
775
777
  try:
@@ -805,8 +807,10 @@ class VideoFile(models.Model):
805
807
  # assert isinstance(frames, list[Path]) #TODO improve TypeCheck
806
808
 
807
809
  # Step 2: Reassemble the video with frames excluding the 'outside' labeled frames
808
- output_video_path = Path(f"/path/to/output/{cls.uuid}_filtered.mp4")
809
- fps = cls.fps if cls.fps else 30.0 # Default to 30 FPS if fps is not set
810
+ output_video_path = Path(f"/path/to/output/{video.video_hash}_filtered.mp4")
811
+ fps = (
812
+ video.fps if video.fps else 30.0
813
+ ) # Default to 30 FPS if fps is not set
810
814
  new_video_file = assemble_video_from_frames(
811
815
  frames, output_video_path, fps, width=video.width, height=video.height
812
816
  )
@@ -814,7 +818,7 @@ class VideoFile(models.Model):
814
818
  return True
815
819
  except AssertionError as ae:
816
820
  logger.error(
817
- f"Assertion error while creating video without 'outside' frames for VideoFile {cls.uuid}: {ae}",
821
+ f"Assertion error while creating video without 'outside' frames for VideoFile {video.video_hash}: {ae}",
818
822
  exc_info=True,
819
823
  )
820
824
  return False
@@ -823,7 +827,7 @@ class VideoFile(models.Model):
823
827
  return False
824
828
  except Exception as e:
825
829
  logger.error(
826
- f"Error creating video without 'outside' frames for VideoFile {cls.uuid}: {e}",
830
+ f"Error creating video without 'outside' frames for VideoFile {video.video_hash}: {e}",
827
831
  exc_info=True,
828
832
  )
829
833
  return False
@@ -884,4 +888,12 @@ class VideoFile(models.Model):
884
888
  Raises:
885
889
  VideoFile.DoesNotExist: If no VideoFile with the given ID exists.
886
890
  """
887
- return VideoFile.objects.get(pk=video_id)
891
+ return VideoFile.objects.get(pk=pk)
892
+
893
+ @staticmethod
894
+ def get_video_by_content_hash(hash: str) -> "VideoFile":
895
+ try:
896
+ return VideoFile.objects.get(video_hash=hash)
897
+ except Exception as e:
898
+ logger.error(f"Video cant be returned for known hash. {e}")
899
+ raise