endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (450) hide show
  1. endoreg_db/admin.py +10 -5
  2. endoreg_db/apps.py +4 -7
  3. endoreg_db/authz/auth.py +1 -0
  4. endoreg_db/authz/backends.py +1 -1
  5. endoreg_db/authz/management/commands/list_routes.py +2 -0
  6. endoreg_db/authz/middleware.py +8 -7
  7. endoreg_db/authz/permissions.py +21 -10
  8. endoreg_db/authz/policy.py +14 -19
  9. endoreg_db/authz/views_auth.py +14 -10
  10. endoreg_db/codemods/rename_datetime_fields.py +8 -1
  11. endoreg_db/exceptions.py +5 -2
  12. endoreg_db/forms/__init__.py +0 -1
  13. endoreg_db/forms/examination_form.py +4 -3
  14. endoreg_db/forms/patient_finding_intervention_form.py +30 -8
  15. endoreg_db/forms/patient_form.py +9 -13
  16. endoreg_db/forms/questionnaires/__init__.py +1 -1
  17. endoreg_db/forms/settings/__init__.py +4 -1
  18. endoreg_db/forms/unit.py +2 -1
  19. endoreg_db/helpers/count_db.py +17 -14
  20. endoreg_db/helpers/default_objects.py +2 -1
  21. endoreg_db/helpers/download_segmentation_model.py +4 -3
  22. endoreg_db/helpers/interact.py +0 -5
  23. endoreg_db/helpers/test_video_helper.py +33 -25
  24. endoreg_db/import_files/__init__.py +1 -1
  25. endoreg_db/import_files/context/__init__.py +1 -1
  26. endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
  27. endoreg_db/import_files/context/ensure_center.py +4 -4
  28. endoreg_db/import_files/context/file_lock.py +3 -3
  29. endoreg_db/import_files/context/import_context.py +11 -12
  30. endoreg_db/import_files/context/validate_directories.py +1 -0
  31. endoreg_db/import_files/file_storage/create_report_file.py +57 -34
  32. endoreg_db/import_files/file_storage/create_video_file.py +64 -35
  33. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
  34. endoreg_db/import_files/file_storage/state_management.py +89 -122
  35. endoreg_db/import_files/file_storage/storage.py +5 -1
  36. endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
  37. endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
  38. endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
  39. endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
  40. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
  41. endoreg_db/import_files/report_import_service.py +36 -30
  42. endoreg_db/import_files/video_import_service.py +27 -23
  43. endoreg_db/logger_conf.py +56 -40
  44. endoreg_db/management/__init__.py +1 -1
  45. endoreg_db/management/commands/__init__.py +1 -1
  46. endoreg_db/management/commands/check_auth.py +45 -38
  47. endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
  48. endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
  49. endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
  50. endoreg_db/management/commands/fix_video_paths.py +75 -54
  51. endoreg_db/management/commands/import_report.py +1 -3
  52. endoreg_db/management/commands/list_routes.py +2 -0
  53. endoreg_db/management/commands/load_ai_model_data.py +8 -2
  54. endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
  55. endoreg_db/management/commands/load_center_data.py +3 -3
  56. endoreg_db/management/commands/load_distribution_data.py +35 -38
  57. endoreg_db/management/commands/load_endoscope_data.py +0 -3
  58. endoreg_db/management/commands/load_examination_data.py +20 -4
  59. endoreg_db/management/commands/load_finding_data.py +18 -3
  60. endoreg_db/management/commands/load_gender_data.py +17 -24
  61. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
  62. endoreg_db/management/commands/load_information_source.py +0 -3
  63. endoreg_db/management/commands/load_lab_value_data.py +14 -3
  64. endoreg_db/management/commands/load_legacy_data.py +303 -0
  65. endoreg_db/management/commands/load_name_data.py +1 -2
  66. endoreg_db/management/commands/load_pdf_type_data.py +4 -8
  67. endoreg_db/management/commands/load_profession_data.py +0 -1
  68. endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
  69. endoreg_db/management/commands/load_requirement_data.py +6 -2
  70. endoreg_db/management/commands/load_unit_data.py +0 -4
  71. endoreg_db/management/commands/load_user_groups.py +5 -7
  72. endoreg_db/management/commands/model_input.py +169 -0
  73. endoreg_db/management/commands/register_ai_model.py +22 -16
  74. endoreg_db/management/commands/setup_endoreg_db.py +110 -32
  75. endoreg_db/management/commands/storage_management.py +14 -8
  76. endoreg_db/management/commands/summarize_db_content.py +154 -63
  77. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  78. endoreg_db/management/commands/validate_video_files.py +82 -50
  79. endoreg_db/management/commands/video_validation.py +4 -6
  80. endoreg_db/migrations/0001_initial.py +112 -63
  81. endoreg_db/models/__init__.py +8 -0
  82. endoreg_db/models/administration/ai/active_model.py +5 -5
  83. endoreg_db/models/administration/ai/ai_model.py +41 -18
  84. endoreg_db/models/administration/ai/model_type.py +1 -0
  85. endoreg_db/models/administration/case/case.py +22 -22
  86. endoreg_db/models/administration/center/__init__.py +5 -5
  87. endoreg_db/models/administration/center/center.py +6 -2
  88. endoreg_db/models/administration/center/center_resource.py +18 -4
  89. endoreg_db/models/administration/center/center_shift.py +3 -1
  90. endoreg_db/models/administration/center/center_waste.py +6 -2
  91. endoreg_db/models/administration/person/__init__.py +1 -1
  92. endoreg_db/models/administration/person/employee/__init__.py +1 -1
  93. endoreg_db/models/administration/person/employee/employee_type.py +3 -1
  94. endoreg_db/models/administration/person/examiner/__init__.py +1 -1
  95. endoreg_db/models/administration/person/examiner/examiner.py +10 -2
  96. endoreg_db/models/administration/person/names/first_name.py +6 -4
  97. endoreg_db/models/administration/person/names/last_name.py +4 -3
  98. endoreg_db/models/administration/person/patient/__init__.py +1 -1
  99. endoreg_db/models/administration/person/patient/patient.py +0 -1
  100. endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
  101. endoreg_db/models/administration/person/person.py +1 -1
  102. endoreg_db/models/administration/product/__init__.py +7 -6
  103. endoreg_db/models/administration/product/product.py +6 -2
  104. endoreg_db/models/administration/product/product_group.py +9 -7
  105. endoreg_db/models/administration/product/product_material.py +9 -2
  106. endoreg_db/models/administration/product/reference_product.py +64 -15
  107. endoreg_db/models/administration/qualification/qualification.py +3 -1
  108. endoreg_db/models/administration/shift/shift.py +3 -1
  109. endoreg_db/models/administration/shift/shift_type.py +12 -4
  110. endoreg_db/models/aidataset/__init__.py +5 -0
  111. endoreg_db/models/aidataset/aidataset.py +193 -0
  112. endoreg_db/models/label/__init__.py +1 -1
  113. endoreg_db/models/label/label.py +10 -2
  114. endoreg_db/models/label/label_set.py +3 -1
  115. endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
  116. endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
  117. endoreg_db/models/media/__init__.py +12 -5
  118. endoreg_db/models/media/frame/__init__.py +1 -1
  119. endoreg_db/models/media/frame/frame.py +34 -8
  120. endoreg_db/models/media/pdf/__init__.py +2 -1
  121. endoreg_db/models/media/pdf/raw_pdf.py +11 -4
  122. endoreg_db/models/media/pdf/report_file.py +6 -2
  123. endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
  124. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
  125. endoreg_db/models/media/video/create_from_file.py +20 -41
  126. endoreg_db/models/media/video/pipe_1.py +75 -30
  127. endoreg_db/models/media/video/pipe_2.py +37 -12
  128. endoreg_db/models/media/video/video_file.py +36 -24
  129. endoreg_db/models/media/video/video_file_ai.py +235 -70
  130. endoreg_db/models/media/video/video_file_anonymize.py +240 -65
  131. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
  132. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
  133. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
  134. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
  135. endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
  136. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
  137. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
  138. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
  139. endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
  140. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
  141. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
  142. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
  143. endoreg_db/models/media/video/video_file_io.py +85 -33
  144. endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
  145. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
  146. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
  147. endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
  148. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
  149. endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
  150. endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
  151. endoreg_db/models/media/video/video_file_segments.py +118 -27
  152. endoreg_db/models/media/video/video_metadata.py +25 -6
  153. endoreg_db/models/media/video/video_processing.py +54 -15
  154. endoreg_db/models/medical/__init__.py +3 -13
  155. endoreg_db/models/medical/contraindication/__init__.py +3 -1
  156. endoreg_db/models/medical/disease.py +18 -6
  157. endoreg_db/models/medical/event.py +6 -2
  158. endoreg_db/models/medical/examination/__init__.py +5 -1
  159. endoreg_db/models/medical/examination/examination.py +22 -6
  160. endoreg_db/models/medical/examination/examination_indication.py +23 -7
  161. endoreg_db/models/medical/examination/examination_time.py +6 -2
  162. endoreg_db/models/medical/finding/__init__.py +3 -1
  163. endoreg_db/models/medical/finding/finding.py +37 -12
  164. endoreg_db/models/medical/finding/finding_classification.py +27 -8
  165. endoreg_db/models/medical/finding/finding_intervention.py +19 -6
  166. endoreg_db/models/medical/finding/finding_type.py +3 -1
  167. endoreg_db/models/medical/hardware/__init__.py +1 -1
  168. endoreg_db/models/medical/hardware/endoscope.py +14 -2
  169. endoreg_db/models/medical/laboratory/__init__.py +1 -1
  170. endoreg_db/models/medical/laboratory/lab_value.py +139 -39
  171. endoreg_db/models/medical/medication/__init__.py +7 -3
  172. endoreg_db/models/medical/medication/medication.py +3 -1
  173. endoreg_db/models/medical/medication/medication_indication.py +3 -1
  174. endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
  175. endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
  176. endoreg_db/models/medical/medication/medication_schedule.py +3 -1
  177. endoreg_db/models/medical/patient/__init__.py +2 -10
  178. endoreg_db/models/medical/patient/medication_examples.py +3 -14
  179. endoreg_db/models/medical/patient/patient_disease.py +17 -5
  180. endoreg_db/models/medical/patient/patient_event.py +12 -4
  181. endoreg_db/models/medical/patient/patient_examination.py +52 -15
  182. endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
  183. endoreg_db/models/medical/patient/patient_finding.py +105 -29
  184. endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
  185. endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
  186. endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
  187. endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
  188. endoreg_db/models/medical/patient/patient_medication.py +25 -7
  189. endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
  190. endoreg_db/models/metadata/model_meta.py +40 -12
  191. endoreg_db/models/metadata/model_meta_logic.py +51 -16
  192. endoreg_db/models/metadata/sensitive_meta.py +65 -28
  193. endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
  194. endoreg_db/models/metadata/video_meta.py +146 -39
  195. endoreg_db/models/metadata/video_prediction_logic.py +70 -21
  196. endoreg_db/models/metadata/video_prediction_meta.py +80 -27
  197. endoreg_db/models/operation_log.py +63 -0
  198. endoreg_db/models/other/__init__.py +10 -10
  199. endoreg_db/models/other/distribution/__init__.py +9 -7
  200. endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
  201. endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
  202. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
  203. endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
  204. endoreg_db/models/other/emission/__init__.py +1 -1
  205. endoreg_db/models/other/emission/emission_factor.py +9 -3
  206. endoreg_db/models/other/information_source.py +15 -5
  207. endoreg_db/models/other/material.py +3 -1
  208. endoreg_db/models/other/transport_route.py +3 -1
  209. endoreg_db/models/other/unit.py +6 -2
  210. endoreg_db/models/report/report.py +0 -1
  211. endoreg_db/models/requirement/requirement.py +84 -27
  212. endoreg_db/models/requirement/requirement_error.py +5 -6
  213. endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
  214. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
  215. endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
  216. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
  217. endoreg_db/models/requirement/requirement_operator.py +28 -8
  218. endoreg_db/models/requirement/requirement_set.py +34 -11
  219. endoreg_db/models/state/__init__.py +1 -0
  220. endoreg_db/models/state/audit_ledger.py +9 -2
  221. endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
  222. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  223. endoreg_db/models/state/raw_pdf.py +0 -1
  224. endoreg_db/models/state/video.py +2 -4
  225. endoreg_db/models/utils.py +4 -2
  226. endoreg_db/queries/__init__.py +2 -6
  227. endoreg_db/queries/annotations/__init__.py +1 -3
  228. endoreg_db/queries/annotations/legacy.py +37 -26
  229. endoreg_db/root_urls.py +3 -4
  230. endoreg_db/schemas/examination_evaluation.py +3 -0
  231. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
  232. endoreg_db/serializers/__init__.py +2 -8
  233. endoreg_db/serializers/administration/__init__.py +1 -2
  234. endoreg_db/serializers/administration/ai/__init__.py +0 -1
  235. endoreg_db/serializers/administration/ai/active_model.py +3 -1
  236. endoreg_db/serializers/administration/ai/ai_model.py +5 -3
  237. endoreg_db/serializers/administration/ai/model_type.py +3 -1
  238. endoreg_db/serializers/administration/center.py +7 -2
  239. endoreg_db/serializers/administration/gender.py +4 -2
  240. endoreg_db/serializers/anonymization.py +13 -13
  241. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
  242. endoreg_db/serializers/examination/__init__.py +1 -1
  243. endoreg_db/serializers/examination/base.py +12 -13
  244. endoreg_db/serializers/examination/dropdown.py +6 -7
  245. endoreg_db/serializers/examination_serializer.py +3 -6
  246. endoreg_db/serializers/finding/__init__.py +1 -1
  247. endoreg_db/serializers/finding/finding.py +14 -7
  248. endoreg_db/serializers/finding_classification/__init__.py +3 -3
  249. endoreg_db/serializers/finding_classification/choice.py +3 -3
  250. endoreg_db/serializers/finding_classification/classification.py +2 -4
  251. endoreg_db/serializers/label_video_segment/__init__.py +5 -3
  252. endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
  253. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  254. endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
  255. endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
  256. endoreg_db/serializers/meta/__init__.py +1 -2
  257. endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
  258. endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
  259. endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
  260. endoreg_db/serializers/misc/__init__.py +2 -2
  261. endoreg_db/serializers/misc/file_overview.py +11 -7
  262. endoreg_db/serializers/misc/stats.py +10 -8
  263. endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
  264. endoreg_db/serializers/misc/upload_job.py +32 -29
  265. endoreg_db/serializers/patient/__init__.py +2 -1
  266. endoreg_db/serializers/patient/patient.py +32 -15
  267. endoreg_db/serializers/patient/patient_dropdown.py +11 -3
  268. endoreg_db/serializers/patient_examination/__init__.py +1 -1
  269. endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
  270. endoreg_db/serializers/patient_finding/__init__.py +1 -1
  271. endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
  272. endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
  273. endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
  274. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
  275. endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
  276. endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
  277. endoreg_db/serializers/pdf/__init__.py +1 -3
  278. endoreg_db/serializers/requirements/requirement_schema.py +1 -6
  279. endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
  280. endoreg_db/serializers/video/__init__.py +2 -2
  281. endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
  282. endoreg_db/serializers/video/video_file_brief.py +6 -2
  283. endoreg_db/serializers/video/video_file_detail.py +36 -23
  284. endoreg_db/serializers/video/video_file_list.py +4 -2
  285. endoreg_db/serializers/video/video_processing_history.py +54 -50
  286. endoreg_db/services/__init__.py +1 -1
  287. endoreg_db/services/anonymization.py +2 -2
  288. endoreg_db/services/examination_evaluation.py +40 -17
  289. endoreg_db/services/model_meta_from_hf.py +76 -0
  290. endoreg_db/services/polling_coordinator.py +101 -70
  291. endoreg_db/services/pseudonym_service.py +27 -22
  292. endoreg_db/services/report_import.py +6 -3
  293. endoreg_db/services/segment_sync.py +75 -59
  294. endoreg_db/services/video_import.py +6 -7
  295. endoreg_db/urls/__init__.py +2 -2
  296. endoreg_db/urls/ai.py +7 -25
  297. endoreg_db/urls/anonymization.py +61 -15
  298. endoreg_db/urls/auth.py +4 -4
  299. endoreg_db/urls/classification.py +4 -9
  300. endoreg_db/urls/examination.py +27 -18
  301. endoreg_db/urls/media.py +27 -34
  302. endoreg_db/urls/patient.py +11 -7
  303. endoreg_db/urls/requirements.py +3 -1
  304. endoreg_db/urls/root_urls.py +2 -3
  305. endoreg_db/urls/stats.py +24 -16
  306. endoreg_db/urls/upload.py +3 -11
  307. endoreg_db/utils/__init__.py +14 -15
  308. endoreg_db/utils/ai/__init__.py +1 -1
  309. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  310. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  311. endoreg_db/utils/ai/get.py +2 -1
  312. endoreg_db/utils/ai/inference_dataset.py +14 -15
  313. endoreg_db/utils/ai/model_training/config.py +117 -0
  314. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  315. endoreg_db/utils/ai/model_training/losses.py +68 -0
  316. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  317. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  318. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  319. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  320. endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
  321. endoreg_db/utils/ai/predict.py +4 -4
  322. endoreg_db/utils/ai/preprocess.py +19 -11
  323. endoreg_db/utils/calc_duration_seconds.py +4 -4
  324. endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
  325. endoreg_db/utils/check_video_files.py +74 -47
  326. endoreg_db/utils/cropping.py +10 -9
  327. endoreg_db/utils/dataloader.py +11 -3
  328. endoreg_db/utils/dates.py +3 -4
  329. endoreg_db/utils/defaults/set_default_center.py +7 -6
  330. endoreg_db/utils/env.py +6 -2
  331. endoreg_db/utils/extract_specific_frames.py +24 -9
  332. endoreg_db/utils/file_operations.py +30 -18
  333. endoreg_db/utils/fix_video_path_direct.py +57 -41
  334. endoreg_db/utils/frame_anonymization_utils.py +157 -157
  335. endoreg_db/utils/hashs.py +3 -18
  336. endoreg_db/utils/links/requirement_link.py +96 -52
  337. endoreg_db/utils/ocr.py +30 -25
  338. endoreg_db/utils/operation_log.py +61 -0
  339. endoreg_db/utils/parse_and_generate_yaml.py +12 -13
  340. endoreg_db/utils/paths.py +6 -6
  341. endoreg_db/utils/permissions.py +40 -24
  342. endoreg_db/utils/pipelines/process_video_dir.py +50 -26
  343. endoreg_db/utils/product/sum_emissions.py +5 -3
  344. endoreg_db/utils/product/sum_weights.py +4 -2
  345. endoreg_db/utils/pydantic_models/__init__.py +3 -4
  346. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
  347. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
  348. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
  349. endoreg_db/utils/setup_config.py +21 -5
  350. endoreg_db/utils/storage.py +3 -1
  351. endoreg_db/utils/translation.py +19 -15
  352. endoreg_db/utils/uuid.py +1 -0
  353. endoreg_db/utils/validate_endo_roi.py +12 -4
  354. endoreg_db/utils/validate_subcategory_dict.py +26 -24
  355. endoreg_db/utils/validate_video_detailed.py +207 -149
  356. endoreg_db/utils/video/__init__.py +7 -3
  357. endoreg_db/utils/video/extract_frames.py +30 -18
  358. endoreg_db/utils/video/names.py +11 -6
  359. endoreg_db/utils/video/streaming_processor.py +175 -101
  360. endoreg_db/utils/video/video_splitter.py +30 -19
  361. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
  362. endoreg_db/views/__init__.py +0 -20
  363. endoreg_db/views/anonymization/__init__.py +6 -2
  364. endoreg_db/views/anonymization/media_management.py +2 -6
  365. endoreg_db/views/anonymization/overview.py +34 -1
  366. endoreg_db/views/anonymization/validate.py +79 -18
  367. endoreg_db/views/auth/__init__.py +1 -1
  368. endoreg_db/views/auth/keycloak.py +16 -14
  369. endoreg_db/views/examination/__init__.py +12 -15
  370. endoreg_db/views/examination/examination.py +5 -5
  371. endoreg_db/views/examination/examination_manifest_cache.py +5 -5
  372. endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
  373. endoreg_db/views/examination/get_finding_classifications.py +9 -7
  374. endoreg_db/views/examination/get_findings.py +8 -10
  375. endoreg_db/views/examination/get_instruments.py +3 -2
  376. endoreg_db/views/examination/get_interventions.py +1 -1
  377. endoreg_db/views/finding/__init__.py +2 -2
  378. endoreg_db/views/finding/finding.py +58 -54
  379. endoreg_db/views/finding/get_classifications.py +1 -1
  380. endoreg_db/views/finding/get_interventions.py +1 -1
  381. endoreg_db/views/finding_classification/__init__.py +5 -5
  382. endoreg_db/views/finding_classification/finding_classification.py +5 -6
  383. endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
  384. endoreg_db/views/media/__init__.py +13 -13
  385. endoreg_db/views/media/pdf_media.py +9 -9
  386. endoreg_db/views/media/sensitive_metadata.py +10 -7
  387. endoreg_db/views/media/video_media.py +4 -4
  388. endoreg_db/views/meta/__init__.py +1 -1
  389. endoreg_db/views/meta/sensitive_meta_list.py +20 -22
  390. endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
  391. endoreg_db/views/misc/__init__.py +6 -34
  392. endoreg_db/views/misc/center.py +2 -1
  393. endoreg_db/views/misc/csrf.py +2 -1
  394. endoreg_db/views/misc/gender.py +2 -1
  395. endoreg_db/views/misc/stats.py +141 -106
  396. endoreg_db/views/patient/__init__.py +1 -3
  397. endoreg_db/views/patient/patient.py +141 -99
  398. endoreg_db/views/patient_examination/__init__.py +5 -5
  399. endoreg_db/views/patient_examination/patient_examination.py +43 -42
  400. endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
  401. endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
  402. endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
  403. endoreg_db/views/patient_examination/video.py +114 -80
  404. endoreg_db/views/patient_finding/__init__.py +1 -1
  405. endoreg_db/views/patient_finding/patient_finding.py +17 -10
  406. endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
  407. endoreg_db/views/patient_finding_classification/__init__.py +1 -1
  408. endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
  409. endoreg_db/views/report/reimport.py +1 -1
  410. endoreg_db/views/report/report_stream.py +5 -8
  411. endoreg_db/views/requirement/__init__.py +2 -1
  412. endoreg_db/views/requirement/evaluate.py +7 -9
  413. endoreg_db/views/requirement/lookup.py +2 -3
  414. endoreg_db/views/requirement/lookup_store.py +0 -1
  415. endoreg_db/views/requirement/requirement_utils.py +2 -4
  416. endoreg_db/views/stats/__init__.py +4 -4
  417. endoreg_db/views/stats/stats_views.py +152 -115
  418. endoreg_db/views/video/__init__.py +18 -27
  419. endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
  420. endoreg_db/views/{ai → video/ai}/label.py +20 -16
  421. endoreg_db/views/video/correction.py +5 -6
  422. endoreg_db/views/video/reimport.py +134 -99
  423. endoreg_db/views/video/segments_crud.py +134 -44
  424. endoreg_db/views/video/video_apply_mask.py +13 -12
  425. endoreg_db/views/video/video_correction.py +2 -1
  426. endoreg_db/views/video/video_download_processed.py +15 -15
  427. endoreg_db/views/video/video_meta_stats.py +7 -6
  428. endoreg_db/views/video/video_processing_history.py +3 -2
  429. endoreg_db/views/video/video_remove_frames.py +13 -12
  430. endoreg_db/views/video/video_stream.py +110 -82
  431. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
  432. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
  433. endoreg_db/management/commands/import_fallback_video.py +0 -203
  434. endoreg_db/management/commands/import_video.py +0 -422
  435. endoreg_db/management/commands/import_video_with_classification.py +0 -367
  436. endoreg_db/models/media/processing_history/processing_history.py +0 -96
  437. endoreg_db/serializers/label/__init__.py +0 -7
  438. endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
  439. endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
  440. endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
  441. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
  442. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
  443. endoreg_db/services/__old/pdf_import.py +0 -1487
  444. endoreg_db/services/__old/video_import.py +0 -1306
  445. endoreg_db/tasks/upload_tasks.py +0 -216
  446. endoreg_db/tasks/video_ingest.py +0 -161
  447. endoreg_db/tasks/video_processing_tasks.py +0 -327
  448. endoreg_db/views/misc/translation.py +0 -182
  449. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
  450. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
@@ -1,34 +1,46 @@
1
- import shutil
2
- from pathlib import Path
3
- from endoreg_db.utils.uuid import get_uuid
1
+ import hashlib
4
2
  import os
3
+ from pathlib import Path
5
4
 
6
- def get_uuid_filename(file:Path) -> tuple[str, str]:
5
+
6
+ def get_content_hash_filename(file: Path) -> tuple[str, str]:
7
7
  """
8
- Returns a new filename with a uuid
8
+ Returns a new filename with a uuid - This is the content hash -
9
+ it is used to identify a raw video before processing when no other
10
+ reliable info exists.
11
+ It gets stored in processing_history model.
9
12
  """
10
13
  # Get the file extension
11
14
  file_extension = file.suffix
12
15
  # Generate a new file name
13
- uuid = get_uuid()
16
+ uuid = sha256_file(file)
14
17
  new_file_name = f"{uuid}{file_extension}"
15
18
  return new_file_name, uuid
16
-
17
- def rename_file_uuid(old_file:Path):
19
+
20
+
21
+ def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str:
18
22
  """
19
- Rename a file by assigning a uuid while preserving file extension. Returns new filepath and uuid
23
+ Compute a SHA-256 hash of the file contents in a streaming manner.
24
+
25
+ Args:
26
+ path: Path to the file on disk.
27
+ chunk_size: Size of the chunks to read (default: 1MB).
28
+
29
+ Returns:
30
+ Hexadecimal SHA-256 digest (64 characters).
20
31
  """
21
- # Get the file extension
22
- file_extension = old_file.suffix
23
- # Generate a new file name
24
- uuid = get_uuid()
25
- new_file_name = f"{uuid}{file_extension}"
32
+ h = hashlib.sha256()
33
+ path_obj = Path(path)
34
+
35
+ with path_obj.open("rb") as f:
36
+ while True:
37
+ chunk = f.read(chunk_size)
38
+ if not chunk:
39
+ break
40
+ h.update(chunk)
26
41
 
27
- # Rename the file
28
- new_file = old_file.with_name(new_file_name)
29
- shutil.move(old_file.as_posix(), new_file.as_posix())
42
+ return h.hexdigest()
30
43
 
31
- return new_file, uuid
32
44
 
33
45
  def copy_with_progress(src: str, dst: str, buffer_size=1024 * 1024):
34
46
  """
@@ -8,54 +8,57 @@ from pathlib import Path
8
8
  import os
9
9
  import argparse
10
10
 
11
+
11
12
  def fix_video_paths_direct(db_path, storage_dir):
12
13
  """Fix video file paths directly in SQLite database."""
13
-
14
+
14
15
  db_path = Path(db_path)
15
16
  storage_dir = Path(storage_dir)
16
-
17
+
17
18
  print("🔧 DIRECT VIDEO PATH FIX")
18
19
  print("=" * 40)
19
-
20
+
20
21
  if not db_path.exists():
21
22
  print(f"❌ Database not found at {db_path}")
22
23
  return
23
-
24
+
24
25
  # Find actual video files
25
26
  print("1. Scanning for video files...")
26
27
  actual_files = {}
27
- for pattern in ['**/*.mp4', '**/*.avi', '**/*.mov', '**/*.mkv']:
28
+ for pattern in ["**/*.mp4", "**/*.avi", "**/*.mov", "**/*.mkv"]:
28
29
  for file_path in storage_dir.glob(pattern):
29
30
  if file_path.is_file() and file_path.stat().st_size > 0:
30
31
  filename = file_path.name
31
- if '_' in filename:
32
- uuid_part = filename.split('_')[0]
32
+ if "_" in filename:
33
+ uuid_part = filename.split("_")[0]
33
34
  else:
34
- uuid_part = filename.split('.')[0]
35
-
35
+ uuid_part = filename.split(".")[0]
36
+
36
37
  relative_path = file_path.relative_to(storage_dir)
37
38
  actual_files[uuid_part] = {
38
- 'absolute_path': str(file_path),
39
- 'relative_path': str(relative_path),
40
- 'size_mb': file_path.stat().st_size / (1024*1024)
39
+ "absolute_path": str(file_path),
40
+ "relative_path": str(relative_path),
41
+ "size_mb": file_path.stat().st_size / (1024 * 1024),
41
42
  }
42
-
43
+
43
44
  print(f"Found {len(actual_files)} video files")
44
-
45
+
45
46
  # Connect to database and check video records
46
47
  print("\n2. Checking database records...")
47
48
  try:
48
49
  conn = sqlite3.connect(db_path)
49
50
  cursor = conn.cursor()
50
-
51
+
51
52
  # Find video file table (might be different names)
52
- cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%video%'")
53
+ cursor.execute(
54
+ "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%video%'"
55
+ )
53
56
  tables = cursor.fetchall()
54
57
  print(f"Found video tables: {[t[0] for t in tables]}")
55
-
58
+
56
59
  # Check for VideoFile records
57
60
  video_table = None
58
- for table in ['endoreg_db_videofile', 'videofile', 'video_file']:
61
+ for table in ["endoreg_db_videofile", "videofile", "video_file"]:
59
62
  try:
60
63
  cursor.execute(f"SELECT COUNT(*) FROM {table}")
61
64
  count = cursor.fetchone()[0]
@@ -65,24 +68,24 @@ def fix_video_paths_direct(db_path, storage_dir):
65
68
  break
66
69
  except sqlite3.OperationalError:
67
70
  continue
68
-
71
+
69
72
  if not video_table:
70
73
  print("❌ No video table found in database")
71
74
  return
72
-
75
+
73
76
  # Get video record 5
74
77
  cursor.execute(f"SELECT id, uuid, raw_file FROM {video_table} WHERE id = 5")
75
78
  record = cursor.fetchone()
76
-
79
+
77
80
  if not record:
78
81
  print("❌ Video ID 5 not found in database")
79
82
  return
80
-
83
+
81
84
  video_id, uuid_str, current_raw_file = record
82
- print(f"\n3. Video ID 5 details:")
85
+ print("\n3. Video ID 5 details:")
83
86
  print(f" UUID: {uuid_str}")
84
87
  print(f" Current raw_file: {current_raw_file}")
85
-
88
+
86
89
  # Check if current file exists
87
90
  if current_raw_file:
88
91
  current_full_path = storage_dir / current_raw_file
@@ -90,30 +93,32 @@ def fix_video_paths_direct(db_path, storage_dir):
90
93
  print(f" File exists: {file_exists}")
91
94
  if not file_exists:
92
95
  print(f" Full path checked: {current_full_path}")
93
-
96
+
94
97
  # Find matching file
95
98
  if uuid_str in actual_files:
96
99
  file_info = actual_files[uuid_str]
97
- print(f"\n4. Found matching file:")
100
+ print("\n4. Found matching file:")
98
101
  print(f" Path: {file_info['absolute_path']}")
99
102
  print(f" Size: {file_info['size_mb']:.1f} MB")
100
103
  print(f" Relative path: {file_info['relative_path']}")
101
-
102
- if current_raw_file != file_info['relative_path']:
103
- print(f"\n5. 🔧 FIXING PATH:")
104
+
105
+ if current_raw_file != file_info["relative_path"]:
106
+ print("\n5. 🔧 FIXING PATH:")
104
107
  print(f" FROM: {current_raw_file}")
105
108
  print(f" TO: {file_info['relative_path']}")
106
-
109
+
107
110
  # Update the database with transaction safety
108
111
  try:
109
- cursor.execute(f"UPDATE {video_table} SET raw_file = ? WHERE id = ?",
110
- (file_info['relative_path'], video_id))
112
+ cursor.execute(
113
+ f"UPDATE {video_table} SET raw_file = ? WHERE id = ?",
114
+ (file_info["relative_path"], video_id),
115
+ )
111
116
  conn.commit()
112
117
  print("✅ Database updated successfully!")
113
118
  except Exception as e:
114
119
  conn.rollback()
115
120
  print(f"❌ Failed to update database: {e}")
116
-
121
+
117
122
  print("\n6. 🎯 NEXT STEPS:")
118
123
  print(" 1. Restart your Django development server")
119
124
  print(" 2. Test video streaming in the frontend")
@@ -125,17 +130,28 @@ def fix_video_paths_direct(db_path, storage_dir):
125
130
  print("Available UUIDs in storage:")
126
131
  for uuid_key in list(actual_files.keys())[:5]:
127
132
  print(f" - {uuid_key}")
128
-
133
+
129
134
  conn.close()
130
-
135
+
131
136
  except Exception as e:
132
137
  print(f"❌ Database error: {e}")
133
138
 
139
+
134
140
  if __name__ == "__main__":
135
- parser = argparse.ArgumentParser(description="Direct fix for video file path issues in SQLite DB.")
136
- parser.add_argument('--db-path', type=str, default=os.environ.get('ENDOREG_DB_PATH', './db.sqlite3'),
137
- help='Path to the SQLite database file (default: ./db.sqlite3 or $ENDOREG_DB_PATH)')
138
- parser.add_argument('--storage-dir', type=str, default=os.environ.get('ENDOREG_STORAGE_DIR', './storage'),
139
- help='Path to the storage directory (default: ./storage or $ENDOREG_STORAGE_DIR)')
141
+ parser = argparse.ArgumentParser(
142
+ description="Direct fix for video file path issues in SQLite DB."
143
+ )
144
+ parser.add_argument(
145
+ "--db-path",
146
+ type=str,
147
+ default=os.environ.get("ENDOREG_DB_PATH", "./db.sqlite3"),
148
+ help="Path to the SQLite database file (default: ./db.sqlite3 or $ENDOREG_DB_PATH)",
149
+ )
150
+ parser.add_argument(
151
+ "--storage-dir",
152
+ type=str,
153
+ default=os.environ.get("ENDOREG_STORAGE_DIR", "./storage"),
154
+ help="Path to the storage directory (default: ./storage or $ENDOREG_STORAGE_DIR)",
155
+ )
140
156
  args = parser.parse_args()
141
- fix_video_paths_direct(args.db_path, args.storage_dir)
157
+ fix_video_paths_direct(args.db_path, args.storage_dir)