endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (450) hide show
  1. endoreg_db/admin.py +10 -5
  2. endoreg_db/apps.py +4 -7
  3. endoreg_db/authz/auth.py +1 -0
  4. endoreg_db/authz/backends.py +1 -1
  5. endoreg_db/authz/management/commands/list_routes.py +2 -0
  6. endoreg_db/authz/middleware.py +8 -7
  7. endoreg_db/authz/permissions.py +21 -10
  8. endoreg_db/authz/policy.py +14 -19
  9. endoreg_db/authz/views_auth.py +14 -10
  10. endoreg_db/codemods/rename_datetime_fields.py +8 -1
  11. endoreg_db/exceptions.py +5 -2
  12. endoreg_db/forms/__init__.py +0 -1
  13. endoreg_db/forms/examination_form.py +4 -3
  14. endoreg_db/forms/patient_finding_intervention_form.py +30 -8
  15. endoreg_db/forms/patient_form.py +9 -13
  16. endoreg_db/forms/questionnaires/__init__.py +1 -1
  17. endoreg_db/forms/settings/__init__.py +4 -1
  18. endoreg_db/forms/unit.py +2 -1
  19. endoreg_db/helpers/count_db.py +17 -14
  20. endoreg_db/helpers/default_objects.py +2 -1
  21. endoreg_db/helpers/download_segmentation_model.py +4 -3
  22. endoreg_db/helpers/interact.py +0 -5
  23. endoreg_db/helpers/test_video_helper.py +33 -25
  24. endoreg_db/import_files/__init__.py +1 -1
  25. endoreg_db/import_files/context/__init__.py +1 -1
  26. endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
  27. endoreg_db/import_files/context/ensure_center.py +4 -4
  28. endoreg_db/import_files/context/file_lock.py +3 -3
  29. endoreg_db/import_files/context/import_context.py +11 -12
  30. endoreg_db/import_files/context/validate_directories.py +1 -0
  31. endoreg_db/import_files/file_storage/create_report_file.py +57 -34
  32. endoreg_db/import_files/file_storage/create_video_file.py +64 -35
  33. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
  34. endoreg_db/import_files/file_storage/state_management.py +89 -122
  35. endoreg_db/import_files/file_storage/storage.py +5 -1
  36. endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
  37. endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
  38. endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
  39. endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
  40. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
  41. endoreg_db/import_files/report_import_service.py +36 -30
  42. endoreg_db/import_files/video_import_service.py +27 -23
  43. endoreg_db/logger_conf.py +56 -40
  44. endoreg_db/management/__init__.py +1 -1
  45. endoreg_db/management/commands/__init__.py +1 -1
  46. endoreg_db/management/commands/check_auth.py +45 -38
  47. endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
  48. endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
  49. endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
  50. endoreg_db/management/commands/fix_video_paths.py +75 -54
  51. endoreg_db/management/commands/import_report.py +1 -3
  52. endoreg_db/management/commands/list_routes.py +2 -0
  53. endoreg_db/management/commands/load_ai_model_data.py +8 -2
  54. endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
  55. endoreg_db/management/commands/load_center_data.py +3 -3
  56. endoreg_db/management/commands/load_distribution_data.py +35 -38
  57. endoreg_db/management/commands/load_endoscope_data.py +0 -3
  58. endoreg_db/management/commands/load_examination_data.py +20 -4
  59. endoreg_db/management/commands/load_finding_data.py +18 -3
  60. endoreg_db/management/commands/load_gender_data.py +17 -24
  61. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
  62. endoreg_db/management/commands/load_information_source.py +0 -3
  63. endoreg_db/management/commands/load_lab_value_data.py +14 -3
  64. endoreg_db/management/commands/load_legacy_data.py +303 -0
  65. endoreg_db/management/commands/load_name_data.py +1 -2
  66. endoreg_db/management/commands/load_pdf_type_data.py +4 -8
  67. endoreg_db/management/commands/load_profession_data.py +0 -1
  68. endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
  69. endoreg_db/management/commands/load_requirement_data.py +6 -2
  70. endoreg_db/management/commands/load_unit_data.py +0 -4
  71. endoreg_db/management/commands/load_user_groups.py +5 -7
  72. endoreg_db/management/commands/model_input.py +169 -0
  73. endoreg_db/management/commands/register_ai_model.py +22 -16
  74. endoreg_db/management/commands/setup_endoreg_db.py +110 -32
  75. endoreg_db/management/commands/storage_management.py +14 -8
  76. endoreg_db/management/commands/summarize_db_content.py +154 -63
  77. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  78. endoreg_db/management/commands/validate_video_files.py +82 -50
  79. endoreg_db/management/commands/video_validation.py +4 -6
  80. endoreg_db/migrations/0001_initial.py +112 -63
  81. endoreg_db/models/__init__.py +8 -0
  82. endoreg_db/models/administration/ai/active_model.py +5 -5
  83. endoreg_db/models/administration/ai/ai_model.py +41 -18
  84. endoreg_db/models/administration/ai/model_type.py +1 -0
  85. endoreg_db/models/administration/case/case.py +22 -22
  86. endoreg_db/models/administration/center/__init__.py +5 -5
  87. endoreg_db/models/administration/center/center.py +6 -2
  88. endoreg_db/models/administration/center/center_resource.py +18 -4
  89. endoreg_db/models/administration/center/center_shift.py +3 -1
  90. endoreg_db/models/administration/center/center_waste.py +6 -2
  91. endoreg_db/models/administration/person/__init__.py +1 -1
  92. endoreg_db/models/administration/person/employee/__init__.py +1 -1
  93. endoreg_db/models/administration/person/employee/employee_type.py +3 -1
  94. endoreg_db/models/administration/person/examiner/__init__.py +1 -1
  95. endoreg_db/models/administration/person/examiner/examiner.py +10 -2
  96. endoreg_db/models/administration/person/names/first_name.py +6 -4
  97. endoreg_db/models/administration/person/names/last_name.py +4 -3
  98. endoreg_db/models/administration/person/patient/__init__.py +1 -1
  99. endoreg_db/models/administration/person/patient/patient.py +0 -1
  100. endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
  101. endoreg_db/models/administration/person/person.py +1 -1
  102. endoreg_db/models/administration/product/__init__.py +7 -6
  103. endoreg_db/models/administration/product/product.py +6 -2
  104. endoreg_db/models/administration/product/product_group.py +9 -7
  105. endoreg_db/models/administration/product/product_material.py +9 -2
  106. endoreg_db/models/administration/product/reference_product.py +64 -15
  107. endoreg_db/models/administration/qualification/qualification.py +3 -1
  108. endoreg_db/models/administration/shift/shift.py +3 -1
  109. endoreg_db/models/administration/shift/shift_type.py +12 -4
  110. endoreg_db/models/aidataset/__init__.py +5 -0
  111. endoreg_db/models/aidataset/aidataset.py +193 -0
  112. endoreg_db/models/label/__init__.py +1 -1
  113. endoreg_db/models/label/label.py +10 -2
  114. endoreg_db/models/label/label_set.py +3 -1
  115. endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
  116. endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
  117. endoreg_db/models/media/__init__.py +12 -5
  118. endoreg_db/models/media/frame/__init__.py +1 -1
  119. endoreg_db/models/media/frame/frame.py +34 -8
  120. endoreg_db/models/media/pdf/__init__.py +2 -1
  121. endoreg_db/models/media/pdf/raw_pdf.py +11 -4
  122. endoreg_db/models/media/pdf/report_file.py +6 -2
  123. endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
  124. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
  125. endoreg_db/models/media/video/create_from_file.py +20 -41
  126. endoreg_db/models/media/video/pipe_1.py +75 -30
  127. endoreg_db/models/media/video/pipe_2.py +37 -12
  128. endoreg_db/models/media/video/video_file.py +36 -24
  129. endoreg_db/models/media/video/video_file_ai.py +235 -70
  130. endoreg_db/models/media/video/video_file_anonymize.py +240 -65
  131. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
  132. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
  133. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
  134. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
  135. endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
  136. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
  137. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
  138. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
  139. endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
  140. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
  141. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
  142. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
  143. endoreg_db/models/media/video/video_file_io.py +85 -33
  144. endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
  145. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
  146. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
  147. endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
  148. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
  149. endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
  150. endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
  151. endoreg_db/models/media/video/video_file_segments.py +118 -27
  152. endoreg_db/models/media/video/video_metadata.py +25 -6
  153. endoreg_db/models/media/video/video_processing.py +54 -15
  154. endoreg_db/models/medical/__init__.py +3 -13
  155. endoreg_db/models/medical/contraindication/__init__.py +3 -1
  156. endoreg_db/models/medical/disease.py +18 -6
  157. endoreg_db/models/medical/event.py +6 -2
  158. endoreg_db/models/medical/examination/__init__.py +5 -1
  159. endoreg_db/models/medical/examination/examination.py +22 -6
  160. endoreg_db/models/medical/examination/examination_indication.py +23 -7
  161. endoreg_db/models/medical/examination/examination_time.py +6 -2
  162. endoreg_db/models/medical/finding/__init__.py +3 -1
  163. endoreg_db/models/medical/finding/finding.py +37 -12
  164. endoreg_db/models/medical/finding/finding_classification.py +27 -8
  165. endoreg_db/models/medical/finding/finding_intervention.py +19 -6
  166. endoreg_db/models/medical/finding/finding_type.py +3 -1
  167. endoreg_db/models/medical/hardware/__init__.py +1 -1
  168. endoreg_db/models/medical/hardware/endoscope.py +14 -2
  169. endoreg_db/models/medical/laboratory/__init__.py +1 -1
  170. endoreg_db/models/medical/laboratory/lab_value.py +139 -39
  171. endoreg_db/models/medical/medication/__init__.py +7 -3
  172. endoreg_db/models/medical/medication/medication.py +3 -1
  173. endoreg_db/models/medical/medication/medication_indication.py +3 -1
  174. endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
  175. endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
  176. endoreg_db/models/medical/medication/medication_schedule.py +3 -1
  177. endoreg_db/models/medical/patient/__init__.py +2 -10
  178. endoreg_db/models/medical/patient/medication_examples.py +3 -14
  179. endoreg_db/models/medical/patient/patient_disease.py +17 -5
  180. endoreg_db/models/medical/patient/patient_event.py +12 -4
  181. endoreg_db/models/medical/patient/patient_examination.py +52 -15
  182. endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
  183. endoreg_db/models/medical/patient/patient_finding.py +105 -29
  184. endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
  185. endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
  186. endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
  187. endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
  188. endoreg_db/models/medical/patient/patient_medication.py +25 -7
  189. endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
  190. endoreg_db/models/metadata/model_meta.py +40 -12
  191. endoreg_db/models/metadata/model_meta_logic.py +51 -16
  192. endoreg_db/models/metadata/sensitive_meta.py +65 -28
  193. endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
  194. endoreg_db/models/metadata/video_meta.py +146 -39
  195. endoreg_db/models/metadata/video_prediction_logic.py +70 -21
  196. endoreg_db/models/metadata/video_prediction_meta.py +80 -27
  197. endoreg_db/models/operation_log.py +63 -0
  198. endoreg_db/models/other/__init__.py +10 -10
  199. endoreg_db/models/other/distribution/__init__.py +9 -7
  200. endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
  201. endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
  202. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
  203. endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
  204. endoreg_db/models/other/emission/__init__.py +1 -1
  205. endoreg_db/models/other/emission/emission_factor.py +9 -3
  206. endoreg_db/models/other/information_source.py +15 -5
  207. endoreg_db/models/other/material.py +3 -1
  208. endoreg_db/models/other/transport_route.py +3 -1
  209. endoreg_db/models/other/unit.py +6 -2
  210. endoreg_db/models/report/report.py +0 -1
  211. endoreg_db/models/requirement/requirement.py +84 -27
  212. endoreg_db/models/requirement/requirement_error.py +5 -6
  213. endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
  214. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
  215. endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
  216. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
  217. endoreg_db/models/requirement/requirement_operator.py +28 -8
  218. endoreg_db/models/requirement/requirement_set.py +34 -11
  219. endoreg_db/models/state/__init__.py +1 -0
  220. endoreg_db/models/state/audit_ledger.py +9 -2
  221. endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
  222. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  223. endoreg_db/models/state/raw_pdf.py +0 -1
  224. endoreg_db/models/state/video.py +2 -4
  225. endoreg_db/models/utils.py +4 -2
  226. endoreg_db/queries/__init__.py +2 -6
  227. endoreg_db/queries/annotations/__init__.py +1 -3
  228. endoreg_db/queries/annotations/legacy.py +37 -26
  229. endoreg_db/root_urls.py +3 -4
  230. endoreg_db/schemas/examination_evaluation.py +3 -0
  231. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
  232. endoreg_db/serializers/__init__.py +2 -8
  233. endoreg_db/serializers/administration/__init__.py +1 -2
  234. endoreg_db/serializers/administration/ai/__init__.py +0 -1
  235. endoreg_db/serializers/administration/ai/active_model.py +3 -1
  236. endoreg_db/serializers/administration/ai/ai_model.py +5 -3
  237. endoreg_db/serializers/administration/ai/model_type.py +3 -1
  238. endoreg_db/serializers/administration/center.py +7 -2
  239. endoreg_db/serializers/administration/gender.py +4 -2
  240. endoreg_db/serializers/anonymization.py +13 -13
  241. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
  242. endoreg_db/serializers/examination/__init__.py +1 -1
  243. endoreg_db/serializers/examination/base.py +12 -13
  244. endoreg_db/serializers/examination/dropdown.py +6 -7
  245. endoreg_db/serializers/examination_serializer.py +3 -6
  246. endoreg_db/serializers/finding/__init__.py +1 -1
  247. endoreg_db/serializers/finding/finding.py +14 -7
  248. endoreg_db/serializers/finding_classification/__init__.py +3 -3
  249. endoreg_db/serializers/finding_classification/choice.py +3 -3
  250. endoreg_db/serializers/finding_classification/classification.py +2 -4
  251. endoreg_db/serializers/label_video_segment/__init__.py +5 -3
  252. endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
  253. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  254. endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
  255. endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
  256. endoreg_db/serializers/meta/__init__.py +1 -2
  257. endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
  258. endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
  259. endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
  260. endoreg_db/serializers/misc/__init__.py +2 -2
  261. endoreg_db/serializers/misc/file_overview.py +11 -7
  262. endoreg_db/serializers/misc/stats.py +10 -8
  263. endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
  264. endoreg_db/serializers/misc/upload_job.py +32 -29
  265. endoreg_db/serializers/patient/__init__.py +2 -1
  266. endoreg_db/serializers/patient/patient.py +32 -15
  267. endoreg_db/serializers/patient/patient_dropdown.py +11 -3
  268. endoreg_db/serializers/patient_examination/__init__.py +1 -1
  269. endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
  270. endoreg_db/serializers/patient_finding/__init__.py +1 -1
  271. endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
  272. endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
  273. endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
  274. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
  275. endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
  276. endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
  277. endoreg_db/serializers/pdf/__init__.py +1 -3
  278. endoreg_db/serializers/requirements/requirement_schema.py +1 -6
  279. endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
  280. endoreg_db/serializers/video/__init__.py +2 -2
  281. endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
  282. endoreg_db/serializers/video/video_file_brief.py +6 -2
  283. endoreg_db/serializers/video/video_file_detail.py +36 -23
  284. endoreg_db/serializers/video/video_file_list.py +4 -2
  285. endoreg_db/serializers/video/video_processing_history.py +54 -50
  286. endoreg_db/services/__init__.py +1 -1
  287. endoreg_db/services/anonymization.py +2 -2
  288. endoreg_db/services/examination_evaluation.py +40 -17
  289. endoreg_db/services/model_meta_from_hf.py +76 -0
  290. endoreg_db/services/polling_coordinator.py +101 -70
  291. endoreg_db/services/pseudonym_service.py +27 -22
  292. endoreg_db/services/report_import.py +6 -3
  293. endoreg_db/services/segment_sync.py +75 -59
  294. endoreg_db/services/video_import.py +6 -7
  295. endoreg_db/urls/__init__.py +2 -2
  296. endoreg_db/urls/ai.py +7 -25
  297. endoreg_db/urls/anonymization.py +61 -15
  298. endoreg_db/urls/auth.py +4 -4
  299. endoreg_db/urls/classification.py +4 -9
  300. endoreg_db/urls/examination.py +27 -18
  301. endoreg_db/urls/media.py +27 -34
  302. endoreg_db/urls/patient.py +11 -7
  303. endoreg_db/urls/requirements.py +3 -1
  304. endoreg_db/urls/root_urls.py +2 -3
  305. endoreg_db/urls/stats.py +24 -16
  306. endoreg_db/urls/upload.py +3 -11
  307. endoreg_db/utils/__init__.py +14 -15
  308. endoreg_db/utils/ai/__init__.py +1 -1
  309. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  310. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  311. endoreg_db/utils/ai/get.py +2 -1
  312. endoreg_db/utils/ai/inference_dataset.py +14 -15
  313. endoreg_db/utils/ai/model_training/config.py +117 -0
  314. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  315. endoreg_db/utils/ai/model_training/losses.py +68 -0
  316. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  317. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  318. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  319. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  320. endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
  321. endoreg_db/utils/ai/predict.py +4 -4
  322. endoreg_db/utils/ai/preprocess.py +19 -11
  323. endoreg_db/utils/calc_duration_seconds.py +4 -4
  324. endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
  325. endoreg_db/utils/check_video_files.py +74 -47
  326. endoreg_db/utils/cropping.py +10 -9
  327. endoreg_db/utils/dataloader.py +11 -3
  328. endoreg_db/utils/dates.py +3 -4
  329. endoreg_db/utils/defaults/set_default_center.py +7 -6
  330. endoreg_db/utils/env.py +6 -2
  331. endoreg_db/utils/extract_specific_frames.py +24 -9
  332. endoreg_db/utils/file_operations.py +30 -18
  333. endoreg_db/utils/fix_video_path_direct.py +57 -41
  334. endoreg_db/utils/frame_anonymization_utils.py +157 -157
  335. endoreg_db/utils/hashs.py +3 -18
  336. endoreg_db/utils/links/requirement_link.py +96 -52
  337. endoreg_db/utils/ocr.py +30 -25
  338. endoreg_db/utils/operation_log.py +61 -0
  339. endoreg_db/utils/parse_and_generate_yaml.py +12 -13
  340. endoreg_db/utils/paths.py +6 -6
  341. endoreg_db/utils/permissions.py +40 -24
  342. endoreg_db/utils/pipelines/process_video_dir.py +50 -26
  343. endoreg_db/utils/product/sum_emissions.py +5 -3
  344. endoreg_db/utils/product/sum_weights.py +4 -2
  345. endoreg_db/utils/pydantic_models/__init__.py +3 -4
  346. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
  347. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
  348. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
  349. endoreg_db/utils/setup_config.py +21 -5
  350. endoreg_db/utils/storage.py +3 -1
  351. endoreg_db/utils/translation.py +19 -15
  352. endoreg_db/utils/uuid.py +1 -0
  353. endoreg_db/utils/validate_endo_roi.py +12 -4
  354. endoreg_db/utils/validate_subcategory_dict.py +26 -24
  355. endoreg_db/utils/validate_video_detailed.py +207 -149
  356. endoreg_db/utils/video/__init__.py +7 -3
  357. endoreg_db/utils/video/extract_frames.py +30 -18
  358. endoreg_db/utils/video/names.py +11 -6
  359. endoreg_db/utils/video/streaming_processor.py +175 -101
  360. endoreg_db/utils/video/video_splitter.py +30 -19
  361. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
  362. endoreg_db/views/__init__.py +0 -20
  363. endoreg_db/views/anonymization/__init__.py +6 -2
  364. endoreg_db/views/anonymization/media_management.py +2 -6
  365. endoreg_db/views/anonymization/overview.py +34 -1
  366. endoreg_db/views/anonymization/validate.py +79 -18
  367. endoreg_db/views/auth/__init__.py +1 -1
  368. endoreg_db/views/auth/keycloak.py +16 -14
  369. endoreg_db/views/examination/__init__.py +12 -15
  370. endoreg_db/views/examination/examination.py +5 -5
  371. endoreg_db/views/examination/examination_manifest_cache.py +5 -5
  372. endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
  373. endoreg_db/views/examination/get_finding_classifications.py +9 -7
  374. endoreg_db/views/examination/get_findings.py +8 -10
  375. endoreg_db/views/examination/get_instruments.py +3 -2
  376. endoreg_db/views/examination/get_interventions.py +1 -1
  377. endoreg_db/views/finding/__init__.py +2 -2
  378. endoreg_db/views/finding/finding.py +58 -54
  379. endoreg_db/views/finding/get_classifications.py +1 -1
  380. endoreg_db/views/finding/get_interventions.py +1 -1
  381. endoreg_db/views/finding_classification/__init__.py +5 -5
  382. endoreg_db/views/finding_classification/finding_classification.py +5 -6
  383. endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
  384. endoreg_db/views/media/__init__.py +13 -13
  385. endoreg_db/views/media/pdf_media.py +9 -9
  386. endoreg_db/views/media/sensitive_metadata.py +10 -7
  387. endoreg_db/views/media/video_media.py +4 -4
  388. endoreg_db/views/meta/__init__.py +1 -1
  389. endoreg_db/views/meta/sensitive_meta_list.py +20 -22
  390. endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
  391. endoreg_db/views/misc/__init__.py +6 -34
  392. endoreg_db/views/misc/center.py +2 -1
  393. endoreg_db/views/misc/csrf.py +2 -1
  394. endoreg_db/views/misc/gender.py +2 -1
  395. endoreg_db/views/misc/stats.py +141 -106
  396. endoreg_db/views/patient/__init__.py +1 -3
  397. endoreg_db/views/patient/patient.py +141 -99
  398. endoreg_db/views/patient_examination/__init__.py +5 -5
  399. endoreg_db/views/patient_examination/patient_examination.py +43 -42
  400. endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
  401. endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
  402. endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
  403. endoreg_db/views/patient_examination/video.py +114 -80
  404. endoreg_db/views/patient_finding/__init__.py +1 -1
  405. endoreg_db/views/patient_finding/patient_finding.py +17 -10
  406. endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
  407. endoreg_db/views/patient_finding_classification/__init__.py +1 -1
  408. endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
  409. endoreg_db/views/report/reimport.py +1 -1
  410. endoreg_db/views/report/report_stream.py +5 -8
  411. endoreg_db/views/requirement/__init__.py +2 -1
  412. endoreg_db/views/requirement/evaluate.py +7 -9
  413. endoreg_db/views/requirement/lookup.py +2 -3
  414. endoreg_db/views/requirement/lookup_store.py +0 -1
  415. endoreg_db/views/requirement/requirement_utils.py +2 -4
  416. endoreg_db/views/stats/__init__.py +4 -4
  417. endoreg_db/views/stats/stats_views.py +152 -115
  418. endoreg_db/views/video/__init__.py +18 -27
  419. endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
  420. endoreg_db/views/{ai → video/ai}/label.py +20 -16
  421. endoreg_db/views/video/correction.py +5 -6
  422. endoreg_db/views/video/reimport.py +134 -99
  423. endoreg_db/views/video/segments_crud.py +134 -44
  424. endoreg_db/views/video/video_apply_mask.py +13 -12
  425. endoreg_db/views/video/video_correction.py +2 -1
  426. endoreg_db/views/video/video_download_processed.py +15 -15
  427. endoreg_db/views/video/video_meta_stats.py +7 -6
  428. endoreg_db/views/video/video_processing_history.py +3 -2
  429. endoreg_db/views/video/video_remove_frames.py +13 -12
  430. endoreg_db/views/video/video_stream.py +110 -82
  431. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
  432. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
  433. endoreg_db/management/commands/import_fallback_video.py +0 -203
  434. endoreg_db/management/commands/import_video.py +0 -422
  435. endoreg_db/management/commands/import_video_with_classification.py +0 -367
  436. endoreg_db/models/media/processing_history/processing_history.py +0 -96
  437. endoreg_db/serializers/label/__init__.py +0 -7
  438. endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
  439. endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
  440. endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
  441. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
  442. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
  443. endoreg_db/services/__old/pdf_import.py +0 -1487
  444. endoreg_db/services/__old/video_import.py +0 -1306
  445. endoreg_db/tasks/upload_tasks.py +0 -216
  446. endoreg_db/tasks/video_ingest.py +0 -161
  447. endoreg_db/tasks/video_processing_tasks.py +0 -327
  448. endoreg_db/views/misc/translation.py +0 -182
  449. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
  450. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
@@ -7,7 +7,8 @@ the DEBUG setting and other environment configurations.
7
7
 
8
8
  from django.conf import settings
9
9
  from rest_framework.permissions import IsAuthenticated, AllowAny, BasePermission
10
- #from rest_framework.decorators import permission_classes as drf_permission_classes
10
+
11
+ # from rest_framework.decorators import permission_classes as drf_permission_classes
11
12
  from functools import wraps
12
13
  import logging
13
14
  import os
@@ -18,20 +19,20 @@ logger = logging.getLogger(__name__)
18
19
  class DynamicAuthPermission:
19
20
  """
20
21
  Permission class that adapts based on environment settings.
21
-
22
+
22
23
  - In DEBUG mode: Allows access without authentication
23
24
  - In production (DEBUG=False): Requires authentication
24
25
  """
25
-
26
+
26
27
  @staticmethod
27
28
  def get_permission_classes():
28
29
  """
29
30
  Returns appropriate permission classes based on current settings.
30
-
31
+
31
32
  Returns:
32
33
  list: Permission classes to use
33
34
  """
34
- if getattr(settings, 'DEBUG', False):
35
+ if getattr(settings, "DEBUG", False):
35
36
  logger.info("DEBUG mode detected - allowing unauthenticated access")
36
37
  return [AllowAny]
37
38
  else:
@@ -42,19 +43,23 @@ class DynamicAuthPermission:
42
43
  def dynamic_permission_classes(force_auth=None):
43
44
  """
44
45
  Decorator that applies permission classes based on environment settings.
45
-
46
+
46
47
  Args:
47
- force_auth (bool, optional):
48
+ force_auth (bool, optional):
48
49
  - True: Always require authentication regardless of DEBUG
49
- - False: Always allow access regardless of DEBUG
50
+ - False: Always allow access regardless of DEBUG
50
51
  - None: Use environment-based logic (default)
51
52
  """
53
+
52
54
  def decorator(view_func):
53
- from rest_framework.decorators import permission_classes as drf_permission_classes
55
+ from rest_framework.decorators import (
56
+ permission_classes as drf_permission_classes,
57
+ )
58
+
54
59
  @wraps(view_func)
55
60
  def wrapper(*args, **kwargs):
56
61
  return view_func(*args, **kwargs)
57
-
62
+
58
63
  # Determine permission classes
59
64
  if force_auth is True:
60
65
  permission_cls = [IsAuthenticated]
@@ -64,22 +69,24 @@ def dynamic_permission_classes(force_auth=None):
64
69
  logger.info(f"View {view_func.__name__} - forced public access")
65
70
  else:
66
71
  permission_cls = DynamicAuthPermission.get_permission_classes()
67
- logger.info(f"View {view_func.__name__} - dynamic permissions: {permission_cls}")
68
-
72
+ logger.info(
73
+ f"View {view_func.__name__} - dynamic permissions: {permission_cls}"
74
+ )
75
+
69
76
  # Apply the permission classes decorator
70
77
  return drf_permission_classes(permission_cls)(wrapper)
71
-
78
+
72
79
  return decorator
73
80
 
74
81
 
75
82
  def get_auth_required():
76
83
  """
77
84
  Simple function to check if authentication is required in current environment.
78
-
85
+
79
86
  Returns:
80
87
  bool: True if authentication is required, False otherwise
81
88
  """
82
- return not getattr(settings, 'DEBUG', False)
89
+ return not getattr(settings, "DEBUG", False)
83
90
 
84
91
 
85
92
  def is_debug_mode():
@@ -89,17 +96,22 @@ def is_debug_mode():
89
96
  """
90
97
  truthy = {"1", "true", "yes", "on"}
91
98
  env_debug = str(os.environ.get("DJANGO_DEBUG", "false")).lower() in truthy
92
- settings_debug = bool(getattr(settings, 'DEBUG', False))
99
+ settings_debug = bool(getattr(settings, "DEBUG", False))
93
100
  pytest_active = "PYTEST_CURRENT_TEST" in os.environ
94
101
  result = settings_debug or env_debug or pytest_active
95
- logger.info(f"is_debug_mode: env={env_debug}, settings={settings_debug}, pytest={pytest_active}, result={result}")
102
+ logger.info(
103
+ f"is_debug_mode: env={env_debug}, settings={settings_debug}, pytest={pytest_active}, result={result}"
104
+ )
96
105
  return result
97
106
 
107
+
98
108
  # Compute default permission classes each call to avoid stale values during tests
99
109
 
110
+
100
111
  def get_debug_permissions():
101
112
  return [AllowAny] if is_debug_mode() else [IsAuthenticated]
102
113
 
114
+
103
115
  # Export a name for convenience, but prefer calling get_debug_permissions() in views
104
116
  DEBUG_PERMISSIONS = get_debug_permissions()
105
117
  ALWAYS_AUTH_PERMISSIONS = [IsAuthenticated]
@@ -117,15 +129,15 @@ class EnvironmentAwarePermission(BasePermission):
117
129
  Custom permission class that can be used directly in DRF views.
118
130
  Honors both Django settings.DEBUG and DJANGO_DEBUG env var.
119
131
  """
120
-
132
+
121
133
  def has_permission(self, request, view):
122
134
  """
123
135
  Check if the request has permission.
124
-
136
+
125
137
  Args:
126
138
  request: The Django request object
127
139
  view: The DRF view
128
-
140
+
129
141
  Returns:
130
142
  bool: True if permission granted, False otherwise
131
143
  """
@@ -133,12 +145,16 @@ class EnvironmentAwarePermission(BasePermission):
133
145
  logger.debug(f"DEBUG mode - granting access to {view.__class__.__name__}")
134
146
  return True
135
147
  # In production, require authentication
136
- is_authenticated = bool(getattr(request, 'user', None) and request.user.is_authenticated)
137
- logger.debug(f"Production mode - authentication check for {view.__class__.__name__}: {is_authenticated}")
148
+ is_authenticated = bool(
149
+ getattr(request, "user", None) and request.user.is_authenticated
150
+ )
151
+ logger.debug(
152
+ f"Production mode - authentication check for {view.__class__.__name__}: {is_authenticated}"
153
+ )
138
154
  return is_authenticated
139
-
155
+
140
156
  def has_object_permission(self, request, view, obj):
141
157
  """
142
158
  Object-level permission check.
143
159
  """
144
- return self.has_permission(request, view)
160
+ return self.has_permission(request, view)
@@ -1,14 +1,16 @@
1
- from endoreg_db.models import (
2
- VideoFile, Center, EndoscopyProcessor, AiModel, ModelMeta,
3
- LabelVideoSegment, SensitiveMeta
4
- )
1
+ import logging
2
+ import os
5
3
  from pathlib import Path
4
+ from django.core.management import call_command
6
5
  from icecream import ic
7
6
  from tqdm import tqdm
8
- import os
9
- from django.core.management import call_command
10
- import logging
11
- from django.core.exceptions import ObjectDoesNotExist
7
+ from endoreg_db.utils.file_operations import sha256_file
8
+ from endoreg_db.models import (
9
+ AiModel,
10
+ LabelVideoSegment,
11
+ ModelMeta,
12
+ VideoFile,
13
+ )
12
14
 
13
15
  logger = logging.getLogger(__name__)
14
16
  # Usage
@@ -24,11 +26,14 @@ MODEL_WEIGHTS_PATH = "./tests/assets/colo_segmentation_RegNetX800MF_6.safetensor
24
26
  ic(DEFAULT_DIR)
25
27
 
26
28
 
27
-
28
- def process_video_dir(video_dir: Path=DEFAULT_DIR, center_name: str = CENTER_NAME, endoscopy_processor_name: str = ENDOSCOPY_PROCESSOR_NAME):
29
+ def process_video_dir(
30
+ video_dir: Path = DEFAULT_DIR,
31
+ center_name: str = CENTER_NAME,
32
+ endoscopy_processor_name: str = ENDOSCOPY_PROCESSOR_NAME,
33
+ ):
29
34
  """
30
35
  Process a directory of video files and create VideoFile objects in the database.
31
-
36
+
32
37
  Args:
33
38
  video_dir (Path): Path to the directory containing video files.
34
39
  center_name (str): Name of the center to associate with the video files.
@@ -50,28 +55,32 @@ def process_video_dir(video_dir: Path=DEFAULT_DIR, center_name: str = CENTER_NAM
50
55
  MODEL_WEIGHTS_PATH,
51
56
  )
52
57
 
53
-
54
-
55
58
  # Get the center and endoscopy processor objects
56
59
 
57
60
  # Iterate through all files in the directory
58
61
  for file_path in tqdm(video_dir.iterdir()):
59
62
  logger.info(f"Processing file: {file_path}")
60
- if file_path.is_file() and file_path.suffix in ['.mp4', '.avi', '.mov']:
63
+ if file_path.is_file() and file_path.suffix in [".mp4", ".avi", ".mov"]:
61
64
  # Create a VideoFile object
62
65
  video_file = VideoFile.create_from_file_initialized(
63
66
  file_path=file_path,
64
67
  center_name=center_name,
65
68
  processor_name=endoscopy_processor_name,
69
+ video_hash=sha256_file(file_path),
66
70
  )
67
71
  logger.warning(f"Processing video file: {video_file}")
68
72
  try:
69
73
  success_pipe_1 = video_file.pipe_1(
70
- model_name = MODEL_NAME,
74
+ model_name=MODEL_NAME,
75
+ )
76
+ assert success_pipe_1, (
77
+ f"Pipe 1 failed for video {video_file.video_hash}"
71
78
  )
72
- assert success_pipe_1, f"Pipe 1 failed for video {video_file.uuid}"
73
79
  except Exception as e:
74
- logger.error(f"Pipe 1 failed for video {video_file.uuid}: {e}", exc_info=True)
80
+ logger.error(
81
+ f"Pipe 1 failed for video {video_file.video_hash}: {e}",
82
+ exc_info=True,
83
+ )
75
84
  raise e
76
85
 
77
86
  video_file.refresh_from_db()
@@ -88,10 +97,14 @@ def process_video_dir(video_dir: Path=DEFAULT_DIR, center_name: str = CENTER_NAM
88
97
  seg_state.save()
89
98
  logger.info(f"Simulated validation for segment {segment.id}")
90
99
  except LabelVideoSegment.state.RelatedObjectDoesNotExist:
91
- logger.warning(f"Cannot simulate validation for segment {segment.id}: Related state object does not exist.")
100
+ logger.warning(
101
+ f"Cannot simulate validation for segment {segment.id}: Related state object does not exist."
102
+ )
92
103
  except Exception as e:
93
- logger.error(f"Error during segment state simulation for segment {segment.id}: {e}", exc_info=True)
94
-
104
+ logger.error(
105
+ f"Error during segment state simulation for segment {segment.id}: {e}",
106
+ exc_info=True,
107
+ )
95
108
 
96
109
  # Simulate the validation process for the video files sensitive meta obj
97
110
  if video_file.sensitive_meta:
@@ -100,12 +113,19 @@ def process_video_dir(video_dir: Path=DEFAULT_DIR, center_name: str = CENTER_NAM
100
113
  sm_state.dob_verified = True
101
114
  sm_state.names_verified = True
102
115
  sm_state.save()
103
- logger.info(f"Simulated validation for sensitive meta of video {video_file.uuid}")
116
+ logger.info(
117
+ f"Simulated validation for sensitive meta of video {video_file.video_hash}"
118
+ )
104
119
  except Exception as e:
105
- logger.error(f"Error during sensitive meta state simulation for video {video_file.uuid}: {e}", exc_info=True)
120
+ logger.error(
121
+ f"Error during sensitive meta state simulation for video {video_file.video_hash}: {e}",
122
+ exc_info=True,
123
+ )
106
124
 
107
125
  else:
108
- logger.warning(f"Cannot simulate validation for sensitive meta of video {video_file.uuid}: SensitiveMeta object does not exist (likely due to pipe_1 failure or incomplete text extraction).")
126
+ logger.warning(
127
+ f"Cannot simulate validation for sensitive meta of video {video_file.video_hash}: SensitiveMeta object does not exist (likely due to pipe_1 failure or incomplete text extraction)."
128
+ )
109
129
 
110
130
  ######## SIMULATION OF VALIDATION ########
111
131
 
@@ -114,7 +134,11 @@ def process_video_dir(video_dir: Path=DEFAULT_DIR, center_name: str = CENTER_NAM
114
134
  try:
115
135
  video_file.pipe_2()
116
136
  except Exception as e:
117
- logger.error(f"Pipe 2 failed for video {video_file.uuid}: {e}", exc_info=True)
137
+ logger.error(
138
+ f"Pipe 2 failed for video {video_file.video_hash}: {e}",
139
+ exc_info=True,
140
+ )
118
141
  else:
119
- logger.warning(f"Skipping Pipe 2 for video {video_file.uuid} because Pipe 1 did not complete successfully.")
120
-
142
+ logger.warning(
143
+ f"Skipping Pipe 2 for video {video_file.video_hash} because Pipe 1 did not complete successfully."
144
+ )
@@ -4,7 +4,7 @@ from endoreg_db.models.administration.product.product_material import ProductMat
4
4
  from typing import List
5
5
 
6
6
 
7
- def sum_emissions(product_materials:List["ProductMaterial"]):
7
+ def sum_emissions(product_materials: List["ProductMaterial"]):
8
8
  # sum up the emissions
9
9
  emission = 0
10
10
  reference_unit = None
@@ -12,9 +12,11 @@ def sum_emissions(product_materials:List["ProductMaterial"]):
12
12
  if not reference_unit:
13
13
  reference_unit = product_material.unit
14
14
  else:
15
- assert reference_unit == product_material.unit, "ProductMaterial units do not match"
15
+ assert reference_unit == product_material.unit, (
16
+ "ProductMaterial units do not match"
17
+ )
16
18
  em_value, emission_unit = product_material.get_emission()
17
19
  assert reference_unit == emission_unit, "ProductMaterial units do not match"
18
20
  emission += em_value
19
21
 
20
- return emission, reference_unit
22
+ return emission, reference_unit
@@ -12,7 +12,9 @@ def sum_weights(product_materials: List["ProductMaterial"]):
12
12
  if not reference_unit:
13
13
  reference_unit = product_material.unit
14
14
  else:
15
- assert reference_unit == product_material.unit, "ProductMaterial units do not match"
15
+ assert reference_unit == product_material.unit, (
16
+ "ProductMaterial units do not match"
17
+ )
16
18
  weight += product_material.quantity
17
19
 
18
- return weight, reference_unit
20
+ return weight, reference_unit
@@ -1,6 +1,5 @@
1
- '''Module for Pydantic models.'''
1
+ """Module for Pydantic models."""
2
+
2
3
  from .db_config import DbConfig
3
4
 
4
- __all__ = [
5
- 'DbConfig'
6
- ]
5
+ __all__ = ["DbConfig"]