endoreg-db 0.8.6.1__py3-none-any.whl → 0.8.8.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (503) hide show
  1. endoreg_db/authz/auth.py +74 -0
  2. endoreg_db/authz/backends.py +168 -0
  3. endoreg_db/authz/management/commands/list_routes.py +18 -0
  4. endoreg_db/authz/middleware.py +83 -0
  5. endoreg_db/authz/permissions.py +127 -0
  6. endoreg_db/authz/policy.py +218 -0
  7. endoreg_db/authz/views_auth.py +66 -0
  8. endoreg_db/config/env.py +13 -8
  9. endoreg_db/data/__init__.py +2 -11
  10. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +3 -3
  11. endoreg_db/data/event_classification/data.yaml +4 -0
  12. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  13. endoreg_db/data/examination/examinations/data.yaml +114 -14
  14. endoreg_db/data/examination/time-type/data.yaml +0 -3
  15. endoreg_db/data/examination_indication/endoscopy.yaml +108 -173
  16. endoreg_db/data/examination_indication_classification/endoscopy.yaml +0 -70
  17. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +33 -37
  18. endoreg_db/data/finding/00_generic.yaml +35 -0
  19. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  20. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  21. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  22. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  23. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  24. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  25. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  26. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  27. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  28. endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
  29. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  30. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  31. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  32. endoreg_db/data/finding_classification_choice/{yes_no_na.yaml → 00_generic.yaml} +5 -1
  33. endoreg_db/data/finding_classification_choice/{examination_setting_generic_types.yaml → 00_generic_baseline.yaml} +10 -2
  34. endoreg_db/data/finding_classification_choice/{complication_generic_types.yaml → 00_generic_complication.yaml} +1 -1
  35. endoreg_db/data/finding_classification_choice/{histology.yaml → 00_generic_histology.yaml} +1 -4
  36. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  37. endoreg_db/data/finding_classification_choice/{bowel_preparation.yaml → 02_colonoscopy_bowel_preparation.yaml} +1 -30
  38. endoreg_db/data/finding_classification_choice/{colonoscopy_not_complete_reason.yaml → 02_colonoscopy_generic.yaml} +1 -1
  39. endoreg_db/data/finding_classification_choice/{histology_polyp.yaml → 02_colonoscopy_histology.yaml} +1 -1
  40. endoreg_db/data/finding_classification_choice/{colonoscopy_location.yaml → 02_colonoscopy_location.yaml} +23 -4
  41. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  42. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  43. endoreg_db/data/finding_classification_choice/{colon_lesion_paris.yaml → 02_colonoscopy_polyp_morphology.yaml} +26 -8
  44. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  45. endoreg_db/data/finding_classification_type/{colonoscopy_basic.yaml → 00_generic.yaml} +18 -13
  46. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  47. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  48. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  49. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  50. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  51. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  52. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  53. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  54. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  55. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  56. endoreg_db/data/finding_type/data.yaml +8 -12
  57. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  58. endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
  59. endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
  60. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  61. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +29 -12
  62. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  63. endoreg_db/data/requirement_set/{endoscopy_bleeding_risk.yaml → 02_endoscopy_bleeding_risk.yaml} +0 -6
  64. endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
  65. endoreg_db/data/requirement_set/_old_ +109 -0
  66. endoreg_db/data/requirement_set_type/data.yaml +21 -0
  67. endoreg_db/data/setup_config.yaml +4 -4
  68. endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
  69. endoreg_db/exceptions.py +4 -2
  70. endoreg_db/forms/examination_form.py +1 -1
  71. endoreg_db/helpers/data_loader.py +125 -53
  72. endoreg_db/helpers/default_objects.py +116 -81
  73. endoreg_db/import_files/__init__.py +27 -0
  74. endoreg_db/import_files/context/__init__.py +7 -0
  75. endoreg_db/import_files/context/default_sensitive_meta.py +81 -0
  76. endoreg_db/import_files/context/ensure_center.py +17 -0
  77. endoreg_db/import_files/context/file_lock.py +66 -0
  78. endoreg_db/import_files/context/import_context.py +43 -0
  79. endoreg_db/import_files/context/validate_directories.py +56 -0
  80. endoreg_db/import_files/file_storage/__init__.py +15 -0
  81. endoreg_db/import_files/file_storage/create_report_file.py +76 -0
  82. endoreg_db/import_files/file_storage/create_video_file.py +75 -0
  83. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +39 -0
  84. endoreg_db/import_files/file_storage/state_management.py +400 -0
  85. endoreg_db/import_files/file_storage/storage.py +36 -0
  86. endoreg_db/import_files/import_service.md +26 -0
  87. endoreg_db/import_files/processing/__init__.py +11 -0
  88. endoreg_db/import_files/processing/report_processing/report_anonymization.py +94 -0
  89. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  90. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  91. endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +119 -0
  92. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  93. endoreg_db/import_files/pseudonymization/k_anonymity.py +182 -0
  94. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +128 -0
  95. endoreg_db/import_files/report_import_service.py +141 -0
  96. endoreg_db/import_files/video_import_service.py +150 -0
  97. endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
  98. endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
  99. endoreg_db/management/commands/import_report.py +130 -65
  100. endoreg_db/management/commands/import_video.py +9 -10
  101. endoreg_db/management/commands/import_video_with_classification.py +2 -2
  102. endoreg_db/management/commands/list_routes.py +18 -0
  103. endoreg_db/management/commands/load_ai_model_data.py +5 -5
  104. endoreg_db/management/commands/load_ai_model_label_data.py +9 -7
  105. endoreg_db/management/commands/load_base_db_data.py +5 -134
  106. endoreg_db/management/commands/load_center_data.py +12 -12
  107. endoreg_db/management/commands/load_contraindication_data.py +14 -16
  108. endoreg_db/management/commands/load_disease_classification_choices_data.py +15 -18
  109. endoreg_db/management/commands/load_disease_classification_data.py +15 -18
  110. endoreg_db/management/commands/load_disease_data.py +25 -28
  111. endoreg_db/management/commands/load_endoscope_data.py +20 -27
  112. endoreg_db/management/commands/load_event_data.py +14 -16
  113. endoreg_db/management/commands/load_examination_data.py +31 -44
  114. endoreg_db/management/commands/load_examination_indication_data.py +20 -21
  115. endoreg_db/management/commands/load_finding_data.py +52 -80
  116. endoreg_db/management/commands/load_information_source.py +21 -23
  117. endoreg_db/management/commands/load_lab_value_data.py +17 -26
  118. endoreg_db/management/commands/load_medication_data.py +13 -12
  119. endoreg_db/management/commands/load_organ_data.py +15 -19
  120. endoreg_db/management/commands/load_pdf_type_data.py +19 -18
  121. endoreg_db/management/commands/load_profession_data.py +14 -17
  122. endoreg_db/management/commands/load_qualification_data.py +20 -23
  123. endoreg_db/management/commands/load_report_reader_flag_data.py +17 -19
  124. endoreg_db/management/commands/load_requirement_data.py +62 -39
  125. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  126. endoreg_db/management/commands/load_risk_data.py +7 -6
  127. endoreg_db/management/commands/load_shift_data.py +20 -23
  128. endoreg_db/management/commands/load_tag_data.py +8 -11
  129. endoreg_db/management/commands/load_unit_data.py +17 -19
  130. endoreg_db/management/commands/setup_endoreg_db.py +3 -3
  131. endoreg_db/management/commands/start_filewatcher.py +46 -37
  132. endoreg_db/management/commands/storage_management.py +271 -203
  133. endoreg_db/management/commands/validate_video_files.py +1 -5
  134. endoreg_db/migrations/0001_initial.py +297 -250
  135. endoreg_db/models/__init__.py +78 -123
  136. endoreg_db/models/administration/__init__.py +21 -42
  137. endoreg_db/models/administration/ai/active_model.py +2 -2
  138. endoreg_db/models/administration/ai/ai_model.py +7 -6
  139. endoreg_db/models/administration/case/__init__.py +1 -15
  140. endoreg_db/models/administration/case/case.py +3 -3
  141. endoreg_db/models/administration/case/case_template/__init__.py +2 -14
  142. endoreg_db/models/administration/case/case_template/case_template.py +2 -124
  143. endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
  144. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
  145. endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
  146. endoreg_db/models/administration/center/center.py +33 -19
  147. endoreg_db/models/administration/center/center_product.py +12 -9
  148. endoreg_db/models/administration/center/center_resource.py +25 -19
  149. endoreg_db/models/administration/center/center_shift.py +21 -17
  150. endoreg_db/models/administration/center/center_waste.py +16 -8
  151. endoreg_db/models/administration/person/__init__.py +2 -0
  152. endoreg_db/models/administration/person/employee/employee.py +10 -5
  153. endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
  154. endoreg_db/models/administration/person/employee/employee_type.py +12 -6
  155. endoreg_db/models/administration/person/examiner/examiner.py +13 -11
  156. endoreg_db/models/administration/person/patient/__init__.py +2 -0
  157. endoreg_db/models/administration/person/patient/patient.py +129 -100
  158. endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
  159. endoreg_db/models/administration/person/person.py +4 -0
  160. endoreg_db/models/administration/person/profession/__init__.py +8 -4
  161. endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
  162. endoreg_db/models/administration/product/product.py +20 -15
  163. endoreg_db/models/administration/product/product_material.py +17 -18
  164. endoreg_db/models/administration/product/product_weight.py +12 -8
  165. endoreg_db/models/administration/product/reference_product.py +23 -55
  166. endoreg_db/models/administration/qualification/qualification.py +7 -3
  167. endoreg_db/models/administration/qualification/qualification_type.py +7 -3
  168. endoreg_db/models/administration/shift/scheduled_days.py +8 -5
  169. endoreg_db/models/administration/shift/shift.py +16 -12
  170. endoreg_db/models/administration/shift/shift_type.py +23 -31
  171. endoreg_db/models/label/__init__.py +8 -9
  172. endoreg_db/models/label/annotation/image_classification.py +10 -9
  173. endoreg_db/models/label/annotation/video_segmentation_annotation.py +23 -28
  174. endoreg_db/models/label/label.py +15 -15
  175. endoreg_db/models/label/label_set.py +19 -6
  176. endoreg_db/models/label/label_type.py +1 -1
  177. endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
  178. endoreg_db/models/label/label_video_segment/label_video_segment.py +98 -102
  179. endoreg_db/models/label/video_segmentation_label.py +4 -0
  180. endoreg_db/models/label/video_segmentation_labelset.py +4 -3
  181. endoreg_db/models/media/frame/frame.py +22 -22
  182. endoreg_db/models/media/pdf/raw_pdf.py +194 -194
  183. endoreg_db/models/media/pdf/report_file.py +25 -29
  184. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +55 -47
  185. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
  186. endoreg_db/models/media/processing_history/__init__.py +5 -0
  187. endoreg_db/models/media/processing_history/processing_history.py +96 -0
  188. endoreg_db/models/media/video/__init__.py +1 -0
  189. endoreg_db/models/media/video/create_from_file.py +139 -77
  190. endoreg_db/models/media/video/pipe_2.py +8 -9
  191. endoreg_db/models/media/video/video_file.py +174 -112
  192. endoreg_db/models/media/video/video_file_ai.py +288 -74
  193. endoreg_db/models/media/video/video_file_anonymize.py +38 -38
  194. endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
  195. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
  196. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
  197. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
  198. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
  199. endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
  200. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
  201. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
  202. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
  203. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
  204. endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
  205. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
  206. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
  207. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
  208. endoreg_db/models/media/video/video_file_io.py +113 -61
  209. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
  210. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
  211. endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
  212. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
  213. endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
  214. endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
  215. endoreg_db/models/media/video/video_file_segments.py +24 -17
  216. endoreg_db/models/media/video/video_metadata.py +19 -35
  217. endoreg_db/models/media/video/video_processing.py +96 -95
  218. endoreg_db/models/medical/contraindication/README.md +1 -0
  219. endoreg_db/models/medical/contraindication/__init__.py +13 -3
  220. endoreg_db/models/medical/disease.py +22 -16
  221. endoreg_db/models/medical/event.py +31 -18
  222. endoreg_db/models/medical/examination/__init__.py +13 -6
  223. endoreg_db/models/medical/examination/examination.py +39 -20
  224. endoreg_db/models/medical/examination/examination_indication.py +30 -95
  225. endoreg_db/models/medical/examination/examination_time.py +23 -8
  226. endoreg_db/models/medical/examination/examination_time_type.py +9 -6
  227. endoreg_db/models/medical/examination/examination_type.py +3 -4
  228. endoreg_db/models/medical/finding/finding.py +32 -40
  229. endoreg_db/models/medical/finding/finding_classification.py +42 -72
  230. endoreg_db/models/medical/finding/finding_intervention.py +25 -22
  231. endoreg_db/models/medical/finding/finding_type.py +13 -12
  232. endoreg_db/models/medical/hardware/endoscope.py +26 -26
  233. endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
  234. endoreg_db/models/medical/laboratory/lab_value.py +62 -91
  235. endoreg_db/models/medical/medication/medication.py +22 -10
  236. endoreg_db/models/medical/medication/medication_indication.py +29 -3
  237. endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
  238. endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
  239. endoreg_db/models/medical/medication/medication_schedule.py +27 -16
  240. endoreg_db/models/medical/organ/__init__.py +15 -12
  241. endoreg_db/models/medical/patient/medication_examples.py +6 -6
  242. endoreg_db/models/medical/patient/patient_disease.py +20 -23
  243. endoreg_db/models/medical/patient/patient_event.py +19 -22
  244. endoreg_db/models/medical/patient/patient_examination.py +48 -54
  245. endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
  246. endoreg_db/models/medical/patient/patient_finding.py +122 -139
  247. endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
  248. endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
  249. endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
  250. endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
  251. endoreg_db/models/medical/patient/patient_medication.py +27 -38
  252. endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
  253. endoreg_db/models/medical/risk/risk.py +7 -6
  254. endoreg_db/models/medical/risk/risk_type.py +8 -5
  255. endoreg_db/models/metadata/model_meta.py +60 -29
  256. endoreg_db/models/metadata/model_meta_logic.py +125 -18
  257. endoreg_db/models/metadata/pdf_meta.py +31 -24
  258. endoreg_db/models/metadata/sensitive_meta.py +105 -85
  259. endoreg_db/models/metadata/sensitive_meta_logic.py +198 -103
  260. endoreg_db/models/metadata/video_meta.py +51 -31
  261. endoreg_db/models/metadata/video_prediction_logic.py +16 -23
  262. endoreg_db/models/metadata/video_prediction_meta.py +29 -33
  263. endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
  264. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
  265. endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
  266. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
  267. endoreg_db/models/other/emission/emission_factor.py +18 -8
  268. endoreg_db/models/other/gender.py +10 -5
  269. endoreg_db/models/other/information_source.py +50 -29
  270. endoreg_db/models/other/material.py +9 -5
  271. endoreg_db/models/other/resource.py +6 -4
  272. endoreg_db/models/other/tag.py +10 -5
  273. endoreg_db/models/other/transport_route.py +13 -8
  274. endoreg_db/models/other/unit.py +10 -6
  275. endoreg_db/models/other/waste.py +6 -5
  276. endoreg_db/models/report/report.py +6 -0
  277. endoreg_db/models/requirement/requirement.py +329 -361
  278. endoreg_db/models/requirement/requirement_error.py +85 -0
  279. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  280. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
  281. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
  282. endoreg_db/models/requirement/requirement_operator.py +103 -112
  283. endoreg_db/models/requirement/requirement_set.py +74 -57
  284. endoreg_db/models/state/__init__.py +4 -4
  285. endoreg_db/models/state/abstract.py +2 -2
  286. endoreg_db/models/state/anonymization.py +12 -0
  287. endoreg_db/models/state/audit_ledger.py +49 -51
  288. endoreg_db/models/state/label_video_segment.py +9 -0
  289. endoreg_db/models/state/raw_pdf.py +101 -68
  290. endoreg_db/models/state/sensitive_meta.py +6 -2
  291. endoreg_db/models/state/video.py +110 -90
  292. endoreg_db/models/upload_job.py +35 -34
  293. endoreg_db/models/utils.py +28 -25
  294. endoreg_db/queries/__init__.py +3 -1
  295. endoreg_db/root_urls.py +21 -2
  296. endoreg_db/schemas/examination_evaluation.py +1 -1
  297. endoreg_db/serializers/__init__.py +2 -10
  298. endoreg_db/serializers/anonymization.py +18 -10
  299. endoreg_db/serializers/label_video_segment/label_video_segment.py +2 -29
  300. endoreg_db/serializers/meta/__init__.py +1 -6
  301. endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
  302. endoreg_db/serializers/misc/file_overview.py +11 -99
  303. endoreg_db/serializers/misc/sensitive_patient_data.py +50 -26
  304. endoreg_db/serializers/patient_examination/patient_examination.py +3 -3
  305. endoreg_db/serializers/pdf/anony_text_validation.py +39 -23
  306. endoreg_db/serializers/requirements/requirement_sets.py +92 -22
  307. endoreg_db/serializers/video/segmentation.py +2 -1
  308. endoreg_db/serializers/video/video_file_list.py +65 -34
  309. endoreg_db/serializers/video/video_processing_history.py +20 -5
  310. endoreg_db/services/__old/pdf_import.py +1487 -0
  311. endoreg_db/services/__old/video_import.py +1306 -0
  312. endoreg_db/services/anonymization.py +128 -89
  313. endoreg_db/services/lookup_service.py +65 -52
  314. endoreg_db/services/lookup_store.py +2 -2
  315. endoreg_db/services/pdf_import.py +0 -1382
  316. endoreg_db/services/report_import.py +10 -0
  317. endoreg_db/services/video_import.py +6 -1255
  318. endoreg_db/tasks/upload_tasks.py +79 -70
  319. endoreg_db/tasks/video_ingest.py +8 -4
  320. endoreg_db/urls/__init__.py +5 -32
  321. endoreg_db/urls/ai.py +32 -0
  322. endoreg_db/urls/media.py +121 -83
  323. endoreg_db/urls/root_urls.py +29 -0
  324. endoreg_db/utils/__init__.py +15 -5
  325. endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
  326. endoreg_db/utils/case_generator/__init__.py +3 -0
  327. endoreg_db/utils/dataloader.py +142 -40
  328. endoreg_db/utils/defaults/set_default_center.py +32 -0
  329. endoreg_db/utils/names.py +22 -16
  330. endoreg_db/utils/paths.py +110 -46
  331. endoreg_db/utils/permissions.py +2 -1
  332. endoreg_db/utils/pipelines/Readme.md +1 -1
  333. endoreg_db/utils/pipelines/process_video_dir.py +1 -1
  334. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +655 -0
  335. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +97 -0
  336. endoreg_db/utils/setup_config.py +8 -5
  337. endoreg_db/utils/storage.py +115 -0
  338. endoreg_db/utils/validate_endo_roi.py +8 -2
  339. endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
  340. endoreg_db/views/__init__.py +85 -183
  341. endoreg_db/views/ai/__init__.py +8 -0
  342. endoreg_db/views/ai/label.py +155 -0
  343. endoreg_db/views/anonymization/media_management.py +202 -166
  344. endoreg_db/views/anonymization/overview.py +99 -67
  345. endoreg_db/views/anonymization/validate.py +182 -44
  346. endoreg_db/views/media/__init__.py +7 -20
  347. endoreg_db/views/media/pdf_media.py +197 -174
  348. endoreg_db/views/media/sensitive_metadata.py +193 -138
  349. endoreg_db/views/media/video_media.py +89 -82
  350. endoreg_db/views/meta/__init__.py +0 -8
  351. endoreg_db/views/misc/__init__.py +1 -7
  352. endoreg_db/views/misc/upload_views.py +94 -93
  353. endoreg_db/views/patient/patient.py +5 -4
  354. endoreg_db/views/report/__init__.py +5 -7
  355. endoreg_db/views/{pdf → report}/reimport.py +22 -22
  356. endoreg_db/views/{pdf/pdf_stream.py → report/report_stream.py} +46 -39
  357. endoreg_db/views/requirement/evaluate.py +188 -187
  358. endoreg_db/views/requirement/lookup.py +17 -3
  359. endoreg_db/views/requirement/lookup_store.py +22 -90
  360. endoreg_db/views/requirement/requirement_utils.py +89 -0
  361. endoreg_db/views/video/__init__.py +23 -24
  362. endoreg_db/views/video/correction.py +201 -172
  363. endoreg_db/views/video/reimport.py +1 -1
  364. endoreg_db/views/{media/video_segments.py → video/segments_crud.py} +77 -40
  365. endoreg_db/views/video/{video_meta.py → video_meta_stats.py} +2 -2
  366. endoreg_db/views/video/video_stream.py +7 -8
  367. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/METADATA +7 -3
  368. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/RECORD +391 -413
  369. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/WHEEL +1 -1
  370. endoreg_db/data/finding/anatomy_colon.yaml +0 -128
  371. endoreg_db/data/finding/colonoscopy.yaml +0 -40
  372. endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +0 -56
  373. endoreg_db/data/finding/complication.yaml +0 -16
  374. endoreg_db/data/finding/data.yaml +0 -105
  375. endoreg_db/data/finding/examination_setting.yaml +0 -16
  376. endoreg_db/data/finding/medication_related.yaml +0 -18
  377. endoreg_db/data/finding/outcome.yaml +0 -12
  378. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +0 -95
  379. endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +0 -22
  380. endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +0 -25
  381. endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
  382. endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
  383. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +0 -68
  384. endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
  385. endoreg_db/data/finding_classification/colonoscopy_location.yaml +0 -80
  386. endoreg_db/data/finding_classification/colonoscopy_lst.yaml +0 -21
  387. endoreg_db/data/finding_classification/colonoscopy_nice.yaml +0 -20
  388. endoreg_db/data/finding_classification/colonoscopy_paris.yaml +0 -26
  389. endoreg_db/data/finding_classification/colonoscopy_sano.yaml +0 -22
  390. endoreg_db/data/finding_classification/colonoscopy_summary.yaml +0 -53
  391. endoreg_db/data/finding_classification/complication_generic.yaml +0 -25
  392. endoreg_db/data/finding_classification/examination_setting_generic.yaml +0 -40
  393. endoreg_db/data/finding_classification/histology_colo.yaml +0 -51
  394. endoreg_db/data/finding_classification/intervention_required.yaml +0 -26
  395. endoreg_db/data/finding_classification/medication_related.yaml +0 -23
  396. endoreg_db/data/finding_classification/visualized.yaml +0 -33
  397. endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
  398. endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
  399. endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
  400. endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +0 -15
  401. endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +0 -17
  402. endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
  403. endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +0 -14
  404. endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
  405. endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +0 -82
  406. endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
  407. endoreg_db/data/finding_classification_choice/outcome.yaml +0 -19
  408. endoreg_db/data/finding_intervention/endoscopy.yaml +0 -43
  409. endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
  410. endoreg_db/data/finding_intervention/endoscopy_egd.yaml +0 -128
  411. endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +0 -32
  412. endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +0 -9
  413. endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +0 -36
  414. endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +0 -79
  415. endoreg_db/data/requirement/age.yaml +0 -26
  416. endoreg_db/data/requirement/gender.yaml +0 -25
  417. endoreg_db/management/commands/init_default_ai_model.py +0 -112
  418. endoreg_db/management/commands/reset_celery_schedule.py +0 -9
  419. endoreg_db/management/commands/validate_video.py +0 -204
  420. endoreg_db/migrations/0002_add_video_correction_models.py +0 -52
  421. endoreg_db/migrations/0003_add_center_display_name.py +0 -30
  422. endoreg_db/models/administration/permissions/__init__.py +0 -44
  423. endoreg_db/models/rule/__init__.py +0 -13
  424. endoreg_db/models/rule/rule.py +0 -27
  425. endoreg_db/models/rule/rule_applicator.py +0 -224
  426. endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
  427. endoreg_db/models/rule/rule_type.py +0 -20
  428. endoreg_db/models/rule/ruleset.py +0 -17
  429. endoreg_db/renames.yml +0 -8
  430. endoreg_db/serializers/_old/raw_pdf_meta_validation.py +0 -223
  431. endoreg_db/serializers/_old/raw_video_meta_validation.py +0 -179
  432. endoreg_db/serializers/_old/video.py +0 -71
  433. endoreg_db/serializers/meta/pdf_file_meta_extraction.py +0 -115
  434. endoreg_db/serializers/meta/report_meta.py +0 -53
  435. endoreg_db/serializers/report/__init__.py +0 -9
  436. endoreg_db/serializers/report/mixins.py +0 -45
  437. endoreg_db/serializers/report/report.py +0 -105
  438. endoreg_db/serializers/report/report_list.py +0 -22
  439. endoreg_db/serializers/report/secure_file_url.py +0 -26
  440. endoreg_db/serializers/video/video_metadata.py +0 -105
  441. endoreg_db/services/requirements_object.py +0 -147
  442. endoreg_db/services/storage_aware_video_processor.py +0 -344
  443. endoreg_db/urls/files.py +0 -6
  444. endoreg_db/urls/label_video_segment_validate.py +0 -33
  445. endoreg_db/urls/label_video_segments.py +0 -46
  446. endoreg_db/urls/report.py +0 -48
  447. endoreg_db/urls/video.py +0 -61
  448. endoreg_db/utils/case_generator/case_generator.py +0 -159
  449. endoreg_db/utils/case_generator/utils.py +0 -30
  450. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +0 -368
  451. endoreg_db/views/label/__init__.py +0 -5
  452. endoreg_db/views/label/label.py +0 -15
  453. endoreg_db/views/label_video_segment/__init__.py +0 -16
  454. endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +0 -44
  455. endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +0 -50
  456. endoreg_db/views/label_video_segment/label_video_segment.py +0 -77
  457. endoreg_db/views/label_video_segment/label_video_segment_by_label.py +0 -174
  458. endoreg_db/views/label_video_segment/label_video_segment_detail.py +0 -73
  459. endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +0 -46
  460. endoreg_db/views/label_video_segment/validate.py +0 -226
  461. endoreg_db/views/media/segments.py +0 -71
  462. endoreg_db/views/meta/available_files_list.py +0 -146
  463. endoreg_db/views/meta/report_meta.py +0 -53
  464. endoreg_db/views/meta/sensitive_meta_detail.py +0 -148
  465. endoreg_db/views/misc/secure_file_serving_view.py +0 -80
  466. endoreg_db/views/misc/secure_file_url_view.py +0 -84
  467. endoreg_db/views/misc/secure_url_validate.py +0 -79
  468. endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +0 -164
  469. endoreg_db/views/patient_finding_location/__init__.py +0 -5
  470. endoreg_db/views/patient_finding_location/pfl_create.py +0 -70
  471. endoreg_db/views/patient_finding_morphology/__init__.py +0 -5
  472. endoreg_db/views/patient_finding_morphology/pfm_create.py +0 -70
  473. endoreg_db/views/pdf/__init__.py +0 -8
  474. endoreg_db/views/report/report_list.py +0 -112
  475. endoreg_db/views/report/report_with_secure_url.py +0 -28
  476. endoreg_db/views/report/start_examination.py +0 -7
  477. endoreg_db/views/video/segmentation.py +0 -274
  478. endoreg_db/views/video/task_status.py +0 -49
  479. endoreg_db/views/video/timeline.py +0 -46
  480. endoreg_db/views/video/video_analyze.py +0 -52
  481. endoreg_db/views.py +0 -0
  482. /endoreg_db/data/requirement/{colonoscopy_baseline_austria.yaml → old/colonoscopy_baseline_austria.yaml} +0 -0
  483. /endoreg_db/data/requirement/{disease_cardiovascular.yaml → old/disease_cardiovascular.yaml} +0 -0
  484. /endoreg_db/data/requirement/{disease_classification_choice_cardiovascular.yaml → old/disease_classification_choice_cardiovascular.yaml} +0 -0
  485. /endoreg_db/data/requirement/{disease_hepatology.yaml → old/disease_hepatology.yaml} +0 -0
  486. /endoreg_db/data/requirement/{disease_misc.yaml → old/disease_misc.yaml} +0 -0
  487. /endoreg_db/data/requirement/{disease_renal.yaml → old/disease_renal.yaml} +0 -0
  488. /endoreg_db/data/requirement/{endoscopy_bleeding_risk.yaml → old/endoscopy_bleeding_risk.yaml} +0 -0
  489. /endoreg_db/data/requirement/{event_cardiology.yaml → old/event_cardiology.yaml} +0 -0
  490. /endoreg_db/data/requirement/{event_requirements.yaml → old/event_requirements.yaml} +0 -0
  491. /endoreg_db/data/requirement/{finding_colon_polyp.yaml → old/finding_colon_polyp.yaml} +0 -0
  492. /endoreg_db/{migrations/__init__.py → data/requirement/old/gender.yaml} +0 -0
  493. /endoreg_db/data/requirement/{lab_value.yaml → old/lab_value.yaml} +0 -0
  494. /endoreg_db/data/requirement/{medication.yaml → old/medication.yaml} +0 -0
  495. /endoreg_db/data/requirement_operator/{age.yaml → _old/age.yaml} +0 -0
  496. /endoreg_db/data/requirement_operator/{lab_operators.yaml → _old/lab_operators.yaml} +0 -0
  497. /endoreg_db/data/requirement_operator/{model_operators.yaml → _old/model_operators.yaml} +0 -0
  498. /endoreg_db/{models/media/video/refactor_plan.md → import_files/pseudonymization/__init__.py} +0 -0
  499. /endoreg_db/{models/media/video/video_file_frames.py → import_files/pseudonymization/pseudonymize.py} +0 -0
  500. /endoreg_db/models/{metadata/frame_ocr_result.py → report/__init__.py} +0 -0
  501. /endoreg_db/{urls/sensitive_meta.py → models/report/images.py} +0 -0
  502. /endoreg_db/utils/requirement_operator_logic/{lab_value_operators.py → _old/lab_value_operators.py} +0 -0
  503. {endoreg_db-0.8.6.1.dist-info → endoreg_db-0.8.8.9.dist-info}/licenses/LICENSE +0 -0
@@ -18,14 +18,14 @@ from endoreg_db.utils.hashs import get_patient_examination_hash, get_patient_has
18
18
  from ..administration import Center, Examiner, FirstName, LastName, Patient
19
19
  from ..medical import PatientExamination
20
20
  from ..other import Gender
21
- from ..state import SensitiveMetaState
22
21
 
23
22
  if TYPE_CHECKING:
24
23
  from .sensitive_meta import SensitiveMeta # Import model for type hinting
25
24
 
26
25
  logger = logging.getLogger(__name__)
27
26
  SECRET_SALT = os.getenv("DJANGO_SALT", "default_salt")
28
- DEFAULT_UNKNOWN_NAME = "unknown"
27
+ DEFAULT_UNKNOWN = "unknown"
28
+
29
29
 
30
30
  # Regex-Pattern für verschiedene Datumsformate
31
31
  ISO_RX = re.compile(r"^\d{4}-\d{2}-\d{2}$")
@@ -192,6 +192,11 @@ def calculate_examination_hash(
192
192
  if not center:
193
193
  raise ValueError("Center is required to calculate examination hash.")
194
194
 
195
+ if not first_name:
196
+ raise ValueError("First name is required to calculate examination hash.")
197
+ if not last_name:
198
+ raise ValueError("Last name is required to calculate examination hash.")
199
+
195
200
  hash_str = get_patient_examination_hash(
196
201
  first_name=first_name,
197
202
  last_name=last_name,
@@ -233,7 +238,7 @@ def create_pseudo_examiner_logic(instance: "SensitiveMeta") -> "Examiner":
233
238
  return examiner
234
239
 
235
240
 
236
- def get_or_create_pseudo_patient_logic(instance: "SensitiveMeta") -> "Patient":
241
+ def get_or_create_pseudo_patient_logic(instance: "SensitiveMeta"):
237
242
  """Gets or creates the pseudo patient based on instance data."""
238
243
  # Ensure necessary fields are set
239
244
  if not instance.patient_hash:
@@ -256,12 +261,12 @@ def get_or_create_pseudo_patient_logic(instance: "SensitiveMeta") -> "Patient":
256
261
  birth_year=year,
257
262
  birth_month=month,
258
263
  )
259
- return patient
264
+ return patient, _created
260
265
 
261
266
 
262
267
  def get_or_create_pseudo_patient_examination_logic(
263
268
  instance: "SensitiveMeta",
264
- ) -> "PatientExamination":
269
+ ):
265
270
  """Gets or creates the pseudo patient examination based on instance data."""
266
271
  # Ensure necessary fields are set
267
272
  if not instance.patient_hash:
@@ -270,9 +275,9 @@ def get_or_create_pseudo_patient_examination_logic(
270
275
  instance.examination_hash = calculate_examination_hash(instance)
271
276
 
272
277
  # Ensure the pseudo patient exists first, as PatientExamination might depend on it
273
- if not instance.pseudo_patient_id:
274
- pseudo_patient = get_or_create_pseudo_patient_logic(instance)
275
- instance.pseudo_patient_id = pseudo_patient.pk # Assign FK directly
278
+ if not instance.pseudo_patient:
279
+ pseudo_patient, _created = get_or_create_pseudo_patient_logic(instance)
280
+ instance.pseudo_patient = pseudo_patient # Assign FK directly
276
281
 
277
282
  patient_examination, _created = (
278
283
  PatientExamination.get_or_create_pseudo_patient_examination_by_hash(
@@ -282,7 +287,7 @@ def get_or_create_pseudo_patient_examination_logic(
282
287
  # pseudo_patient=instance.pseudo_patient
283
288
  )
284
289
  )
285
- return patient_examination
290
+ return patient_examination, _created
286
291
 
287
292
 
288
293
  @transaction.atomic # Ensure all operations within save succeed or fail together
@@ -294,7 +299,7 @@ def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
294
299
  This function is called on every save() operation and implements a two-phase approach:
295
300
 
296
301
  **Phase 1: Initial Creation (with defaults)**
297
- - When a SensitiveMeta is first created (e.g., via get_or_create_sensitive_meta()),
302
+ - When a SensitiveMeta is first created (e.g., via create_from_dict),
298
303
  it may have missing patient data (names, DOB, etc.)
299
304
  - Default values are set to prevent hash calculation errors:
300
305
  * patient_first_name: "unknown"
@@ -381,19 +386,19 @@ def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
381
386
  # Updated: patient_first_name = "Max" → hash = sha256("Max Mustermann...")
382
387
  #
383
388
  if not instance.patient_first_name:
384
- instance.patient_first_name = DEFAULT_UNKNOWN_NAME
389
+ instance.patient_first_name = DEFAULT_UNKNOWN
385
390
  logger.debug(
386
391
  "SensitiveMeta (pk=%s): Patient first name missing, set to default '%s'.",
387
392
  instance.pk or "new",
388
- DEFAULT_UNKNOWN_NAME,
393
+ DEFAULT_UNKNOWN,
389
394
  )
390
395
 
391
396
  if not instance.patient_last_name:
392
- instance.patient_last_name = DEFAULT_UNKNOWN_NAME
397
+ instance.patient_last_name = DEFAULT_UNKNOWN
393
398
  logger.debug(
394
399
  "SensitiveMeta (pk=%s): Patient last name missing, set to default '%s'.",
395
400
  instance.pk or "new",
396
- DEFAULT_UNKNOWN_NAME,
401
+ DEFAULT_UNKNOWN,
397
402
  )
398
403
 
399
404
  # 3. Ensure Gender exists (should be set before calling save, e.g., during creation/update)
@@ -426,13 +431,15 @@ def perform_save_logic(instance: "SensitiveMeta") -> "Examiner":
426
431
 
427
432
  # 5. Get or Create Pseudo Patient (depends on hash, center, gender, dob)
428
433
  # Assign directly to the FK field to avoid premature saving issues
429
- pseudo_patient = get_or_create_pseudo_patient_logic(instance)
430
- instance.pseudo_patient_id = pseudo_patient.pk
434
+ pseudo_patient, _created = get_or_create_pseudo_patient_logic(instance)
435
+ instance.pseudo_patient = pseudo_patient
431
436
 
432
437
  # 6. Get or Create Pseudo Examination (depends on hashes)
433
438
  # Assign directly to the FK field
434
- pseudo_examination = get_or_create_pseudo_patient_examination_logic(instance)
435
- instance.pseudo_examination_id = pseudo_examination.pk
439
+ pseudo_examination, _created = get_or_create_pseudo_patient_examination_logic(
440
+ instance
441
+ )
442
+ instance.pseudo_examination = pseudo_examination
436
443
 
437
444
  # 7. Get or Create Pseudo Examiner (depends on names, center)
438
445
  # This needs to happen *after* the main instance has a PK for M2M linking.
@@ -467,6 +474,8 @@ def create_sensitive_meta_from_dict(
467
474
  "patient_dob": date(1990, 1, 1),
468
475
  "examination_date": date.today(),
469
476
  "center": center_obj, # ← Center object
477
+ "text": text #from extraction
478
+
470
479
  }
471
480
  sm = SensitiveMeta.create_from_dict(data)
472
481
 
@@ -477,6 +486,7 @@ def create_sensitive_meta_from_dict(
477
486
  "patient_dob": date(1990, 1, 1),
478
487
  "examination_date": date.today(),
479
488
  "center_name": "university_hospital_wuerzburg", # ← String
489
+ "anonymized_text": "anonymized text"
480
490
  }
481
491
  sm = SensitiveMeta.create_from_dict(data)
482
492
  ```
@@ -620,6 +630,7 @@ def create_sensitive_meta_from_dict(
620
630
  exam_date,
621
631
  )
622
632
  selected_data.pop("examination_date", None)
633
+
623
634
  except Exception as e:
624
635
  logger.warning(
625
636
  "Error parsing examination_date string '%s': %s, removing from data",
@@ -653,8 +664,8 @@ def create_sensitive_meta_from_dict(
653
664
  )
654
665
 
655
666
  # Handle Names and Gender
656
- first_name = selected_data.get("patient_first_name") or DEFAULT_UNKNOWN_NAME
657
- last_name = selected_data.get("patient_last_name") or DEFAULT_UNKNOWN_NAME
667
+ first_name = selected_data.get("patient_first_name") or DEFAULT_UNKNOWN
668
+ last_name = selected_data.get("patient_last_name") or DEFAULT_UNKNOWN
658
669
  selected_data["patient_first_name"] = first_name # Ensure defaults are set
659
670
  selected_data["patient_last_name"] = last_name
660
671
 
@@ -674,7 +685,18 @@ def create_sensitive_meta_from_dict(
674
685
  f"Gender with name '{patient_gender_input}' provided but not found. Attempting to guess or use default."
675
686
  )
676
687
  # Fall through to guessing logic if provided string name is invalid
677
- patient_gender_input = None # Reset to trigger guessing
688
+ normalized = (patient_gender_input or "").lower()
689
+ if normalized in {"male", "female", "unknown"}:
690
+ gender_obj, _ = Gender.objects.get_or_create(
691
+ name=normalized,
692
+ defaults={
693
+ "abbreviation": normalized[:1].upper() or None,
694
+ "description": "Auto-created default gender entry",
695
+ },
696
+ )
697
+ selected_data["patient_gender"] = gender_obj
698
+ else:
699
+ patient_gender_input = None # Reset to trigger guessing
678
700
 
679
701
  if not isinstance(
680
702
  selected_data.get("patient_gender"), Gender
@@ -690,10 +712,63 @@ def create_sensitive_meta_from_dict(
690
712
  name=gender_name_to_use
691
713
  )
692
714
  except Gender.DoesNotExist:
693
- # This should ideally not happen if "unknown" gender is guaranteed to exist
694
- raise ValueError(
695
- f"Default or guessed gender '{gender_name_to_use}' does not exist in Gender table."
715
+ gender_obj, _ = Gender.objects.get_or_create(
716
+ name=gender_name_to_use,
717
+ defaults={
718
+ "abbreviation": gender_name_to_use[:1].upper() or None,
719
+ "description": "Auto-created default gender entry",
720
+ },
696
721
  )
722
+ selected_data["patient_gender"] = gender_obj
723
+
724
+ # Handle Text
725
+ selected_data["text"] = data.get("text") or DEFAULT_UNKNOWN
726
+
727
+ # --- Add missing optional fields safely ---
728
+ file_path = data.get("file_path")
729
+ if file_path:
730
+ selected_data["file_path"] = str(file_path)
731
+ logger.debug(f"Set file_path: {file_path}")
732
+
733
+ casenumber = data.get("casenumber")
734
+ if casenumber:
735
+ selected_data["casenumber"] = str(casenumber).strip()
736
+ logger.debug(f"Set casenumber: {casenumber}")
737
+
738
+ exam_time = data.get("examination_time")
739
+ if exam_time:
740
+ try:
741
+ from datetime import time as dt_time
742
+
743
+ # Accepts strings like "14:35" or full datetime
744
+ if isinstance(exam_time, str):
745
+ h, m = exam_time.strip().split(":")[:2]
746
+ selected_data["examination_time"] = dt_time(int(h), int(m))
747
+ elif isinstance(exam_time, datetime):
748
+ selected_data["examination_time"] = exam_time.time()
749
+ elif isinstance(exam_time, date):
750
+ # no time info — ignore
751
+ logger.debug(
752
+ f"examination_time value {exam_time} has no time component; skipping"
753
+ )
754
+ else:
755
+ selected_data["examination_time"] = exam_time
756
+ except Exception as e:
757
+ logger.warning(f"Invalid examination_time '{exam_time}': {e}")
758
+
759
+ anonymized_text = data.get("anonymized_text") or data.get("anonym_text")
760
+ if anonymized_text:
761
+ if isinstance(anonymized_text, (str, bytes)):
762
+ selected_data["anonymized_text"] = (
763
+ anonymized_text.decode()
764
+ if isinstance(anonymized_text, bytes)
765
+ else anonymized_text
766
+ )
767
+ else:
768
+ selected_data["anonymized_text"] = str(anonymized_text)
769
+ logger.debug(
770
+ "Set anonymized_text (length=%d)", len(selected_data["anonymized_text"])
771
+ )
697
772
 
698
773
  # Update name DB
699
774
  update_name_db(first_name, last_name)
@@ -715,7 +790,7 @@ def update_sensitive_meta_from_dict(
715
790
 
716
791
  **Integration with two-phase save pattern:**
717
792
  This function is typically called after initial SensitiveMeta creation when real
718
- patient data becomes available (e.g., extracted from video OCR, PDF parsing, or
793
+ patient data becomes available (e.g., extracted from video OCR, report parsing, or
719
794
  manual annotation).
720
795
 
721
796
  **Example workflow:**
@@ -875,6 +950,36 @@ def update_sensitive_meta_from_dict(
875
950
  )
876
951
  selected_data.pop("patient_gender", None)
877
952
 
953
+ # TODO Review: Handle new optional fields on update
954
+ for key in (
955
+ "file_path",
956
+ "casenumber",
957
+ "examination_time",
958
+ "anonymized_text",
959
+ "anonym_text",
960
+ ):
961
+ if key in data and data[key] is not None:
962
+ val = data[key]
963
+ if key in ("file_path", "casenumber"):
964
+ setattr(instance, key, str(val))
965
+ elif key in ("anonymized_text", "anonym_text"):
966
+ setattr(
967
+ instance,
968
+ "anonymized_text",
969
+ val if isinstance(val, str) else str(val),
970
+ )
971
+ elif key == "examination_time":
972
+ try:
973
+ from datetime import time as dt_time
974
+
975
+ if isinstance(val, str) and ":" in val:
976
+ h, m = val.strip().split(":")[:2]
977
+ setattr(instance, "examination_time", dt_time(int(h), int(m)))
978
+ elif isinstance(val, datetime):
979
+ setattr(instance, "examination_time", val.time())
980
+ except Exception as e:
981
+ logger.warning(f"Skipping invalid examination_time '{val}': {e}")
982
+
878
983
  # Update other attributes from selected_data
879
984
  patient_name_changed = False
880
985
  for k, v in selected_data.items():
@@ -903,70 +1008,28 @@ def update_sensitive_meta_from_dict(
903
1008
  aware_dob,
904
1009
  )
905
1010
  elif isinstance(v, str):
906
- # Handle string DOB - check if it's a field name or actual date
907
- if v == "patient_dob" or v in [
908
- "patient_first_name",
909
- "patient_last_name",
910
- "examination_date",
911
- ]:
912
- logger.warning(
913
- "Skipping invalid patient_dob value '%s' during update - appears to be field name",
1011
+ parsed = parse_any_date(v)
1012
+ if parsed:
1013
+ aware_dob = timezone.make_aware(
1014
+ datetime.combine(parsed, datetime.min.time())
1015
+ )
1016
+ value_to_set = aware_dob
1017
+ logger.debug(
1018
+ "Parsed string patient_dob '%s' during update to aware datetime: %s",
914
1019
  v,
1020
+ aware_dob,
915
1021
  )
916
- continue # Skip this field
917
1022
  else:
918
- # Try to parse as date string
919
- try:
920
- import dateparser
921
-
922
- parsed_dob = dateparser.parse(
923
- v, languages=["de"], settings={"DATE_ORDER": "DMY"}
924
- )
925
- if parsed_dob:
926
- value_to_set = timezone.make_aware(
927
- parsed_dob.replace(
928
- hour=0, minute=0, second=0, microsecond=0
929
- )
930
- )
931
- logger.debug(
932
- "Parsed string patient_dob '%s' during update to aware datetime: %s",
933
- v,
934
- value_to_set,
935
- )
936
- else:
937
- logger.warning(
938
- "Could not parse patient_dob string '%s' during update, skipping",
939
- v,
940
- )
941
- continue
942
- except Exception as e:
943
- logger.warning(
944
- "Error parsing patient_dob string '%s' during update: %s, skipping",
945
- v,
946
- e,
947
- )
948
- continue
949
- elif k == "examination_date" and isinstance(v, str):
950
- if v == "examination_date" or v in [
951
- "patient_first_name",
952
- "patient_last_name",
953
- "patient_dob",
954
- ]:
955
- logger.warning(
956
- "Skipping invalid examination_date value '%s' during update - appears to be field name",
957
- v,
958
- )
959
- continue
960
- else:
961
- # Try to parse as date string
962
- try:
963
- import dateparser
964
-
965
- parsed_date = dateparser.parse(
966
- v, languages=["de"], settings={"DATE_ORDER": "DMY"}
1023
+ logger.warning(
1024
+ "Could not parse patient_dob string '%s' during update, skipping",
1025
+ v,
967
1026
  )
968
- if parsed_date:
969
- value_to_set = parsed_date.date()
1027
+ continue
1028
+ elif k == "examination_date":
1029
+ if isinstance(v, str):
1030
+ parsed = parse_any_date(v)
1031
+ if parsed:
1032
+ value_to_set = parsed # field is DateField, so keep it as date
970
1033
  logger.debug(
971
1034
  "Parsed string examination_date '%s' during update to date: %s",
972
1035
  v,
@@ -978,23 +1041,19 @@ def update_sensitive_meta_from_dict(
978
1041
  v,
979
1042
  )
980
1043
  continue
981
- except Exception as e:
982
- logger.warning(
983
- "Error parsing examination_date string '%s' during update: %s, skipping",
984
- v,
985
- e,
986
- )
987
- continue
988
- # --- End Conversion ---
1044
+ elif isinstance(v, date):
1045
+ value_to_set = v
989
1046
 
990
- # Check if patient name is changing
991
- if (
992
- k in ["patient_first_name", "patient_last_name"]
993
- and getattr(instance, k) != value_to_set
994
- ):
995
- patient_name_changed = True
1047
+ # --- End Conversion ---
996
1048
 
997
- setattr(instance, k, value_to_set) # Use value_to_set
1049
+ # Check if patient name is changing
1050
+ if (
1051
+ k in ["patient_first_name", "patient_last_name"]
1052
+ and getattr(instance, k) != value_to_set
1053
+ ):
1054
+ patient_name_changed = True
1055
+
1056
+ setattr(instance, k, value_to_set) # Use value_to_set
998
1057
 
999
1058
  except Exception as e:
1000
1059
  logger.error(
@@ -1023,15 +1082,21 @@ def update_or_create_sensitive_meta_from_dict(
1023
1082
  cls: Type["SensitiveMeta"],
1024
1083
  data: Dict[str, Any],
1025
1084
  instance: Optional["SensitiveMeta"] = None,
1026
- ) -> "SensitiveMeta":
1085
+ ):
1027
1086
  """Logic to update or create a SensitiveMeta instance from a dictionary."""
1028
1087
  # Check if the instance already exists based on unique fields
1088
+ sensitive_meta: "SensitiveMeta"
1089
+ _created: bool
1029
1090
  if instance:
1030
1091
  # Update the existing instance
1031
- return update_sensitive_meta_from_dict(instance, data), False
1092
+ sensitive_meta = update_sensitive_meta_from_dict(instance, data)
1093
+ _created = False
1094
+
1032
1095
  else:
1033
1096
  # Create a new instance
1034
- return create_sensitive_meta_from_dict(cls, data), True
1097
+ sensitive_meta = create_sensitive_meta_from_dict(cls, data)
1098
+ _created = True
1099
+ return sensitive_meta, _created
1035
1100
 
1036
1101
 
1037
1102
  def _map_gender_string_to_standard(gender_str: str) -> Optional[str]:
@@ -1046,3 +1111,33 @@ def _map_gender_string_to_standard(gender_str: str) -> Optional[str]:
1046
1111
  if gender_lower in variants:
1047
1112
  return standard
1048
1113
  return None
1114
+
1115
+
1116
+ def _create_anonymized_record(
1117
+ instance: "SensitiveMeta",
1118
+ DEFAULT_ANONYMIZED=None,
1119
+ DEFAULT_ANONYMIZED_DATE=timezone.make_aware(datetime(1900, 1, 1)),
1120
+ ) -> None:
1121
+ """
1122
+ Create a SensitiveMeta instance with all sensitive fields set to anonymized defaults.
1123
+ This is only called after anonymization and will delete all data that can identify a patient from the database.
1124
+ What is left will only be the patient hash.
1125
+
1126
+ Args:
1127
+ instance: The existing SensitiveMeta instance to anonymize
1128
+ DEFAULT_ANONYMIZED: Usually None, The default string to use for anonymized fields (e.g., "anonymized,")
1129
+ """
1130
+
1131
+ instance.refresh_from_db()
1132
+ instance.get_patient_hash()
1133
+ instance.get_patient_examination_hash()
1134
+
1135
+ anonymized_data = {
1136
+ "patient_first_name": DEFAULT_ANONYMIZED,
1137
+ "patient_last_name": DEFAULT_ANONYMIZED,
1138
+ "patient_dob": DEFAULT_ANONYMIZED_DATE,
1139
+ "examination_date": DEFAULT_ANONYMIZED_DATE,
1140
+ }
1141
+ sensitive_meta = update_sensitive_meta_from_dict(instance, anonymized_data)
1142
+
1143
+ sensitive_meta.save()
@@ -1,10 +1,10 @@
1
- from django.db import models
2
1
  import logging
3
2
  from pathlib import Path
4
- from typing import Optional, TYPE_CHECKING
3
+ from typing import TYPE_CHECKING, Optional
5
4
 
6
5
  # import endoreg_center_id from django settings
7
6
  from django.conf import settings
7
+ from django.db import models
8
8
 
9
9
  # check if endoreg_center_id is set
10
10
  if not hasattr(settings, "ENDOREG_CENTER_ID"):
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
19
19
 
20
20
  if TYPE_CHECKING:
21
21
  from ..administration import Center
22
- from ..medical.hardware import EndoscopyProcessor, Endoscope
22
+ from ..medical.hardware import Endoscope, EndoscopyProcessor
23
23
 
24
24
 
25
25
  # VideoMeta
@@ -29,19 +29,40 @@ class VideoMeta(models.Model):
29
29
 
30
30
  Links to hardware (processor, endoscope), center, import details, and FFmpeg technical specs.
31
31
  """
32
- processor = models.ForeignKey(
33
- "EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
34
- )
35
- endoscope = models.ForeignKey(
36
- "Endoscope", on_delete=models.CASCADE, blank=True, null=True
37
- )
32
+
33
+ processor = models.ForeignKey("EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True)
34
+ endoscope = models.ForeignKey("Endoscope", on_delete=models.CASCADE, blank=True, null=True)
38
35
  center = models.ForeignKey("Center", on_delete=models.CASCADE)
39
- import_meta = models.OneToOneField(
40
- "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
41
- )
42
- ffmpeg_meta = models.OneToOneField(
43
- "FFMpegMeta", on_delete=models.CASCADE, blank=True, null=True
44
- )
36
+ import_meta = models.OneToOneField("VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True)
37
+ ffmpeg_meta = models.OneToOneField("FFMpegMeta", on_delete=models.CASCADE, blank=True, null=True)
38
+
39
+ if TYPE_CHECKING:
40
+ processor: models.ForeignKey["EndoscopyProcessor|None"]
41
+ endoscope: models.ForeignKey["Endoscope|None"]
42
+ center: models.ForeignKey["Center|None"]
43
+ import_meta: models.OneToOneField["VideoImportMeta|None"]
44
+ ffmpeg_meta: models.OneToOneField["FFMpegMeta|None"]
45
+
46
+ @property
47
+ def center_safe(self) -> "Center":
48
+ center = self.center
49
+ if not center:
50
+ raise Center.DoesNotExist("Center does not exist for this VideoMeta instance.")
51
+ return center
52
+
53
+ @property
54
+ def processor_safe(self) -> "EndoscopyProcessor":
55
+ processor = self.processor
56
+ if not processor:
57
+ raise EndoscopyProcessor.DoesNotExist("EndoscopyProcessor does not exist for this VideoMeta instance.")
58
+ return processor
59
+
60
+ @property
61
+ def ffmpeg_meta_safe(self) -> "FFMpegMeta":
62
+ ffmpeg_meta = self.ffmpeg_meta
63
+ if not ffmpeg_meta:
64
+ raise FFMpegMeta.DoesNotExist("FFMpegMeta does not exist for this VideoMeta instance.")
65
+ return ffmpeg_meta
45
66
 
46
67
  @classmethod
47
68
  def create_from_file(
@@ -71,7 +92,7 @@ class VideoMeta(models.Model):
71
92
  raise RuntimeError(f"Failed to initialize FFMpeg metadata for {video_path.name}") from e
72
93
 
73
94
  if save_instance:
74
- meta.save() # This ensures VideoImportMeta is created too
95
+ meta.save() # This ensures VideoImportMeta is created too
75
96
  logger.info("Created and saved VideoMeta instance PK %s from %s", meta.pk, video_path.name)
76
97
  else:
77
98
  logger.info("Instantiated VideoMeta from %s (not saved yet)", video_path.name)
@@ -120,8 +141,8 @@ class VideoMeta(models.Model):
120
141
  # If the VideoMeta instance is already saved, save the link immediately.
121
142
  # Otherwise, the link will be saved when VideoMeta itself is saved.
122
143
  if self.pk:
123
- self.save(update_fields=['ffmpeg_meta'])
124
- logger.info("Successfully created and linked FFMpegMeta PK %s", self.ffmpeg_meta.pk)
144
+ self.save(update_fields=["ffmpeg_meta"])
145
+ logger.info("Successfully created and linked FFMpegMeta PK %s", self.ffmpeg_meta_safe.pk)
125
146
 
126
147
  except Exception as e:
127
148
  # Log the error and re-raise it
@@ -140,8 +161,8 @@ class VideoMeta(models.Model):
140
161
  logger.debug("Deleting existing FFMpegMeta PK %s before update.", existing_ffmpeg_pk)
141
162
  # Nullify the relation first before deleting the related object
142
163
  self.ffmpeg_meta = None
143
- self.save(update_fields=['ffmpeg_meta']) # Save the null relation
144
- FFMpegMeta.objects.filter(pk=existing_ffmpeg_pk).delete() # Delete the old object
164
+ self.save(update_fields=["ffmpeg_meta"]) # Save the null relation
165
+ FFMpegMeta.objects.filter(pk=existing_ffmpeg_pk).delete() # Delete the old object
145
166
 
146
167
  # initialize_ffmpeg_meta handles creation, linking, saving the link, and raises exceptions
147
168
  self.initialize_ffmpeg_meta(video_path)
@@ -150,7 +171,7 @@ class VideoMeta(models.Model):
150
171
  """Retrieves the endoscope region of interest (ROI) from the associated processor."""
151
172
  from ..medical.hardware import EndoscopyProcessor
152
173
 
153
- processor: EndoscopyProcessor = self.processor
174
+ processor: EndoscopyProcessor = self.processor_safe
154
175
  endo_roi = processor.get_roi_endoscope_image()
155
176
  return endo_roi
156
177
 
@@ -189,6 +210,7 @@ class FFMpegMeta(models.Model):
189
210
  """
190
211
  Stores technical video stream information extracted using FFmpeg (ffprobe).
191
212
  """
213
+
192
214
  width = models.IntegerField(null=True, blank=True)
193
215
  height = models.IntegerField(null=True, blank=True)
194
216
  duration = models.FloatField(null=True, blank=True) # Duration in seconds
@@ -219,7 +241,6 @@ class FFMpegMeta(models.Model):
219
241
  logger.error("ffprobe execution failed for %s: %s", file_path, probe_err, exc_info=True)
220
242
  raise RuntimeError(f"ffprobe execution failed for {file_path}") from probe_err
221
243
 
222
-
223
244
  if not probe_data or "streams" not in probe_data:
224
245
  logger.error("Failed to get valid stream info from ffprobe for %s", file_path)
225
246
  # Raise exception instead of returning None
@@ -237,8 +258,8 @@ class FFMpegMeta(models.Model):
237
258
  height = video_stream.get("height")
238
259
  duration_str = video_stream.get("duration")
239
260
  # --- FIX: Handle potential format key ---
240
- if duration_str is None and 'format' in probe_data and 'duration' in probe_data['format']:
241
- duration_str = probe_data['format']['duration']
261
+ if duration_str is None and "format" in probe_data and "duration" in probe_data["format"]:
262
+ duration_str = probe_data["format"]["duration"]
242
263
  logger.debug("Using duration from format block: %s", duration_str)
243
264
  # --- End Fix ---
244
265
  duration = float(duration_str) if duration_str else None
@@ -253,10 +274,10 @@ class FFMpegMeta(models.Model):
253
274
  frame_rate_num, frame_rate_den = None, None
254
275
  if frame_rate_str and "/" in frame_rate_str:
255
276
  try:
256
- num_str, den_str = frame_rate_str.split('/')
277
+ num_str, den_str = frame_rate_str.split("/")
257
278
  frame_rate_num = int(num_str)
258
279
  frame_rate_den = int(den_str)
259
- if frame_rate_den == 0: # Avoid division by zero
280
+ if frame_rate_den == 0: # Avoid division by zero
260
281
  logger.warning("Invalid frame rate denominator (0) for %s", file_path)
261
282
  frame_rate_num, frame_rate_den = None, None
262
283
  except ValueError:
@@ -267,8 +288,8 @@ class FFMpegMeta(models.Model):
267
288
  pixel_format = video_stream.get("pix_fmt")
268
289
  bit_rate_str = video_stream.get("bit_rate")
269
290
  # --- FIX: Handle potential format key for bit_rate ---
270
- if bit_rate_str is None and 'format' in probe_data and 'bit_rate' in probe_data['format']:
271
- bit_rate_str = probe_data['format']['bit_rate']
291
+ if bit_rate_str is None and "format" in probe_data and "bit_rate" in probe_data["format"]:
292
+ bit_rate_str = probe_data["format"]["bit_rate"]
272
293
  logger.debug("Using bit_rate from format block: %s", bit_rate_str)
273
294
  # --- End Fix ---
274
295
  bit_rate = int(bit_rate_str) if bit_rate_str else None
@@ -311,6 +332,7 @@ class VideoImportMeta(models.Model):
311
332
  """
312
333
  Stores metadata related to the import and processing status of a video.
313
334
  """
335
+
314
336
  file_name = models.CharField(max_length=255, blank=True, null=True)
315
337
  video_anonymized = models.BooleanField(default=False)
316
338
  video_patient_data_detected = models.BooleanField(default=False)
@@ -323,9 +345,7 @@ class VideoImportMeta(models.Model):
323
345
  result_html = ""
324
346
 
325
347
  result_html += f"Video anonymized: {self.video_anonymized}\n"
326
- result_html += (
327
- f"Video patient data detected: {self.video_patient_data_detected}\n"
328
- )
348
+ result_html += f"Video patient data detected: {self.video_patient_data_detected}\n"
329
349
  result_html += f"Outside detected: {self.outside_detected}\n"
330
350
  result_html += f"Patient data removed: {self.patient_data_removed}\n"
331
351
  result_html += f"Outside removed: {self.outside_removed}\n"