endoreg-db 0.4.5__py3-none-any.whl → 0.8.6.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (846) hide show
  1. endoreg_db/admin.py +90 -1
  2. endoreg_db/api_urls.py +4 -0
  3. endoreg_db/apps.py +12 -0
  4. endoreg_db/assets/dummy_model.ckpt +1 -0
  5. endoreg_db/codemods/readme.md +88 -0
  6. endoreg_db/codemods/rename_datetime_fields.py +92 -0
  7. endoreg_db/config/env.py +101 -0
  8. endoreg_db/data/__init__.py +76 -4
  9. endoreg_db/data/ai_model/data.yaml +7 -0
  10. endoreg_db/data/{label → ai_model_label}/label/data.yaml +27 -1
  11. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  12. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  13. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  14. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  15. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  16. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  17. endoreg_db/data/center/data.yaml +40 -9
  18. endoreg_db/data/center_shift/ukw.yaml +9 -0
  19. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  20. endoreg_db/data/db_summary.csv +58 -0
  21. endoreg_db/data/db_summary.xlsx +0 -0
  22. endoreg_db/data/disease/misc.yaml +1 -2
  23. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +2 -2
  24. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +6 -6
  25. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  26. endoreg_db/data/endoscope/data.yaml +93 -0
  27. endoreg_db/data/endoscopy_processor/data.yaml +3 -0
  28. endoreg_db/data/event/cardiology.yaml +0 -13
  29. endoreg_db/data/examination/examinations/data.yaml +34 -28
  30. endoreg_db/data/examination/type/data.yaml +12 -0
  31. endoreg_db/data/examination_indication/endoscopy.yaml +424 -0
  32. endoreg_db/data/examination_indication_classification/endoscopy.yaml +160 -0
  33. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +101 -0
  34. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  35. endoreg_db/data/finding/anatomy_colon.yaml +128 -0
  36. endoreg_db/data/finding/colonoscopy.yaml +40 -0
  37. endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +56 -0
  38. endoreg_db/data/finding/complication.yaml +16 -0
  39. endoreg_db/data/finding/data.yaml +105 -0
  40. endoreg_db/data/finding/examination_setting.yaml +16 -0
  41. endoreg_db/data/finding/medication_related.yaml +18 -0
  42. endoreg_db/data/finding/outcome.yaml +12 -0
  43. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +95 -0
  44. endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +22 -0
  45. endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +25 -0
  46. endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  47. endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  48. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  49. endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  50. endoreg_db/data/finding_classification/colonoscopy_location.yaml +80 -0
  51. endoreg_db/data/finding_classification/colonoscopy_lst.yaml +21 -0
  52. endoreg_db/data/finding_classification/colonoscopy_nice.yaml +20 -0
  53. endoreg_db/data/finding_classification/colonoscopy_paris.yaml +26 -0
  54. endoreg_db/data/finding_classification/colonoscopy_sano.yaml +22 -0
  55. endoreg_db/data/finding_classification/colonoscopy_summary.yaml +53 -0
  56. endoreg_db/data/finding_classification/complication_generic.yaml +25 -0
  57. endoreg_db/data/finding_classification/examination_setting_generic.yaml +40 -0
  58. endoreg_db/data/finding_classification/histology_colo.yaml +51 -0
  59. endoreg_db/data/finding_classification/intervention_required.yaml +26 -0
  60. endoreg_db/data/finding_classification/medication_related.yaml +23 -0
  61. endoreg_db/data/finding_classification/visualized.yaml +33 -0
  62. endoreg_db/data/finding_classification_choice/bowel_preparation.yaml +78 -0
  63. endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
  64. endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  65. endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  66. endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  67. endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +17 -0
  68. endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +57 -0
  69. endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
  70. endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  71. endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
  72. endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +229 -0
  73. endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  74. endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +82 -0
  75. endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  76. endoreg_db/data/finding_classification_choice/complication_generic_types.yaml +15 -0
  77. endoreg_db/data/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  78. endoreg_db/data/finding_classification_choice/histology.yaml +24 -0
  79. endoreg_db/data/finding_classification_choice/histology_polyp.yaml +20 -0
  80. endoreg_db/data/finding_classification_choice/outcome.yaml +19 -0
  81. endoreg_db/data/finding_classification_choice/yes_no_na.yaml +11 -0
  82. endoreg_db/data/finding_classification_type/colonoscopy_basic.yaml +48 -0
  83. endoreg_db/data/finding_intervention/endoscopy.yaml +43 -0
  84. endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
  85. endoreg_db/data/finding_intervention/endoscopy_egd.yaml +128 -0
  86. endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +32 -0
  87. endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +9 -0
  88. endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +36 -0
  89. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  90. endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +79 -0
  91. endoreg_db/data/finding_type/data.yaml +43 -0
  92. endoreg_db/data/gender/data.yaml +24 -0
  93. endoreg_db/data/information_source/annotation.yaml +6 -0
  94. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  95. endoreg_db/data/information_source/prediction.yaml +7 -0
  96. endoreg_db/data/information_source_type/data.yaml +8 -0
  97. endoreg_db/data/lab_value/cardiac_enzymes.yaml +7 -1
  98. endoreg_db/data/lab_value/coagulation.yaml +6 -1
  99. endoreg_db/data/lab_value/electrolytes.yaml +39 -1
  100. endoreg_db/data/lab_value/gastrointestinal_function.yaml +12 -0
  101. endoreg_db/data/lab_value/hematology.yaml +17 -2
  102. endoreg_db/data/lab_value/hormones.yaml +6 -0
  103. endoreg_db/data/lab_value/lipids.yaml +12 -3
  104. endoreg_db/data/lab_value/misc.yaml +48 -2
  105. endoreg_db/data/lab_value/renal_function.yaml +2 -1
  106. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  107. endoreg_db/data/lx_client_type/base.yaml +30 -0
  108. endoreg_db/data/lx_permission/base.yaml +24 -0
  109. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  110. endoreg_db/data/medication/anticoagulation.yaml +5 -5
  111. endoreg_db/data/medication/tah.yaml +5 -5
  112. endoreg_db/data/medication_indication/anticoagulation.yaml +48 -53
  113. endoreg_db/data/medication_intake_time/base.yaml +4 -4
  114. endoreg_db/data/names_first/first_names.yaml +54 -0
  115. endoreg_db/data/names_last/last_names.yaml +51 -0
  116. endoreg_db/data/network_device/data.yaml +30 -0
  117. endoreg_db/data/organ/data.yaml +29 -0
  118. endoreg_db/data/pdf_type/data.yaml +27 -9
  119. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  120. endoreg_db/data/qualification/m2.yaml +39 -0
  121. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  122. endoreg_db/data/qualification/sonography.yaml +36 -0
  123. endoreg_db/data/qualification_type/base.yaml +29 -0
  124. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  125. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +4 -0
  126. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +5 -0
  127. endoreg_db/data/requirement/age.yaml +26 -0
  128. endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +45 -0
  129. endoreg_db/data/requirement/disease_cardiovascular.yaml +79 -0
  130. endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
  131. endoreg_db/data/requirement/disease_hepatology.yaml +12 -0
  132. endoreg_db/data/requirement/disease_misc.yaml +12 -0
  133. endoreg_db/data/requirement/disease_renal.yaml +96 -0
  134. endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +59 -0
  135. endoreg_db/data/requirement/event_cardiology.yaml +251 -0
  136. endoreg_db/data/requirement/event_requirements.yaml +145 -0
  137. endoreg_db/data/requirement/finding_colon_polyp.yaml +50 -0
  138. endoreg_db/data/requirement/gender.yaml +25 -0
  139. endoreg_db/data/requirement/lab_value.yaml +441 -0
  140. endoreg_db/data/requirement/medication.yaml +93 -0
  141. endoreg_db/data/requirement_operator/age.yaml +13 -0
  142. endoreg_db/data/requirement_operator/lab_operators.yaml +129 -0
  143. endoreg_db/data/requirement_operator/model_operators.yaml +96 -0
  144. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +48 -0
  145. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  146. endoreg_db/data/requirement_set/endoscopy_bleeding_risk.yaml +52 -0
  147. endoreg_db/data/requirement_set_type/data.yaml +20 -0
  148. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  149. endoreg_db/data/risk/bleeding.yaml +26 -0
  150. endoreg_db/data/risk/thrombosis.yaml +37 -0
  151. endoreg_db/data/risk_type/data.yaml +27 -0
  152. endoreg_db/data/setup_config.yaml +38 -0
  153. endoreg_db/data/shift/endoscopy.yaml +21 -0
  154. endoreg_db/data/shift_type/base.yaml +35 -0
  155. endoreg_db/data/tag/requirement_set_tags.yaml +11 -0
  156. endoreg_db/data/unit/concentration.yaml +23 -0
  157. endoreg_db/data/unit/time.yaml +36 -1
  158. endoreg_db/exceptions.py +19 -0
  159. endoreg_db/forms/__init__.py +3 -1
  160. endoreg_db/forms/examination_form.py +11 -0
  161. endoreg_db/forms/patient_finding_intervention_form.py +18 -0
  162. endoreg_db/forms/patient_form.py +27 -0
  163. endoreg_db/forms/questionnaires/__init__.py +1 -1
  164. endoreg_db/forms/questionnaires/tto_questionnaire.py +19 -19
  165. endoreg_db/helpers/count_db.py +45 -0
  166. endoreg_db/helpers/data_loader.py +208 -0
  167. endoreg_db/helpers/default_objects.py +378 -0
  168. endoreg_db/helpers/download_segmentation_model.py +31 -0
  169. endoreg_db/helpers/interact.py +6 -0
  170. endoreg_db/helpers/test_video_helper.py +119 -0
  171. endoreg_db/logger_conf.py +140 -0
  172. endoreg_db/management/__init__.py +1 -0
  173. endoreg_db/management/commands/__init__.py +1 -0
  174. endoreg_db/management/commands/anonymize_video.py +0 -0
  175. endoreg_db/management/commands/check_auth.py +125 -0
  176. endoreg_db/management/commands/create_model_meta_from_huggingface.py +115 -0
  177. endoreg_db/management/commands/create_multilabel_model_meta.py +214 -0
  178. endoreg_db/management/commands/fix_missing_patient_data.py +172 -0
  179. endoreg_db/management/commands/fix_video_paths.py +165 -0
  180. endoreg_db/management/commands/import_fallback_video.py +203 -0
  181. endoreg_db/management/commands/import_report.py +298 -0
  182. endoreg_db/management/commands/import_video.py +423 -0
  183. endoreg_db/management/commands/import_video_with_classification.py +367 -0
  184. endoreg_db/management/commands/init_default_ai_model.py +112 -0
  185. endoreg_db/management/commands/load_ai_model_data.py +58 -26
  186. endoreg_db/management/commands/load_ai_model_label_data.py +59 -0
  187. endoreg_db/management/commands/load_base_db_data.py +174 -118
  188. endoreg_db/management/commands/load_center_data.py +46 -21
  189. endoreg_db/management/commands/{load_logging_data.py → load_contraindication_data.py} +4 -2
  190. endoreg_db/management/commands/load_disease_data.py +29 -7
  191. endoreg_db/management/commands/{load_endoscope_type_data.py → load_endoscope_data.py} +30 -7
  192. endoreg_db/management/commands/load_examination_indication_data.py +86 -0
  193. endoreg_db/management/commands/load_finding_data.py +128 -0
  194. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +0 -1
  195. endoreg_db/management/commands/load_information_source.py +13 -7
  196. endoreg_db/management/commands/load_lab_value_data.py +3 -3
  197. endoreg_db/management/commands/load_medication_data.py +83 -21
  198. endoreg_db/management/commands/load_name_data.py +37 -0
  199. endoreg_db/management/commands/{load_medication_intake_time_data.py → load_organ_data.py} +7 -5
  200. endoreg_db/management/commands/load_qualification_data.py +59 -0
  201. endoreg_db/management/commands/load_requirement_data.py +180 -0
  202. endoreg_db/management/commands/load_risk_data.py +56 -0
  203. endoreg_db/management/commands/load_shift_data.py +60 -0
  204. endoreg_db/management/commands/load_tag_data.py +57 -0
  205. endoreg_db/management/commands/register_ai_model.py +1 -1
  206. endoreg_db/management/commands/setup_endoreg_db.py +381 -0
  207. endoreg_db/management/commands/start_filewatcher.py +106 -0
  208. endoreg_db/management/commands/storage_management.py +548 -0
  209. endoreg_db/management/commands/summarize_db_content.py +189 -0
  210. endoreg_db/management/commands/validate_video.py +204 -0
  211. endoreg_db/management/commands/validate_video_files.py +161 -0
  212. endoreg_db/management/commands/video_validation.py +22 -0
  213. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  214. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  215. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  216. endoreg_db/mermaid/classification.md +8 -0
  217. endoreg_db/mermaid/examination.md +8 -0
  218. endoreg_db/mermaid/findings.md +7 -0
  219. endoreg_db/mermaid/image_classification.md +28 -0
  220. endoreg_db/mermaid/interventions.md +8 -0
  221. endoreg_db/mermaid/morphology.md +8 -0
  222. endoreg_db/mermaid/patient_creation.md +14 -0
  223. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  224. endoreg_db/migrations/0001_initial.py +1234 -944
  225. endoreg_db/migrations/0002_add_video_correction_models.py +52 -0
  226. endoreg_db/migrations/0003_add_center_display_name.py +30 -0
  227. endoreg_db/models/__init__.py +339 -53
  228. endoreg_db/models/administration/__init__.py +116 -0
  229. endoreg_db/models/administration/ai/__init__.py +9 -0
  230. endoreg_db/models/administration/ai/active_model.py +35 -0
  231. endoreg_db/models/administration/ai/ai_model.py +156 -0
  232. endoreg_db/models/{ai_model → administration/ai}/model_type.py +19 -4
  233. endoreg_db/models/administration/case/__init__.py +19 -0
  234. endoreg_db/models/administration/case/case.py +114 -0
  235. endoreg_db/models/{case_template → administration/case/case_template}/__init__.py +10 -1
  236. endoreg_db/models/{case_template → administration/case/case_template}/case_template.py +60 -16
  237. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule.py +6 -13
  238. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule_value.py +21 -8
  239. endoreg_db/models/{case_template → administration/case/case_template}/case_template_type.py +1 -3
  240. endoreg_db/models/{center → administration/center}/__init__.py +9 -0
  241. endoreg_db/models/administration/center/center.py +67 -0
  242. endoreg_db/models/administration/center/center_product.py +64 -0
  243. endoreg_db/models/administration/center/center_resource.py +49 -0
  244. endoreg_db/models/administration/center/center_shift.py +88 -0
  245. endoreg_db/models/administration/center/center_waste.py +30 -0
  246. endoreg_db/models/administration/permissions/__init__.py +44 -0
  247. endoreg_db/models/administration/person/__init__.py +24 -0
  248. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  249. endoreg_db/models/administration/person/employee/employee.py +35 -0
  250. endoreg_db/models/administration/person/employee/employee_qualification.py +39 -0
  251. endoreg_db/models/administration/person/employee/employee_type.py +42 -0
  252. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  253. endoreg_db/models/administration/person/examiner/examiner.py +54 -0
  254. endoreg_db/models/administration/person/names/__init__.py +0 -0
  255. endoreg_db/models/{persons → administration/person/names}/first_name.py +1 -1
  256. endoreg_db/models/{persons → administration/person/names}/last_name.py +2 -3
  257. endoreg_db/models/administration/person/patient/__init__.py +5 -0
  258. endoreg_db/models/administration/person/patient/patient.py +460 -0
  259. endoreg_db/models/administration/person/profession/__init__.py +24 -0
  260. endoreg_db/models/administration/person/user/__init__.py +5 -0
  261. endoreg_db/models/administration/person/user/portal_user_information.py +37 -0
  262. endoreg_db/models/administration/product/__init__.py +14 -0
  263. endoreg_db/models/administration/product/product.py +97 -0
  264. endoreg_db/models/administration/product/product_group.py +39 -0
  265. endoreg_db/models/administration/product/product_material.py +54 -0
  266. endoreg_db/models/{product → administration/product}/product_weight.py +21 -0
  267. endoreg_db/models/{product → administration/product}/reference_product.py +44 -13
  268. endoreg_db/models/administration/qualification/__init__.py +7 -0
  269. endoreg_db/models/administration/qualification/qualification.py +37 -0
  270. endoreg_db/models/administration/qualification/qualification_type.py +35 -0
  271. endoreg_db/models/administration/shift/__init__.py +9 -0
  272. endoreg_db/models/administration/shift/scheduled_days.py +69 -0
  273. endoreg_db/models/administration/shift/shift.py +51 -0
  274. endoreg_db/models/administration/shift/shift_type.py +108 -0
  275. endoreg_db/models/label/__init__.py +24 -1
  276. endoreg_db/models/label/annotation/__init__.py +12 -0
  277. endoreg_db/models/label/annotation/image_classification.py +84 -0
  278. endoreg_db/models/label/annotation/video_segmentation_annotation.py +66 -0
  279. endoreg_db/models/label/label.py +53 -54
  280. endoreg_db/models/label/label_set.py +53 -0
  281. endoreg_db/models/label/label_type.py +29 -0
  282. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  283. endoreg_db/models/label/label_video_segment/_create_from_video.py +41 -0
  284. endoreg_db/models/label/label_video_segment/label_video_segment.py +511 -0
  285. endoreg_db/models/label/video_segmentation_label.py +31 -0
  286. endoreg_db/models/label/video_segmentation_labelset.py +27 -0
  287. endoreg_db/models/media/__init__.py +16 -0
  288. endoreg_db/models/media/frame/__init__.py +3 -0
  289. endoreg_db/models/media/frame/frame.py +111 -0
  290. endoreg_db/models/media/pdf/__init__.py +11 -0
  291. endoreg_db/models/media/pdf/raw_pdf.py +757 -0
  292. endoreg_db/models/media/pdf/report_file.py +162 -0
  293. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  294. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +77 -0
  295. endoreg_db/models/media/video/__init__.py +8 -0
  296. endoreg_db/models/media/video/create_from_file.py +358 -0
  297. endoreg_db/models/media/video/pipe_1.py +213 -0
  298. endoreg_db/models/media/video/pipe_2.py +105 -0
  299. endoreg_db/models/media/video/refactor_plan.md +0 -0
  300. endoreg_db/models/media/video/video_file.py +825 -0
  301. endoreg_db/models/media/video/video_file_ai.py +443 -0
  302. endoreg_db/models/media/video/video_file_anonymize.py +349 -0
  303. endoreg_db/models/media/video/video_file_frames/__init__.py +47 -0
  304. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +22 -0
  305. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  306. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +104 -0
  307. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +174 -0
  308. endoreg_db/models/media/video/video_file_frames/_get_frame.py +28 -0
  309. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +27 -0
  310. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +20 -0
  311. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +27 -0
  312. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +34 -0
  313. endoreg_db/models/media/video/video_file_frames/_get_frames.py +27 -0
  314. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +129 -0
  315. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +141 -0
  316. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +65 -0
  317. endoreg_db/models/media/video/video_file_frames.py +0 -0
  318. endoreg_db/models/media/video/video_file_io.py +168 -0
  319. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  320. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +45 -0
  321. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +39 -0
  322. endoreg_db/models/media/video/video_file_meta/get_fps.py +147 -0
  323. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +143 -0
  324. endoreg_db/models/media/video/video_file_meta/text_meta.py +134 -0
  325. endoreg_db/models/media/video/video_file_meta/video_meta.py +70 -0
  326. endoreg_db/models/media/video/video_file_segments.py +209 -0
  327. endoreg_db/models/media/video/video_metadata.py +65 -0
  328. endoreg_db/models/media/video/video_processing.py +152 -0
  329. endoreg_db/models/medical/__init__.py +146 -0
  330. endoreg_db/models/medical/contraindication/__init__.py +17 -0
  331. endoreg_db/models/medical/disease.py +156 -0
  332. endoreg_db/models/medical/event.py +137 -0
  333. endoreg_db/models/medical/examination/__init__.py +9 -0
  334. endoreg_db/models/medical/examination/examination.py +148 -0
  335. endoreg_db/models/medical/examination/examination_indication.py +278 -0
  336. endoreg_db/models/medical/examination/examination_time.py +49 -0
  337. endoreg_db/models/medical/examination/examination_time_type.py +41 -0
  338. endoreg_db/models/medical/examination/examination_type.py +48 -0
  339. endoreg_db/models/medical/finding/__init__.py +18 -0
  340. endoreg_db/models/medical/finding/finding.py +96 -0
  341. endoreg_db/models/medical/finding/finding_classification.py +142 -0
  342. endoreg_db/models/medical/finding/finding_intervention.py +52 -0
  343. endoreg_db/models/medical/finding/finding_type.py +35 -0
  344. endoreg_db/models/medical/hardware/__init__.py +8 -0
  345. endoreg_db/models/medical/hardware/endoscope.py +65 -0
  346. endoreg_db/models/{hardware → medical/hardware}/endoscopy_processor.py +68 -29
  347. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  348. endoreg_db/models/medical/laboratory/lab_value.py +419 -0
  349. endoreg_db/models/medical/medication/__init__.py +19 -0
  350. endoreg_db/models/medical/medication/medication.py +31 -0
  351. endoreg_db/models/medical/medication/medication_indication.py +50 -0
  352. endoreg_db/models/medical/medication/medication_indication_type.py +39 -0
  353. endoreg_db/models/medical/medication/medication_intake_time.py +44 -0
  354. endoreg_db/models/medical/medication/medication_schedule.py +45 -0
  355. endoreg_db/models/medical/organ/__init__.py +35 -0
  356. endoreg_db/models/medical/patient/__init__.py +56 -0
  357. endoreg_db/models/medical/patient/medication_examples.py +38 -0
  358. endoreg_db/models/medical/patient/patient_disease.py +63 -0
  359. endoreg_db/models/medical/patient/patient_event.py +75 -0
  360. endoreg_db/models/medical/patient/patient_examination.py +249 -0
  361. endoreg_db/models/medical/patient/patient_examination_indication.py +44 -0
  362. endoreg_db/models/medical/patient/patient_finding.py +357 -0
  363. endoreg_db/models/medical/patient/patient_finding_classification.py +207 -0
  364. endoreg_db/models/medical/patient/patient_finding_intervention.py +40 -0
  365. endoreg_db/models/medical/patient/patient_lab_sample.py +148 -0
  366. endoreg_db/models/{persons → medical}/patient/patient_lab_value.py +68 -22
  367. endoreg_db/models/medical/patient/patient_medication.py +104 -0
  368. endoreg_db/models/medical/patient/patient_medication_schedule.py +136 -0
  369. endoreg_db/models/medical/risk/__init__.py +7 -0
  370. endoreg_db/models/medical/risk/risk.py +72 -0
  371. endoreg_db/models/medical/risk/risk_type.py +51 -0
  372. endoreg_db/models/metadata/__init__.py +19 -0
  373. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  374. endoreg_db/models/metadata/model_meta.py +206 -0
  375. endoreg_db/models/metadata/model_meta_logic.py +343 -0
  376. endoreg_db/models/{data_file/metadata → metadata}/pdf_meta.py +32 -13
  377. endoreg_db/models/metadata/sensitive_meta.py +288 -0
  378. endoreg_db/models/metadata/sensitive_meta_logic.py +1048 -0
  379. endoreg_db/models/metadata/video_meta.py +332 -0
  380. endoreg_db/models/metadata/video_prediction_logic.py +190 -0
  381. endoreg_db/models/metadata/video_prediction_meta.py +270 -0
  382. endoreg_db/models/other/__init__.py +36 -1
  383. endoreg_db/models/other/distribution/__init__.py +44 -0
  384. endoreg_db/models/other/distribution/base_value_distribution.py +20 -0
  385. endoreg_db/models/other/distribution/date_value_distribution.py +89 -0
  386. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +32 -0
  387. endoreg_db/models/other/distribution/numeric_value_distribution.py +125 -0
  388. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +22 -0
  389. endoreg_db/models/other/emission/__init__.py +5 -0
  390. endoreg_db/models/other/emission/emission_factor.py +94 -0
  391. endoreg_db/models/{persons → other}/gender.py +8 -3
  392. endoreg_db/models/other/information_source.py +159 -0
  393. endoreg_db/models/other/material.py +14 -2
  394. endoreg_db/models/other/resource.py +6 -2
  395. endoreg_db/models/other/tag.py +27 -0
  396. endoreg_db/models/other/transport_route.py +15 -3
  397. endoreg_db/models/{unit.py → other/unit.py} +16 -6
  398. endoreg_db/models/other/waste.py +10 -3
  399. endoreg_db/models/requirement/__init__.py +11 -0
  400. endoreg_db/models/requirement/requirement.py +767 -0
  401. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  402. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  403. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +9 -0
  404. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +95 -0
  405. endoreg_db/models/requirement/requirement_operator.py +176 -0
  406. endoreg_db/models/requirement/requirement_set.py +287 -0
  407. endoreg_db/models/rule/__init__.py +13 -0
  408. endoreg_db/models/{rules → rule}/rule.py +6 -3
  409. endoreg_db/models/{rules → rule}/rule_attribute_dtype.py +0 -2
  410. endoreg_db/models/{rules → rule}/rule_type.py +0 -2
  411. endoreg_db/models/{rules → rule}/ruleset.py +0 -2
  412. endoreg_db/models/state/__init__.py +12 -0
  413. endoreg_db/models/state/abstract.py +11 -0
  414. endoreg_db/models/state/audit_ledger.py +150 -0
  415. endoreg_db/models/state/label_video_segment.py +22 -0
  416. endoreg_db/models/state/raw_pdf.py +187 -0
  417. endoreg_db/models/state/sensitive_meta.py +46 -0
  418. endoreg_db/models/state/video.py +232 -0
  419. endoreg_db/models/upload_job.py +99 -0
  420. endoreg_db/models/utils.py +135 -0
  421. endoreg_db/renames.yml +8 -0
  422. endoreg_db/root_urls.py +9 -0
  423. endoreg_db/schemas/__init__.py +0 -0
  424. endoreg_db/schemas/examination_evaluation.py +27 -0
  425. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +775 -0
  426. endoreg_db/serializers/__init__.py +118 -10
  427. endoreg_db/serializers/_old/raw_pdf_meta_validation.py +223 -0
  428. endoreg_db/serializers/_old/raw_video_meta_validation.py +179 -0
  429. endoreg_db/serializers/_old/video.py +71 -0
  430. endoreg_db/serializers/administration/__init__.py +14 -0
  431. endoreg_db/serializers/administration/ai/__init__.py +10 -0
  432. endoreg_db/serializers/administration/ai/active_model.py +10 -0
  433. endoreg_db/serializers/administration/ai/ai_model.py +18 -0
  434. endoreg_db/serializers/administration/ai/model_type.py +10 -0
  435. endoreg_db/serializers/administration/center.py +9 -0
  436. endoreg_db/serializers/administration/gender.py +9 -0
  437. endoreg_db/serializers/anonymization.py +69 -0
  438. endoreg_db/serializers/evaluation/examination_evaluation.py +1 -0
  439. endoreg_db/serializers/examination/__init__.py +10 -0
  440. endoreg_db/serializers/examination/base.py +46 -0
  441. endoreg_db/serializers/examination/dropdown.py +21 -0
  442. endoreg_db/serializers/examination_serializer.py +12 -0
  443. endoreg_db/serializers/finding/__init__.py +5 -0
  444. endoreg_db/serializers/finding/finding.py +54 -0
  445. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  446. endoreg_db/serializers/finding_classification/choice.py +19 -0
  447. endoreg_db/serializers/finding_classification/classification.py +13 -0
  448. endoreg_db/serializers/label/__init__.py +7 -0
  449. endoreg_db/serializers/label/image_classification_annotation.py +62 -0
  450. endoreg_db/serializers/label/label.py +15 -0
  451. endoreg_db/serializers/label_video_segment/__init__.py +7 -0
  452. endoreg_db/serializers/label_video_segment/_lvs_create.py +149 -0
  453. endoreg_db/serializers/label_video_segment/_lvs_update.py +138 -0
  454. endoreg_db/serializers/label_video_segment/_lvs_validate.py +149 -0
  455. endoreg_db/serializers/label_video_segment/label_video_segment.py +344 -0
  456. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +99 -0
  457. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +163 -0
  458. endoreg_db/serializers/meta/__init__.py +19 -0
  459. endoreg_db/serializers/meta/pdf_file_meta_extraction.py +115 -0
  460. endoreg_db/serializers/meta/report_meta.py +53 -0
  461. endoreg_db/serializers/meta/sensitive_meta_detail.py +162 -0
  462. endoreg_db/serializers/meta/sensitive_meta_update.py +148 -0
  463. endoreg_db/serializers/meta/sensitive_meta_verification.py +59 -0
  464. endoreg_db/serializers/meta/video_meta.py +39 -0
  465. endoreg_db/serializers/misc/__init__.py +14 -0
  466. endoreg_db/serializers/misc/file_overview.py +182 -0
  467. endoreg_db/serializers/misc/sensitive_patient_data.py +120 -0
  468. endoreg_db/serializers/misc/stats.py +33 -0
  469. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  470. endoreg_db/serializers/misc/upload_job.py +71 -0
  471. endoreg_db/serializers/patient/__init__.py +11 -0
  472. endoreg_db/serializers/patient/patient.py +86 -0
  473. endoreg_db/serializers/patient/patient_dropdown.py +27 -0
  474. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  475. endoreg_db/serializers/patient_examination/patient_examination.py +141 -0
  476. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  477. endoreg_db/serializers/patient_finding/patient_finding.py +31 -0
  478. endoreg_db/serializers/patient_finding/patient_finding_classification.py +39 -0
  479. endoreg_db/serializers/patient_finding/patient_finding_detail.py +53 -0
  480. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +26 -0
  481. endoreg_db/serializers/patient_finding/patient_finding_list.py +41 -0
  482. endoreg_db/serializers/patient_finding/patient_finding_write.py +126 -0
  483. endoreg_db/serializers/pdf/__init__.py +5 -0
  484. endoreg_db/serializers/pdf/anony_text_validation.py +85 -0
  485. endoreg_db/serializers/report/__init__.py +9 -0
  486. endoreg_db/serializers/report/mixins.py +45 -0
  487. endoreg_db/serializers/report/report.py +105 -0
  488. endoreg_db/serializers/report/report_list.py +22 -0
  489. endoreg_db/serializers/report/secure_file_url.py +26 -0
  490. endoreg_db/serializers/requirements/requirement_schema.py +25 -0
  491. endoreg_db/serializers/requirements/requirement_sets.py +29 -0
  492. endoreg_db/serializers/sensitive_meta_serializer.py +282 -0
  493. endoreg_db/serializers/video/__init__.py +7 -0
  494. endoreg_db/serializers/video/segmentation.py +263 -0
  495. endoreg_db/serializers/video/video_file_brief.py +10 -0
  496. endoreg_db/serializers/video/video_file_detail.py +83 -0
  497. endoreg_db/serializers/video/video_file_list.py +67 -0
  498. endoreg_db/serializers/video/video_metadata.py +105 -0
  499. endoreg_db/serializers/video/video_processing_history.py +153 -0
  500. endoreg_db/serializers/video_examination.py +198 -0
  501. endoreg_db/services/__init__.py +5 -0
  502. endoreg_db/services/anonymization.py +223 -0
  503. endoreg_db/services/examination_evaluation.py +149 -0
  504. endoreg_db/services/finding_description_service.py +0 -0
  505. endoreg_db/services/lookup_service.py +411 -0
  506. endoreg_db/services/lookup_store.py +266 -0
  507. endoreg_db/services/pdf_import.py +1382 -0
  508. endoreg_db/services/polling_coordinator.py +288 -0
  509. endoreg_db/services/pseudonym_service.py +89 -0
  510. endoreg_db/services/requirements_object.py +147 -0
  511. endoreg_db/services/segment_sync.py +155 -0
  512. endoreg_db/services/storage_aware_video_processor.py +344 -0
  513. endoreg_db/services/video_import.py +1259 -0
  514. endoreg_db/tasks/upload_tasks.py +207 -0
  515. endoreg_db/tasks/video_ingest.py +157 -0
  516. endoreg_db/tasks/video_processing_tasks.py +327 -0
  517. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  518. endoreg_db/templates/admin/start_examination.html +12 -0
  519. endoreg_db/templates/timeline.html +176 -0
  520. endoreg_db/urls/__init__.py +83 -0
  521. endoreg_db/urls/anonymization.py +32 -0
  522. endoreg_db/urls/auth.py +16 -0
  523. endoreg_db/urls/classification.py +39 -0
  524. endoreg_db/urls/examination.py +54 -0
  525. endoreg_db/urls/files.py +6 -0
  526. endoreg_db/urls/label_video_segment_validate.py +33 -0
  527. endoreg_db/urls/label_video_segments.py +46 -0
  528. endoreg_db/urls/media.py +227 -0
  529. endoreg_db/urls/patient.py +19 -0
  530. endoreg_db/urls/report.py +48 -0
  531. endoreg_db/urls/requirements.py +13 -0
  532. endoreg_db/urls/sensitive_meta.py +0 -0
  533. endoreg_db/urls/stats.py +46 -0
  534. endoreg_db/urls/upload.py +20 -0
  535. endoreg_db/urls/video.py +61 -0
  536. endoreg_db/urls.py +9 -0
  537. endoreg_db/utils/__init__.py +88 -1
  538. endoreg_db/utils/ai/__init__.py +9 -0
  539. endoreg_db/{models/ai_model/utils.py → utils/ai/get.py} +1 -4
  540. endoreg_db/utils/ai/inference_dataset.py +52 -0
  541. endoreg_db/utils/ai/multilabel_classification_net.py +159 -0
  542. endoreg_db/utils/ai/postprocess.py +63 -0
  543. endoreg_db/utils/ai/predict.py +291 -0
  544. endoreg_db/utils/ai/preprocess.py +68 -0
  545. endoreg_db/utils/calc_duration_seconds.py +24 -0
  546. endoreg_db/utils/case_generator/__init__.py +0 -0
  547. endoreg_db/utils/case_generator/case_generator.py +159 -0
  548. endoreg_db/utils/case_generator/lab_sample_factory.py +33 -0
  549. endoreg_db/utils/case_generator/utils.py +30 -0
  550. endoreg_db/utils/check_video_files.py +148 -0
  551. endoreg_db/utils/dataloader.py +118 -35
  552. endoreg_db/utils/dates.py +60 -0
  553. endoreg_db/utils/env.py +33 -0
  554. endoreg_db/utils/extract_specific_frames.py +72 -0
  555. endoreg_db/utils/file_operations.py +29 -1
  556. endoreg_db/utils/fix_video_path_direct.py +141 -0
  557. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  558. endoreg_db/utils/hashs.py +123 -4
  559. endoreg_db/utils/links/__init__.py +0 -0
  560. endoreg_db/utils/links/requirement_link.py +193 -0
  561. endoreg_db/utils/mime_types.py +0 -0
  562. endoreg_db/utils/names.py +76 -0
  563. endoreg_db/utils/parse_and_generate_yaml.py +46 -0
  564. endoreg_db/utils/paths.py +95 -0
  565. endoreg_db/utils/permissions.py +143 -0
  566. endoreg_db/utils/pipelines/Readme.md +235 -0
  567. endoreg_db/utils/pipelines/__init__.py +0 -0
  568. endoreg_db/utils/pipelines/process_video_dir.py +120 -0
  569. endoreg_db/utils/product/__init__.py +0 -0
  570. endoreg_db/utils/product/sum_emissions.py +20 -0
  571. endoreg_db/utils/product/sum_weights.py +18 -0
  572. endoreg_db/utils/pydantic_models/__init__.py +6 -0
  573. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  574. endoreg_db/utils/requirement_helpers.py +0 -0
  575. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  576. endoreg_db/utils/requirement_operator_logic/lab_value_operators.py +578 -0
  577. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +368 -0
  578. endoreg_db/utils/setup_config.py +177 -0
  579. endoreg_db/utils/translation.py +27 -0
  580. endoreg_db/utils/validate_endo_roi.py +19 -0
  581. endoreg_db/utils/validate_subcategory_dict.py +91 -0
  582. endoreg_db/utils/validate_video_detailed.py +357 -0
  583. endoreg_db/utils/video/__init__.py +26 -0
  584. endoreg_db/utils/video/extract_frames.py +88 -0
  585. endoreg_db/utils/video/ffmpeg_wrapper.py +835 -0
  586. endoreg_db/utils/video/names.py +42 -0
  587. endoreg_db/utils/video/streaming_processor.py +312 -0
  588. endoreg_db/utils/video/video_splitter.py +94 -0
  589. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +238 -0
  590. endoreg_db/views/__init__.py +274 -0
  591. endoreg_db/views/anonymization/__init__.py +27 -0
  592. endoreg_db/views/anonymization/media_management.py +454 -0
  593. endoreg_db/views/anonymization/overview.py +216 -0
  594. endoreg_db/views/anonymization/validate.py +107 -0
  595. endoreg_db/views/auth/__init__.py +13 -0
  596. endoreg_db/views/auth/keycloak.py +113 -0
  597. endoreg_db/views/examination/__init__.py +33 -0
  598. endoreg_db/views/examination/examination.py +37 -0
  599. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  600. endoreg_db/views/examination/get_finding_classification_choices.py +59 -0
  601. endoreg_db/views/examination/get_finding_classifications.py +36 -0
  602. endoreg_db/views/examination/get_findings.py +41 -0
  603. endoreg_db/views/examination/get_instruments.py +18 -0
  604. endoreg_db/views/examination/get_interventions.py +14 -0
  605. endoreg_db/views/finding/__init__.py +9 -0
  606. endoreg_db/views/finding/finding.py +112 -0
  607. endoreg_db/views/finding/get_classifications.py +14 -0
  608. endoreg_db/views/finding/get_interventions.py +17 -0
  609. endoreg_db/views/finding_classification/__init__.py +13 -0
  610. endoreg_db/views/finding_classification/base.py +0 -0
  611. endoreg_db/views/finding_classification/finding_classification.py +42 -0
  612. endoreg_db/views/finding_classification/get_classification_choices.py +55 -0
  613. endoreg_db/views/label/__init__.py +5 -0
  614. endoreg_db/views/label/label.py +15 -0
  615. endoreg_db/views/label_video_segment/__init__.py +16 -0
  616. endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +44 -0
  617. endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +50 -0
  618. endoreg_db/views/label_video_segment/label_video_segment.py +77 -0
  619. endoreg_db/views/label_video_segment/label_video_segment_by_label.py +174 -0
  620. endoreg_db/views/label_video_segment/label_video_segment_detail.py +73 -0
  621. endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +46 -0
  622. endoreg_db/views/label_video_segment/validate.py +226 -0
  623. endoreg_db/views/media/__init__.py +45 -0
  624. endoreg_db/views/media/pdf_media.py +388 -0
  625. endoreg_db/views/media/segments.py +71 -0
  626. endoreg_db/views/media/sensitive_metadata.py +314 -0
  627. endoreg_db/views/media/video_media.py +272 -0
  628. endoreg_db/views/media/video_segments.py +524 -0
  629. endoreg_db/views/meta/__init__.py +15 -0
  630. endoreg_db/views/meta/available_files_list.py +146 -0
  631. endoreg_db/views/meta/report_meta.py +53 -0
  632. endoreg_db/views/meta/sensitive_meta_detail.py +148 -0
  633. endoreg_db/views/meta/sensitive_meta_list.py +104 -0
  634. endoreg_db/views/meta/sensitive_meta_verification.py +71 -0
  635. endoreg_db/views/misc/__init__.py +63 -0
  636. endoreg_db/views/misc/center.py +13 -0
  637. endoreg_db/views/misc/csrf.py +7 -0
  638. endoreg_db/views/misc/gender.py +14 -0
  639. endoreg_db/views/misc/secure_file_serving_view.py +80 -0
  640. endoreg_db/views/misc/secure_file_url_view.py +84 -0
  641. endoreg_db/views/misc/secure_url_validate.py +79 -0
  642. endoreg_db/views/misc/stats.py +220 -0
  643. endoreg_db/views/misc/translation.py +182 -0
  644. endoreg_db/views/misc/upload_views.py +240 -0
  645. endoreg_db/views/patient/__init__.py +5 -0
  646. endoreg_db/views/patient/patient.py +210 -0
  647. endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +164 -0
  648. endoreg_db/views/patient_examination/__init__.py +11 -0
  649. endoreg_db/views/patient_examination/patient_examination.py +140 -0
  650. endoreg_db/views/patient_examination/patient_examination_create.py +63 -0
  651. endoreg_db/views/patient_examination/patient_examination_detail.py +66 -0
  652. endoreg_db/views/patient_examination/patient_examination_list.py +68 -0
  653. endoreg_db/views/patient_examination/video.py +194 -0
  654. endoreg_db/views/patient_finding/__init__.py +7 -0
  655. endoreg_db/views/patient_finding/base.py +0 -0
  656. endoreg_db/views/patient_finding/patient_finding.py +64 -0
  657. endoreg_db/views/patient_finding/patient_finding_optimized.py +259 -0
  658. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  659. endoreg_db/views/patient_finding_classification/pfc_create.py +67 -0
  660. endoreg_db/views/patient_finding_location/__init__.py +5 -0
  661. endoreg_db/views/patient_finding_location/pfl_create.py +70 -0
  662. endoreg_db/views/patient_finding_morphology/__init__.py +5 -0
  663. endoreg_db/views/patient_finding_morphology/pfm_create.py +70 -0
  664. endoreg_db/views/pdf/__init__.py +8 -0
  665. endoreg_db/views/pdf/pdf_stream.py +187 -0
  666. endoreg_db/views/pdf/reimport.py +177 -0
  667. endoreg_db/views/report/__init__.py +9 -0
  668. endoreg_db/views/report/report_list.py +112 -0
  669. endoreg_db/views/report/report_with_secure_url.py +28 -0
  670. endoreg_db/views/report/start_examination.py +7 -0
  671. endoreg_db/views/requirement/__init__.py +10 -0
  672. endoreg_db/views/requirement/evaluate.py +279 -0
  673. endoreg_db/views/requirement/lookup.py +367 -0
  674. endoreg_db/views/requirement/lookup_store.py +252 -0
  675. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  676. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  677. endoreg_db/views/stats/__init__.py +13 -0
  678. endoreg_db/views/stats/stats_views.py +229 -0
  679. endoreg_db/views/video/__init__.py +59 -0
  680. endoreg_db/views/video/correction.py +530 -0
  681. endoreg_db/views/video/reimport.py +195 -0
  682. endoreg_db/views/video/segmentation.py +274 -0
  683. endoreg_db/views/video/task_status.py +49 -0
  684. endoreg_db/views/video/timeline.py +46 -0
  685. endoreg_db/views/video/video_analyze.py +52 -0
  686. endoreg_db/views/video/video_apply_mask.py +48 -0
  687. endoreg_db/views/video/video_correction.py +21 -0
  688. endoreg_db/views/video/video_download_processed.py +58 -0
  689. endoreg_db/views/video/video_examination_viewset.py +242 -0
  690. endoreg_db/views/video/video_meta.py +29 -0
  691. endoreg_db/views/video/video_processing_history.py +24 -0
  692. endoreg_db/views/video/video_remove_frames.py +48 -0
  693. endoreg_db/views/video/video_stream.py +306 -0
  694. endoreg_db/views.py +0 -3
  695. endoreg_db-0.8.6.3.dist-info/METADATA +383 -0
  696. endoreg_db-0.8.6.3.dist-info/RECORD +793 -0
  697. {endoreg_db-0.4.5.dist-info → endoreg_db-0.8.6.3.dist-info}/WHEEL +1 -1
  698. endoreg_db/data/active_model/data.yaml +0 -3
  699. endoreg_db/data/agl_service/data.yaml +0 -19
  700. endoreg_db/data/label/label-set/data.yaml +0 -18
  701. endoreg_db/management/commands/_load_model_template.py +0 -41
  702. endoreg_db/management/commands/delete_all.py +0 -18
  703. endoreg_db/management/commands/delete_legacy_images.py +0 -19
  704. endoreg_db/management/commands/delete_legacy_videos.py +0 -17
  705. endoreg_db/management/commands/extract_legacy_video_frames.py +0 -18
  706. endoreg_db/management/commands/fetch_legacy_image_dataset.py +0 -32
  707. endoreg_db/management/commands/fix_auth_permission.py +0 -20
  708. endoreg_db/management/commands/import_legacy_images.py +0 -94
  709. endoreg_db/management/commands/import_legacy_videos.py +0 -76
  710. endoreg_db/management/commands/load_active_model_data.py +0 -45
  711. endoreg_db/management/commands/load_endoscopy_processor_data.py +0 -45
  712. endoreg_db/management/commands/load_g_play_data.py +0 -113
  713. endoreg_db/management/commands/load_label_data.py +0 -67
  714. endoreg_db/management/commands/load_medication_indication_data.py +0 -63
  715. endoreg_db/management/commands/load_medication_indication_type_data.py +0 -41
  716. endoreg_db/management/commands/load_medication_schedule_data.py +0 -55
  717. endoreg_db/management/commands/load_network_data.py +0 -57
  718. endoreg_db/migrations/0002_anonymizedimagelabel_anonymousimageannotation_and_more.py +0 -55
  719. endoreg_db/migrations/0003_anonymousimageannotation_original_image_url_and_more.py +0 -39
  720. endoreg_db/migrations/0004_alter_rawpdffile_file.py +0 -20
  721. endoreg_db/migrations/0005_uploadedfile_alter_rawpdffile_file_anonymizedfile.py +0 -40
  722. endoreg_db/migrations/0006_alter_rawpdffile_file.py +0 -20
  723. endoreg_db/migrations/0007_networkdevicelogentry_datetime_and_more.py +0 -43
  724. endoreg_db/models/ai_model/__init__.py +0 -3
  725. endoreg_db/models/ai_model/active_model.py +0 -9
  726. endoreg_db/models/ai_model/model_meta.py +0 -24
  727. endoreg_db/models/annotation/__init__.py +0 -3
  728. endoreg_db/models/annotation/anonymized_image_annotation.py +0 -60
  729. endoreg_db/models/annotation/binary_classification_annotation_task.py +0 -80
  730. endoreg_db/models/annotation/image_classification.py +0 -27
  731. endoreg_db/models/center/center.py +0 -25
  732. endoreg_db/models/center/center_product.py +0 -34
  733. endoreg_db/models/center/center_resource.py +0 -19
  734. endoreg_db/models/center/center_waste.py +0 -11
  735. endoreg_db/models/data_file/__init__.py +0 -6
  736. endoreg_db/models/data_file/base_classes/__init__.py +0 -2
  737. endoreg_db/models/data_file/base_classes/abstract_frame.py +0 -51
  738. endoreg_db/models/data_file/base_classes/abstract_video.py +0 -201
  739. endoreg_db/models/data_file/frame.py +0 -45
  740. endoreg_db/models/data_file/import_classes/__init__.py +0 -32
  741. endoreg_db/models/data_file/import_classes/processing_functions/__init__.py +0 -35
  742. endoreg_db/models/data_file/import_classes/processing_functions/pdf.py +0 -28
  743. endoreg_db/models/data_file/import_classes/processing_functions/video.py +0 -260
  744. endoreg_db/models/data_file/import_classes/raw_pdf.py +0 -188
  745. endoreg_db/models/data_file/import_classes/raw_video.py +0 -343
  746. endoreg_db/models/data_file/metadata/__init__.py +0 -3
  747. endoreg_db/models/data_file/metadata/sensitive_meta.py +0 -31
  748. endoreg_db/models/data_file/metadata/video_meta.py +0 -133
  749. endoreg_db/models/data_file/report_file.py +0 -89
  750. endoreg_db/models/data_file/video/__init__.py +0 -7
  751. endoreg_db/models/data_file/video/import_meta.py +0 -25
  752. endoreg_db/models/data_file/video/video.py +0 -25
  753. endoreg_db/models/data_file/video_segment.py +0 -107
  754. endoreg_db/models/disease.py +0 -56
  755. endoreg_db/models/emission/__init__.py +0 -1
  756. endoreg_db/models/emission/emission_factor.py +0 -20
  757. endoreg_db/models/event.py +0 -22
  758. endoreg_db/models/examination/__init__.py +0 -4
  759. endoreg_db/models/examination/examination.py +0 -26
  760. endoreg_db/models/examination/examination_time.py +0 -27
  761. endoreg_db/models/examination/examination_time_type.py +0 -24
  762. endoreg_db/models/examination/examination_type.py +0 -18
  763. endoreg_db/models/hardware/__init__.py +0 -2
  764. endoreg_db/models/hardware/endoscope.py +0 -44
  765. endoreg_db/models/information_source.py +0 -29
  766. endoreg_db/models/laboratory/__init__.py +0 -1
  767. endoreg_db/models/laboratory/lab_value.py +0 -102
  768. endoreg_db/models/legacy_data/__init__.py +0 -3
  769. endoreg_db/models/legacy_data/image.py +0 -34
  770. endoreg_db/models/logging/__init__.py +0 -4
  771. endoreg_db/models/logging/agl_service.py +0 -19
  772. endoreg_db/models/logging/base.py +0 -22
  773. endoreg_db/models/logging/log_type.py +0 -23
  774. endoreg_db/models/logging/network_device.py +0 -24
  775. endoreg_db/models/medication/__init__.py +0 -1
  776. endoreg_db/models/medication/medication.py +0 -148
  777. endoreg_db/models/network/__init__.py +0 -3
  778. endoreg_db/models/network/agl_service.py +0 -38
  779. endoreg_db/models/network/network_device.py +0 -53
  780. endoreg_db/models/network/network_device_type.py +0 -23
  781. endoreg_db/models/other/distribution.py +0 -215
  782. endoreg_db/models/patient_examination/__init__.py +0 -35
  783. endoreg_db/models/permissions/__init__.py +0 -44
  784. endoreg_db/models/persons/__init__.py +0 -7
  785. endoreg_db/models/persons/examiner/__init__.py +0 -2
  786. endoreg_db/models/persons/examiner/examiner.py +0 -16
  787. endoreg_db/models/persons/examiner/examiner_type.py +0 -2
  788. endoreg_db/models/persons/patient/__init__.py +0 -8
  789. endoreg_db/models/persons/patient/case/case.py +0 -30
  790. endoreg_db/models/persons/patient/patient.py +0 -216
  791. endoreg_db/models/persons/patient/patient_disease.py +0 -16
  792. endoreg_db/models/persons/patient/patient_event.py +0 -22
  793. endoreg_db/models/persons/patient/patient_lab_sample.py +0 -106
  794. endoreg_db/models/persons/patient/patient_medication.py +0 -44
  795. endoreg_db/models/persons/patient/patient_medication_schedule.py +0 -28
  796. endoreg_db/models/persons/portal_user_information.py +0 -27
  797. endoreg_db/models/prediction/__init__.py +0 -2
  798. endoreg_db/models/prediction/image_classification.py +0 -37
  799. endoreg_db/models/prediction/video_prediction_meta.py +0 -244
  800. endoreg_db/models/product/__init__.py +0 -5
  801. endoreg_db/models/product/product.py +0 -97
  802. endoreg_db/models/product/product_group.py +0 -19
  803. endoreg_db/models/product/product_material.py +0 -24
  804. endoreg_db/models/questionnaires/__init__.py +0 -114
  805. endoreg_db/models/quiz/__init__.py +0 -2
  806. endoreg_db/models/quiz/quiz_answer.py +0 -41
  807. endoreg_db/models/quiz/quiz_question.py +0 -54
  808. endoreg_db/models/report_reader/__init__.py +0 -2
  809. endoreg_db/models/report_reader/report_reader_config.py +0 -53
  810. endoreg_db/models/rules/__init__.py +0 -5
  811. endoreg_db/queries/get/__init__.py +0 -6
  812. endoreg_db/queries/get/center.py +0 -42
  813. endoreg_db/queries/get/model.py +0 -13
  814. endoreg_db/queries/get/patient.py +0 -14
  815. endoreg_db/queries/get/patient_examination.py +0 -20
  816. endoreg_db/queries/get/report_file.py +0 -33
  817. endoreg_db/queries/get/video.py +0 -31
  818. endoreg_db/serializers/ai_model.py +0 -19
  819. endoreg_db/serializers/annotation.py +0 -17
  820. endoreg_db/serializers/center.py +0 -11
  821. endoreg_db/serializers/examination.py +0 -33
  822. endoreg_db/serializers/frame.py +0 -13
  823. endoreg_db/serializers/hardware.py +0 -21
  824. endoreg_db/serializers/label.py +0 -22
  825. endoreg_db/serializers/patient.py +0 -10
  826. endoreg_db/serializers/prediction.py +0 -15
  827. endoreg_db/serializers/report_file.py +0 -7
  828. endoreg_db/serializers/video.py +0 -27
  829. endoreg_db/tests.py +0 -3
  830. endoreg_db/utils/legacy_ocr.py +0 -201
  831. endoreg_db/utils/video_metadata.py +0 -87
  832. endoreg_db-0.4.5.dist-info/METADATA +0 -34
  833. endoreg_db-0.4.5.dist-info/RECORD +0 -316
  834. /endoreg_db/{data/distribution/numeric/.init → api/serializers/finding_descriptions.py} +0 -0
  835. /endoreg_db/{models/persons/patient/case/__init__.py → api/views/finding_descriptions.py} +0 -0
  836. /endoreg_db/{queries/get/annotation.py → config/__init__.py} +0 -0
  837. /endoreg_db/data/{label → ai_model_label}/label-type/data.yaml +0 -0
  838. /endoreg_db/data/{model_type → ai_model_type}/data.yaml +0 -0
  839. /endoreg_db/{queries/get/prediction.py → data/shift/m2.yaml} +0 -0
  840. /endoreg_db/{queries/get/video_import_meta.py → factories/__init__.py} +0 -0
  841. /endoreg_db/{queries/get/video_prediction_meta.py → helpers/__init__.py} +0 -0
  842. /endoreg_db/management/commands/{load_report_reader_flag.py → load_report_reader_flag_data.py} +0 -0
  843. /endoreg_db/models/{persons → administration/person}/person.py +0 -0
  844. /endoreg_db/models/{report_reader → media/pdf/report_reader}/report_reader_flag.py +0 -0
  845. /endoreg_db/models/{rules → rule}/rule_applicator.py +0 -0
  846. {endoreg_db-0.4.5.dist-info → endoreg_db-0.8.6.3.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,378 @@
1
+ import random
2
+ from typing import Optional
3
+ from endoreg_db.models import (
4
+ Center,
5
+ Gender,
6
+ Patient,
7
+ Examination,
8
+ ExaminationIndication,
9
+ RawPdfFile,
10
+ EndoscopyProcessor,
11
+ ModelMeta,
12
+ InformationSource,
13
+ AiModel,
14
+ )
15
+ from logging import getLogger
16
+ from datetime import date
17
+ import shutil
18
+ from pathlib import Path
19
+ from django.conf import settings # Import settings
20
+ from django.core.files.storage import default_storage # Import default storage
21
+ from django.db.models.fields.files import FieldFile
22
+
23
+ from endoreg_db.utils import (
24
+ create_mock_patient_name,
25
+ )
26
+
27
+ logger = getLogger("default_objects")
28
+
29
+
30
+ DEFAULT_CENTER_NAME = "university_hospital_wuerzburg"
31
+ DEFAULT_ENDOSCOPE_NAME = "test_endoscope"
32
+ DEFAULT_ENDOSCOPY_PROCESSOR_NAME = "olympus_cv_1500"
33
+
34
+ DEFAULT_EGD_PATH = Path("tests/assets/lux-gastro-report.pdf")
35
+ DEFAULT_GENDERS = ["male","female","unknown"]
36
+ DEFAULT_EXAMINATIONS = ["colonoscopy"]
37
+ DEFAULT_INDICATIONS = [
38
+ "colonoscopy",
39
+ "colonoscopy_screening",
40
+ "colonoscopy_lesion_removal_small",
41
+ "colonoscopy_lesion_removal_emr",
42
+ "colonoscopy_lesion_removal_large",
43
+ "colonoscopy_diagnostic_acute_symptomatic",
44
+ ]
45
+
46
+ DEFAULT_SEGMENTATION_MODEL_NAME = "image_multilabel_classification_colonoscopy_default"
47
+
48
+ DEFAULT_GENDER = "unknown"
49
+ DEFAULT_PATIENT_FIRST_NAME = "TestFirst"
50
+ DEFAULT_PATIENT_LAST_NAME = "TestLast"
51
+ DEFAULT_PATIENT_GENDER_NAME = "female"
52
+ DEFAULT_PATIENT_BIRTH_DATE = date(1970, 1, 1)
53
+
54
+ def get_information_source_prediction():
55
+ """
56
+ Retrieves the InformationSource object with the name "prediction".
57
+
58
+ Loads information source data if needed and returns the corresponding InformationSource instance. Raises a ValueError if the object is not found or is not an InformationSource.
59
+ """
60
+ from .data_loader import load_information_source
61
+ load_information_source()
62
+ source = InformationSource.objects.get(name="prediction")
63
+ if not isinstance(source, InformationSource):
64
+ raise ValueError("No InformationSource found in the database.")
65
+ return source
66
+
67
+ def get_latest_segmentation_model(model_name:str=DEFAULT_SEGMENTATION_MODEL_NAME) -> ModelMeta:
68
+
69
+ """
70
+ Retrieves the latest metadata for a segmentation model by name.
71
+
72
+ Loads necessary data and returns the most recent ModelMeta instance for the specified AI model. If no metadata exists, attempts to initialize it automatically; if initialization fails, raises a ValueError with instructions for manual setup.
73
+
74
+ Args:
75
+ model_name: The name of the segmentation model to retrieve.
76
+
77
+ Returns:
78
+ The latest ModelMeta instance for the specified model.
79
+
80
+ Raises:
81
+ ValueError: If the AI model does not exist, or if model metadata cannot be found or initialized.
82
+ """
83
+ from .data_loader import (
84
+ load_center_data,
85
+ load_ai_model_label_data,
86
+ load_ai_model_data,
87
+ )
88
+ load_center_data()
89
+ load_ai_model_label_data()
90
+ load_ai_model_data()
91
+
92
+ try:
93
+ ai_model = AiModel.objects.get(name=model_name)
94
+ except AiModel.DoesNotExist:
95
+ raise ValueError(f"AI model '{model_name}' not found. Run 'python manage.py load_ai_model_data' first.")
96
+
97
+ try:
98
+ latest_meta = ai_model.get_latest_version()
99
+ return latest_meta
100
+ except ValueError as e:
101
+ if "No model metadata found" in str(e):
102
+ logger.warning(f"No ModelMeta found for {model_name}. Attempting to initialize default model metadata...")
103
+
104
+ # Try to initialize the default model metadata
105
+ try:
106
+ from django.core.management import call_command
107
+ call_command('init_default_ai_model')
108
+ # Try again after initialization
109
+ latest_meta = ai_model.get_latest_version()
110
+ return latest_meta
111
+ except Exception as init_error:
112
+ raise ValueError(
113
+ f"No model metadata found for AI model '{model_name}' and failed to auto-initialize. "
114
+ f"Please run 'python manage.py init_default_ai_model' manually. "
115
+ f"Original error: {e}. Initialization error: {init_error}"
116
+ ) from e
117
+ else:
118
+ raise
119
+
120
+
121
+ def get_default_gender() -> Gender:
122
+ """
123
+ Retrieves the Gender object representing the default "unknown" gender.
124
+
125
+ Returns:
126
+ The Gender instance with the name "unknown".
127
+ """
128
+ return Gender.objects.get(name=DEFAULT_GENDER)
129
+
130
+ def get_gender_m_or_f() -> Gender:
131
+ """
132
+ Returns a randomly selected Gender object representing either male or female.
133
+ """
134
+ gender_name = random.choice(["male", "female"])
135
+ return Gender.objects.get(name=gender_name)
136
+
137
+ def get_random_gender() -> Gender:
138
+ """
139
+ Returns a randomly selected Gender object from the available default genders.
140
+ """
141
+ gender_name = random.choice(DEFAULT_GENDERS)
142
+ return Gender.objects.get(name=gender_name) # Fetch and return the Gender object
143
+
144
+ def get_default_processor() -> EndoscopyProcessor:
145
+ """
146
+ Retrieves the default EndoscopyProcessor object by its predefined name.
147
+
148
+ Raises:
149
+ ValueError: If no EndoscopyProcessor with the default name exists.
150
+
151
+ Returns:
152
+ The EndoscopyProcessor instance with the default name.
153
+ """
154
+ processor = EndoscopyProcessor.objects.get(name=DEFAULT_ENDOSCOPY_PROCESSOR_NAME)
155
+ if not isinstance(processor, EndoscopyProcessor):
156
+ raise ValueError(f"No EndoscopyProcessor found with name {DEFAULT_ENDOSCOPY_PROCESSOR_NAME}")
157
+ return processor
158
+
159
+
160
+ def get_default_center() -> Center:
161
+ """
162
+ Retrieves the default Center object with the predefined name.
163
+
164
+ Raises:
165
+ ValueError: If no Center with the default name exists.
166
+
167
+ Returns:
168
+ The Center instance with the default name.
169
+ """
170
+ center = Center.objects.get(
171
+ name=DEFAULT_CENTER_NAME,
172
+ )
173
+ if not isinstance(center, Center):
174
+ raise ValueError(f"No Center found with name {DEFAULT_CENTER_NAME}")
175
+
176
+ return center
177
+
178
+ def generate_patient(**kwargs) -> Patient:
179
+ """Create a Patient with deterministic defaults unless ``randomize=True`` is supplied."""
180
+
181
+ randomize = kwargs.pop("randomize", False)
182
+
183
+ gender = kwargs.get("gender")
184
+ if gender is None:
185
+ if randomize:
186
+ gender = get_random_gender()
187
+ else:
188
+ gender = Gender.objects.get(name=DEFAULT_PATIENT_GENDER_NAME)
189
+ elif not isinstance(gender, Gender):
190
+ gender = Gender.objects.get(name=gender)
191
+
192
+ first_name = kwargs.get("first_name")
193
+ last_name = kwargs.get("last_name")
194
+ if first_name is None or last_name is None:
195
+ if randomize:
196
+ generated_first, generated_last = create_mock_patient_name(gender=gender.name)
197
+ else:
198
+ generated_first, generated_last = DEFAULT_PATIENT_FIRST_NAME, DEFAULT_PATIENT_LAST_NAME
199
+ first_name = first_name or generated_first
200
+ last_name = last_name or generated_last
201
+
202
+ dob = kwargs.get("dob")
203
+ if dob is None:
204
+ birth_date = kwargs.get("birth_date", DEFAULT_PATIENT_BIRTH_DATE)
205
+ if isinstance(birth_date, date):
206
+ dob = birth_date
207
+ else:
208
+ dob = date.fromisoformat(str(birth_date))
209
+
210
+ center = kwargs.get("center")
211
+ if center is None:
212
+ center = get_default_center()
213
+ elif not isinstance(center, Center):
214
+ center = Center.objects.get(name=center)
215
+
216
+ patient = Patient(
217
+ first_name=first_name,
218
+ last_name=last_name,
219
+ dob=dob,
220
+ center = center,
221
+ gender = gender,
222
+ )
223
+
224
+ return patient
225
+
226
+ def get_random_default_examination():
227
+ """
228
+ Retrieves a random Examination object from the default examination names.
229
+
230
+ Returns:
231
+ Examination: A randomly selected Examination instance from the defaults.
232
+ """
233
+ examination_name = random.choice(DEFAULT_EXAMINATIONS)
234
+
235
+ examination = Examination.objects.get(name=examination_name)
236
+ return examination
237
+
238
+ def get_random_default_examination_indication():
239
+ """
240
+ Returns a random ExaminationIndication object from the default indications list.
241
+
242
+ Selects a random indication name from the predefined defaults and retrieves the corresponding ExaminationIndication instance from the database.
243
+ """
244
+ examination_indication = random.choice(DEFAULT_INDICATIONS)
245
+ all_examination_indications = ExaminationIndication.objects.all()
246
+ try:
247
+ examination_indication = ExaminationIndication.objects.get(name=examination_indication)
248
+
249
+ except Exception as e:
250
+ logger.info(f"examination_indication: {examination_indication}")
251
+ logger.info(f"all_examination_indications: {all_examination_indications}")
252
+ raise e
253
+ return examination_indication
254
+
255
+ def get_default_egd_pdf():
256
+ """
257
+ Creates and processes a default EGD PDF file for testing purposes.
258
+
259
+ This function copies a default EGD PDF to a temporary location, creates a RawPdfFile instance from it, processes the file to generate associated metadata, and ensures cleanup of the temporary file. The resulting RawPdfFile instance is returned for use in tests.
260
+
261
+ Returns:
262
+ RawPdfFile: The created and processed RawPdfFile instance.
263
+ """
264
+ egd_path = DEFAULT_EGD_PATH
265
+ center = get_default_center()
266
+ center_name = center.name
267
+
268
+ # Create a temporary file path within the test's media root if possible,
269
+ # otherwise use the source directory. Using MEDIA_ROOT is safer.
270
+ # Ensure MEDIA_ROOT is configured correctly in test settings.
271
+ temp_dir = Path(settings.MEDIA_ROOT) / "temp_test_files"
272
+ temp_dir.mkdir(parents=True, exist_ok=True)
273
+ temp_file_path = temp_dir / f"temp_{egd_path.name}"
274
+
275
+ shutil.copy(egd_path, temp_file_path)
276
+
277
+ pdf_file = None
278
+ file_field: Optional[FieldFile] = None
279
+ try:
280
+ # Create the PDF record using the temporary file.
281
+ # delete_source=True will ensure temp_file_path is deleted by create_from_file
282
+ pdf_file = RawPdfFile.create_from_file(
283
+ file_path=temp_file_path,
284
+ center_name=center_name,
285
+ save=True, # save=True is default and handled internally now
286
+ delete_source=True,
287
+ )
288
+
289
+ if pdf_file is None:
290
+ raise RuntimeError("Failed to create PDF file object")
291
+
292
+ # Use storage API to check existence
293
+ file_field = pdf_file.file
294
+ if not isinstance(file_field, FieldFile):
295
+ raise RuntimeError("RawPdfFile.file did not return a FieldFile instance")
296
+ if not default_storage.exists(file_field.path):
297
+ raise RuntimeError(f"PDF file does not exist in storage at {file_field.path}")
298
+
299
+ # Check that the source temp file was deleted
300
+ if temp_file_path.exists():
301
+ raise RuntimeError(f"Temporary source file {temp_file_path} still exists after creation")
302
+
303
+ # Prepare a minimal report_meta for SensitiveMeta creation
304
+ default_report_meta = {
305
+ "patient_first_name": "DefaultFirstName",
306
+ "patient_last_name": "DefaultLastName",
307
+ "patient_dob": date(1980, 1, 1), # Pass date object directly
308
+ "examination_date": date(2024, 1, 1), # Pass date object directly
309
+ # center_name will be added by process_file using pdf_file.center.name
310
+ }
311
+
312
+ # Call process_file to create SensitiveMeta and extract other info
313
+ pdf_file.process_file(
314
+ text="Default PDF text content.",
315
+ anonymized_text="Default anonymized PDF text content.",
316
+ report_meta=default_report_meta,
317
+ verbose=False
318
+ )
319
+ # process_file calls sensitive_meta.save() and self.save() (for RawPdfFile)
320
+
321
+ except Exception as e:
322
+ # Clean up temp file in case of error before deletion could occur
323
+ if temp_file_path.exists():
324
+ temp_file_path.unlink()
325
+ raise e # Re-raise the exception
326
+
327
+ # pdf_file.file.path might fail if storage doesn't support direct paths (like S3)
328
+ # Prefer using storage API for checks. Logging path if available.
329
+ if file_field is not None:
330
+ try:
331
+ logger.info(f"PDF file created: {file_field.name}, Path: {file_field.path}")
332
+ except NotImplementedError:
333
+ logger.info(f"PDF file created: {file_field.name}, Path: (Not available from storage)")
334
+
335
+
336
+ return pdf_file
337
+
338
+ def get_default_video_file():
339
+ """
340
+ Creates and returns a VideoFile instance using a randomly selected EGD examination video.
341
+
342
+ Loads all necessary data dependencies, selects a random video file for the 'egd' examination, and initializes a VideoFile object with the default center and processor names. The original video file is retained after creation.
343
+
344
+ Returns:
345
+ VideoFile: The created and initialized VideoFile instance.
346
+ """
347
+ from .test_video_helper import get_random_video_path_by_examination_alias
348
+ from endoreg_db.models import VideoFile
349
+ from .data_loader import (
350
+ load_disease_data,
351
+ load_event_data,
352
+ load_information_source,
353
+ load_examination_data,
354
+ load_center_data,
355
+ load_endoscope_data,
356
+ load_ai_model_label_data,
357
+ load_ai_model_data,
358
+ )
359
+ load_disease_data()
360
+ load_event_data()
361
+ load_information_source()
362
+ load_examination_data()
363
+ load_center_data()
364
+ load_endoscope_data()
365
+ load_ai_model_label_data()
366
+ load_ai_model_data()
367
+ video_path = get_random_video_path_by_examination_alias(
368
+ examination_alias='egd', is_anonymous=False
369
+ )
370
+
371
+ video_file = VideoFile.create_from_file_initialized(
372
+ file_path=video_path,
373
+ center_name=DEFAULT_CENTER_NAME, # Pass center name as expected by _create_from_file
374
+ delete_source=False, # Keep the original asset for other tests
375
+ processor_name = DEFAULT_ENDOSCOPY_PROCESSOR_NAME,
376
+ )
377
+
378
+ return video_file
@@ -0,0 +1,31 @@
1
+ import huggingface_hub
2
+ from typing import Optional
3
+
4
+ def download_segmentation_model(
5
+ repo_id: str = "wg-lux/colo_segmentation_RegNetX800MF_base",
6
+ filename: str = "model.safetensors",
7
+ cache_dir: Optional[str] = None
8
+ ) -> str:
9
+ """
10
+ Downloads a segmentation model from Hugging Face and caches it locally.
11
+
12
+ Args:
13
+ repo_id (str): The Hugging Face repository ID (default: wg-lux model).
14
+ filename (str): The specific file to download from the repo (default: model.safetensors).
15
+ cache_dir (str): The directory to cache the downloaded model. If None, uses HF default cache.
16
+
17
+ Returns:
18
+ str: The local path to the downloaded model.
19
+
20
+ Example:
21
+ >>> model_path = download_segmentation_model()
22
+ >>> # Downloads from wg-lux/colo_segmentation_RegNetX800MF_base
23
+ """
24
+ local_path = huggingface_hub.hf_hub_download(
25
+ repo_id=repo_id,
26
+ filename=filename,
27
+ cache_dir=cache_dir,
28
+ force_download=False,
29
+ resume_download=True,
30
+ )
31
+ return local_path
@@ -0,0 +1,6 @@
1
+ from endoreg_db.models import EndoscopyProcessor
2
+ from django.core.management.base import BaseCommand
3
+ from typing import TYPE_CHECKING
4
+ from django.db.models import QuerySet
5
+
6
+
@@ -0,0 +1,119 @@
1
+ from pathlib import Path
2
+ import random
3
+ import re
4
+ from typing import Optional
5
+
6
+ from logging import getLogger
7
+ logger = getLogger(__name__)
8
+
9
+
10
+ from endoreg_db.utils.video.names import (
11
+ get_video_key_regex_by_examination_alias
12
+ )
13
+
14
+ from django.conf import settings
15
+
16
+
17
+ ASSET_DIR:Path = settings.ASSET_DIR
18
+ assert ASSET_DIR.exists(), f"ASSET_DIR does not exist: {ASSET_DIR}"
19
+
20
+ _TEST_VIDEOS = {
21
+ # "egd-instrument-non_anonymous": ASSET_DIR / "test_instrument.mp4", # No detected segments
22
+ "egd-endoscope-non_anonymous": ASSET_DIR / "test_endoscope.mp4",
23
+ "egd-nbi-non_anonymous": ASSET_DIR / "test_nbi.mp4",
24
+ # "egd-outside-non_anonymous": ASSET_DIR / "test_outside.mp4",
25
+ "egd-small_intestine-non_anonymous": ASSET_DIR / "test_small_intestine.mp4",
26
+ }
27
+
28
+ # Only keep entries that actually exist on disk to avoid returning None paths
29
+ TEST_VIDEOS = {key: value for key, value in _TEST_VIDEOS.items() if value.exists()}
30
+
31
+
32
+ def get_video_path(video_key:str) -> Path:
33
+ """
34
+ Returns the file path associated with the given video key.
35
+
36
+ Raises:
37
+ ValueError: If the video key does not exist in the available test videos.
38
+ """
39
+ if video_key in TEST_VIDEOS:
40
+ return TEST_VIDEOS[video_key]
41
+ else:
42
+ raise ValueError(f"Video key '{video_key}' not found in TEST_VIDEOS.")
43
+
44
+ def get_video_keys(
45
+ examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
46
+ ):
47
+ """
48
+ Returns a list of video keys matching the specified examination alias, content type, and anonymity status.
49
+
50
+ If no direct matches are found, falls back to suffix-based filtering for anonymity. Logs warnings and errors when fallback logic is used or no matches are found.
51
+
52
+ Args:
53
+ examination_alias: The examination alias to filter by, or None for any.
54
+ content: The content type to filter by, or None for any.
55
+ is_anonymous: Whether to filter for anonymous, non-anonymous, or both.
56
+
57
+ Returns:
58
+ A list of matching video keys.
59
+ """
60
+ # Build a more flexible pattern locally if content is None
61
+ if content is None:
62
+ pattern_parts = ["^"]
63
+ if examination_alias:
64
+ pattern_parts.append(re.escape(examination_alias))
65
+ pattern_parts.append("-.*-") # Match any content
66
+ else:
67
+ pattern_parts.append(".*-") # Match any examination alias and content
68
+
69
+ if is_anonymous is True:
70
+ pattern_parts.append("anonymous$")
71
+ elif is_anonymous is False:
72
+ pattern_parts.append("non_anonymous$")
73
+ else: # is_anonymous is None
74
+ pattern_parts.append("(non_)?anonymous$") # Match either
75
+
76
+ pattern = "".join(pattern_parts)
77
+ else:
78
+ # Use the imported function if content is specified
79
+ pattern = get_video_key_regex_by_examination_alias(examination_alias, content, is_anonymous)
80
+ logger.warning(f"Generated pattern (from imported function): {pattern}")
81
+
82
+
83
+ # Only consider keys for which the file actually exists
84
+ keys_to_check = list(TEST_VIDEOS.keys())
85
+ matched_keys = [key for key in keys_to_check if re.match(pattern, key)]
86
+
87
+ # Fallback logic remains as a safety net, but ideally shouldn't be needed now for this case
88
+ if not matched_keys and is_anonymous is False:
89
+ logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=False. Falling back to suffix check '-non_anonymous'.")
90
+ matched_keys = [key for key in keys_to_check if key.endswith("-non_anonymous")]
91
+ elif not matched_keys and is_anonymous is True:
92
+ logger.warning(f"Pattern '{pattern}' yielded no results for is_anonymous=True. Falling back to suffix check '-anonymous'.")
93
+ matched_keys = [key for key in keys_to_check if key.endswith("-anonymous")]
94
+
95
+ if not matched_keys:
96
+ logger.error(f"No keys found matching pattern '{pattern}' or fallback logic for keys: {keys_to_check}")
97
+
98
+
99
+ return matched_keys
100
+
101
+
102
+ def get_random_video_path_by_examination_alias(
103
+ examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
104
+ ):
105
+ """
106
+ Returns the file path of a randomly selected video matching the specified examination alias, content type, and anonymity status.
107
+
108
+ Raises:
109
+ ValueError: If no matching video keys are found for the given criteria.
110
+ """
111
+ keys = get_video_keys(examination_alias, content, is_anonymous)
112
+ if keys:
113
+ random_video_key = random.choice(keys)
114
+ video_path = get_video_path(random_video_key)
115
+ return video_path # Return the first match for simplicity
116
+ else:
117
+ raise ValueError("No matching video keys found for the given criteria.")
118
+
119
+
@@ -0,0 +1,140 @@
1
+ import logging
2
+ from pathlib import Path
3
+ import os
4
+
5
+ LOG_DIR = Path("data/logs")
6
+ DEFAULT_FILE_LOG_LEVEL = "INFO"
7
+ DEFAULT_CONSOLE_LOG_LEVEL = "WARNING"
8
+
9
+ def clear_log_files(logger_names, log_dir=None):
10
+ """
11
+ Clears specified log files in the log directory.
12
+
13
+ Args:
14
+ logger_names (list): A list of strings, each being a logger name.
15
+ log_dir (Path, optional): The directory containing log files. Defaults to LOG_DIR.
16
+ """
17
+ log_dir = log_dir or LOG_DIR
18
+
19
+ # Ensure log directory exists (though clearing implies it might)
20
+ log_dir.mkdir(parents=True, exist_ok=True)
21
+
22
+ # Files to manage
23
+ log_files_to_clear = [log_dir / f"{logger_name}.log" for logger_name in logger_names]
24
+ log_files_to_clear.append(log_dir / "root.log") # Add root log file
25
+
26
+ # Clear existing log files in the directory managed by this config
27
+ for log_file in log_files_to_clear:
28
+ if log_file.exists():
29
+ try:
30
+ log_file.unlink()
31
+ print(f"Removed log file: {log_file}")
32
+ except OSError as e:
33
+ print(f"Error removing log file {log_file}: {e}")
34
+ # Optionally, create the log file to ensure it exists after clearing
35
+ # log_file.touch() # Removed touch to just clear
36
+
37
+ def get_logging_config(logger_names, file_log_level=None, console_log_level=None, log_dir=None):
38
+ """
39
+ Generates Django LOGGING configuration dynamically.
40
+
41
+ Args:
42
+ logger_names (list): A list of strings, each being a logger name.
43
+ file_log_level (str, optional): The default log level for file handlers. Defaults to DEFAULT_FILE_LOG_LEVEL.
44
+ console_log_level (str, optional): The default log level for the console handler. Defaults to DEFAULT_CONSOLE_LOG_LEVEL.
45
+ log_dir (Path, optional): The directory to store log files. Defaults to LOG_DIR.
46
+
47
+ Returns:
48
+ dict: The Django LOGGING configuration dictionary.
49
+ """
50
+ file_log_level = file_log_level or os.environ.get("FILE_LOG_LEVEL", DEFAULT_FILE_LOG_LEVEL)
51
+ console_log_level = console_log_level or os.environ.get("CONSOLE_LOG_LEVEL", DEFAULT_CONSOLE_LOG_LEVEL)
52
+ log_dir = log_dir or LOG_DIR
53
+
54
+ # Ensure log directory exists
55
+ log_dir.mkdir(parents=True, exist_ok=True)
56
+
57
+ handlers = {
58
+ 'console': {
59
+ 'level': console_log_level, # Use console level
60
+ 'class': 'logging.StreamHandler',
61
+ 'formatter': 'standard',
62
+ },
63
+ 'file_root': { # Handler for the root logger's file
64
+ 'level': file_log_level, # Use file level
65
+ 'class': 'logging.FileHandler',
66
+ 'filename': log_dir / "root.log",
67
+ 'formatter': 'standard',
68
+ },
69
+ }
70
+ loggers = {
71
+ # Root logger configuration - logs INFO+ to file, WARNING+ to console
72
+ '': {
73
+ 'handlers': ['console', 'file_root'], # Use both handlers
74
+ 'level': file_log_level, # Set to lowest level needed (INFO for file)
75
+ 'propagate': False, # Root logger doesn't propagate further
76
+ },
77
+ # Django's default logger
78
+ 'django': {
79
+ 'handlers': ['console', 'file_root'], # Log to console and root file
80
+ 'level': file_log_level, # Or your preferred level, ensure it's captured by root file
81
+ 'propagate': False, # Don't propagate Django logs to avoid double handling by root
82
+ },
83
+ }
84
+
85
+ # Dynamically create file handlers and logger configurations
86
+ for name in logger_names:
87
+ handler_name = f"file_{name}"
88
+ handlers[handler_name] = {
89
+ 'level': file_log_level, # Use file level
90
+ 'class': 'logging.FileHandler',
91
+ 'filename': log_dir / f"{name}.log",
92
+ 'formatter': 'standard',
93
+ }
94
+ loggers[name] = {
95
+ 'handlers': [handler_name], # Log only to its own file directly
96
+ 'level': file_log_level, # Use file level
97
+ 'propagate': True, # Propagate to root logger (which handles console and root.log)
98
+ }
99
+
100
+ return {
101
+ 'version': 1,
102
+ 'disable_existing_loggers': False,
103
+ 'formatters': {
104
+ 'standard': {
105
+ 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
106
+ },
107
+ },
108
+ 'handlers': handlers,
109
+ 'loggers': loggers,
110
+ }
111
+
112
+ # Example usage (optional, for testing the function itself)
113
+ if __name__ == "__main__":
114
+ test_loggers = ["app1", "app2", "database"]
115
+ log_directory = Path("data/logs_test") # Use a test directory
116
+
117
+ # Optionally clear logs before configuring logging
118
+ print("Clearing logs...")
119
+ clear_log_files(test_loggers, log_dir=log_directory)
120
+ print("Log clearing complete.")
121
+
122
+ # Get logging configuration
123
+ logging_config = get_logging_config(test_loggers, file_log_level="DEBUG", console_log_level="WARNING", log_dir=log_directory)
124
+ import json
125
+ print("\nGenerated Logging Configuration:")
126
+ print(json.dumps(logging_config, indent=4))
127
+
128
+ # Example of how to use it with Python's logging
129
+ # import logging.config
130
+ # logging.config.dictConfig(logging_config)
131
+ # logger1 = logging.getLogger("app1")
132
+ # logger1.debug("This is a debug message for app1.")
133
+ # logger1.info("This is an info message for app1.")
134
+
135
+ # logger_db = logging.getLogger("database")
136
+ # logger_db.warning("This is a warning for database.")
137
+
138
+ # root_logger = logging.getLogger()
139
+ # root_logger.info("This is an info message from root.") # Should go to root.log and console (if level >= WARNING)
140
+ # root_logger.warning("This is a warning message from root.") # Should go to root.log and console
@@ -0,0 +1 @@
1
+ # Management commands package
@@ -0,0 +1 @@
1
+ # Management commands
File without changes