endoreg-db 0.8.9.32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (787) hide show
  1. endoreg_db/__init__.py +0 -0
  2. endoreg_db/_version.py +34 -0
  3. endoreg_db/admin.py +97 -0
  4. endoreg_db/api/serializers/finding_descriptions.py +0 -0
  5. endoreg_db/api/views/finding_descriptions.py +0 -0
  6. endoreg_db/api_urls.py +4 -0
  7. endoreg_db/apps.py +17 -0
  8. endoreg_db/assets/dummy_model.ckpt +1 -0
  9. endoreg_db/authz/auth.py +78 -0
  10. endoreg_db/authz/backends.py +168 -0
  11. endoreg_db/authz/management/commands/list_routes.py +20 -0
  12. endoreg_db/authz/middleware.py +84 -0
  13. endoreg_db/authz/permissions.py +138 -0
  14. endoreg_db/authz/policy.py +224 -0
  15. endoreg_db/authz/settings.py +64 -0
  16. endoreg_db/authz/views_auth.py +70 -0
  17. endoreg_db/codemods/readme.md +88 -0
  18. endoreg_db/codemods/rename_datetime_fields.py +99 -0
  19. endoreg_db/config/__init__.py +0 -0
  20. endoreg_db/config/env.py +106 -0
  21. endoreg_db/config/settings/__init__.py +6 -0
  22. endoreg_db/config/settings/base.py +148 -0
  23. endoreg_db/config/settings/case_gen.py +32 -0
  24. endoreg_db/config/settings/dev.py +108 -0
  25. endoreg_db/config/settings/keycloak.py +177 -0
  26. endoreg_db/config/settings/prod.py +66 -0
  27. endoreg_db/config/settings/test.py +72 -0
  28. endoreg_db/data/__init__.py +135 -0
  29. endoreg_db/data/ai_model/data.yaml +7 -0
  30. endoreg_db/data/ai_model_label/label/data.yaml +88 -0
  31. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  32. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  33. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  34. endoreg_db/data/ai_model_label/label-type/data.yaml +7 -0
  35. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  36. endoreg_db/data/ai_model_type/data.yaml +7 -0
  37. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  38. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  39. endoreg_db/data/case_template/rule/00_patient_lab_sample_add_default_value.yaml +167 -0
  40. endoreg_db/data/case_template/rule/01_patient-set-age.yaml +8 -0
  41. endoreg_db/data/case_template/rule/01_patient-set-gender.yaml +9 -0
  42. endoreg_db/data/case_template/rule/11_create_patient_lab_sample.yaml +23 -0
  43. endoreg_db/data/case_template/rule/12_create-patient_medication-anticoagulation.yaml +19 -0
  44. endoreg_db/data/case_template/rule/13_create-patient_medication_schedule-anticoagulation.yaml +19 -0
  45. endoreg_db/data/case_template/rule/19_create_patient.yaml +17 -0
  46. endoreg_db/data/case_template/rule_type/base_types.yaml +35 -0
  47. endoreg_db/data/case_template/rule_value/.init +0 -0
  48. endoreg_db/data/case_template/rule_value_type/base_types.yaml +59 -0
  49. endoreg_db/data/case_template/template/base.yaml +8 -0
  50. endoreg_db/data/case_template/template_type/pre_endoscopy.yaml +3 -0
  51. endoreg_db/data/case_template/tmp/_rule_value +13 -0
  52. endoreg_db/data/case_template/tmp/rule/01_atrial_fibrillation.yaml +21 -0
  53. endoreg_db/data/case_template/tmp/rule/02_create_object.yaml +10 -0
  54. endoreg_db/data/case_template/tmp/template/atrial_fibrillation_low_risk.yaml +7 -0
  55. endoreg_db/data/center/data.yaml +99 -0
  56. endoreg_db/data/center_resource/green_endoscopy_dashboard_CenterResource.yaml +144 -0
  57. endoreg_db/data/center_shift/ukw.yaml +9 -0
  58. endoreg_db/data/center_waste/green_endoscopy_dashboard_CenterWaste.yaml +48 -0
  59. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  60. endoreg_db/data/db_summary.csv +58 -0
  61. endoreg_db/data/db_summary.xlsx +0 -0
  62. endoreg_db/data/disease/cardiovascular.yaml +37 -0
  63. endoreg_db/data/disease/hepatology.yaml +5 -0
  64. endoreg_db/data/disease/misc.yaml +5 -0
  65. endoreg_db/data/disease/renal.yaml +5 -0
  66. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +6 -0
  67. endoreg_db/data/disease_classification/coronary_vessel_disease.yaml +6 -0
  68. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +41 -0
  69. endoreg_db/data/disease_classification_choice/coronary_vessel_disease.yaml +20 -0
  70. endoreg_db/data/distribution/date/patient.yaml +7 -0
  71. endoreg_db/data/distribution/multiple_categorical/.init +0 -0
  72. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  73. endoreg_db/data/distribution/single_categorical/patient.yaml +7 -0
  74. endoreg_db/data/emission_factor/green_endoscopy_dashboard_EmissionFactor.yaml +132 -0
  75. endoreg_db/data/endoscope/data.yaml +93 -0
  76. endoreg_db/data/endoscope_type/data.yaml +11 -0
  77. endoreg_db/data/endoscopy_processor/data.yaml +50 -0
  78. endoreg_db/data/event/cardiology.yaml +15 -0
  79. endoreg_db/data/event/neurology.yaml +14 -0
  80. endoreg_db/data/event/surgery.yaml +13 -0
  81. endoreg_db/data/event/thrombembolism.yaml +20 -0
  82. endoreg_db/data/event_classification/data.yaml +4 -0
  83. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  84. endoreg_db/data/examination/examinations/data.yaml +172 -0
  85. endoreg_db/data/examination/time/data.yaml +48 -0
  86. endoreg_db/data/examination/time-type/data.yaml +5 -0
  87. endoreg_db/data/examination/type/data.yaml +17 -0
  88. endoreg_db/data/examination_indication/endoscopy.yaml +359 -0
  89. endoreg_db/data/examination_indication_classification/endoscopy.yaml +90 -0
  90. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +97 -0
  91. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  92. endoreg_db/data/finding/00_generic.yaml +35 -0
  93. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  94. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  95. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  96. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  97. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  98. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  99. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  100. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  101. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  102. endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
  103. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  104. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  105. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  106. endoreg_db/data/finding_classification_choice/00_generic.yaml +15 -0
  107. endoreg_db/data/finding_classification_choice/00_generic_baseline.yaml +23 -0
  108. endoreg_db/data/finding_classification_choice/00_generic_complication.yaml +15 -0
  109. endoreg_db/data/finding_classification_choice/00_generic_histology.yaml +21 -0
  110. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  111. endoreg_db/data/finding_classification_choice/02_colonoscopy_bowel_preparation.yaml +49 -0
  112. endoreg_db/data/finding_classification_choice/02_colonoscopy_generic.yaml +19 -0
  113. endoreg_db/data/finding_classification_choice/02_colonoscopy_histology.yaml +20 -0
  114. endoreg_db/data/finding_classification_choice/02_colonoscopy_location.yaml +248 -0
  115. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  116. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  117. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_morphology.yaml +75 -0
  118. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  119. endoreg_db/data/finding_classification_type/00_generic.yaml +53 -0
  120. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  121. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  122. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  123. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  124. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  125. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  126. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  127. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  128. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  129. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  130. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  131. endoreg_db/data/finding_type/data.yaml +39 -0
  132. endoreg_db/data/gender/data.yaml +42 -0
  133. endoreg_db/data/information_source/annotation.yaml +6 -0
  134. endoreg_db/data/information_source/data.yaml +30 -0
  135. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  136. endoreg_db/data/information_source/medication.yaml +6 -0
  137. endoreg_db/data/information_source/prediction.yaml +7 -0
  138. endoreg_db/data/information_source_type/data.yaml +8 -0
  139. endoreg_db/data/lab_value/cardiac_enzymes.yaml +37 -0
  140. endoreg_db/data/lab_value/coagulation.yaml +54 -0
  141. endoreg_db/data/lab_value/electrolytes.yaml +228 -0
  142. endoreg_db/data/lab_value/gastrointestinal_function.yaml +133 -0
  143. endoreg_db/data/lab_value/hematology.yaml +184 -0
  144. endoreg_db/data/lab_value/hormones.yaml +59 -0
  145. endoreg_db/data/lab_value/lipids.yaml +53 -0
  146. endoreg_db/data/lab_value/misc.yaml +76 -0
  147. endoreg_db/data/lab_value/renal_function.yaml +12 -0
  148. endoreg_db/data/log_type/data.yaml +57 -0
  149. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  150. endoreg_db/data/lx_client_type/base.yaml +30 -0
  151. endoreg_db/data/lx_permission/base.yaml +24 -0
  152. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  153. endoreg_db/data/material/material.yaml +91 -0
  154. endoreg_db/data/medication/anticoagulation.yaml +65 -0
  155. endoreg_db/data/medication/tah.yaml +70 -0
  156. endoreg_db/data/medication_indication/anticoagulation.yaml +115 -0
  157. endoreg_db/data/medication_indication_type/data.yaml +11 -0
  158. endoreg_db/data/medication_indication_type/thrombembolism.yaml +41 -0
  159. endoreg_db/data/medication_intake_time/base.yaml +31 -0
  160. endoreg_db/data/medication_schedule/apixaban.yaml +95 -0
  161. endoreg_db/data/medication_schedule/ass.yaml +12 -0
  162. endoreg_db/data/medication_schedule/enoxaparin.yaml +26 -0
  163. endoreg_db/data/names_first/first_names.yaml +54 -0
  164. endoreg_db/data/names_last/last_names.yaml +51 -0
  165. endoreg_db/data/network_device/data.yaml +59 -0
  166. endoreg_db/data/network_device_type/data.yaml +12 -0
  167. endoreg_db/data/organ/data.yaml +29 -0
  168. endoreg_db/data/patient_lab_sample_type/generic.yaml +6 -0
  169. endoreg_db/data/pdf_type/data.yaml +46 -0
  170. endoreg_db/data/product/green_endoscopy_dashboard_Product.yaml +66 -0
  171. endoreg_db/data/product_group/green_endoscopy_dashboard_ProductGroup.yaml +33 -0
  172. endoreg_db/data/product_material/green_endoscopy_dashboard_ProductMaterial.yaml +308 -0
  173. endoreg_db/data/product_weight/green_endoscopy_dashboard_ProductWeight.yaml +88 -0
  174. endoreg_db/data/profession/data.yaml +70 -0
  175. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  176. endoreg_db/data/qualification/m2.yaml +39 -0
  177. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  178. endoreg_db/data/qualification/sonography.yaml +36 -0
  179. endoreg_db/data/qualification_type/base.yaml +29 -0
  180. endoreg_db/data/reference_product/green_endoscopy_dashboard_ReferenceProduct.yaml +55 -0
  181. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  182. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +30 -0
  183. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +24 -0
  184. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  185. endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
  186. endoreg_db/data/requirement/old/colonoscopy_baseline_austria.yaml +45 -0
  187. endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
  188. endoreg_db/data/requirement/old/disease_cardiovascular.yaml +79 -0
  189. endoreg_db/data/requirement/old/disease_classification_choice_cardiovascular.yaml +41 -0
  190. endoreg_db/data/requirement/old/disease_hepatology.yaml +12 -0
  191. endoreg_db/data/requirement/old/disease_misc.yaml +12 -0
  192. endoreg_db/data/requirement/old/disease_renal.yaml +96 -0
  193. endoreg_db/data/requirement/old/endoscopy_bleeding_risk.yaml +59 -0
  194. endoreg_db/data/requirement/old/event_cardiology.yaml +251 -0
  195. endoreg_db/data/requirement/old/event_requirements.yaml +145 -0
  196. endoreg_db/data/requirement/old/finding_colon_polyp.yaml +50 -0
  197. endoreg_db/data/requirement/old/gender.yaml +0 -0
  198. endoreg_db/data/requirement/old/lab_value.yaml +441 -0
  199. endoreg_db/data/requirement/old/medication.yaml +93 -0
  200. endoreg_db/data/requirement_operator/_old/age.yaml +13 -0
  201. endoreg_db/data/requirement_operator/_old/lab_operators.yaml +129 -0
  202. endoreg_db/data/requirement_operator/_old/model_operators.yaml +96 -0
  203. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  204. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +65 -0
  205. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  206. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  207. endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
  208. endoreg_db/data/requirement_set/_old_ +109 -0
  209. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  210. endoreg_db/data/requirement_set_type/data.yaml +41 -0
  211. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  212. endoreg_db/data/resource/green_endoscopy_dashboard_Resource.yaml +15 -0
  213. endoreg_db/data/risk/bleeding.yaml +26 -0
  214. endoreg_db/data/risk/thrombosis.yaml +37 -0
  215. endoreg_db/data/risk_type/data.yaml +27 -0
  216. endoreg_db/data/setup_config.yaml +38 -0
  217. endoreg_db/data/shift/endoscopy.yaml +21 -0
  218. endoreg_db/data/shift/m2.yaml +0 -0
  219. endoreg_db/data/shift_type/base.yaml +35 -0
  220. endoreg_db/data/tag/requirement_set_tags.yaml +32 -0
  221. endoreg_db/data/tmp/chronic_kidney_disease.yaml +0 -0
  222. endoreg_db/data/tmp/congestive_heart_failure.yaml +0 -0
  223. endoreg_db/data/transport_route/green_endoscopy_dashboard_TransportRoute.yaml +12 -0
  224. endoreg_db/data/unit/concentration.yaml +115 -0
  225. endoreg_db/data/unit/data.yaml +17 -0
  226. endoreg_db/data/unit/length.yaml +31 -0
  227. endoreg_db/data/unit/misc.yaml +20 -0
  228. endoreg_db/data/unit/rate.yaml +6 -0
  229. endoreg_db/data/unit/time.yaml +48 -0
  230. endoreg_db/data/unit/volume.yaml +35 -0
  231. endoreg_db/data/unit/weight.yaml +38 -0
  232. endoreg_db/data/waste/data.yaml +12 -0
  233. endoreg_db/exceptions.py +24 -0
  234. endoreg_db/export/frames/export.py +6 -0
  235. endoreg_db/export/frames/export_frames_with_labels.py +616 -0
  236. endoreg_db/factories/__init__.py +0 -0
  237. endoreg_db/forms/__init__.py +4 -0
  238. endoreg_db/forms/examination_form.py +12 -0
  239. endoreg_db/forms/patient_finding_intervention_form.py +40 -0
  240. endoreg_db/forms/patient_form.py +23 -0
  241. endoreg_db/forms/questionnaires/__init__.py +1 -0
  242. endoreg_db/forms/questionnaires/tto_questionnaire.py +23 -0
  243. endoreg_db/forms/settings/__init__.py +11 -0
  244. endoreg_db/forms/unit.py +7 -0
  245. endoreg_db/helpers/__init__.py +0 -0
  246. endoreg_db/helpers/count_db.py +48 -0
  247. endoreg_db/helpers/data_loader.py +280 -0
  248. endoreg_db/helpers/default_objects.py +414 -0
  249. endoreg_db/helpers/download_segmentation_model.py +32 -0
  250. endoreg_db/helpers/interact.py +1 -0
  251. endoreg_db/helpers/test_video_helper.py +127 -0
  252. endoreg_db/import_files/__init__.py +27 -0
  253. endoreg_db/import_files/context/__init__.py +7 -0
  254. endoreg_db/import_files/context/default_sensitive_meta.py +83 -0
  255. endoreg_db/import_files/context/ensure_center.py +17 -0
  256. endoreg_db/import_files/context/file_lock.py +66 -0
  257. endoreg_db/import_files/context/import_context.py +42 -0
  258. endoreg_db/import_files/context/validate_directories.py +57 -0
  259. endoreg_db/import_files/file_storage/__init__.py +15 -0
  260. endoreg_db/import_files/file_storage/create_report_file.py +99 -0
  261. endoreg_db/import_files/file_storage/create_video_file.py +104 -0
  262. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +42 -0
  263. endoreg_db/import_files/file_storage/state_management.py +463 -0
  264. endoreg_db/import_files/file_storage/storage.py +42 -0
  265. endoreg_db/import_files/import_service.md +26 -0
  266. endoreg_db/import_files/processing/__init__.py +11 -0
  267. endoreg_db/import_files/processing/report_processing/report_anonymization.py +99 -0
  268. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  269. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  270. endoreg_db/import_files/pseudonymization/__init__.py +0 -0
  271. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  272. endoreg_db/import_files/pseudonymization/k_anonymity.py +181 -0
  273. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +139 -0
  274. endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
  275. endoreg_db/import_files/report_import_service.py +147 -0
  276. endoreg_db/import_files/video_import_service.py +154 -0
  277. endoreg_db/logger_conf.py +156 -0
  278. endoreg_db/management/__init__.py +1 -0
  279. endoreg_db/management/commands/__init__.py +1 -0
  280. endoreg_db/management/commands/anonymize_video.py +0 -0
  281. endoreg_db/management/commands/check_auth.py +132 -0
  282. endoreg_db/management/commands/create_model_meta_from_huggingface.py +177 -0
  283. endoreg_db/management/commands/create_multilabel_model_meta.py +419 -0
  284. endoreg_db/management/commands/export_frame_annot.py +196 -0
  285. endoreg_db/management/commands/fix_missing_patient_data.py +206 -0
  286. endoreg_db/management/commands/fix_video_paths.py +186 -0
  287. endoreg_db/management/commands/import_report.py +361 -0
  288. endoreg_db/management/commands/list_routes.py +20 -0
  289. endoreg_db/management/commands/load_ai_model_data.py +83 -0
  290. endoreg_db/management/commands/load_ai_model_label_data.py +60 -0
  291. endoreg_db/management/commands/load_base_db_data.py +63 -0
  292. endoreg_db/management/commands/load_center_data.py +68 -0
  293. endoreg_db/management/commands/load_contraindication_data.py +39 -0
  294. endoreg_db/management/commands/load_disease_classification_choices_data.py +38 -0
  295. endoreg_db/management/commands/load_disease_classification_data.py +38 -0
  296. endoreg_db/management/commands/load_disease_data.py +59 -0
  297. endoreg_db/management/commands/load_distribution_data.py +63 -0
  298. endoreg_db/management/commands/load_endoscope_data.py +58 -0
  299. endoreg_db/management/commands/load_event_data.py +39 -0
  300. endoreg_db/management/commands/load_examination_data.py +78 -0
  301. endoreg_db/management/commands/load_examination_indication_data.py +85 -0
  302. endoreg_db/management/commands/load_finding_data.py +115 -0
  303. endoreg_db/management/commands/load_gender_data.py +37 -0
  304. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +142 -0
  305. endoreg_db/management/commands/load_information_source.py +46 -0
  306. endoreg_db/management/commands/load_lab_value_data.py +52 -0
  307. endoreg_db/management/commands/load_legacy_data.py +303 -0
  308. endoreg_db/management/commands/load_medication_data.py +104 -0
  309. endoreg_db/management/commands/load_name_data.py +36 -0
  310. endoreg_db/management/commands/load_organ_data.py +39 -0
  311. endoreg_db/management/commands/load_pdf_type_data.py +58 -0
  312. endoreg_db/management/commands/load_profession_data.py +40 -0
  313. endoreg_db/management/commands/load_qualification_data.py +56 -0
  314. endoreg_db/management/commands/load_report_reader_flag_data.py +40 -0
  315. endoreg_db/management/commands/load_requirement_data.py +207 -0
  316. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  317. endoreg_db/management/commands/load_risk_data.py +57 -0
  318. endoreg_db/management/commands/load_shift_data.py +57 -0
  319. endoreg_db/management/commands/load_tag_data.py +54 -0
  320. endoreg_db/management/commands/load_unit_data.py +40 -0
  321. endoreg_db/management/commands/load_user_groups.py +26 -0
  322. endoreg_db/management/commands/model_input.py +169 -0
  323. endoreg_db/management/commands/register_ai_model.py +70 -0
  324. endoreg_db/management/commands/setup_endoreg_db.py +459 -0
  325. endoreg_db/management/commands/start_filewatcher.py +115 -0
  326. endoreg_db/management/commands/storage_management.py +622 -0
  327. endoreg_db/management/commands/summarize_db_content.py +280 -0
  328. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  329. endoreg_db/management/commands/validate_video_files.py +189 -0
  330. endoreg_db/management/commands/video_validation.py +20 -0
  331. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  332. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  333. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  334. endoreg_db/mermaid/classification.md +8 -0
  335. endoreg_db/mermaid/examination.md +8 -0
  336. endoreg_db/mermaid/findings.md +7 -0
  337. endoreg_db/mermaid/image_classification.md +28 -0
  338. endoreg_db/mermaid/interventions.md +8 -0
  339. endoreg_db/mermaid/morphology.md +8 -0
  340. endoreg_db/mermaid/patient_creation.md +14 -0
  341. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  342. endoreg_db/migrations/0001_initial.py +1953 -0
  343. endoreg_db/migrations/__init__.py +0 -0
  344. endoreg_db/models/__init__.py +322 -0
  345. endoreg_db/models/administration/__init__.py +95 -0
  346. endoreg_db/models/administration/ai/__init__.py +9 -0
  347. endoreg_db/models/administration/ai/active_model.py +35 -0
  348. endoreg_db/models/administration/ai/ai_model.py +180 -0
  349. endoreg_db/models/administration/ai/model_type.py +42 -0
  350. endoreg_db/models/administration/case/__init__.py +5 -0
  351. endoreg_db/models/administration/case/case.py +114 -0
  352. endoreg_db/models/administration/case/case_template/__init__.py +3 -0
  353. endoreg_db/models/administration/case/case_template/case_template.py +3 -0
  354. endoreg_db/models/administration/case/case_template/case_template_rule.py +3 -0
  355. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +3 -0
  356. endoreg_db/models/administration/case/case_template/case_template_type.py +3 -0
  357. endoreg_db/models/administration/center/__init__.py +13 -0
  358. endoreg_db/models/administration/center/center.py +85 -0
  359. endoreg_db/models/administration/center/center_product.py +67 -0
  360. endoreg_db/models/administration/center/center_resource.py +69 -0
  361. endoreg_db/models/administration/center/center_shift.py +94 -0
  362. endoreg_db/models/administration/center/center_waste.py +42 -0
  363. endoreg_db/models/administration/person/__init__.py +26 -0
  364. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  365. endoreg_db/models/administration/person/employee/employee.py +40 -0
  366. endoreg_db/models/administration/person/employee/employee_qualification.py +44 -0
  367. endoreg_db/models/administration/person/employee/employee_type.py +50 -0
  368. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  369. endoreg_db/models/administration/person/examiner/examiner.py +64 -0
  370. endoreg_db/models/administration/person/names/__init__.py +0 -0
  371. endoreg_db/models/administration/person/names/first_name.py +20 -0
  372. endoreg_db/models/administration/person/names/last_name.py +20 -0
  373. endoreg_db/models/administration/person/patient/__init__.py +7 -0
  374. endoreg_db/models/administration/person/patient/patient.py +488 -0
  375. endoreg_db/models/administration/person/patient/patient_external_id.py +36 -0
  376. endoreg_db/models/administration/person/person.py +35 -0
  377. endoreg_db/models/administration/person/profession/__init__.py +28 -0
  378. endoreg_db/models/administration/person/user/__init__.py +5 -0
  379. endoreg_db/models/administration/person/user/portal_user_information.py +41 -0
  380. endoreg_db/models/administration/product/__init__.py +15 -0
  381. endoreg_db/models/administration/product/product.py +106 -0
  382. endoreg_db/models/administration/product/product_group.py +41 -0
  383. endoreg_db/models/administration/product/product_material.py +60 -0
  384. endoreg_db/models/administration/product/product_weight.py +51 -0
  385. endoreg_db/models/administration/product/reference_product.py +147 -0
  386. endoreg_db/models/administration/qualification/__init__.py +7 -0
  387. endoreg_db/models/administration/qualification/qualification.py +43 -0
  388. endoreg_db/models/administration/qualification/qualification_type.py +39 -0
  389. endoreg_db/models/administration/shift/__init__.py +9 -0
  390. endoreg_db/models/administration/shift/scheduled_days.py +72 -0
  391. endoreg_db/models/administration/shift/shift.py +57 -0
  392. endoreg_db/models/administration/shift/shift_type.py +108 -0
  393. endoreg_db/models/aidataset/__init__.py +5 -0
  394. endoreg_db/models/aidataset/aidataset.py +193 -0
  395. endoreg_db/models/label/__init__.py +23 -0
  396. endoreg_db/models/label/annotation/__init__.py +12 -0
  397. endoreg_db/models/label/annotation/image_classification.py +85 -0
  398. endoreg_db/models/label/annotation/video_segmentation_annotation.py +61 -0
  399. endoreg_db/models/label/label.py +91 -0
  400. endoreg_db/models/label/label_set.py +68 -0
  401. endoreg_db/models/label/label_type.py +29 -0
  402. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  403. endoreg_db/models/label/label_video_segment/_create_from_video.py +42 -0
  404. endoreg_db/models/label/label_video_segment/label_video_segment.py +611 -0
  405. endoreg_db/models/label/video_segmentation_label.py +35 -0
  406. endoreg_db/models/label/video_segmentation_labelset.py +28 -0
  407. endoreg_db/models/media/__init__.py +23 -0
  408. endoreg_db/models/media/frame/__init__.py +3 -0
  409. endoreg_db/models/media/frame/frame.py +137 -0
  410. endoreg_db/models/media/pdf/__init__.py +12 -0
  411. endoreg_db/models/media/pdf/raw_pdf.py +764 -0
  412. endoreg_db/models/media/pdf/report_file.py +162 -0
  413. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  414. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +85 -0
  415. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +46 -0
  416. endoreg_db/models/media/video/__init__.py +9 -0
  417. endoreg_db/models/media/video/create_from_file.py +402 -0
  418. endoreg_db/models/media/video/pipe_1.py +258 -0
  419. endoreg_db/models/media/video/pipe_2.py +129 -0
  420. endoreg_db/models/media/video/video_file.py +907 -0
  421. endoreg_db/models/media/video/video_file_ai.py +828 -0
  422. endoreg_db/models/media/video/video_file_anonymize.py +524 -0
  423. endoreg_db/models/media/video/video_file_frames/__init__.py +49 -0
  424. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +25 -0
  425. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  426. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +126 -0
  427. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +233 -0
  428. endoreg_db/models/media/video/video_file_frames/_get_frame.py +36 -0
  429. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +13 -0
  430. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +24 -0
  431. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +40 -0
  432. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +44 -0
  433. endoreg_db/models/media/video/video_file_frames/_get_frames.py +30 -0
  434. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +205 -0
  435. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +228 -0
  436. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +107 -0
  437. endoreg_db/models/media/video/video_file_io.py +272 -0
  438. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  439. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +58 -0
  440. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +62 -0
  441. endoreg_db/models/media/video/video_file_meta/get_fps.py +183 -0
  442. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +198 -0
  443. endoreg_db/models/media/video/video_file_meta/text_meta.py +178 -0
  444. endoreg_db/models/media/video/video_file_meta/video_meta.py +105 -0
  445. endoreg_db/models/media/video/video_file_segments.py +317 -0
  446. endoreg_db/models/media/video/video_metadata.py +67 -0
  447. endoreg_db/models/media/video/video_processing.py +192 -0
  448. endoreg_db/models/medical/__init__.py +136 -0
  449. endoreg_db/models/medical/contraindication/README.md +1 -0
  450. endoreg_db/models/medical/contraindication/__init__.py +29 -0
  451. endoreg_db/models/medical/disease.py +174 -0
  452. endoreg_db/models/medical/event.py +154 -0
  453. endoreg_db/models/medical/examination/__init__.py +20 -0
  454. endoreg_db/models/medical/examination/examination.py +183 -0
  455. endoreg_db/models/medical/examination/examination_indication.py +229 -0
  456. endoreg_db/models/medical/examination/examination_time.py +68 -0
  457. endoreg_db/models/medical/examination/examination_time_type.py +44 -0
  458. endoreg_db/models/medical/examination/examination_type.py +47 -0
  459. endoreg_db/models/medical/finding/__init__.py +20 -0
  460. endoreg_db/models/medical/finding/finding.py +113 -0
  461. endoreg_db/models/medical/finding/finding_classification.py +131 -0
  462. endoreg_db/models/medical/finding/finding_intervention.py +68 -0
  463. endoreg_db/models/medical/finding/finding_type.py +38 -0
  464. endoreg_db/models/medical/hardware/__init__.py +8 -0
  465. endoreg_db/models/medical/hardware/endoscope.py +77 -0
  466. endoreg_db/models/medical/hardware/endoscopy_processor.py +182 -0
  467. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  468. endoreg_db/models/medical/laboratory/lab_value.py +490 -0
  469. endoreg_db/models/medical/medication/__init__.py +23 -0
  470. endoreg_db/models/medical/medication/medication.py +45 -0
  471. endoreg_db/models/medical/medication/medication_indication.py +78 -0
  472. endoreg_db/models/medical/medication/medication_indication_type.py +58 -0
  473. endoreg_db/models/medical/medication/medication_intake_time.py +58 -0
  474. endoreg_db/models/medical/medication/medication_schedule.py +58 -0
  475. endoreg_db/models/medical/organ/__init__.py +38 -0
  476. endoreg_db/models/medical/patient/__init__.py +48 -0
  477. endoreg_db/models/medical/patient/medication_examples.py +56 -0
  478. endoreg_db/models/medical/patient/patient_disease.py +72 -0
  479. endoreg_db/models/medical/patient/patient_event.py +80 -0
  480. endoreg_db/models/medical/patient/patient_examination.py +280 -0
  481. endoreg_db/models/medical/patient/patient_examination_indication.py +57 -0
  482. endoreg_db/models/medical/patient/patient_finding.py +416 -0
  483. endoreg_db/models/medical/patient/patient_finding_classification.py +231 -0
  484. endoreg_db/models/medical/patient/patient_finding_intervention.py +37 -0
  485. endoreg_db/models/medical/patient/patient_lab_sample.py +157 -0
  486. endoreg_db/models/medical/patient/patient_lab_value.py +247 -0
  487. endoreg_db/models/medical/patient/patient_medication.py +111 -0
  488. endoreg_db/models/medical/patient/patient_medication_schedule.py +152 -0
  489. endoreg_db/models/medical/risk/__init__.py +7 -0
  490. endoreg_db/models/medical/risk/risk.py +73 -0
  491. endoreg_db/models/medical/risk/risk_type.py +54 -0
  492. endoreg_db/models/metadata/__init__.py +19 -0
  493. endoreg_db/models/metadata/model_meta.py +266 -0
  494. endoreg_db/models/metadata/model_meta_logic.py +485 -0
  495. endoreg_db/models/metadata/pdf_meta.py +96 -0
  496. endoreg_db/models/metadata/sensitive_meta.py +345 -0
  497. endoreg_db/models/metadata/sensitive_meta_logic.py +1161 -0
  498. endoreg_db/models/metadata/video_meta.py +459 -0
  499. endoreg_db/models/metadata/video_prediction_logic.py +232 -0
  500. endoreg_db/models/metadata/video_prediction_meta.py +319 -0
  501. endoreg_db/models/operation_log.py +63 -0
  502. endoreg_db/models/other/__init__.py +40 -0
  503. endoreg_db/models/other/distribution/__init__.py +46 -0
  504. endoreg_db/models/other/distribution/base_value_distribution.py +22 -0
  505. endoreg_db/models/other/distribution/date_value_distribution.py +163 -0
  506. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +50 -0
  507. endoreg_db/models/other/distribution/numeric_value_distribution.py +211 -0
  508. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +23 -0
  509. endoreg_db/models/other/emission/__init__.py +5 -0
  510. endoreg_db/models/other/emission/emission_factor.py +110 -0
  511. endoreg_db/models/other/gender.py +32 -0
  512. endoreg_db/models/other/information_source.py +190 -0
  513. endoreg_db/models/other/material.py +34 -0
  514. endoreg_db/models/other/resource.py +24 -0
  515. endoreg_db/models/other/tag.py +32 -0
  516. endoreg_db/models/other/transport_route.py +40 -0
  517. endoreg_db/models/other/unit.py +40 -0
  518. endoreg_db/models/other/waste.py +28 -0
  519. endoreg_db/models/report/__init__.py +0 -0
  520. endoreg_db/models/report/images.py +0 -0
  521. endoreg_db/models/report/report.py +5 -0
  522. endoreg_db/models/requirement/__init__.py +11 -0
  523. endoreg_db/models/requirement/requirement.py +792 -0
  524. endoreg_db/models/requirement/requirement_error.py +84 -0
  525. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  526. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  527. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  528. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +6 -0
  529. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +137 -0
  530. endoreg_db/models/requirement/requirement_operator.py +187 -0
  531. endoreg_db/models/requirement/requirement_set.py +327 -0
  532. endoreg_db/models/state/__init__.py +13 -0
  533. endoreg_db/models/state/abstract.py +11 -0
  534. endoreg_db/models/state/anonymization.py +30 -0
  535. endoreg_db/models/state/audit_ledger.py +155 -0
  536. endoreg_db/models/state/label_video_segment.py +31 -0
  537. endoreg_db/models/state/processing_history/__init__.py +3 -0
  538. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  539. endoreg_db/models/state/raw_pdf.py +219 -0
  540. endoreg_db/models/state/sensitive_meta.py +50 -0
  541. endoreg_db/models/state/video.py +251 -0
  542. endoreg_db/models/upload_job.py +100 -0
  543. endoreg_db/models/utils.py +138 -0
  544. endoreg_db/queries/__init__.py +3 -0
  545. endoreg_db/queries/annotations/__init__.py +1 -0
  546. endoreg_db/queries/annotations/legacy.py +169 -0
  547. endoreg_db/queries/sanity/__init_.py +0 -0
  548. endoreg_db/root_urls.py +27 -0
  549. endoreg_db/schemas/__init__.py +0 -0
  550. endoreg_db/schemas/examination_evaluation.py +30 -0
  551. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +861 -0
  552. endoreg_db/serializers/__init__.py +104 -0
  553. endoreg_db/serializers/administration/__init__.py +13 -0
  554. endoreg_db/serializers/administration/ai/__init__.py +9 -0
  555. endoreg_db/serializers/administration/ai/active_model.py +12 -0
  556. endoreg_db/serializers/administration/ai/ai_model.py +20 -0
  557. endoreg_db/serializers/administration/ai/model_type.py +12 -0
  558. endoreg_db/serializers/administration/center.py +14 -0
  559. endoreg_db/serializers/administration/gender.py +11 -0
  560. endoreg_db/serializers/anonymization.py +77 -0
  561. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -0
  562. endoreg_db/serializers/examination/__init__.py +10 -0
  563. endoreg_db/serializers/examination/base.py +45 -0
  564. endoreg_db/serializers/examination/dropdown.py +20 -0
  565. endoreg_db/serializers/examination_serializer.py +9 -0
  566. endoreg_db/serializers/finding/__init__.py +5 -0
  567. endoreg_db/serializers/finding/finding.py +61 -0
  568. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  569. endoreg_db/serializers/finding_classification/choice.py +19 -0
  570. endoreg_db/serializers/finding_classification/classification.py +11 -0
  571. endoreg_db/serializers/label_video_segment/__init__.py +9 -0
  572. endoreg_db/serializers/label_video_segment/image_classification_annotation.py +62 -0
  573. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  574. endoreg_db/serializers/label_video_segment/label/label.py +15 -0
  575. endoreg_db/serializers/label_video_segment/label_video_segment.py +427 -0
  576. endoreg_db/serializers/meta/__init__.py +13 -0
  577. endoreg_db/serializers/meta/sensitive_meta_detail.py +122 -0
  578. endoreg_db/serializers/meta/sensitive_meta_update.py +153 -0
  579. endoreg_db/serializers/meta/sensitive_meta_verification.py +62 -0
  580. endoreg_db/serializers/meta/video_meta.py +39 -0
  581. endoreg_db/serializers/misc/__init__.py +14 -0
  582. endoreg_db/serializers/misc/file_overview.py +72 -0
  583. endoreg_db/serializers/misc/sensitive_patient_data.py +144 -0
  584. endoreg_db/serializers/misc/stats.py +35 -0
  585. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  586. endoreg_db/serializers/misc/upload_job.py +74 -0
  587. endoreg_db/serializers/patient/__init__.py +12 -0
  588. endoreg_db/serializers/patient/patient.py +103 -0
  589. endoreg_db/serializers/patient/patient_dropdown.py +35 -0
  590. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  591. endoreg_db/serializers/patient_examination/patient_examination.py +168 -0
  592. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  593. endoreg_db/serializers/patient_finding/patient_finding.py +32 -0
  594. endoreg_db/serializers/patient_finding/patient_finding_classification.py +47 -0
  595. endoreg_db/serializers/patient_finding/patient_finding_detail.py +62 -0
  596. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +28 -0
  597. endoreg_db/serializers/patient_finding/patient_finding_list.py +40 -0
  598. endoreg_db/serializers/patient_finding/patient_finding_write.py +135 -0
  599. endoreg_db/serializers/pdf/__init__.py +3 -0
  600. endoreg_db/serializers/pdf/anony_text_validation.py +101 -0
  601. endoreg_db/serializers/requirements/requirement_schema.py +20 -0
  602. endoreg_db/serializers/requirements/requirement_sets.py +99 -0
  603. endoreg_db/serializers/sensitive_meta_serializer.py +301 -0
  604. endoreg_db/serializers/video/__init__.py +7 -0
  605. endoreg_db/serializers/video/video_file.py +283 -0
  606. endoreg_db/serializers/video/video_file_brief.py +14 -0
  607. endoreg_db/serializers/video/video_file_detail.py +96 -0
  608. endoreg_db/serializers/video/video_file_list.py +100 -0
  609. endoreg_db/serializers/video/video_processing_history.py +172 -0
  610. endoreg_db/serializers/video_examination.py +198 -0
  611. endoreg_db/services/__init__.py +5 -0
  612. endoreg_db/services/anonymization.py +274 -0
  613. endoreg_db/services/examination_evaluation.py +172 -0
  614. endoreg_db/services/finding_description_service.py +0 -0
  615. endoreg_db/services/lookup_service.py +424 -0
  616. endoreg_db/services/lookup_store.py +266 -0
  617. endoreg_db/services/model_meta_from_hf.py +76 -0
  618. endoreg_db/services/pdf_import.py +0 -0
  619. endoreg_db/services/polling_coordinator.py +319 -0
  620. endoreg_db/services/pseudonym_service.py +94 -0
  621. endoreg_db/services/report_import.py +13 -0
  622. endoreg_db/services/segment_sync.py +171 -0
  623. endoreg_db/services/video_import.py +9 -0
  624. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  625. endoreg_db/templates/admin/start_examination.html +12 -0
  626. endoreg_db/templates/timeline.html +176 -0
  627. endoreg_db/urls/__init__.py +56 -0
  628. endoreg_db/urls/ai.py +14 -0
  629. endoreg_db/urls/anonymization.py +78 -0
  630. endoreg_db/urls/auth.py +16 -0
  631. endoreg_db/urls/classification.py +34 -0
  632. endoreg_db/urls/examination.py +63 -0
  633. endoreg_db/urls/media.py +251 -0
  634. endoreg_db/urls/patient.py +23 -0
  635. endoreg_db/urls/requirements.py +15 -0
  636. endoreg_db/urls/root_urls.py +28 -0
  637. endoreg_db/urls/stats.py +54 -0
  638. endoreg_db/urls/upload.py +12 -0
  639. endoreg_db/urls.py +9 -0
  640. endoreg_db/utils/__init__.py +97 -0
  641. endoreg_db/utils/ai/__init__.py +9 -0
  642. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  643. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  644. endoreg_db/utils/ai/get.py +6 -0
  645. endoreg_db/utils/ai/inference_dataset.py +51 -0
  646. endoreg_db/utils/ai/model_training/config.py +117 -0
  647. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  648. endoreg_db/utils/ai/model_training/losses.py +68 -0
  649. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  650. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  651. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  652. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  653. endoreg_db/utils/ai/multilabel_classification_net.py +270 -0
  654. endoreg_db/utils/ai/postprocess.py +63 -0
  655. endoreg_db/utils/ai/predict.py +293 -0
  656. endoreg_db/utils/ai/preprocess.py +76 -0
  657. endoreg_db/utils/calc_duration_seconds.py +24 -0
  658. endoreg_db/utils/case_generator/__init__.py +3 -0
  659. endoreg_db/utils/case_generator/lab_sample_factory.py +32 -0
  660. endoreg_db/utils/check_video_files.py +175 -0
  661. endoreg_db/utils/cropping.py +30 -0
  662. endoreg_db/utils/dataloader.py +285 -0
  663. endoreg_db/utils/dates.py +59 -0
  664. endoreg_db/utils/defaults/set_default_center.py +33 -0
  665. endoreg_db/utils/env.py +37 -0
  666. endoreg_db/utils/extract_specific_frames.py +87 -0
  667. endoreg_db/utils/file_operations.py +70 -0
  668. endoreg_db/utils/fix_video_path_direct.py +157 -0
  669. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  670. endoreg_db/utils/hashs.py +138 -0
  671. endoreg_db/utils/links/__init__.py +0 -0
  672. endoreg_db/utils/links/requirement_link.py +237 -0
  673. endoreg_db/utils/mime_types.py +0 -0
  674. endoreg_db/utils/names.py +82 -0
  675. endoreg_db/utils/ocr.py +195 -0
  676. endoreg_db/utils/operation_log.py +87 -0
  677. endoreg_db/utils/parse_and_generate_yaml.py +45 -0
  678. endoreg_db/utils/paths.py +159 -0
  679. endoreg_db/utils/permissions.py +160 -0
  680. endoreg_db/utils/pipelines/Readme.md +235 -0
  681. endoreg_db/utils/pipelines/__init__.py +0 -0
  682. endoreg_db/utils/pipelines/process_video_dir.py +144 -0
  683. endoreg_db/utils/product/__init__.py +0 -0
  684. endoreg_db/utils/product/sum_emissions.py +22 -0
  685. endoreg_db/utils/product/sum_weights.py +20 -0
  686. endoreg_db/utils/pydantic_models/__init__.py +5 -0
  687. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  688. endoreg_db/utils/requirement_helpers.py +0 -0
  689. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  690. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +678 -0
  691. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +842 -0
  692. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +114 -0
  693. endoreg_db/utils/setup_config.py +196 -0
  694. endoreg_db/utils/storage.py +117 -0
  695. endoreg_db/utils/translation.py +31 -0
  696. endoreg_db/utils/uuid.py +5 -0
  697. endoreg_db/utils/validate_endo_roi.py +33 -0
  698. endoreg_db/utils/validate_subcategory_dict.py +93 -0
  699. endoreg_db/utils/validate_video_detailed.py +415 -0
  700. endoreg_db/utils/video/__init__.py +30 -0
  701. endoreg_db/utils/video/extract_frames.py +100 -0
  702. endoreg_db/utils/video/ffmpeg_wrapper.py +996 -0
  703. endoreg_db/utils/video/names.py +47 -0
  704. endoreg_db/utils/video/streaming_processor.py +386 -0
  705. endoreg_db/utils/video/video_splitter.py +105 -0
  706. endoreg_db/versioning.md +79 -0
  707. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +247 -0
  708. endoreg_db/views/__init__.py +157 -0
  709. endoreg_db/views/anonymization/__init__.py +31 -0
  710. endoreg_db/views/anonymization/media_management.py +486 -0
  711. endoreg_db/views/anonymization/overview.py +307 -0
  712. endoreg_db/views/anonymization/validate.py +310 -0
  713. endoreg_db/views/auth/__init__.py +13 -0
  714. endoreg_db/views/auth/keycloak.py +146 -0
  715. endoreg_db/views/examination/__init__.py +30 -0
  716. endoreg_db/views/examination/examination.py +37 -0
  717. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  718. endoreg_db/views/examination/get_finding_classification_choices.py +62 -0
  719. endoreg_db/views/examination/get_finding_classifications.py +38 -0
  720. endoreg_db/views/examination/get_findings.py +39 -0
  721. endoreg_db/views/examination/get_instruments.py +19 -0
  722. endoreg_db/views/examination/get_interventions.py +14 -0
  723. endoreg_db/views/finding/__init__.py +9 -0
  724. endoreg_db/views/finding/finding.py +116 -0
  725. endoreg_db/views/finding/get_classifications.py +14 -0
  726. endoreg_db/views/finding/get_interventions.py +17 -0
  727. endoreg_db/views/finding_classification/__init__.py +13 -0
  728. endoreg_db/views/finding_classification/base.py +0 -0
  729. endoreg_db/views/finding_classification/finding_classification.py +41 -0
  730. endoreg_db/views/finding_classification/get_classification_choices.py +54 -0
  731. endoreg_db/views/media/__init__.py +32 -0
  732. endoreg_db/views/media/pdf_media.py +411 -0
  733. endoreg_db/views/media/sensitive_metadata.py +372 -0
  734. endoreg_db/views/media/video_media.py +275 -0
  735. endoreg_db/views/meta/__init__.py +7 -0
  736. endoreg_db/views/meta/sensitive_meta_list.py +102 -0
  737. endoreg_db/views/meta/sensitive_meta_verification.py +74 -0
  738. endoreg_db/views/misc/__init__.py +29 -0
  739. endoreg_db/views/misc/center.py +14 -0
  740. endoreg_db/views/misc/csrf.py +8 -0
  741. endoreg_db/views/misc/gender.py +15 -0
  742. endoreg_db/views/misc/stats.py +255 -0
  743. endoreg_db/views/misc/upload_views.py +241 -0
  744. endoreg_db/views/patient/__init__.py +3 -0
  745. endoreg_db/views/patient/patient.py +253 -0
  746. endoreg_db/views/patient_examination/__init__.py +11 -0
  747. endoreg_db/views/patient_examination/patient_examination.py +141 -0
  748. endoreg_db/views/patient_examination/patient_examination_create.py +58 -0
  749. endoreg_db/views/patient_examination/patient_examination_detail.py +63 -0
  750. endoreg_db/views/patient_examination/patient_examination_list.py +72 -0
  751. endoreg_db/views/patient_examination/video.py +228 -0
  752. endoreg_db/views/patient_finding/__init__.py +7 -0
  753. endoreg_db/views/patient_finding/base.py +0 -0
  754. endoreg_db/views/patient_finding/patient_finding.py +71 -0
  755. endoreg_db/views/patient_finding/patient_finding_optimized.py +291 -0
  756. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  757. endoreg_db/views/patient_finding_classification/pfc_create.py +75 -0
  758. endoreg_db/views/report/__init__.py +7 -0
  759. endoreg_db/views/report/reimport.py +177 -0
  760. endoreg_db/views/report/report_stream.py +191 -0
  761. endoreg_db/views/requirement/__init__.py +11 -0
  762. endoreg_db/views/requirement/evaluate.py +278 -0
  763. endoreg_db/views/requirement/lookup.py +380 -0
  764. endoreg_db/views/requirement/lookup_store.py +183 -0
  765. endoreg_db/views/requirement/requirement_utils.py +87 -0
  766. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  767. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  768. endoreg_db/views/stats/__init__.py +13 -0
  769. endoreg_db/views/stats/stats_views.py +266 -0
  770. endoreg_db/views/video/__init__.py +49 -0
  771. endoreg_db/views/video/ai/__init__.py +8 -0
  772. endoreg_db/views/video/ai/label.py +159 -0
  773. endoreg_db/views/video/correction.py +529 -0
  774. endoreg_db/views/video/reimport.py +230 -0
  775. endoreg_db/views/video/segments_crud.py +709 -0
  776. endoreg_db/views/video/video_apply_mask.py +49 -0
  777. endoreg_db/views/video/video_correction.py +22 -0
  778. endoreg_db/views/video/video_download_processed.py +58 -0
  779. endoreg_db/views/video/video_examination_viewset.py +242 -0
  780. endoreg_db/views/video/video_metadata.py +101 -0
  781. endoreg_db/views/video/video_processing_history.py +25 -0
  782. endoreg_db/views/video/video_remove_frames.py +49 -0
  783. endoreg_db/views/video/video_stream.py +334 -0
  784. endoreg_db-0.8.9.32.dist-info/METADATA +404 -0
  785. endoreg_db-0.8.9.32.dist-info/RECORD +787 -0
  786. endoreg_db-0.8.9.32.dist-info/WHEEL +4 -0
  787. endoreg_db-0.8.9.32.dist-info/licenses/LICENSE +674 -0
@@ -0,0 +1,764 @@
1
+ # models/data_file/import_classes/raw_pdf.py
2
+ # django db model "RawPdf"
3
+ # Class to store raw pdf file using django file field
4
+ # Class contains classmethod to create object from pdf file
5
+ # objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
6
+ # ------------------------------------------------------------------------------
7
+ from typing import TYPE_CHECKING, Optional, cast
8
+
9
+ from django.core.exceptions import ValidationError
10
+ from django.core.files import File
11
+ from django.core.validators import FileExtensionValidator
12
+ from django.db import models
13
+
14
+ from endoreg_db.utils.file_operations import get_content_hash_filename
15
+ from endoreg_db.utils.hashs import get_pdf_hash
16
+ from endoreg_db.utils.paths import (
17
+ ANONYM_REPORT_DIR,
18
+ IMPORT_REPORT_DIR,
19
+ SENSITIVE_REPORT_DIR,
20
+ )
21
+ from endoreg_db.utils.storage import (
22
+ delete_field_file,
23
+ ensure_local_file,
24
+ file_exists,
25
+ save_local_file,
26
+ )
27
+
28
+ if TYPE_CHECKING:
29
+ from django.db.models.fields.files import FieldFile
30
+
31
+ from endoreg_db.models.state import RawPdfState
32
+
33
+ import logging
34
+ from pathlib import Path
35
+
36
+ from ...metadata import SensitiveMeta
37
+
38
+ logger = logging.getLogger("raw_pdf")
39
+
40
+
41
+ class RawPdfFile(models.Model):
42
+ objects = models.Manager()
43
+ # Fields from AbstractPdfFile
44
+ pdf_hash = models.CharField(max_length=255, unique=True)
45
+ pdf_type = models.ForeignKey(
46
+ "PdfType",
47
+ on_delete=models.SET_NULL,
48
+ blank=True,
49
+ null=True,
50
+ )
51
+ center = models.ForeignKey(
52
+ "Center",
53
+ on_delete=models.SET_NULL,
54
+ blank=True,
55
+ null=True,
56
+ )
57
+ examination = models.ForeignKey(
58
+ "PatientExamination",
59
+ on_delete=models.SET_NULL,
60
+ blank=True,
61
+ null=True,
62
+ related_name="raw_pdf_files",
63
+ )
64
+ examiner = models.ForeignKey(
65
+ "Examiner",
66
+ on_delete=models.SET_NULL,
67
+ blank=True,
68
+ null=True,
69
+ )
70
+ text = models.TextField(blank=True, null=True)
71
+ date_created = models.DateTimeField(auto_now_add=True)
72
+ date_modified = models.DateTimeField(auto_now=True)
73
+
74
+ file = models.FileField(
75
+ # Use the relative path from the specific REPORT_DIR
76
+ upload_to=SENSITIVE_REPORT_DIR.name,
77
+ validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
78
+ )
79
+ processed_file = models.FileField(
80
+ upload_to=ANONYM_REPORT_DIR.name,
81
+ validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
82
+ null=True,
83
+ blank=True,
84
+ )
85
+ state = models.OneToOneField(
86
+ "RawPdfState",
87
+ on_delete=models.SET_NULL,
88
+ blank=True,
89
+ null=True,
90
+ related_name="raw_pdf_file",
91
+ )
92
+ patient = models.ForeignKey(
93
+ "Patient",
94
+ on_delete=models.SET_NULL,
95
+ blank=True,
96
+ null=True,
97
+ related_name="raw_pdf_files",
98
+ )
99
+ sensitive_meta = models.ForeignKey(
100
+ "SensitiveMeta",
101
+ on_delete=models.SET_NULL,
102
+ related_name="raw_pdf_files",
103
+ null=True,
104
+ blank=True,
105
+ )
106
+ state_report_processing_required = models.BooleanField(default=True)
107
+ state_report_processed = models.BooleanField(default=False)
108
+ raw_meta = models.JSONField(blank=True, null=True)
109
+ anonym_examination_report = models.OneToOneField(
110
+ "AnonymExaminationReport",
111
+ on_delete=models.SET_NULL,
112
+ blank=True,
113
+ null=True,
114
+ related_name="raw_pdf_file",
115
+ )
116
+ anonymized_text = models.TextField(blank=True, null=True)
117
+
118
+ # Type hinting is needed, improve and use correct django types
119
+ if TYPE_CHECKING:
120
+ from endoreg_db.models import (
121
+ AnonymExaminationReport,
122
+ Center,
123
+ Examiner,
124
+ Patient,
125
+ PatientExamination,
126
+ RawPdfState,
127
+ SensitiveMeta,
128
+ )
129
+
130
+ center: models.ForeignKey["Center | None"]
131
+ examination: models.ForeignKey["PatientExamination | None"]
132
+ examiner: models.ForeignKey["Examiner | None"]
133
+ state: models.ForeignKey["RawPdfState | None"]
134
+ patient: models.ForeignKey["Patient | None"]
135
+ sensitive_meta: models.ForeignKey["SensitiveMeta | None"]
136
+ anonym_examination_report: models.OneToOneField[
137
+ "AnonymExaminationReport | None"
138
+ ]
139
+ file = cast(FieldFile, file)
140
+ processed_file = cast(FieldFile, processed_file)
141
+
142
+ @property
143
+ def uuid(self):
144
+ """
145
+ Compatibility property - returns pdf_hash as UUID-like identifier.
146
+
147
+ Note: RawPdfFile uses pdf_hash instead of UUID for identification.
148
+ This property exists for API backward compatibility.
149
+ """
150
+ return self.pdf_hash
151
+
152
+ @property
153
+ def file_path(self) -> Path | None:
154
+ """
155
+ Returns the file path of the stored report file if available; otherwise, returns None.
156
+ """
157
+ from django.db.models.fields.files import FieldFile
158
+
159
+ # assert self.file has path attribute
160
+ assert isinstance(self.file, FieldFile)
161
+ if self.file and self.file.name:
162
+ try:
163
+ return Path(self.file.path)
164
+ except (ValueError, AttributeError, NotImplementedError):
165
+ return None
166
+ return None
167
+
168
+ def set_file_path(self, file_path: Path):
169
+ """
170
+ Sets the file path of the stored report file.
171
+ """
172
+ if not file_path.exists():
173
+ raise FileNotFoundError(f"File path does not exist: {file_path}")
174
+
175
+ save_local_file(self.file, file_path, name=file_path.name, save=False)
176
+ self.save(update_fields=["file"])
177
+
178
+ @property
179
+ def anonymized_file_path(self) -> Path | None:
180
+ """
181
+ Returns the file path of the anonymized report file if available; otherwise, returns None.
182
+ """
183
+ if self.processed_file and self.processed_file.name:
184
+ try:
185
+ return Path(self.processed_file.path)
186
+ except (ValueError, AttributeError, NotImplementedError):
187
+ return None
188
+ return None
189
+
190
+ def set_anonymized_file_path(self, file_path: Path):
191
+ """
192
+ Sets the file path of the anonymized report file.
193
+ """
194
+ if not file_path.exists():
195
+ raise FileNotFoundError(f"File path does not exist: {file_path}")
196
+
197
+ save_local_file(self.processed_file, file_path, name=file_path.name, save=False)
198
+ self.save(update_fields=["processed_file"])
199
+
200
+ def get_raw_file_path(self) -> Optional[Path]:
201
+ """
202
+ Get the path to the raw report file, searching common locations.
203
+
204
+ This method attempts to find the original raw report file by checking:
205
+ 1. Direct hash-based path in raw_pdfs/
206
+ 2. Scanning raw_pdfs/ directory for files matching the hash
207
+ 3. Checking the file field if it exists
208
+
209
+ Returns:
210
+ Path to raw file if it exists, None otherwise
211
+ """
212
+ from django.conf import settings
213
+
214
+ # Check if file field already points to a valid file
215
+ if self.file and self.file.name:
216
+ try:
217
+ file_path = Path(self.file.path)
218
+ if file_path.exists():
219
+ logger.debug(f"Found raw report via file field: {file_path}")
220
+ return file_path
221
+ except (ValueError, AttributeError, NotImplementedError):
222
+ pass
223
+
224
+ # Define potential raw directories
225
+ raw_dirs = [
226
+ SENSITIVE_REPORT_DIR, # Files might be in sensitive dir
227
+ Path(settings.BASE_DIR) / "data" / "temporary_reports",
228
+ Path(settings.BASE_DIR) / "data" / "pdfs" / "raw",
229
+ IMPORT_REPORT_DIR, # General report directory
230
+ ]
231
+
232
+ # Check direct hash-based name in each directory
233
+ for raw_dir in raw_dirs:
234
+ if not raw_dir.exists():
235
+ continue
236
+
237
+ hash_path = raw_dir / f"{self.pdf_hash}.pdf"
238
+ if hash_path.exists():
239
+ logger.debug(f"Found raw report at: {hash_path}")
240
+ return hash_path
241
+
242
+ # Scan directories for matching hash
243
+ for raw_dir in raw_dirs:
244
+ if not raw_dir.exists():
245
+ continue
246
+
247
+ for file_path in raw_dir.glob("*.pdf"):
248
+ try:
249
+ file_hash = get_pdf_hash(file_path)
250
+ if file_hash == self.pdf_hash:
251
+ logger.debug(f"Found matching report by hash: {file_path}")
252
+ return file_path
253
+ except Exception as e:
254
+ logger.debug(f"Error checking {file_path}: {e}")
255
+ continue
256
+
257
+ logger.warning(f"No raw file found for report hash: {self.pdf_hash}")
258
+ return None
259
+
260
+ @property
261
+ def file_url(self):
262
+ """
263
+ Returns the URL of the stored report file if available; otherwise, returns None.
264
+ """
265
+ try:
266
+ return self.file.url if self.file and self.file.name else None
267
+ except (ValueError, AttributeError):
268
+ return None
269
+
270
+ @property
271
+ def anonymized_file_url(self):
272
+ """
273
+ Returns the URL of the stored report file if available; otherwise, returns None.
274
+ """
275
+ try:
276
+ return (
277
+ self.processed_file.url
278
+ if self.processed_file and self.processed_file.name
279
+ else None
280
+ )
281
+ except (ValueError, AttributeError):
282
+ return None
283
+
284
+ def __str__(self):
285
+ """
286
+ Return a string representation of the RawPdfFile, including its report hash, type, and center.
287
+ """
288
+ str_repr = f"{self.pdf_hash} ({self.pdf_type}, {self.center})"
289
+ return str_repr
290
+
291
+ def delete(self, *args, **kwargs):
292
+ """
293
+ Deletes the RawPdfFile instance from the database and removes the associated file from storage if it exists.
294
+
295
+ This method ensures that the physical report file is deleted from the file system after the database record is removed. Logs warnings or errors if the file cannot be found or deleted.
296
+ """
297
+ primary_name = self.file.name if self.file and self.file.name else None
298
+ anonymized_name = (
299
+ self.processed_file.name
300
+ if self.processed_file and self.processed_file.name
301
+ else None
302
+ )
303
+
304
+ if delete_field_file(self.file, missing_ok=True, save=False):
305
+ logger.info("Original file removed from storage: %s", primary_name)
306
+ if delete_field_file(self.processed_file, missing_ok=True, save=False):
307
+ logger.info("Anonymized file removed from storage: %s", anonymized_name)
308
+
309
+ super().delete(*args, **kwargs)
310
+
311
+ # --- Convenience state/meta helpers used in tests and admin workflows ---
312
+
313
+ def mark_sensitive_meta_processed(self, *, save: bool = True) -> "RawPdfFile":
314
+ """
315
+ Mark this video's processing state as having its sensitive meta fully processed.
316
+ This proxies to the related VideoState and persists by default.
317
+ """
318
+ sm = self.sensitive_meta
319
+ from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
320
+
321
+ if not isinstance(sm, SensitiveMeta):
322
+ raise AttributeError()
323
+ state = self.get_or_create_state()
324
+ state.mark_sensitive_meta_processed(save=save)
325
+ return self
326
+
327
+ def mark_sensitive_meta_verified(self) -> "RawPdfFile":
328
+ """
329
+ Mark the associated SensitiveMeta as verified by setting both DOB and names as verified.
330
+ Ensures the SensitiveMeta and its state exist.
331
+ """
332
+ sm = self.sensitive_meta
333
+ # Use SensitiveMeta methods to update underlying SensitiveMetaState
334
+ from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
335
+
336
+ if not isinstance(sm, SensitiveMeta):
337
+ raise AttributeError()
338
+
339
+ sm.mark_dob_verified()
340
+ sm.mark_names_verified()
341
+ return self
342
+
343
+ def validate_metadata_annotation(
344
+ self, extracted_data_dict: Optional[dict] = None
345
+ ) -> bool:
346
+ """
347
+ Validate the metadata of the RawPdf instance.
348
+
349
+ Called after annotation in the frontend, this method deletes the associated active file, updates the sensitive meta data with the user annotated data.
350
+ It also ensures the video file is properly saved after the metadata update.
351
+ """
352
+
353
+ self.mark_sensitive_meta_processed()
354
+ self.mark_sensitive_meta_verified()
355
+
356
+ if not extracted_data_dict:
357
+ logger.error("No extracted data provided for validation.")
358
+ return False
359
+
360
+ if extracted_data_dict:
361
+ self.sensitive_meta.update_from_dict(extracted_data_dict)
362
+ else:
363
+ return False
364
+
365
+ # Save the sensitive meta to ensure changes are persisted
366
+ self.sensitive_meta.save()
367
+
368
+ # Save the RawPdfFile instance to ensure all changes are saved
369
+ self.save()
370
+
371
+ logger.info(
372
+ f"Metadata for report {self.pk} validated and updated successfully."
373
+ )
374
+
375
+ deleted_original = delete_field_file(self.file, missing_ok=True, save=False)
376
+ deleted_anonymized = delete_field_file(
377
+ self.processed_file, missing_ok=True, save=False
378
+ )
379
+ self.get_or_create_state().mark_anonymization_validated()
380
+
381
+ if deleted_original or deleted_anonymized:
382
+ self.save(
383
+ update_fields=["file", "processed_file"]
384
+ ) # Persist cleared fields
385
+
386
+ logger.info(f"Files for report {self.pk} deleted successfully.")
387
+ return True
388
+
389
+ @classmethod
390
+ def create_from_file_initialized(
391
+ cls,
392
+ file_path: Path,
393
+ center_name: str,
394
+ delete_source: bool = True,
395
+ ):
396
+ """
397
+ Creates a RawPdfFile instance from a file and center name, ensuring an associated RawPdfState exists.
398
+
399
+ Parameters:
400
+ file_path (Path): Path to the source report file.
401
+ center_name (str): Name of the center to associate with the report.
402
+ delete_source (bool): Whether to delete the source file after processing. Defaults to True.
403
+
404
+ Returns:
405
+ RawPdfFile: The created or retrieved RawPdfFile instance with an associated RawPdfState.
406
+ """
407
+ raw_pdf = cls.create_from_file(
408
+ file_path=file_path,
409
+ center_name=center_name,
410
+ delete_source=delete_source,
411
+ )
412
+ _state = raw_pdf.get_or_create_state()
413
+
414
+ return raw_pdf
415
+
416
+ @classmethod
417
+ def create_from_file(
418
+ cls,
419
+ file_path: Path,
420
+ center_name,
421
+ save=True, # Parameter kept for compatibility, but save now happens internally
422
+ delete_source=True,
423
+ ):
424
+ """
425
+ Creates or retrieves a RawPdfFile instance from a given report file path and center name.
426
+
427
+ If a RawPdfFile with the same report hash already exists, verifies the file exists in storage and restores it if missing. Otherwise, creates a new RawPdfFile, assigns the file, and saves it to storage. Optionally deletes the source file after processing.
428
+
429
+ Parameters:
430
+ file_path (Path): Path to the source report file.
431
+ center_name (str): Name of the center to associate with the file.
432
+ save (bool, optional): Deprecated; saving occurs internally.
433
+ delete_source (bool, optional): Whether to delete the source file after processing (default True).
434
+
435
+ Returns:
436
+ RawPdfFile: The created or retrieved RawPdfFile instance.
437
+
438
+ Raises:
439
+ FileNotFoundError: If the source file does not exist.
440
+ Center.DoesNotExist: If the specified center is not found.
441
+ ValueError: If the report hash cannot be calculated.
442
+ IOError: If the file fails to save to storage.
443
+ """
444
+ from endoreg_db.models.administration import Center
445
+
446
+ if not file_path.exists():
447
+ logger.error(f"Source file does not exist: {file_path}")
448
+ raise FileNotFoundError(f"Source file not found: {file_path}")
449
+
450
+ # 1. Calculate hash from source file
451
+ try:
452
+ pdf_hash = get_pdf_hash(file_path)
453
+ logger.info(pdf_hash)
454
+ except Exception as e:
455
+ logger.error(f"Could not calculate hash for {file_path}: {e}")
456
+ raise ValueError(f"Could not calculate hash for {file_path}") from e
457
+
458
+ # 2. Check if record with this hash already exists
459
+ existing_pdf_file = cls.objects.filter(pdf_hash=pdf_hash).first()
460
+ if existing_pdf_file:
461
+ logger.warning(
462
+ "RawPdfFile with hash %s already exists (ID: %s)",
463
+ pdf_hash,
464
+ existing_pdf_file.pk,
465
+ )
466
+
467
+ # Verify physical file exists for the existing record
468
+ try:
469
+ if existing_pdf_file is not None and isinstance(existing_pdf_file, cls):
470
+ # Use storage API to check existence
471
+ _file = existing_pdf_file.file
472
+ assert _file is not None
473
+ if not _file.storage.exists(_file.name):
474
+ logger.warning(
475
+ "File for existing RawPdfFile %s not found in storage at %s. Attempting to restore from source %s",
476
+ pdf_hash,
477
+ _file.name,
478
+ file_path,
479
+ )
480
+ # Re-save the file from the source to potentially fix it
481
+ with file_path.open("rb") as f:
482
+ django_file = File(
483
+ f, name=Path(_file.name).name
484
+ ) # Use existing name if possible
485
+ existing_pdf_file.file = django_file
486
+ existing_pdf_file.save(
487
+ update_fields=["file"]
488
+ ) # Only update file field
489
+ else:
490
+ pass
491
+ # logger.debug("File for existing RawPdfFile %s already exists in storage.", pdf_hash)
492
+ except Exception as e:
493
+ logger.error(
494
+ "Error verifying/restoring file for existing record %s: %s",
495
+ pdf_hash,
496
+ e,
497
+ )
498
+
499
+ # Delete the source temp file if requested
500
+ if delete_source:
501
+ try:
502
+ file_path.unlink()
503
+ # logger.info("Deleted source file %s after finding existing record.", file_path)
504
+ except OSError as e:
505
+ logger.error("Error deleting source file %s: %s", file_path, e)
506
+
507
+ return existing_pdf_file
508
+
509
+ # --- Create new record if not existing ---
510
+ assert center_name is not None, "center_name is required"
511
+ try:
512
+ center = Center.objects.get(name=center_name)
513
+ except Center.DoesNotExist:
514
+ logger.error(f"Center with name '{center_name}' not found.")
515
+ raise
516
+
517
+ # Generate a unique filename (e.g., using UUID)
518
+ new_file_name, _uuid = get_content_hash_filename(file_path)
519
+ logger.info(f"Generated new filename: {new_file_name}")
520
+
521
+ # Create model instance via manager so creation can be intercepted/mocked during tests
522
+ try:
523
+ with file_path.open("rb") as f:
524
+ django_file = File(f, name=new_file_name)
525
+ raw_pdf = cls.objects.create(
526
+ pdf_hash=pdf_hash,
527
+ center=center,
528
+ file=django_file,
529
+ )
530
+
531
+ _file = raw_pdf.file
532
+ assert _file is not None
533
+ logger.info(
534
+ "Created and saved new RawPdfFile %s with file %s",
535
+ raw_pdf.pk,
536
+ _file.name,
537
+ )
538
+
539
+ if not _file.storage.exists(_file.name):
540
+ logger.error(
541
+ "File was not saved correctly to storage path %s after model save.",
542
+ _file.name,
543
+ )
544
+ raise IOError(
545
+ f"File not found at expected storage path after save: {_file.name}"
546
+ )
547
+
548
+ try:
549
+ logger.info("File saved to absolute path: %s", _file.path)
550
+ except NotImplementedError:
551
+ logger.info(
552
+ "File saved to storage path: %s (Absolute path not available from storage)",
553
+ _file.name,
554
+ )
555
+
556
+ except Exception as e:
557
+ logger.error(
558
+ "Error processing or saving file %s for new record: %s", file_path, e
559
+ )
560
+ raise
561
+
562
+ # Delete source file *after* successful save and verification
563
+ if delete_source:
564
+ try:
565
+ file_path.unlink()
566
+ logger.info(
567
+ "Deleted source file %s after creating new record.", file_path
568
+ )
569
+ except OSError as e:
570
+ logger.error("Error deleting source file %s: %s", file_path, e)
571
+
572
+ # raw_pdf.save() # unnecessary?
573
+ return raw_pdf
574
+
575
+ def save(self, *args, **kwargs):
576
+ # Ensure hash is calculated before the first save if possible and not already set
577
+ # This is primarily a fallback if instance created manually without using create_from_file
578
+ """
579
+ Saves the RawPdfFile instance, ensuring the report hash is set and related fields are derived from metadata.
580
+
581
+ If the report hash is missing, attempts to calculate it from the file before saving. Validates that the file has a `.pdf` extension. If related fields such as patient, examination, center, or examiner are unset but available in the associated sensitive metadata, they are populated accordingly before saving.
582
+ """
583
+ if not self.pk and not self.pdf_hash and self.file:
584
+ try:
585
+ with ensure_local_file(self.file) as local_path:
586
+ self.pdf_hash = get_pdf_hash(local_path)
587
+ logger.info(
588
+ "Calculated hash during pre-save for %s", self.file.name
589
+ )
590
+ except Exception as exc:
591
+ logger.warning(
592
+ "Could not calculate hash before initial save for %s: %s",
593
+ self.file.name,
594
+ exc,
595
+ )
596
+
597
+ if self.file and not self.file.name.endswith(".pdf"):
598
+ raise ValidationError("Only report files are allowed")
599
+
600
+ # If hash is still missing after potential creation logic (e.g., direct instantiation)
601
+ # and the file exists in storage, try calculating it from storage path.
602
+ # This is less ideal as it requires the file to be saved first.
603
+ if not self.pdf_hash and self.pk and self.file and file_exists(self.file):
604
+ try:
605
+ with ensure_local_file(self.file) as local_path:
606
+ logger.warning(
607
+ "Hash missing for saved file %s. Recalculating.", self.file.name
608
+ )
609
+ self.pdf_hash = get_pdf_hash(local_path)
610
+ except Exception as exc:
611
+ logger.error(
612
+ "Could not calculate hash during save for existing file %s: %s",
613
+ self.file.name,
614
+ exc,
615
+ )
616
+
617
+ # Derive related fields from sensitive_meta if available
618
+ if not self.patient and self.sensitive_meta:
619
+ self.patient = self.sensitive_meta.pseudo_patient
620
+ if not self.examination and self.sensitive_meta:
621
+ self.examination = self.sensitive_meta.pseudo_examination
622
+ if not self.center and self.sensitive_meta:
623
+ self.center = self.sensitive_meta.center
624
+ # TODO Outdated?
625
+ # if not self.examiner and self.sensitive_meta and hasattr(self.sensitive_meta, 'pseudo_examiner'):
626
+ # self.examiner = self.sensitive_meta.pseudo_examiner
627
+
628
+ super().save(*args, **kwargs)
629
+
630
+ def get_or_create_state(self) -> "RawPdfState":
631
+ """
632
+ Retrieve the associated RawPdfState for this RawPdfFile, creating and linking a new one if none exists.
633
+
634
+ Returns:
635
+ RawPdfState: The existing or newly created RawPdfState instance linked to this RawPdfFile.
636
+ """
637
+ from endoreg_db.models.state import RawPdfState
638
+
639
+ if self.state:
640
+ return self.state
641
+
642
+ # Create a new RawPdfState instance directly and assign it
643
+ state = RawPdfState()
644
+ state.save()
645
+ self.state = state
646
+ self.save(update_fields=["state"]) # Save the RawPdfFile to link the state
647
+ logger.info("Created new RawPdfState for RawPdfFile %s", self.pk)
648
+ return state
649
+
650
+ def verify_existing_file(self, fallback_file):
651
+ # This method might still be useful if called explicitly, but create_from_file now handles restoration
652
+ # Ensure fallback_file is a Path object.
653
+ """
654
+ Checks if the stored report file exists in storage and attempts to restore it from a fallback file path if missing.
655
+
656
+ Parameters:
657
+ fallback_file: Path or string representing the fallback file location to restore from if the stored file is missing.
658
+ """
659
+ if not isinstance(fallback_file, Path):
660
+ fallback_file = Path(fallback_file)
661
+
662
+ _file = self.file
663
+ assert _file is not None
664
+ try:
665
+ if not _file.field.storage.exists(_file.name):
666
+ logger.warning(
667
+ f"File missing at storage path {_file.name}. Attempting copy from fallback {fallback_file}"
668
+ )
669
+ if fallback_file.exists():
670
+ with fallback_file.open("rb") as f:
671
+ # Use save method which handles storage backend
672
+ _file.save(
673
+ Path(_file.name).name, File(f), save=True
674
+ ) # Re-save the file content
675
+ logger.info(
676
+ f"Successfully restored file from fallback {fallback_file} to {_file.name}"
677
+ )
678
+ else:
679
+ logger.error(f"Fallback file {fallback_file} does not exist.")
680
+ except Exception as e:
681
+ logger.error(f"Error during verify_existing_file for {_file.name}: {e}")
682
+
683
+ def process_file(self, text, anonymized_text, report_meta, verbose):
684
+ self.text = text
685
+ self.anonymized_text = anonymized_text
686
+
687
+ assert self.center is not None, "Center must be set before processing file"
688
+
689
+ report_meta["center_name"] = self.center.name
690
+ if not self.sensitive_meta:
691
+ # Pass the original report_meta with date objects to SensitiveMeta logic
692
+ sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
693
+ self.sensitive_meta = sensitive_meta
694
+ else:
695
+ sensitive_meta = self.sensitive_meta
696
+ # Pass the original report_meta with date objects to SensitiveMeta logic
697
+ sensitive_meta.update_from_dict(report_meta)
698
+
699
+ # For storing in raw_meta (JSONField), dates need to be strings.
700
+ # Create a serializable version of report_meta for raw_meta.
701
+ import copy
702
+ from datetime import date, datetime
703
+
704
+ serializable_report_meta = copy.deepcopy(report_meta)
705
+ for key, value in serializable_report_meta.items():
706
+ if isinstance(value, (datetime, date)):
707
+ serializable_report_meta[key] = value.isoformat()
708
+
709
+ self.raw_meta = serializable_report_meta # Assign the version with string dates
710
+
711
+ sensitive_meta.save() # Save SensitiveMeta first
712
+ self.save() # Then save RawPdfFile
713
+
714
+ return text, anonymized_text, report_meta
715
+
716
+ def get_report_reader_config(self):
717
+ from warnings import warn
718
+
719
+ from ...administration import Center
720
+ from ...metadata.pdf_meta import PdfType
721
+
722
+ _center = self.center
723
+ assert _center is not None, "Center must be set to get report reader config"
724
+
725
+ if not self.pdf_type:
726
+ warn("PdfType not set, using default settings")
727
+ pdf_type = PdfType.default_pdf_type()
728
+ else:
729
+ pdf_type: PdfType = self.pdf_type
730
+ center: Center = _center
731
+ if pdf_type.endoscope_info_line:
732
+ endoscope_info_line = pdf_type.endoscope_info_line.value
733
+
734
+ else:
735
+ endoscope_info_line = None
736
+ settings_dict = {
737
+ "locale": "de_DE",
738
+ "employee_first_names": [_.name for _ in center.first_names.all()],
739
+ "employee_last_names": [_.name for _ in center.last_names.all()],
740
+ "text_date_format": "%d.%m.%Y",
741
+ "flags": {
742
+ "patient_info_line": pdf_type.patient_info_line.value,
743
+ "endoscope_info_line": endoscope_info_line,
744
+ "examiner_info_line": pdf_type.examiner_info_line.value,
745
+ "cut_off_below": [_.value for _ in pdf_type.cut_off_below_lines.all()],
746
+ "cut_off_above": [_.value for _ in pdf_type.cut_off_above_lines.all()],
747
+ },
748
+ }
749
+
750
+ return settings_dict
751
+
752
+ @staticmethod
753
+ def get_report_by_pk(pk: int) -> "RawPdfFile":
754
+ try:
755
+ return RawPdfFile.objects.get(pk=pk)
756
+ except RawPdfFile.DoesNotExist:
757
+ raise ValueError(f"report with ID {pk} does not exist.")
758
+
759
+ @staticmethod
760
+ def get_report_by_hash(hash: str) -> "RawPdfFile":
761
+ try:
762
+ return RawPdfFile.objects.get(pdf_hash=hash)
763
+ except RawPdfFile.DoesNotExist:
764
+ raise ValueError(f"report with ID {hash} does not exist.")