endoreg-db 0.8.9.32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (787) hide show
  1. endoreg_db/__init__.py +0 -0
  2. endoreg_db/_version.py +34 -0
  3. endoreg_db/admin.py +97 -0
  4. endoreg_db/api/serializers/finding_descriptions.py +0 -0
  5. endoreg_db/api/views/finding_descriptions.py +0 -0
  6. endoreg_db/api_urls.py +4 -0
  7. endoreg_db/apps.py +17 -0
  8. endoreg_db/assets/dummy_model.ckpt +1 -0
  9. endoreg_db/authz/auth.py +78 -0
  10. endoreg_db/authz/backends.py +168 -0
  11. endoreg_db/authz/management/commands/list_routes.py +20 -0
  12. endoreg_db/authz/middleware.py +84 -0
  13. endoreg_db/authz/permissions.py +138 -0
  14. endoreg_db/authz/policy.py +224 -0
  15. endoreg_db/authz/settings.py +64 -0
  16. endoreg_db/authz/views_auth.py +70 -0
  17. endoreg_db/codemods/readme.md +88 -0
  18. endoreg_db/codemods/rename_datetime_fields.py +99 -0
  19. endoreg_db/config/__init__.py +0 -0
  20. endoreg_db/config/env.py +106 -0
  21. endoreg_db/config/settings/__init__.py +6 -0
  22. endoreg_db/config/settings/base.py +148 -0
  23. endoreg_db/config/settings/case_gen.py +32 -0
  24. endoreg_db/config/settings/dev.py +108 -0
  25. endoreg_db/config/settings/keycloak.py +177 -0
  26. endoreg_db/config/settings/prod.py +66 -0
  27. endoreg_db/config/settings/test.py +72 -0
  28. endoreg_db/data/__init__.py +135 -0
  29. endoreg_db/data/ai_model/data.yaml +7 -0
  30. endoreg_db/data/ai_model_label/label/data.yaml +88 -0
  31. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  32. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  33. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  34. endoreg_db/data/ai_model_label/label-type/data.yaml +7 -0
  35. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  36. endoreg_db/data/ai_model_type/data.yaml +7 -0
  37. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  38. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  39. endoreg_db/data/case_template/rule/00_patient_lab_sample_add_default_value.yaml +167 -0
  40. endoreg_db/data/case_template/rule/01_patient-set-age.yaml +8 -0
  41. endoreg_db/data/case_template/rule/01_patient-set-gender.yaml +9 -0
  42. endoreg_db/data/case_template/rule/11_create_patient_lab_sample.yaml +23 -0
  43. endoreg_db/data/case_template/rule/12_create-patient_medication-anticoagulation.yaml +19 -0
  44. endoreg_db/data/case_template/rule/13_create-patient_medication_schedule-anticoagulation.yaml +19 -0
  45. endoreg_db/data/case_template/rule/19_create_patient.yaml +17 -0
  46. endoreg_db/data/case_template/rule_type/base_types.yaml +35 -0
  47. endoreg_db/data/case_template/rule_value/.init +0 -0
  48. endoreg_db/data/case_template/rule_value_type/base_types.yaml +59 -0
  49. endoreg_db/data/case_template/template/base.yaml +8 -0
  50. endoreg_db/data/case_template/template_type/pre_endoscopy.yaml +3 -0
  51. endoreg_db/data/case_template/tmp/_rule_value +13 -0
  52. endoreg_db/data/case_template/tmp/rule/01_atrial_fibrillation.yaml +21 -0
  53. endoreg_db/data/case_template/tmp/rule/02_create_object.yaml +10 -0
  54. endoreg_db/data/case_template/tmp/template/atrial_fibrillation_low_risk.yaml +7 -0
  55. endoreg_db/data/center/data.yaml +99 -0
  56. endoreg_db/data/center_resource/green_endoscopy_dashboard_CenterResource.yaml +144 -0
  57. endoreg_db/data/center_shift/ukw.yaml +9 -0
  58. endoreg_db/data/center_waste/green_endoscopy_dashboard_CenterWaste.yaml +48 -0
  59. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  60. endoreg_db/data/db_summary.csv +58 -0
  61. endoreg_db/data/db_summary.xlsx +0 -0
  62. endoreg_db/data/disease/cardiovascular.yaml +37 -0
  63. endoreg_db/data/disease/hepatology.yaml +5 -0
  64. endoreg_db/data/disease/misc.yaml +5 -0
  65. endoreg_db/data/disease/renal.yaml +5 -0
  66. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +6 -0
  67. endoreg_db/data/disease_classification/coronary_vessel_disease.yaml +6 -0
  68. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +41 -0
  69. endoreg_db/data/disease_classification_choice/coronary_vessel_disease.yaml +20 -0
  70. endoreg_db/data/distribution/date/patient.yaml +7 -0
  71. endoreg_db/data/distribution/multiple_categorical/.init +0 -0
  72. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  73. endoreg_db/data/distribution/single_categorical/patient.yaml +7 -0
  74. endoreg_db/data/emission_factor/green_endoscopy_dashboard_EmissionFactor.yaml +132 -0
  75. endoreg_db/data/endoscope/data.yaml +93 -0
  76. endoreg_db/data/endoscope_type/data.yaml +11 -0
  77. endoreg_db/data/endoscopy_processor/data.yaml +50 -0
  78. endoreg_db/data/event/cardiology.yaml +15 -0
  79. endoreg_db/data/event/neurology.yaml +14 -0
  80. endoreg_db/data/event/surgery.yaml +13 -0
  81. endoreg_db/data/event/thrombembolism.yaml +20 -0
  82. endoreg_db/data/event_classification/data.yaml +4 -0
  83. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  84. endoreg_db/data/examination/examinations/data.yaml +172 -0
  85. endoreg_db/data/examination/time/data.yaml +48 -0
  86. endoreg_db/data/examination/time-type/data.yaml +5 -0
  87. endoreg_db/data/examination/type/data.yaml +17 -0
  88. endoreg_db/data/examination_indication/endoscopy.yaml +359 -0
  89. endoreg_db/data/examination_indication_classification/endoscopy.yaml +90 -0
  90. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +97 -0
  91. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  92. endoreg_db/data/finding/00_generic.yaml +35 -0
  93. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  94. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  95. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  96. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  97. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  98. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  99. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  100. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  101. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  102. endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
  103. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  104. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  105. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  106. endoreg_db/data/finding_classification_choice/00_generic.yaml +15 -0
  107. endoreg_db/data/finding_classification_choice/00_generic_baseline.yaml +23 -0
  108. endoreg_db/data/finding_classification_choice/00_generic_complication.yaml +15 -0
  109. endoreg_db/data/finding_classification_choice/00_generic_histology.yaml +21 -0
  110. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  111. endoreg_db/data/finding_classification_choice/02_colonoscopy_bowel_preparation.yaml +49 -0
  112. endoreg_db/data/finding_classification_choice/02_colonoscopy_generic.yaml +19 -0
  113. endoreg_db/data/finding_classification_choice/02_colonoscopy_histology.yaml +20 -0
  114. endoreg_db/data/finding_classification_choice/02_colonoscopy_location.yaml +248 -0
  115. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  116. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  117. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_morphology.yaml +75 -0
  118. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  119. endoreg_db/data/finding_classification_type/00_generic.yaml +53 -0
  120. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  121. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  122. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  123. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  124. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  125. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  126. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  127. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  128. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  129. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  130. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  131. endoreg_db/data/finding_type/data.yaml +39 -0
  132. endoreg_db/data/gender/data.yaml +42 -0
  133. endoreg_db/data/information_source/annotation.yaml +6 -0
  134. endoreg_db/data/information_source/data.yaml +30 -0
  135. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  136. endoreg_db/data/information_source/medication.yaml +6 -0
  137. endoreg_db/data/information_source/prediction.yaml +7 -0
  138. endoreg_db/data/information_source_type/data.yaml +8 -0
  139. endoreg_db/data/lab_value/cardiac_enzymes.yaml +37 -0
  140. endoreg_db/data/lab_value/coagulation.yaml +54 -0
  141. endoreg_db/data/lab_value/electrolytes.yaml +228 -0
  142. endoreg_db/data/lab_value/gastrointestinal_function.yaml +133 -0
  143. endoreg_db/data/lab_value/hematology.yaml +184 -0
  144. endoreg_db/data/lab_value/hormones.yaml +59 -0
  145. endoreg_db/data/lab_value/lipids.yaml +53 -0
  146. endoreg_db/data/lab_value/misc.yaml +76 -0
  147. endoreg_db/data/lab_value/renal_function.yaml +12 -0
  148. endoreg_db/data/log_type/data.yaml +57 -0
  149. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  150. endoreg_db/data/lx_client_type/base.yaml +30 -0
  151. endoreg_db/data/lx_permission/base.yaml +24 -0
  152. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  153. endoreg_db/data/material/material.yaml +91 -0
  154. endoreg_db/data/medication/anticoagulation.yaml +65 -0
  155. endoreg_db/data/medication/tah.yaml +70 -0
  156. endoreg_db/data/medication_indication/anticoagulation.yaml +115 -0
  157. endoreg_db/data/medication_indication_type/data.yaml +11 -0
  158. endoreg_db/data/medication_indication_type/thrombembolism.yaml +41 -0
  159. endoreg_db/data/medication_intake_time/base.yaml +31 -0
  160. endoreg_db/data/medication_schedule/apixaban.yaml +95 -0
  161. endoreg_db/data/medication_schedule/ass.yaml +12 -0
  162. endoreg_db/data/medication_schedule/enoxaparin.yaml +26 -0
  163. endoreg_db/data/names_first/first_names.yaml +54 -0
  164. endoreg_db/data/names_last/last_names.yaml +51 -0
  165. endoreg_db/data/network_device/data.yaml +59 -0
  166. endoreg_db/data/network_device_type/data.yaml +12 -0
  167. endoreg_db/data/organ/data.yaml +29 -0
  168. endoreg_db/data/patient_lab_sample_type/generic.yaml +6 -0
  169. endoreg_db/data/pdf_type/data.yaml +46 -0
  170. endoreg_db/data/product/green_endoscopy_dashboard_Product.yaml +66 -0
  171. endoreg_db/data/product_group/green_endoscopy_dashboard_ProductGroup.yaml +33 -0
  172. endoreg_db/data/product_material/green_endoscopy_dashboard_ProductMaterial.yaml +308 -0
  173. endoreg_db/data/product_weight/green_endoscopy_dashboard_ProductWeight.yaml +88 -0
  174. endoreg_db/data/profession/data.yaml +70 -0
  175. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  176. endoreg_db/data/qualification/m2.yaml +39 -0
  177. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  178. endoreg_db/data/qualification/sonography.yaml +36 -0
  179. endoreg_db/data/qualification_type/base.yaml +29 -0
  180. endoreg_db/data/reference_product/green_endoscopy_dashboard_ReferenceProduct.yaml +55 -0
  181. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  182. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +30 -0
  183. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +24 -0
  184. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  185. endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
  186. endoreg_db/data/requirement/old/colonoscopy_baseline_austria.yaml +45 -0
  187. endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
  188. endoreg_db/data/requirement/old/disease_cardiovascular.yaml +79 -0
  189. endoreg_db/data/requirement/old/disease_classification_choice_cardiovascular.yaml +41 -0
  190. endoreg_db/data/requirement/old/disease_hepatology.yaml +12 -0
  191. endoreg_db/data/requirement/old/disease_misc.yaml +12 -0
  192. endoreg_db/data/requirement/old/disease_renal.yaml +96 -0
  193. endoreg_db/data/requirement/old/endoscopy_bleeding_risk.yaml +59 -0
  194. endoreg_db/data/requirement/old/event_cardiology.yaml +251 -0
  195. endoreg_db/data/requirement/old/event_requirements.yaml +145 -0
  196. endoreg_db/data/requirement/old/finding_colon_polyp.yaml +50 -0
  197. endoreg_db/data/requirement/old/gender.yaml +0 -0
  198. endoreg_db/data/requirement/old/lab_value.yaml +441 -0
  199. endoreg_db/data/requirement/old/medication.yaml +93 -0
  200. endoreg_db/data/requirement_operator/_old/age.yaml +13 -0
  201. endoreg_db/data/requirement_operator/_old/lab_operators.yaml +129 -0
  202. endoreg_db/data/requirement_operator/_old/model_operators.yaml +96 -0
  203. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  204. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +65 -0
  205. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  206. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  207. endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
  208. endoreg_db/data/requirement_set/_old_ +109 -0
  209. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  210. endoreg_db/data/requirement_set_type/data.yaml +41 -0
  211. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  212. endoreg_db/data/resource/green_endoscopy_dashboard_Resource.yaml +15 -0
  213. endoreg_db/data/risk/bleeding.yaml +26 -0
  214. endoreg_db/data/risk/thrombosis.yaml +37 -0
  215. endoreg_db/data/risk_type/data.yaml +27 -0
  216. endoreg_db/data/setup_config.yaml +38 -0
  217. endoreg_db/data/shift/endoscopy.yaml +21 -0
  218. endoreg_db/data/shift/m2.yaml +0 -0
  219. endoreg_db/data/shift_type/base.yaml +35 -0
  220. endoreg_db/data/tag/requirement_set_tags.yaml +32 -0
  221. endoreg_db/data/tmp/chronic_kidney_disease.yaml +0 -0
  222. endoreg_db/data/tmp/congestive_heart_failure.yaml +0 -0
  223. endoreg_db/data/transport_route/green_endoscopy_dashboard_TransportRoute.yaml +12 -0
  224. endoreg_db/data/unit/concentration.yaml +115 -0
  225. endoreg_db/data/unit/data.yaml +17 -0
  226. endoreg_db/data/unit/length.yaml +31 -0
  227. endoreg_db/data/unit/misc.yaml +20 -0
  228. endoreg_db/data/unit/rate.yaml +6 -0
  229. endoreg_db/data/unit/time.yaml +48 -0
  230. endoreg_db/data/unit/volume.yaml +35 -0
  231. endoreg_db/data/unit/weight.yaml +38 -0
  232. endoreg_db/data/waste/data.yaml +12 -0
  233. endoreg_db/exceptions.py +24 -0
  234. endoreg_db/export/frames/export.py +6 -0
  235. endoreg_db/export/frames/export_frames_with_labels.py +616 -0
  236. endoreg_db/factories/__init__.py +0 -0
  237. endoreg_db/forms/__init__.py +4 -0
  238. endoreg_db/forms/examination_form.py +12 -0
  239. endoreg_db/forms/patient_finding_intervention_form.py +40 -0
  240. endoreg_db/forms/patient_form.py +23 -0
  241. endoreg_db/forms/questionnaires/__init__.py +1 -0
  242. endoreg_db/forms/questionnaires/tto_questionnaire.py +23 -0
  243. endoreg_db/forms/settings/__init__.py +11 -0
  244. endoreg_db/forms/unit.py +7 -0
  245. endoreg_db/helpers/__init__.py +0 -0
  246. endoreg_db/helpers/count_db.py +48 -0
  247. endoreg_db/helpers/data_loader.py +280 -0
  248. endoreg_db/helpers/default_objects.py +414 -0
  249. endoreg_db/helpers/download_segmentation_model.py +32 -0
  250. endoreg_db/helpers/interact.py +1 -0
  251. endoreg_db/helpers/test_video_helper.py +127 -0
  252. endoreg_db/import_files/__init__.py +27 -0
  253. endoreg_db/import_files/context/__init__.py +7 -0
  254. endoreg_db/import_files/context/default_sensitive_meta.py +83 -0
  255. endoreg_db/import_files/context/ensure_center.py +17 -0
  256. endoreg_db/import_files/context/file_lock.py +66 -0
  257. endoreg_db/import_files/context/import_context.py +42 -0
  258. endoreg_db/import_files/context/validate_directories.py +57 -0
  259. endoreg_db/import_files/file_storage/__init__.py +15 -0
  260. endoreg_db/import_files/file_storage/create_report_file.py +99 -0
  261. endoreg_db/import_files/file_storage/create_video_file.py +104 -0
  262. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +42 -0
  263. endoreg_db/import_files/file_storage/state_management.py +463 -0
  264. endoreg_db/import_files/file_storage/storage.py +42 -0
  265. endoreg_db/import_files/import_service.md +26 -0
  266. endoreg_db/import_files/processing/__init__.py +11 -0
  267. endoreg_db/import_files/processing/report_processing/report_anonymization.py +99 -0
  268. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  269. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  270. endoreg_db/import_files/pseudonymization/__init__.py +0 -0
  271. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  272. endoreg_db/import_files/pseudonymization/k_anonymity.py +181 -0
  273. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +139 -0
  274. endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
  275. endoreg_db/import_files/report_import_service.py +147 -0
  276. endoreg_db/import_files/video_import_service.py +154 -0
  277. endoreg_db/logger_conf.py +156 -0
  278. endoreg_db/management/__init__.py +1 -0
  279. endoreg_db/management/commands/__init__.py +1 -0
  280. endoreg_db/management/commands/anonymize_video.py +0 -0
  281. endoreg_db/management/commands/check_auth.py +132 -0
  282. endoreg_db/management/commands/create_model_meta_from_huggingface.py +177 -0
  283. endoreg_db/management/commands/create_multilabel_model_meta.py +419 -0
  284. endoreg_db/management/commands/export_frame_annot.py +196 -0
  285. endoreg_db/management/commands/fix_missing_patient_data.py +206 -0
  286. endoreg_db/management/commands/fix_video_paths.py +186 -0
  287. endoreg_db/management/commands/import_report.py +361 -0
  288. endoreg_db/management/commands/list_routes.py +20 -0
  289. endoreg_db/management/commands/load_ai_model_data.py +83 -0
  290. endoreg_db/management/commands/load_ai_model_label_data.py +60 -0
  291. endoreg_db/management/commands/load_base_db_data.py +63 -0
  292. endoreg_db/management/commands/load_center_data.py +68 -0
  293. endoreg_db/management/commands/load_contraindication_data.py +39 -0
  294. endoreg_db/management/commands/load_disease_classification_choices_data.py +38 -0
  295. endoreg_db/management/commands/load_disease_classification_data.py +38 -0
  296. endoreg_db/management/commands/load_disease_data.py +59 -0
  297. endoreg_db/management/commands/load_distribution_data.py +63 -0
  298. endoreg_db/management/commands/load_endoscope_data.py +58 -0
  299. endoreg_db/management/commands/load_event_data.py +39 -0
  300. endoreg_db/management/commands/load_examination_data.py +78 -0
  301. endoreg_db/management/commands/load_examination_indication_data.py +85 -0
  302. endoreg_db/management/commands/load_finding_data.py +115 -0
  303. endoreg_db/management/commands/load_gender_data.py +37 -0
  304. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +142 -0
  305. endoreg_db/management/commands/load_information_source.py +46 -0
  306. endoreg_db/management/commands/load_lab_value_data.py +52 -0
  307. endoreg_db/management/commands/load_legacy_data.py +303 -0
  308. endoreg_db/management/commands/load_medication_data.py +104 -0
  309. endoreg_db/management/commands/load_name_data.py +36 -0
  310. endoreg_db/management/commands/load_organ_data.py +39 -0
  311. endoreg_db/management/commands/load_pdf_type_data.py +58 -0
  312. endoreg_db/management/commands/load_profession_data.py +40 -0
  313. endoreg_db/management/commands/load_qualification_data.py +56 -0
  314. endoreg_db/management/commands/load_report_reader_flag_data.py +40 -0
  315. endoreg_db/management/commands/load_requirement_data.py +207 -0
  316. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  317. endoreg_db/management/commands/load_risk_data.py +57 -0
  318. endoreg_db/management/commands/load_shift_data.py +57 -0
  319. endoreg_db/management/commands/load_tag_data.py +54 -0
  320. endoreg_db/management/commands/load_unit_data.py +40 -0
  321. endoreg_db/management/commands/load_user_groups.py +26 -0
  322. endoreg_db/management/commands/model_input.py +169 -0
  323. endoreg_db/management/commands/register_ai_model.py +70 -0
  324. endoreg_db/management/commands/setup_endoreg_db.py +459 -0
  325. endoreg_db/management/commands/start_filewatcher.py +115 -0
  326. endoreg_db/management/commands/storage_management.py +622 -0
  327. endoreg_db/management/commands/summarize_db_content.py +280 -0
  328. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  329. endoreg_db/management/commands/validate_video_files.py +189 -0
  330. endoreg_db/management/commands/video_validation.py +20 -0
  331. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  332. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  333. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  334. endoreg_db/mermaid/classification.md +8 -0
  335. endoreg_db/mermaid/examination.md +8 -0
  336. endoreg_db/mermaid/findings.md +7 -0
  337. endoreg_db/mermaid/image_classification.md +28 -0
  338. endoreg_db/mermaid/interventions.md +8 -0
  339. endoreg_db/mermaid/morphology.md +8 -0
  340. endoreg_db/mermaid/patient_creation.md +14 -0
  341. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  342. endoreg_db/migrations/0001_initial.py +1953 -0
  343. endoreg_db/migrations/__init__.py +0 -0
  344. endoreg_db/models/__init__.py +322 -0
  345. endoreg_db/models/administration/__init__.py +95 -0
  346. endoreg_db/models/administration/ai/__init__.py +9 -0
  347. endoreg_db/models/administration/ai/active_model.py +35 -0
  348. endoreg_db/models/administration/ai/ai_model.py +180 -0
  349. endoreg_db/models/administration/ai/model_type.py +42 -0
  350. endoreg_db/models/administration/case/__init__.py +5 -0
  351. endoreg_db/models/administration/case/case.py +114 -0
  352. endoreg_db/models/administration/case/case_template/__init__.py +3 -0
  353. endoreg_db/models/administration/case/case_template/case_template.py +3 -0
  354. endoreg_db/models/administration/case/case_template/case_template_rule.py +3 -0
  355. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +3 -0
  356. endoreg_db/models/administration/case/case_template/case_template_type.py +3 -0
  357. endoreg_db/models/administration/center/__init__.py +13 -0
  358. endoreg_db/models/administration/center/center.py +85 -0
  359. endoreg_db/models/administration/center/center_product.py +67 -0
  360. endoreg_db/models/administration/center/center_resource.py +69 -0
  361. endoreg_db/models/administration/center/center_shift.py +94 -0
  362. endoreg_db/models/administration/center/center_waste.py +42 -0
  363. endoreg_db/models/administration/person/__init__.py +26 -0
  364. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  365. endoreg_db/models/administration/person/employee/employee.py +40 -0
  366. endoreg_db/models/administration/person/employee/employee_qualification.py +44 -0
  367. endoreg_db/models/administration/person/employee/employee_type.py +50 -0
  368. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  369. endoreg_db/models/administration/person/examiner/examiner.py +64 -0
  370. endoreg_db/models/administration/person/names/__init__.py +0 -0
  371. endoreg_db/models/administration/person/names/first_name.py +20 -0
  372. endoreg_db/models/administration/person/names/last_name.py +20 -0
  373. endoreg_db/models/administration/person/patient/__init__.py +7 -0
  374. endoreg_db/models/administration/person/patient/patient.py +488 -0
  375. endoreg_db/models/administration/person/patient/patient_external_id.py +36 -0
  376. endoreg_db/models/administration/person/person.py +35 -0
  377. endoreg_db/models/administration/person/profession/__init__.py +28 -0
  378. endoreg_db/models/administration/person/user/__init__.py +5 -0
  379. endoreg_db/models/administration/person/user/portal_user_information.py +41 -0
  380. endoreg_db/models/administration/product/__init__.py +15 -0
  381. endoreg_db/models/administration/product/product.py +106 -0
  382. endoreg_db/models/administration/product/product_group.py +41 -0
  383. endoreg_db/models/administration/product/product_material.py +60 -0
  384. endoreg_db/models/administration/product/product_weight.py +51 -0
  385. endoreg_db/models/administration/product/reference_product.py +147 -0
  386. endoreg_db/models/administration/qualification/__init__.py +7 -0
  387. endoreg_db/models/administration/qualification/qualification.py +43 -0
  388. endoreg_db/models/administration/qualification/qualification_type.py +39 -0
  389. endoreg_db/models/administration/shift/__init__.py +9 -0
  390. endoreg_db/models/administration/shift/scheduled_days.py +72 -0
  391. endoreg_db/models/administration/shift/shift.py +57 -0
  392. endoreg_db/models/administration/shift/shift_type.py +108 -0
  393. endoreg_db/models/aidataset/__init__.py +5 -0
  394. endoreg_db/models/aidataset/aidataset.py +193 -0
  395. endoreg_db/models/label/__init__.py +23 -0
  396. endoreg_db/models/label/annotation/__init__.py +12 -0
  397. endoreg_db/models/label/annotation/image_classification.py +85 -0
  398. endoreg_db/models/label/annotation/video_segmentation_annotation.py +61 -0
  399. endoreg_db/models/label/label.py +91 -0
  400. endoreg_db/models/label/label_set.py +68 -0
  401. endoreg_db/models/label/label_type.py +29 -0
  402. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  403. endoreg_db/models/label/label_video_segment/_create_from_video.py +42 -0
  404. endoreg_db/models/label/label_video_segment/label_video_segment.py +611 -0
  405. endoreg_db/models/label/video_segmentation_label.py +35 -0
  406. endoreg_db/models/label/video_segmentation_labelset.py +28 -0
  407. endoreg_db/models/media/__init__.py +23 -0
  408. endoreg_db/models/media/frame/__init__.py +3 -0
  409. endoreg_db/models/media/frame/frame.py +137 -0
  410. endoreg_db/models/media/pdf/__init__.py +12 -0
  411. endoreg_db/models/media/pdf/raw_pdf.py +764 -0
  412. endoreg_db/models/media/pdf/report_file.py +162 -0
  413. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  414. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +85 -0
  415. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +46 -0
  416. endoreg_db/models/media/video/__init__.py +9 -0
  417. endoreg_db/models/media/video/create_from_file.py +402 -0
  418. endoreg_db/models/media/video/pipe_1.py +258 -0
  419. endoreg_db/models/media/video/pipe_2.py +129 -0
  420. endoreg_db/models/media/video/video_file.py +907 -0
  421. endoreg_db/models/media/video/video_file_ai.py +828 -0
  422. endoreg_db/models/media/video/video_file_anonymize.py +524 -0
  423. endoreg_db/models/media/video/video_file_frames/__init__.py +49 -0
  424. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +25 -0
  425. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  426. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +126 -0
  427. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +233 -0
  428. endoreg_db/models/media/video/video_file_frames/_get_frame.py +36 -0
  429. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +13 -0
  430. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +24 -0
  431. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +40 -0
  432. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +44 -0
  433. endoreg_db/models/media/video/video_file_frames/_get_frames.py +30 -0
  434. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +205 -0
  435. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +228 -0
  436. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +107 -0
  437. endoreg_db/models/media/video/video_file_io.py +272 -0
  438. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  439. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +58 -0
  440. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +62 -0
  441. endoreg_db/models/media/video/video_file_meta/get_fps.py +183 -0
  442. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +198 -0
  443. endoreg_db/models/media/video/video_file_meta/text_meta.py +178 -0
  444. endoreg_db/models/media/video/video_file_meta/video_meta.py +105 -0
  445. endoreg_db/models/media/video/video_file_segments.py +317 -0
  446. endoreg_db/models/media/video/video_metadata.py +67 -0
  447. endoreg_db/models/media/video/video_processing.py +192 -0
  448. endoreg_db/models/medical/__init__.py +136 -0
  449. endoreg_db/models/medical/contraindication/README.md +1 -0
  450. endoreg_db/models/medical/contraindication/__init__.py +29 -0
  451. endoreg_db/models/medical/disease.py +174 -0
  452. endoreg_db/models/medical/event.py +154 -0
  453. endoreg_db/models/medical/examination/__init__.py +20 -0
  454. endoreg_db/models/medical/examination/examination.py +183 -0
  455. endoreg_db/models/medical/examination/examination_indication.py +229 -0
  456. endoreg_db/models/medical/examination/examination_time.py +68 -0
  457. endoreg_db/models/medical/examination/examination_time_type.py +44 -0
  458. endoreg_db/models/medical/examination/examination_type.py +47 -0
  459. endoreg_db/models/medical/finding/__init__.py +20 -0
  460. endoreg_db/models/medical/finding/finding.py +113 -0
  461. endoreg_db/models/medical/finding/finding_classification.py +131 -0
  462. endoreg_db/models/medical/finding/finding_intervention.py +68 -0
  463. endoreg_db/models/medical/finding/finding_type.py +38 -0
  464. endoreg_db/models/medical/hardware/__init__.py +8 -0
  465. endoreg_db/models/medical/hardware/endoscope.py +77 -0
  466. endoreg_db/models/medical/hardware/endoscopy_processor.py +182 -0
  467. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  468. endoreg_db/models/medical/laboratory/lab_value.py +490 -0
  469. endoreg_db/models/medical/medication/__init__.py +23 -0
  470. endoreg_db/models/medical/medication/medication.py +45 -0
  471. endoreg_db/models/medical/medication/medication_indication.py +78 -0
  472. endoreg_db/models/medical/medication/medication_indication_type.py +58 -0
  473. endoreg_db/models/medical/medication/medication_intake_time.py +58 -0
  474. endoreg_db/models/medical/medication/medication_schedule.py +58 -0
  475. endoreg_db/models/medical/organ/__init__.py +38 -0
  476. endoreg_db/models/medical/patient/__init__.py +48 -0
  477. endoreg_db/models/medical/patient/medication_examples.py +56 -0
  478. endoreg_db/models/medical/patient/patient_disease.py +72 -0
  479. endoreg_db/models/medical/patient/patient_event.py +80 -0
  480. endoreg_db/models/medical/patient/patient_examination.py +280 -0
  481. endoreg_db/models/medical/patient/patient_examination_indication.py +57 -0
  482. endoreg_db/models/medical/patient/patient_finding.py +416 -0
  483. endoreg_db/models/medical/patient/patient_finding_classification.py +231 -0
  484. endoreg_db/models/medical/patient/patient_finding_intervention.py +37 -0
  485. endoreg_db/models/medical/patient/patient_lab_sample.py +157 -0
  486. endoreg_db/models/medical/patient/patient_lab_value.py +247 -0
  487. endoreg_db/models/medical/patient/patient_medication.py +111 -0
  488. endoreg_db/models/medical/patient/patient_medication_schedule.py +152 -0
  489. endoreg_db/models/medical/risk/__init__.py +7 -0
  490. endoreg_db/models/medical/risk/risk.py +73 -0
  491. endoreg_db/models/medical/risk/risk_type.py +54 -0
  492. endoreg_db/models/metadata/__init__.py +19 -0
  493. endoreg_db/models/metadata/model_meta.py +266 -0
  494. endoreg_db/models/metadata/model_meta_logic.py +485 -0
  495. endoreg_db/models/metadata/pdf_meta.py +96 -0
  496. endoreg_db/models/metadata/sensitive_meta.py +345 -0
  497. endoreg_db/models/metadata/sensitive_meta_logic.py +1161 -0
  498. endoreg_db/models/metadata/video_meta.py +459 -0
  499. endoreg_db/models/metadata/video_prediction_logic.py +232 -0
  500. endoreg_db/models/metadata/video_prediction_meta.py +319 -0
  501. endoreg_db/models/operation_log.py +63 -0
  502. endoreg_db/models/other/__init__.py +40 -0
  503. endoreg_db/models/other/distribution/__init__.py +46 -0
  504. endoreg_db/models/other/distribution/base_value_distribution.py +22 -0
  505. endoreg_db/models/other/distribution/date_value_distribution.py +163 -0
  506. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +50 -0
  507. endoreg_db/models/other/distribution/numeric_value_distribution.py +211 -0
  508. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +23 -0
  509. endoreg_db/models/other/emission/__init__.py +5 -0
  510. endoreg_db/models/other/emission/emission_factor.py +110 -0
  511. endoreg_db/models/other/gender.py +32 -0
  512. endoreg_db/models/other/information_source.py +190 -0
  513. endoreg_db/models/other/material.py +34 -0
  514. endoreg_db/models/other/resource.py +24 -0
  515. endoreg_db/models/other/tag.py +32 -0
  516. endoreg_db/models/other/transport_route.py +40 -0
  517. endoreg_db/models/other/unit.py +40 -0
  518. endoreg_db/models/other/waste.py +28 -0
  519. endoreg_db/models/report/__init__.py +0 -0
  520. endoreg_db/models/report/images.py +0 -0
  521. endoreg_db/models/report/report.py +5 -0
  522. endoreg_db/models/requirement/__init__.py +11 -0
  523. endoreg_db/models/requirement/requirement.py +792 -0
  524. endoreg_db/models/requirement/requirement_error.py +84 -0
  525. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  526. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  527. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  528. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +6 -0
  529. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +137 -0
  530. endoreg_db/models/requirement/requirement_operator.py +187 -0
  531. endoreg_db/models/requirement/requirement_set.py +327 -0
  532. endoreg_db/models/state/__init__.py +13 -0
  533. endoreg_db/models/state/abstract.py +11 -0
  534. endoreg_db/models/state/anonymization.py +30 -0
  535. endoreg_db/models/state/audit_ledger.py +155 -0
  536. endoreg_db/models/state/label_video_segment.py +31 -0
  537. endoreg_db/models/state/processing_history/__init__.py +3 -0
  538. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  539. endoreg_db/models/state/raw_pdf.py +219 -0
  540. endoreg_db/models/state/sensitive_meta.py +50 -0
  541. endoreg_db/models/state/video.py +251 -0
  542. endoreg_db/models/upload_job.py +100 -0
  543. endoreg_db/models/utils.py +138 -0
  544. endoreg_db/queries/__init__.py +3 -0
  545. endoreg_db/queries/annotations/__init__.py +1 -0
  546. endoreg_db/queries/annotations/legacy.py +169 -0
  547. endoreg_db/queries/sanity/__init_.py +0 -0
  548. endoreg_db/root_urls.py +27 -0
  549. endoreg_db/schemas/__init__.py +0 -0
  550. endoreg_db/schemas/examination_evaluation.py +30 -0
  551. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +861 -0
  552. endoreg_db/serializers/__init__.py +104 -0
  553. endoreg_db/serializers/administration/__init__.py +13 -0
  554. endoreg_db/serializers/administration/ai/__init__.py +9 -0
  555. endoreg_db/serializers/administration/ai/active_model.py +12 -0
  556. endoreg_db/serializers/administration/ai/ai_model.py +20 -0
  557. endoreg_db/serializers/administration/ai/model_type.py +12 -0
  558. endoreg_db/serializers/administration/center.py +14 -0
  559. endoreg_db/serializers/administration/gender.py +11 -0
  560. endoreg_db/serializers/anonymization.py +77 -0
  561. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -0
  562. endoreg_db/serializers/examination/__init__.py +10 -0
  563. endoreg_db/serializers/examination/base.py +45 -0
  564. endoreg_db/serializers/examination/dropdown.py +20 -0
  565. endoreg_db/serializers/examination_serializer.py +9 -0
  566. endoreg_db/serializers/finding/__init__.py +5 -0
  567. endoreg_db/serializers/finding/finding.py +61 -0
  568. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  569. endoreg_db/serializers/finding_classification/choice.py +19 -0
  570. endoreg_db/serializers/finding_classification/classification.py +11 -0
  571. endoreg_db/serializers/label_video_segment/__init__.py +9 -0
  572. endoreg_db/serializers/label_video_segment/image_classification_annotation.py +62 -0
  573. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  574. endoreg_db/serializers/label_video_segment/label/label.py +15 -0
  575. endoreg_db/serializers/label_video_segment/label_video_segment.py +427 -0
  576. endoreg_db/serializers/meta/__init__.py +13 -0
  577. endoreg_db/serializers/meta/sensitive_meta_detail.py +122 -0
  578. endoreg_db/serializers/meta/sensitive_meta_update.py +153 -0
  579. endoreg_db/serializers/meta/sensitive_meta_verification.py +62 -0
  580. endoreg_db/serializers/meta/video_meta.py +39 -0
  581. endoreg_db/serializers/misc/__init__.py +14 -0
  582. endoreg_db/serializers/misc/file_overview.py +72 -0
  583. endoreg_db/serializers/misc/sensitive_patient_data.py +144 -0
  584. endoreg_db/serializers/misc/stats.py +35 -0
  585. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  586. endoreg_db/serializers/misc/upload_job.py +74 -0
  587. endoreg_db/serializers/patient/__init__.py +12 -0
  588. endoreg_db/serializers/patient/patient.py +103 -0
  589. endoreg_db/serializers/patient/patient_dropdown.py +35 -0
  590. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  591. endoreg_db/serializers/patient_examination/patient_examination.py +168 -0
  592. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  593. endoreg_db/serializers/patient_finding/patient_finding.py +32 -0
  594. endoreg_db/serializers/patient_finding/patient_finding_classification.py +47 -0
  595. endoreg_db/serializers/patient_finding/patient_finding_detail.py +62 -0
  596. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +28 -0
  597. endoreg_db/serializers/patient_finding/patient_finding_list.py +40 -0
  598. endoreg_db/serializers/patient_finding/patient_finding_write.py +135 -0
  599. endoreg_db/serializers/pdf/__init__.py +3 -0
  600. endoreg_db/serializers/pdf/anony_text_validation.py +101 -0
  601. endoreg_db/serializers/requirements/requirement_schema.py +20 -0
  602. endoreg_db/serializers/requirements/requirement_sets.py +99 -0
  603. endoreg_db/serializers/sensitive_meta_serializer.py +301 -0
  604. endoreg_db/serializers/video/__init__.py +7 -0
  605. endoreg_db/serializers/video/video_file.py +283 -0
  606. endoreg_db/serializers/video/video_file_brief.py +14 -0
  607. endoreg_db/serializers/video/video_file_detail.py +96 -0
  608. endoreg_db/serializers/video/video_file_list.py +100 -0
  609. endoreg_db/serializers/video/video_processing_history.py +172 -0
  610. endoreg_db/serializers/video_examination.py +198 -0
  611. endoreg_db/services/__init__.py +5 -0
  612. endoreg_db/services/anonymization.py +274 -0
  613. endoreg_db/services/examination_evaluation.py +172 -0
  614. endoreg_db/services/finding_description_service.py +0 -0
  615. endoreg_db/services/lookup_service.py +424 -0
  616. endoreg_db/services/lookup_store.py +266 -0
  617. endoreg_db/services/model_meta_from_hf.py +76 -0
  618. endoreg_db/services/pdf_import.py +0 -0
  619. endoreg_db/services/polling_coordinator.py +319 -0
  620. endoreg_db/services/pseudonym_service.py +94 -0
  621. endoreg_db/services/report_import.py +13 -0
  622. endoreg_db/services/segment_sync.py +171 -0
  623. endoreg_db/services/video_import.py +9 -0
  624. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  625. endoreg_db/templates/admin/start_examination.html +12 -0
  626. endoreg_db/templates/timeline.html +176 -0
  627. endoreg_db/urls/__init__.py +56 -0
  628. endoreg_db/urls/ai.py +14 -0
  629. endoreg_db/urls/anonymization.py +78 -0
  630. endoreg_db/urls/auth.py +16 -0
  631. endoreg_db/urls/classification.py +34 -0
  632. endoreg_db/urls/examination.py +63 -0
  633. endoreg_db/urls/media.py +251 -0
  634. endoreg_db/urls/patient.py +23 -0
  635. endoreg_db/urls/requirements.py +15 -0
  636. endoreg_db/urls/root_urls.py +28 -0
  637. endoreg_db/urls/stats.py +54 -0
  638. endoreg_db/urls/upload.py +12 -0
  639. endoreg_db/urls.py +9 -0
  640. endoreg_db/utils/__init__.py +97 -0
  641. endoreg_db/utils/ai/__init__.py +9 -0
  642. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  643. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  644. endoreg_db/utils/ai/get.py +6 -0
  645. endoreg_db/utils/ai/inference_dataset.py +51 -0
  646. endoreg_db/utils/ai/model_training/config.py +117 -0
  647. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  648. endoreg_db/utils/ai/model_training/losses.py +68 -0
  649. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  650. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  651. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  652. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  653. endoreg_db/utils/ai/multilabel_classification_net.py +270 -0
  654. endoreg_db/utils/ai/postprocess.py +63 -0
  655. endoreg_db/utils/ai/predict.py +293 -0
  656. endoreg_db/utils/ai/preprocess.py +76 -0
  657. endoreg_db/utils/calc_duration_seconds.py +24 -0
  658. endoreg_db/utils/case_generator/__init__.py +3 -0
  659. endoreg_db/utils/case_generator/lab_sample_factory.py +32 -0
  660. endoreg_db/utils/check_video_files.py +175 -0
  661. endoreg_db/utils/cropping.py +30 -0
  662. endoreg_db/utils/dataloader.py +285 -0
  663. endoreg_db/utils/dates.py +59 -0
  664. endoreg_db/utils/defaults/set_default_center.py +33 -0
  665. endoreg_db/utils/env.py +37 -0
  666. endoreg_db/utils/extract_specific_frames.py +87 -0
  667. endoreg_db/utils/file_operations.py +70 -0
  668. endoreg_db/utils/fix_video_path_direct.py +157 -0
  669. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  670. endoreg_db/utils/hashs.py +138 -0
  671. endoreg_db/utils/links/__init__.py +0 -0
  672. endoreg_db/utils/links/requirement_link.py +237 -0
  673. endoreg_db/utils/mime_types.py +0 -0
  674. endoreg_db/utils/names.py +82 -0
  675. endoreg_db/utils/ocr.py +195 -0
  676. endoreg_db/utils/operation_log.py +87 -0
  677. endoreg_db/utils/parse_and_generate_yaml.py +45 -0
  678. endoreg_db/utils/paths.py +159 -0
  679. endoreg_db/utils/permissions.py +160 -0
  680. endoreg_db/utils/pipelines/Readme.md +235 -0
  681. endoreg_db/utils/pipelines/__init__.py +0 -0
  682. endoreg_db/utils/pipelines/process_video_dir.py +144 -0
  683. endoreg_db/utils/product/__init__.py +0 -0
  684. endoreg_db/utils/product/sum_emissions.py +22 -0
  685. endoreg_db/utils/product/sum_weights.py +20 -0
  686. endoreg_db/utils/pydantic_models/__init__.py +5 -0
  687. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  688. endoreg_db/utils/requirement_helpers.py +0 -0
  689. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  690. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +678 -0
  691. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +842 -0
  692. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +114 -0
  693. endoreg_db/utils/setup_config.py +196 -0
  694. endoreg_db/utils/storage.py +117 -0
  695. endoreg_db/utils/translation.py +31 -0
  696. endoreg_db/utils/uuid.py +5 -0
  697. endoreg_db/utils/validate_endo_roi.py +33 -0
  698. endoreg_db/utils/validate_subcategory_dict.py +93 -0
  699. endoreg_db/utils/validate_video_detailed.py +415 -0
  700. endoreg_db/utils/video/__init__.py +30 -0
  701. endoreg_db/utils/video/extract_frames.py +100 -0
  702. endoreg_db/utils/video/ffmpeg_wrapper.py +996 -0
  703. endoreg_db/utils/video/names.py +47 -0
  704. endoreg_db/utils/video/streaming_processor.py +386 -0
  705. endoreg_db/utils/video/video_splitter.py +105 -0
  706. endoreg_db/versioning.md +79 -0
  707. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +247 -0
  708. endoreg_db/views/__init__.py +157 -0
  709. endoreg_db/views/anonymization/__init__.py +31 -0
  710. endoreg_db/views/anonymization/media_management.py +486 -0
  711. endoreg_db/views/anonymization/overview.py +307 -0
  712. endoreg_db/views/anonymization/validate.py +310 -0
  713. endoreg_db/views/auth/__init__.py +13 -0
  714. endoreg_db/views/auth/keycloak.py +146 -0
  715. endoreg_db/views/examination/__init__.py +30 -0
  716. endoreg_db/views/examination/examination.py +37 -0
  717. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  718. endoreg_db/views/examination/get_finding_classification_choices.py +62 -0
  719. endoreg_db/views/examination/get_finding_classifications.py +38 -0
  720. endoreg_db/views/examination/get_findings.py +39 -0
  721. endoreg_db/views/examination/get_instruments.py +19 -0
  722. endoreg_db/views/examination/get_interventions.py +14 -0
  723. endoreg_db/views/finding/__init__.py +9 -0
  724. endoreg_db/views/finding/finding.py +116 -0
  725. endoreg_db/views/finding/get_classifications.py +14 -0
  726. endoreg_db/views/finding/get_interventions.py +17 -0
  727. endoreg_db/views/finding_classification/__init__.py +13 -0
  728. endoreg_db/views/finding_classification/base.py +0 -0
  729. endoreg_db/views/finding_classification/finding_classification.py +41 -0
  730. endoreg_db/views/finding_classification/get_classification_choices.py +54 -0
  731. endoreg_db/views/media/__init__.py +32 -0
  732. endoreg_db/views/media/pdf_media.py +411 -0
  733. endoreg_db/views/media/sensitive_metadata.py +372 -0
  734. endoreg_db/views/media/video_media.py +275 -0
  735. endoreg_db/views/meta/__init__.py +7 -0
  736. endoreg_db/views/meta/sensitive_meta_list.py +102 -0
  737. endoreg_db/views/meta/sensitive_meta_verification.py +74 -0
  738. endoreg_db/views/misc/__init__.py +29 -0
  739. endoreg_db/views/misc/center.py +14 -0
  740. endoreg_db/views/misc/csrf.py +8 -0
  741. endoreg_db/views/misc/gender.py +15 -0
  742. endoreg_db/views/misc/stats.py +255 -0
  743. endoreg_db/views/misc/upload_views.py +241 -0
  744. endoreg_db/views/patient/__init__.py +3 -0
  745. endoreg_db/views/patient/patient.py +253 -0
  746. endoreg_db/views/patient_examination/__init__.py +11 -0
  747. endoreg_db/views/patient_examination/patient_examination.py +141 -0
  748. endoreg_db/views/patient_examination/patient_examination_create.py +58 -0
  749. endoreg_db/views/patient_examination/patient_examination_detail.py +63 -0
  750. endoreg_db/views/patient_examination/patient_examination_list.py +72 -0
  751. endoreg_db/views/patient_examination/video.py +228 -0
  752. endoreg_db/views/patient_finding/__init__.py +7 -0
  753. endoreg_db/views/patient_finding/base.py +0 -0
  754. endoreg_db/views/patient_finding/patient_finding.py +71 -0
  755. endoreg_db/views/patient_finding/patient_finding_optimized.py +291 -0
  756. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  757. endoreg_db/views/patient_finding_classification/pfc_create.py +75 -0
  758. endoreg_db/views/report/__init__.py +7 -0
  759. endoreg_db/views/report/reimport.py +177 -0
  760. endoreg_db/views/report/report_stream.py +191 -0
  761. endoreg_db/views/requirement/__init__.py +11 -0
  762. endoreg_db/views/requirement/evaluate.py +278 -0
  763. endoreg_db/views/requirement/lookup.py +380 -0
  764. endoreg_db/views/requirement/lookup_store.py +183 -0
  765. endoreg_db/views/requirement/requirement_utils.py +87 -0
  766. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  767. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  768. endoreg_db/views/stats/__init__.py +13 -0
  769. endoreg_db/views/stats/stats_views.py +266 -0
  770. endoreg_db/views/video/__init__.py +49 -0
  771. endoreg_db/views/video/ai/__init__.py +8 -0
  772. endoreg_db/views/video/ai/label.py +159 -0
  773. endoreg_db/views/video/correction.py +529 -0
  774. endoreg_db/views/video/reimport.py +230 -0
  775. endoreg_db/views/video/segments_crud.py +709 -0
  776. endoreg_db/views/video/video_apply_mask.py +49 -0
  777. endoreg_db/views/video/video_correction.py +22 -0
  778. endoreg_db/views/video/video_download_processed.py +58 -0
  779. endoreg_db/views/video/video_examination_viewset.py +242 -0
  780. endoreg_db/views/video/video_metadata.py +101 -0
  781. endoreg_db/views/video/video_processing_history.py +25 -0
  782. endoreg_db/views/video/video_remove_frames.py +49 -0
  783. endoreg_db/views/video/video_stream.py +334 -0
  784. endoreg_db-0.8.9.32.dist-info/METADATA +404 -0
  785. endoreg_db-0.8.9.32.dist-info/RECORD +787 -0
  786. endoreg_db-0.8.9.32.dist-info/WHEEL +4 -0
  787. endoreg_db-0.8.9.32.dist-info/licenses/LICENSE +674 -0
@@ -0,0 +1,262 @@
1
+ # endoreg_db/utils/ai/data_loader_for_model_training.py
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import defaultdict
6
+ from pathlib import Path
7
+ from typing import Dict, List, Optional, TypedDict
8
+
9
+ from django.db import models
10
+
11
+ from endoreg_db.models import (
12
+ AIDataSet,
13
+ Frame,
14
+ ImageClassificationAnnotation,
15
+ Label,
16
+ LabelSet,
17
+ )
18
+
19
+
20
+ class ImageMultilabelDataset(TypedDict):
21
+ """
22
+ In-memory representation of an image multi-label training dataset.
23
+
24
+ All lists are aligned by index:
25
+
26
+ image_paths[i] -> path to image file for sample i
27
+ label_vectors[i] -> list[int|None] of length == len(labels)
28
+ label_masks[i] -> list[int] of length == len(labels)
29
+
30
+ Where:
31
+ - label_vectors[i][j] is:
32
+ 1 -> positive annotation (value=True)
33
+ 0 -> negative annotation (value=False)
34
+ None -> UNKNOWN (no annotation for that (frame, label))
35
+
36
+ - label_masks[i][j] is:
37
+ 1 -> this entry participates in the loss (0 or 1 is known)
38
+ 0 -> IGNORE in the loss (value was None)
39
+ """
40
+
41
+ # type description of the returned dict.
42
+
43
+ image_paths: List[str]
44
+ label_vectors: List[List[Optional[int]]]
45
+ label_masks: List[List[int]]
46
+ labels: List[Label]
47
+ labelset: LabelSet
48
+
49
+ # New: keep track of which DB rows were used, and their legacy exam ids
50
+ frame_ids: List[int] # Frame.pk for each sample
51
+ old_examination_ids: List[Optional[int]] # may be None if not set
52
+
53
+
54
+ def _infer_labelset_from_annotations(
55
+ annotations_qs: models.QuerySet[ImageClassificationAnnotation],
56
+ ) -> LabelSet:
57
+ """
58
+ Try to infer a unique LabelSet from the labels used in the annotations.
59
+
60
+ Strategy:
61
+ 1. Collect all distinct label_ids from the annotations.
62
+ 2. Fetch all Label objects + their label_sets.
63
+ 3. Compute the intersection of all label_sets across all labels.
64
+ 4. If there is exactly ONE common LabelSet, return it.
65
+ Otherwise, raise NotImplementedError for now.
66
+ """
67
+ label_ids = list(annotations_qs.values_list("label_id", flat=True).distinct())
68
+ if not label_ids:
69
+ raise ValueError("Cannot infer LabelSet: annotations queryset has no labels.")
70
+
71
+ labels_qs = Label.objects.filter(id__in=label_ids).prefetch_related("label_sets")
72
+ labelsets_for_each_label = []
73
+
74
+ for lbl in labels_qs:
75
+ # lbl.label_sets is the reverse of LabelSet.labels M2M
76
+ ls_ids = list(lbl.label_sets.values_list("id", flat=True))
77
+ if not ls_ids:
78
+ # This label is not part of any LabelSet -> ambiguous
79
+ raise NotImplementedError(
80
+ f"Label id={lbl.id}, name='{lbl.name}' is not part of any LabelSet. "
81
+ "Explicit LabelSet selection is required."
82
+ )
83
+ labelsets_for_each_label.append(set(ls_ids))
84
+
85
+ # Intersection of all labelset id sets
86
+ common_ids = set.intersection(*labelsets_for_each_label)
87
+ if not common_ids:
88
+ raise NotImplementedError(
89
+ "No common LabelSet across all labels in this AIDataSet. "
90
+ "Please specify a LabelSet explicitly."
91
+ )
92
+ if len(common_ids) > 1:
93
+ raise NotImplementedError(
94
+ "More than one common LabelSet found for the labels in this AIDataSet. "
95
+ "Please specify a LabelSet explicitly to disambiguate."
96
+ )
97
+
98
+ ls_id = next(iter(common_ids))
99
+ return LabelSet.objects.get(id=ls_id)
100
+
101
+
102
+ def build_image_multilabel_dataset_from_db(
103
+ dataset: AIDataSet,
104
+ labelset: Optional[LabelSet] = None,
105
+ ) -> ImageMultilabelDataset:
106
+ """
107
+ Build an in-memory multilabel dataset for an IMAGE-based AIDataSet.
108
+
109
+ Steps:
110
+ 1. Take all ImageClassificationAnnotation rows linked to this AIDataSet
111
+ (via dataset.image_annotations M2M).
112
+ 2. Determine the LabelSet (either explicitly given or inferred).
113
+ 3. For each used Frame, build:
114
+ - an image path
115
+ - a label vector (1, 0, or None for each label in LabelSet)
116
+ - a mask vector (1 where known, 0 where unknown)
117
+ 4. Return a dict that can be wrapped in a torch/tf Dataset.
118
+
119
+ NOTE:
120
+ - This function does NOT write anything to the DB.
121
+ - It only reads DB rows and returns Python structures.
122
+ """
123
+ if dataset.dataset_type != AIDataSet.DATASET_TYPE_IMAGE:
124
+ raise ValueError(
125
+ f"build_image_multilabel_dataset_from_db expected dataset_type='image', "
126
+ f"got '{dataset.dataset_type}' for AIDataSet id={dataset.id}."
127
+ )
128
+
129
+ # Get the annotation relation dynamically (for future video/text types)
130
+ annotations_qs = dataset.get_annotations_queryset().select_related("frame", "label")
131
+
132
+ if annotations_qs.count() == 0:
133
+ raise ValueError(
134
+ f"AIDataSet id={dataset.id} has no annotations attached. "
135
+ "Make sure your import script populated image_annotations."
136
+ )
137
+
138
+ # Decide which LabelSet to use
139
+ if labelset is None:
140
+ labelset = _infer_labelset_from_annotations(annotations_qs)
141
+
142
+ # Fixed label order (= fixed column order for the label vectors)
143
+ labels_in_order: List[Label] = labelset.get_labels_in_order()
144
+ if not labels_in_order:
145
+ raise ValueError(
146
+ f"LabelSet id={labelset.id}, name='{labelset.name}' has no labels."
147
+ )
148
+
149
+ num_labels = len(labels_in_order)
150
+ label_index: Dict[int, int] = {
151
+ lbl.id: idx for idx, lbl in enumerate(labels_in_order)
152
+ }
153
+
154
+ # Group annotations by frame
155
+ anns_by_frame: Dict[int, List[ImageClassificationAnnotation]] = defaultdict(list)
156
+ frames_order: List[int] = []
157
+
158
+ for ann in annotations_qs:
159
+ frame_id = ann.frame_id
160
+ if frame_id not in anns_by_frame:
161
+ frames_order.append(frame_id)
162
+ anns_by_frame[frame_id].append(ann)
163
+
164
+ # Build vectors
165
+ image_paths: List[str] = []
166
+ label_vectors: List[List[Optional[int]]] = []
167
+ label_masks: List[List[int]] = []
168
+
169
+ # New: id tracking for splitting / logging
170
+ frame_ids: List[int] = []
171
+ old_examination_ids: List[Optional[int]] = []
172
+
173
+ # Cache frames to avoid repeated DB hits
174
+ frame_obj_by_id: Dict[int, Frame] = {}
175
+
176
+ for frame_id in frames_order:
177
+ frame_annotations = anns_by_frame[frame_id]
178
+
179
+ # Resolve frame object (from first annotation of this frame)
180
+ frame = frame_obj_by_id.get(frame_id)
181
+ if frame is None:
182
+ frame = frame_annotations[0].frame
183
+ frame_obj_by_id[frame_id] = frame
184
+
185
+ # New: remember DB ids for this sample
186
+ frame_ids.append(frame_id)
187
+ old_examination_ids.append(getattr(frame, "old_examination_id", None))
188
+
189
+ # Start with unknown for all labels
190
+ vec: List[Optional[int]] = [None] * num_labels
191
+
192
+ # Fill with 1/0 where we have annotations
193
+ for ann in frame_annotations:
194
+ idx = label_index.get(ann.label_id)
195
+ if idx is None:
196
+ # Label not part of this LabelSet: ignore
197
+ continue
198
+ vec[idx] = 1 if ann.value else 0
199
+
200
+ # Build mask: 1 where vec is known, 0 where unknown
201
+ mask: List[int] = [0 if v is None else 1 for v in vec]
202
+
203
+ # Resolve absolute image path from the Frame model
204
+ file_path: Path = frame.file_path
205
+ image_paths.append(str(file_path))
206
+ label_vectors.append(vec)
207
+ label_masks.append(mask)
208
+
209
+ return ImageMultilabelDataset(
210
+ image_paths=image_paths,
211
+ label_vectors=label_vectors,
212
+ label_masks=label_masks,
213
+ labels=labels_in_order,
214
+ labelset=labelset,
215
+ frame_ids=frame_ids,
216
+ old_examination_ids=old_examination_ids,
217
+ )
218
+
219
+
220
+ def build_dataset_for_training(
221
+ dataset: AIDataSet,
222
+ labelset: Optional[LabelSet] = None,
223
+ ):
224
+ """
225
+ High-level entry point to build a training dataset from an AIDataSet row.
226
+
227
+ It inspects:
228
+ - dataset.dataset_type
229
+ - dataset.ai_model_type
230
+
231
+ and dispatches to the appropriate builder.
232
+
233
+ For now, we support:
234
+ - dataset_type = "image"
235
+ - ai_model_type = "image_multilabel_classification"
236
+
237
+ Later, you can extend this to:
238
+ - video segmentation
239
+ - text classification
240
+ etc.
241
+ """
242
+ # IMAGE MULTILABEL CASE
243
+ if (
244
+ dataset.dataset_type == AIDataSet.DATASET_TYPE_IMAGE
245
+ and dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_IMAGE_MULTILABEL
246
+ ):
247
+ return build_image_multilabel_dataset_from_db(dataset, labelset=labelset)
248
+
249
+ # FUTURE EXTENSIONS (example structure, not yet implemented):
250
+ # if dataset.dataset_type == AIDataSet.DATASET_TYPE_VIDEO and \
251
+ # dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_VIDEO_SEGMENTATION:
252
+ # return build_video_segmentation_dataset_from_db(dataset, labelset=labelset)
253
+ #
254
+ # if dataset.dataset_type == AIDataSet.DATASET_TYPE_TEXT and \
255
+ # dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_TEXT_CLASSIFICATION:
256
+ # return build_text_classification_dataset_from_db(dataset, labelset=labelset)
257
+
258
+ raise NotImplementedError(
259
+ f"No dataset builder implemented for "
260
+ f"dataset_type='{dataset.dataset_type}', "
261
+ f"ai_model_type='{dataset.ai_model_type}'."
262
+ )
@@ -0,0 +1,262 @@
1
+ # endoreg_db/utils/ai/data_loader_for_model_training.py
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import defaultdict
6
+ from pathlib import Path
7
+ from typing import Dict, List, Optional, TypedDict
8
+
9
+ from django.db import models
10
+
11
+ from endoreg_db.models import (
12
+ AIDataSet,
13
+ Frame,
14
+ ImageClassificationAnnotation,
15
+ Label,
16
+ LabelSet,
17
+ )
18
+
19
+
20
+ class ImageMultilabelDataset(TypedDict):
21
+ """
22
+ In-memory representation of an image multi-label training dataset.
23
+
24
+ All lists are aligned by index:
25
+
26
+ image_paths[i] -> path to image file for sample i
27
+ label_vectors[i] -> list[int|None] of length == len(labels)
28
+ label_masks[i] -> list[int] of length == len(labels)
29
+
30
+ Where:
31
+ - label_vectors[i][j] is:
32
+ 1 -> positive annotation (value=True)
33
+ 0 -> negative annotation (value=False)
34
+ None -> UNKNOWN (no annotation for that (frame, label))
35
+
36
+ - label_masks[i][j] is:
37
+ 1 -> this entry participates in the loss (0 or 1 is known)
38
+ 0 -> IGNORE in the loss (value was None)
39
+ """
40
+
41
+ # type description of the returned dict.
42
+
43
+ image_paths: List[str]
44
+ label_vectors: List[List[Optional[int]]]
45
+ label_masks: List[List[int]]
46
+ labels: List[Label]
47
+ labelset: LabelSet
48
+
49
+ # New: keep track of which DB rows were used, and their legacy exam ids
50
+ frame_ids: List[int] # Frame.pk for each sample
51
+ old_examination_ids: List[Optional[int]] # may be None if not set
52
+
53
+
54
+ def _infer_labelset_from_annotations(
55
+ annotations_qs: models.QuerySet[ImageClassificationAnnotation],
56
+ ) -> LabelSet:
57
+ """
58
+ Try to infer a unique LabelSet from the labels used in the annotations.
59
+
60
+ Strategy:
61
+ 1. Collect all distinct label_ids from the annotations.
62
+ 2. Fetch all Label objects + their label_sets.
63
+ 3. Compute the intersection of all label_sets across all labels.
64
+ 4. If there is exactly ONE common LabelSet, return it.
65
+ Otherwise, raise NotImplementedError for now.
66
+ """
67
+ label_ids = list(annotations_qs.values_list("label_id", flat=True).distinct())
68
+ if not label_ids:
69
+ raise ValueError("Cannot infer LabelSet: annotations queryset has no labels.")
70
+
71
+ labels_qs = Label.objects.filter(id__in=label_ids).prefetch_related("label_sets")
72
+ labelsets_for_each_label = []
73
+
74
+ for lbl in labels_qs:
75
+ # lbl.label_sets is the reverse of LabelSet.labels M2M
76
+ ls_ids = list(lbl.label_sets.values_list("id", flat=True))
77
+ if not ls_ids:
78
+ # This label is not part of any LabelSet -> ambiguous
79
+ raise NotImplementedError(
80
+ f"Label id={lbl.id}, name='{lbl.name}' is not part of any LabelSet. "
81
+ "Explicit LabelSet selection is required."
82
+ )
83
+ labelsets_for_each_label.append(set(ls_ids))
84
+
85
+ # Intersection of all labelset id sets
86
+ common_ids = set.intersection(*labelsets_for_each_label)
87
+ if not common_ids:
88
+ raise NotImplementedError(
89
+ "No common LabelSet across all labels in this AIDataSet. "
90
+ "Please specify a LabelSet explicitly."
91
+ )
92
+ if len(common_ids) > 1:
93
+ raise NotImplementedError(
94
+ "More than one common LabelSet found for the labels in this AIDataSet. "
95
+ "Please specify a LabelSet explicitly to disambiguate."
96
+ )
97
+
98
+ ls_id = next(iter(common_ids))
99
+ return LabelSet.objects.get(id=ls_id)
100
+
101
+
102
+ def build_image_multilabel_dataset_from_db(
103
+ dataset: AIDataSet,
104
+ labelset: Optional[LabelSet] = None,
105
+ ) -> ImageMultilabelDataset:
106
+ """
107
+ Build an in-memory multilabel dataset for an IMAGE-based AIDataSet.
108
+
109
+ Steps:
110
+ 1. Take all ImageClassificationAnnotation rows linked to this AIDataSet
111
+ (via dataset.image_annotations M2M).
112
+ 2. Determine the LabelSet (either explicitly given or inferred).
113
+ 3. For each used Frame, build:
114
+ - an image path
115
+ - a label vector (1, 0, or None for each label in LabelSet)
116
+ - a mask vector (1 where known, 0 where unknown)
117
+ 4. Return a dict that can be wrapped in a torch/tf Dataset.
118
+
119
+ NOTE:
120
+ - This function does NOT write anything to the DB.
121
+ - It only reads DB rows and returns Python structures.
122
+ """
123
+ if dataset.dataset_type != AIDataSet.DATASET_TYPE_IMAGE:
124
+ raise ValueError(
125
+ f"build_image_multilabel_dataset_from_db expected dataset_type='image', "
126
+ f"got '{dataset.dataset_type}' for AIDataSet id={dataset.id}."
127
+ )
128
+
129
+ # Get the annotation relation dynamically (for future video/text types)
130
+ annotations_qs = dataset.get_annotations_queryset().select_related("frame", "label")
131
+
132
+ if annotations_qs.count() == 0:
133
+ raise ValueError(
134
+ f"AIDataSet id={dataset.id} has no annotations attached. "
135
+ "Make sure your import script populated image_annotations."
136
+ )
137
+
138
+ # Decide which LabelSet to use
139
+ if labelset is None:
140
+ labelset = _infer_labelset_from_annotations(annotations_qs)
141
+
142
+ # Fixed label order (= fixed column order for the label vectors)
143
+ labels_in_order: List[Label] = labelset.get_labels_in_order()
144
+ if not labels_in_order:
145
+ raise ValueError(
146
+ f"LabelSet id={labelset.id}, name='{labelset.name}' has no labels."
147
+ )
148
+
149
+ num_labels = len(labels_in_order)
150
+ label_index: Dict[int, int] = {
151
+ lbl.id: idx for idx, lbl in enumerate(labels_in_order)
152
+ }
153
+
154
+ # Group annotations by frame
155
+ anns_by_frame: Dict[int, List[ImageClassificationAnnotation]] = defaultdict(list)
156
+ frames_order: List[int] = []
157
+
158
+ for ann in annotations_qs:
159
+ frame_id = ann.frame_id
160
+ if frame_id not in anns_by_frame:
161
+ frames_order.append(frame_id)
162
+ anns_by_frame[frame_id].append(ann)
163
+
164
+ # Build vectors
165
+ image_paths: List[str] = []
166
+ label_vectors: List[List[Optional[int]]] = []
167
+ label_masks: List[List[int]] = []
168
+
169
+ # New: id tracking for splitting / logging
170
+ frame_ids: List[int] = []
171
+ old_examination_ids: List[Optional[int]] = []
172
+
173
+ # Cache frames to avoid repeated DB hits
174
+ frame_obj_by_id: Dict[int, Frame] = {}
175
+
176
+ for frame_id in frames_order:
177
+ frame_annotations = anns_by_frame[frame_id]
178
+
179
+ # Resolve frame object (from first annotation of this frame)
180
+ frame = frame_obj_by_id.get(frame_id)
181
+ if frame is None:
182
+ frame = frame_annotations[0].frame
183
+ frame_obj_by_id[frame_id] = frame
184
+
185
+ # New: remember DB ids for this sample
186
+ frame_ids.append(frame_id)
187
+ old_examination_ids.append(getattr(frame, "old_examination_id", None))
188
+
189
+ # Start with unknown for all labels
190
+ vec: List[Optional[int]] = [None] * num_labels
191
+
192
+ # Fill with 1/0 where we have annotations
193
+ for ann in frame_annotations:
194
+ idx = label_index.get(ann.label_id)
195
+ if idx is None:
196
+ # Label not part of this LabelSet: ignore
197
+ continue
198
+ vec[idx] = 1 if ann.value else 0
199
+
200
+ # Build mask: 1 where vec is known, 0 where unknown
201
+ mask: List[int] = [0 if v is None else 1 for v in vec]
202
+
203
+ # Resolve absolute image path from the Frame model
204
+ file_path: Path = frame.file_path
205
+ image_paths.append(str(file_path))
206
+ label_vectors.append(vec)
207
+ label_masks.append(mask)
208
+
209
+ return ImageMultilabelDataset(
210
+ image_paths=image_paths,
211
+ label_vectors=label_vectors,
212
+ label_masks=label_masks,
213
+ labels=labels_in_order,
214
+ labelset=labelset,
215
+ frame_ids=frame_ids,
216
+ old_examination_ids=old_examination_ids,
217
+ )
218
+
219
+
220
+ def build_dataset_for_training(
221
+ dataset: AIDataSet,
222
+ labelset: Optional[LabelSet] = None,
223
+ ):
224
+ """
225
+ High-level entry point to build a training dataset from an AIDataSet row.
226
+
227
+ It inspects:
228
+ - dataset.dataset_type
229
+ - dataset.ai_model_type
230
+
231
+ and dispatches to the appropriate builder.
232
+
233
+ For now, we support:
234
+ - dataset_type = "image"
235
+ - ai_model_type = "image_multilabel_classification"
236
+
237
+ Later, you can extend this to:
238
+ - video segmentation
239
+ - text classification
240
+ etc.
241
+ """
242
+ # IMAGE MULTILABEL CASE
243
+ if (
244
+ dataset.dataset_type == AIDataSet.DATASET_TYPE_IMAGE
245
+ and dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_IMAGE_MULTILABEL
246
+ ):
247
+ return build_image_multilabel_dataset_from_db(dataset, labelset=labelset)
248
+
249
+ # FUTURE EXTENSIONS (example structure, not yet implemented):
250
+ # if dataset.dataset_type == AIDataSet.DATASET_TYPE_VIDEO and \
251
+ # dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_VIDEO_SEGMENTATION:
252
+ # return build_video_segmentation_dataset_from_db(dataset, labelset=labelset)
253
+ #
254
+ # if dataset.dataset_type == AIDataSet.DATASET_TYPE_TEXT and \
255
+ # dataset.ai_model_type == AIDataSet.AI_MODEL_TYPE_TEXT_CLASSIFICATION:
256
+ # return build_text_classification_dataset_from_db(dataset, labelset=labelset)
257
+
258
+ raise NotImplementedError(
259
+ f"No dataset builder implemented for "
260
+ f"dataset_type='{dataset.dataset_type}', "
261
+ f"ai_model_type='{dataset.ai_model_type}'."
262
+ )
@@ -0,0 +1,6 @@
1
+ from endoreg_db.models import ModelMeta
2
+
3
+
4
+ def get_latest_model_meta_by_model_name(model_name):
5
+ model_meta = ModelMeta.objects.filter(name=model_name).order_by("-version").first()
6
+ return model_meta
@@ -0,0 +1,51 @@
1
+ from torch.utils.data import Dataset
2
+ import numpy as np
3
+ from PIL import Image
4
+ from torchvision import transforms
5
+ from .preprocess import Cropper
6
+
7
+
8
+ class InferenceDataset(Dataset):
9
+ def __init__(self, paths, crops, config):
10
+ self.paths = paths
11
+ self.crops = crops
12
+ self.cropper = Cropper() # Assuming Cropper can work with NumPy arrays
13
+ self.config = config
14
+
15
+ # Initialize the image transformations using torchvision
16
+ self.transforms = transforms.Compose(
17
+ [
18
+ # Convert PIL image to PyTorch tensor
19
+ transforms.ToTensor(),
20
+ # Normalize the image using the provided mean and std
21
+ transforms.Normalize(mean=self.config["mean"], std=self.config["std"]),
22
+ ]
23
+ )
24
+
25
+ def __len__(self):
26
+ # Returns the total number of samples
27
+ return len(self.paths)
28
+
29
+ def __getitem__(self, idx):
30
+ # Open the image with Pillow
31
+ with Image.open(self.paths[idx]) as pil_image:
32
+ # Convert the image to RGB to ensure 3 channels
33
+ pil_image = pil_image.convert("RGB")
34
+
35
+ # Get the corresponding crop for the current image
36
+ crop = self.crops[idx]
37
+
38
+ # Crop the image based on the provided crop parameters and convert to numpy for cropping
39
+ cropped = self.cropper(
40
+ np.array(pil_image), # Convert PIL image to numpy array for cropping
41
+ crop,
42
+ scale=[self.config["size_x"], self.config["size_y"]],
43
+ )
44
+
45
+ # Convert cropped numpy array back to PIL image for torchvision transforms
46
+ cropped_pil = Image.fromarray(cropped.astype("uint8"), "RGB")
47
+
48
+ # Apply the transformations
49
+ img = self.transforms(cropped_pil)
50
+
51
+ return img
@@ -0,0 +1,117 @@
1
+ # endoreg_db/utils/ai/model_training/config.py
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Optional
8
+
9
+ from django.conf import settings
10
+
11
+
12
+ # ---------------------------------------------------------------------
13
+ # PATHS
14
+ # ---------------------------------------------------------------------
15
+
16
+ # Base project directory (e.g. /home/admin/dev/endoreg-db)
17
+ BASE_DIR = Path(getattr(settings, "BASE_DIR", Path(__file__).resolve().parents[4]))
18
+
19
+ # All training artifacts go here:
20
+ # /home/admin/dev/endoreg-db/data/model_training/
21
+ TRAINING_ROOT = BASE_DIR / "data" / "model_training"
22
+ CHECKPOINTS_DIR = TRAINING_ROOT / "checkpoints"
23
+ RUNS_DIR = TRAINING_ROOT / "runs"
24
+
25
+ for d in (TRAINING_ROOT, CHECKPOINTS_DIR, RUNS_DIR):
26
+ d.mkdir(parents=True, exist_ok=True)
27
+
28
+ # Which LabelSet.version we train on (for label filtering)
29
+ DEFAULT_LABELSET_VERSION_TO_TRAIN: int = 2
30
+
31
+
32
+ # ---------------------------------------------------------------------
33
+ # TRAINING CONFIG
34
+ # ---------------------------------------------------------------------
35
+
36
+
37
+ @dataclass
38
+ class TrainingConfig:
39
+ """
40
+ Configuration for GastroNet multi-label training.
41
+
42
+ Most important knobs:
43
+ - dataset_id: which AIDataSet row to use from the database
44
+ - labelset_version_to_train: only labels belonging to LabelSet.version == this
45
+ are used for training (e.g. 2).
46
+ - treat_unlabeled_as_negative:
47
+ True -> Option A: for v2 labels, if not annotated in a frame, we
48
+ assume "absent" (0) and include it in the loss.
49
+ False -> keep "unknown" semantics (mask = 0, ignored in loss/metrics).
50
+
51
+ Learning rate schedule:
52
+ - lr_head / lr_backbone: base learning rates
53
+ - use_scheduler: if True, we use warm-up + cosine decay
54
+ - warmup_epochs: how many epochs to linearly increase LR from 0 → base LR
55
+ - min_lr: lowest LR reached at the end of cosine schedule
56
+ """
57
+
58
+ # --- WHAT TO TRAIN ON -------------------------------------------------
59
+ dataset_id: int
60
+
61
+ # Train only on labels belonging to ANY LabelSet with this version.
62
+ labelset_version_to_train: int = DEFAULT_LABELSET_VERSION_TO_TRAIN
63
+
64
+ # Path to GastroNet RN50 checkpoint (.pth); if None, backbone is random.
65
+ backbone_checkpoint: Optional[str] = None
66
+
67
+ # --- EPOCHS / BATCHING -----------------------------------------------
68
+ num_epochs: int = 5
69
+ batch_size: int = 32
70
+
71
+ # Split ratios (by colonoscopy exam groups, not by individual frames)
72
+ val_split: float = 0.2
73
+ test_split: float = 0.1
74
+
75
+ # --- LEARNING RATES --------------------------------------------------
76
+ # Base learning rates for classifier head and backbone.
77
+ lr_head: float = 1e-3 # usually larger (newly initialized layer)
78
+ lr_backbone: float = 1e-4 # smaller (pretrained GastroNet backbone)
79
+
80
+ # --- FOCAL LOSS HYPERPARAMETERS -------------------------------------
81
+ gamma_focal: float = 2.0 # how strongly to focus on hard examples
82
+ alpha_focal: float = 0.25 # weight for positives vs negatives
83
+
84
+ # --- DEVICE & SEED ---------------------------------------------------
85
+ device: str = "auto" # "auto", "cpu", or "cuda"
86
+ random_seed: int = 42
87
+
88
+ # --- LABEL SEMANTICS -------------------------------------------------
89
+ # For the filtered labels (LabelSet.version == labelset_version_to_train):
90
+ # True -> Option A: unlabeled => negative (0) and mask=1 (supervised)
91
+ # False -> keep unlabeled as unknown (mask=0, ignored)
92
+ treat_unlabeled_as_negative: bool = True
93
+
94
+ # --- LR SCHEDULER: WARM-UP + COSINE DECAY ----------------------------
95
+ # If True, we apply:
96
+ # - linear warm-up for 'warmup_epochs'
97
+ # - then CosineAnnealingLR for the remaining epochs
98
+ use_scheduler: bool = True
99
+
100
+ # Number of warm-up epochs (can be 0 for "no warm-up").
101
+ warmup_epochs: int = 3
102
+
103
+ # Minimum learning rate at the end of cosine decay for all param groups.
104
+ # (Both head and backbone decay towards this value.)
105
+ min_lr: float = 1e-6
106
+
107
+ # which CNN backbone / weights to use
108
+ # "gastro_rn50" → current behavior (ResNet50 + GastroNet checkpoint)
109
+ # "resnet50_imagenet" → ResNet50 with ImageNet weights
110
+ # "resnet50_random" → ResNet50 with random initialization
111
+ # (later) "efficientnet_b0_imagenet", etc.
112
+ backbone_name: str = "gastro_rn50"
113
+
114
+ # whether to freeze backbone (feature extractor)
115
+ freeze_backbone: bool = True
116
+
117
+ # backbone_name: str = "gastro_rn50"