endoreg-db 0.8.9.32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (787) hide show
  1. endoreg_db/__init__.py +0 -0
  2. endoreg_db/_version.py +34 -0
  3. endoreg_db/admin.py +97 -0
  4. endoreg_db/api/serializers/finding_descriptions.py +0 -0
  5. endoreg_db/api/views/finding_descriptions.py +0 -0
  6. endoreg_db/api_urls.py +4 -0
  7. endoreg_db/apps.py +17 -0
  8. endoreg_db/assets/dummy_model.ckpt +1 -0
  9. endoreg_db/authz/auth.py +78 -0
  10. endoreg_db/authz/backends.py +168 -0
  11. endoreg_db/authz/management/commands/list_routes.py +20 -0
  12. endoreg_db/authz/middleware.py +84 -0
  13. endoreg_db/authz/permissions.py +138 -0
  14. endoreg_db/authz/policy.py +224 -0
  15. endoreg_db/authz/settings.py +64 -0
  16. endoreg_db/authz/views_auth.py +70 -0
  17. endoreg_db/codemods/readme.md +88 -0
  18. endoreg_db/codemods/rename_datetime_fields.py +99 -0
  19. endoreg_db/config/__init__.py +0 -0
  20. endoreg_db/config/env.py +106 -0
  21. endoreg_db/config/settings/__init__.py +6 -0
  22. endoreg_db/config/settings/base.py +148 -0
  23. endoreg_db/config/settings/case_gen.py +32 -0
  24. endoreg_db/config/settings/dev.py +108 -0
  25. endoreg_db/config/settings/keycloak.py +177 -0
  26. endoreg_db/config/settings/prod.py +66 -0
  27. endoreg_db/config/settings/test.py +72 -0
  28. endoreg_db/data/__init__.py +135 -0
  29. endoreg_db/data/ai_model/data.yaml +7 -0
  30. endoreg_db/data/ai_model_label/label/data.yaml +88 -0
  31. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  32. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  33. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  34. endoreg_db/data/ai_model_label/label-type/data.yaml +7 -0
  35. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  36. endoreg_db/data/ai_model_type/data.yaml +7 -0
  37. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  38. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  39. endoreg_db/data/case_template/rule/00_patient_lab_sample_add_default_value.yaml +167 -0
  40. endoreg_db/data/case_template/rule/01_patient-set-age.yaml +8 -0
  41. endoreg_db/data/case_template/rule/01_patient-set-gender.yaml +9 -0
  42. endoreg_db/data/case_template/rule/11_create_patient_lab_sample.yaml +23 -0
  43. endoreg_db/data/case_template/rule/12_create-patient_medication-anticoagulation.yaml +19 -0
  44. endoreg_db/data/case_template/rule/13_create-patient_medication_schedule-anticoagulation.yaml +19 -0
  45. endoreg_db/data/case_template/rule/19_create_patient.yaml +17 -0
  46. endoreg_db/data/case_template/rule_type/base_types.yaml +35 -0
  47. endoreg_db/data/case_template/rule_value/.init +0 -0
  48. endoreg_db/data/case_template/rule_value_type/base_types.yaml +59 -0
  49. endoreg_db/data/case_template/template/base.yaml +8 -0
  50. endoreg_db/data/case_template/template_type/pre_endoscopy.yaml +3 -0
  51. endoreg_db/data/case_template/tmp/_rule_value +13 -0
  52. endoreg_db/data/case_template/tmp/rule/01_atrial_fibrillation.yaml +21 -0
  53. endoreg_db/data/case_template/tmp/rule/02_create_object.yaml +10 -0
  54. endoreg_db/data/case_template/tmp/template/atrial_fibrillation_low_risk.yaml +7 -0
  55. endoreg_db/data/center/data.yaml +99 -0
  56. endoreg_db/data/center_resource/green_endoscopy_dashboard_CenterResource.yaml +144 -0
  57. endoreg_db/data/center_shift/ukw.yaml +9 -0
  58. endoreg_db/data/center_waste/green_endoscopy_dashboard_CenterWaste.yaml +48 -0
  59. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  60. endoreg_db/data/db_summary.csv +58 -0
  61. endoreg_db/data/db_summary.xlsx +0 -0
  62. endoreg_db/data/disease/cardiovascular.yaml +37 -0
  63. endoreg_db/data/disease/hepatology.yaml +5 -0
  64. endoreg_db/data/disease/misc.yaml +5 -0
  65. endoreg_db/data/disease/renal.yaml +5 -0
  66. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +6 -0
  67. endoreg_db/data/disease_classification/coronary_vessel_disease.yaml +6 -0
  68. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +41 -0
  69. endoreg_db/data/disease_classification_choice/coronary_vessel_disease.yaml +20 -0
  70. endoreg_db/data/distribution/date/patient.yaml +7 -0
  71. endoreg_db/data/distribution/multiple_categorical/.init +0 -0
  72. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  73. endoreg_db/data/distribution/single_categorical/patient.yaml +7 -0
  74. endoreg_db/data/emission_factor/green_endoscopy_dashboard_EmissionFactor.yaml +132 -0
  75. endoreg_db/data/endoscope/data.yaml +93 -0
  76. endoreg_db/data/endoscope_type/data.yaml +11 -0
  77. endoreg_db/data/endoscopy_processor/data.yaml +50 -0
  78. endoreg_db/data/event/cardiology.yaml +15 -0
  79. endoreg_db/data/event/neurology.yaml +14 -0
  80. endoreg_db/data/event/surgery.yaml +13 -0
  81. endoreg_db/data/event/thrombembolism.yaml +20 -0
  82. endoreg_db/data/event_classification/data.yaml +4 -0
  83. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  84. endoreg_db/data/examination/examinations/data.yaml +172 -0
  85. endoreg_db/data/examination/time/data.yaml +48 -0
  86. endoreg_db/data/examination/time-type/data.yaml +5 -0
  87. endoreg_db/data/examination/type/data.yaml +17 -0
  88. endoreg_db/data/examination_indication/endoscopy.yaml +359 -0
  89. endoreg_db/data/examination_indication_classification/endoscopy.yaml +90 -0
  90. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +97 -0
  91. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  92. endoreg_db/data/finding/00_generic.yaml +35 -0
  93. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  94. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  95. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  96. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  97. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  98. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  99. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  100. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  101. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  102. endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
  103. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  104. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  105. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  106. endoreg_db/data/finding_classification_choice/00_generic.yaml +15 -0
  107. endoreg_db/data/finding_classification_choice/00_generic_baseline.yaml +23 -0
  108. endoreg_db/data/finding_classification_choice/00_generic_complication.yaml +15 -0
  109. endoreg_db/data/finding_classification_choice/00_generic_histology.yaml +21 -0
  110. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  111. endoreg_db/data/finding_classification_choice/02_colonoscopy_bowel_preparation.yaml +49 -0
  112. endoreg_db/data/finding_classification_choice/02_colonoscopy_generic.yaml +19 -0
  113. endoreg_db/data/finding_classification_choice/02_colonoscopy_histology.yaml +20 -0
  114. endoreg_db/data/finding_classification_choice/02_colonoscopy_location.yaml +248 -0
  115. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  116. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  117. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_morphology.yaml +75 -0
  118. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  119. endoreg_db/data/finding_classification_type/00_generic.yaml +53 -0
  120. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  121. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  122. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  123. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  124. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  125. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  126. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  127. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  128. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  129. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  130. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  131. endoreg_db/data/finding_type/data.yaml +39 -0
  132. endoreg_db/data/gender/data.yaml +42 -0
  133. endoreg_db/data/information_source/annotation.yaml +6 -0
  134. endoreg_db/data/information_source/data.yaml +30 -0
  135. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  136. endoreg_db/data/information_source/medication.yaml +6 -0
  137. endoreg_db/data/information_source/prediction.yaml +7 -0
  138. endoreg_db/data/information_source_type/data.yaml +8 -0
  139. endoreg_db/data/lab_value/cardiac_enzymes.yaml +37 -0
  140. endoreg_db/data/lab_value/coagulation.yaml +54 -0
  141. endoreg_db/data/lab_value/electrolytes.yaml +228 -0
  142. endoreg_db/data/lab_value/gastrointestinal_function.yaml +133 -0
  143. endoreg_db/data/lab_value/hematology.yaml +184 -0
  144. endoreg_db/data/lab_value/hormones.yaml +59 -0
  145. endoreg_db/data/lab_value/lipids.yaml +53 -0
  146. endoreg_db/data/lab_value/misc.yaml +76 -0
  147. endoreg_db/data/lab_value/renal_function.yaml +12 -0
  148. endoreg_db/data/log_type/data.yaml +57 -0
  149. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  150. endoreg_db/data/lx_client_type/base.yaml +30 -0
  151. endoreg_db/data/lx_permission/base.yaml +24 -0
  152. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  153. endoreg_db/data/material/material.yaml +91 -0
  154. endoreg_db/data/medication/anticoagulation.yaml +65 -0
  155. endoreg_db/data/medication/tah.yaml +70 -0
  156. endoreg_db/data/medication_indication/anticoagulation.yaml +115 -0
  157. endoreg_db/data/medication_indication_type/data.yaml +11 -0
  158. endoreg_db/data/medication_indication_type/thrombembolism.yaml +41 -0
  159. endoreg_db/data/medication_intake_time/base.yaml +31 -0
  160. endoreg_db/data/medication_schedule/apixaban.yaml +95 -0
  161. endoreg_db/data/medication_schedule/ass.yaml +12 -0
  162. endoreg_db/data/medication_schedule/enoxaparin.yaml +26 -0
  163. endoreg_db/data/names_first/first_names.yaml +54 -0
  164. endoreg_db/data/names_last/last_names.yaml +51 -0
  165. endoreg_db/data/network_device/data.yaml +59 -0
  166. endoreg_db/data/network_device_type/data.yaml +12 -0
  167. endoreg_db/data/organ/data.yaml +29 -0
  168. endoreg_db/data/patient_lab_sample_type/generic.yaml +6 -0
  169. endoreg_db/data/pdf_type/data.yaml +46 -0
  170. endoreg_db/data/product/green_endoscopy_dashboard_Product.yaml +66 -0
  171. endoreg_db/data/product_group/green_endoscopy_dashboard_ProductGroup.yaml +33 -0
  172. endoreg_db/data/product_material/green_endoscopy_dashboard_ProductMaterial.yaml +308 -0
  173. endoreg_db/data/product_weight/green_endoscopy_dashboard_ProductWeight.yaml +88 -0
  174. endoreg_db/data/profession/data.yaml +70 -0
  175. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  176. endoreg_db/data/qualification/m2.yaml +39 -0
  177. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  178. endoreg_db/data/qualification/sonography.yaml +36 -0
  179. endoreg_db/data/qualification_type/base.yaml +29 -0
  180. endoreg_db/data/reference_product/green_endoscopy_dashboard_ReferenceProduct.yaml +55 -0
  181. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  182. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +30 -0
  183. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +24 -0
  184. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  185. endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
  186. endoreg_db/data/requirement/old/colonoscopy_baseline_austria.yaml +45 -0
  187. endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
  188. endoreg_db/data/requirement/old/disease_cardiovascular.yaml +79 -0
  189. endoreg_db/data/requirement/old/disease_classification_choice_cardiovascular.yaml +41 -0
  190. endoreg_db/data/requirement/old/disease_hepatology.yaml +12 -0
  191. endoreg_db/data/requirement/old/disease_misc.yaml +12 -0
  192. endoreg_db/data/requirement/old/disease_renal.yaml +96 -0
  193. endoreg_db/data/requirement/old/endoscopy_bleeding_risk.yaml +59 -0
  194. endoreg_db/data/requirement/old/event_cardiology.yaml +251 -0
  195. endoreg_db/data/requirement/old/event_requirements.yaml +145 -0
  196. endoreg_db/data/requirement/old/finding_colon_polyp.yaml +50 -0
  197. endoreg_db/data/requirement/old/gender.yaml +0 -0
  198. endoreg_db/data/requirement/old/lab_value.yaml +441 -0
  199. endoreg_db/data/requirement/old/medication.yaml +93 -0
  200. endoreg_db/data/requirement_operator/_old/age.yaml +13 -0
  201. endoreg_db/data/requirement_operator/_old/lab_operators.yaml +129 -0
  202. endoreg_db/data/requirement_operator/_old/model_operators.yaml +96 -0
  203. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  204. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +65 -0
  205. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  206. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  207. endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
  208. endoreg_db/data/requirement_set/_old_ +109 -0
  209. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  210. endoreg_db/data/requirement_set_type/data.yaml +41 -0
  211. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  212. endoreg_db/data/resource/green_endoscopy_dashboard_Resource.yaml +15 -0
  213. endoreg_db/data/risk/bleeding.yaml +26 -0
  214. endoreg_db/data/risk/thrombosis.yaml +37 -0
  215. endoreg_db/data/risk_type/data.yaml +27 -0
  216. endoreg_db/data/setup_config.yaml +38 -0
  217. endoreg_db/data/shift/endoscopy.yaml +21 -0
  218. endoreg_db/data/shift/m2.yaml +0 -0
  219. endoreg_db/data/shift_type/base.yaml +35 -0
  220. endoreg_db/data/tag/requirement_set_tags.yaml +32 -0
  221. endoreg_db/data/tmp/chronic_kidney_disease.yaml +0 -0
  222. endoreg_db/data/tmp/congestive_heart_failure.yaml +0 -0
  223. endoreg_db/data/transport_route/green_endoscopy_dashboard_TransportRoute.yaml +12 -0
  224. endoreg_db/data/unit/concentration.yaml +115 -0
  225. endoreg_db/data/unit/data.yaml +17 -0
  226. endoreg_db/data/unit/length.yaml +31 -0
  227. endoreg_db/data/unit/misc.yaml +20 -0
  228. endoreg_db/data/unit/rate.yaml +6 -0
  229. endoreg_db/data/unit/time.yaml +48 -0
  230. endoreg_db/data/unit/volume.yaml +35 -0
  231. endoreg_db/data/unit/weight.yaml +38 -0
  232. endoreg_db/data/waste/data.yaml +12 -0
  233. endoreg_db/exceptions.py +24 -0
  234. endoreg_db/export/frames/export.py +6 -0
  235. endoreg_db/export/frames/export_frames_with_labels.py +616 -0
  236. endoreg_db/factories/__init__.py +0 -0
  237. endoreg_db/forms/__init__.py +4 -0
  238. endoreg_db/forms/examination_form.py +12 -0
  239. endoreg_db/forms/patient_finding_intervention_form.py +40 -0
  240. endoreg_db/forms/patient_form.py +23 -0
  241. endoreg_db/forms/questionnaires/__init__.py +1 -0
  242. endoreg_db/forms/questionnaires/tto_questionnaire.py +23 -0
  243. endoreg_db/forms/settings/__init__.py +11 -0
  244. endoreg_db/forms/unit.py +7 -0
  245. endoreg_db/helpers/__init__.py +0 -0
  246. endoreg_db/helpers/count_db.py +48 -0
  247. endoreg_db/helpers/data_loader.py +280 -0
  248. endoreg_db/helpers/default_objects.py +414 -0
  249. endoreg_db/helpers/download_segmentation_model.py +32 -0
  250. endoreg_db/helpers/interact.py +1 -0
  251. endoreg_db/helpers/test_video_helper.py +127 -0
  252. endoreg_db/import_files/__init__.py +27 -0
  253. endoreg_db/import_files/context/__init__.py +7 -0
  254. endoreg_db/import_files/context/default_sensitive_meta.py +83 -0
  255. endoreg_db/import_files/context/ensure_center.py +17 -0
  256. endoreg_db/import_files/context/file_lock.py +66 -0
  257. endoreg_db/import_files/context/import_context.py +42 -0
  258. endoreg_db/import_files/context/validate_directories.py +57 -0
  259. endoreg_db/import_files/file_storage/__init__.py +15 -0
  260. endoreg_db/import_files/file_storage/create_report_file.py +99 -0
  261. endoreg_db/import_files/file_storage/create_video_file.py +104 -0
  262. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +42 -0
  263. endoreg_db/import_files/file_storage/state_management.py +463 -0
  264. endoreg_db/import_files/file_storage/storage.py +42 -0
  265. endoreg_db/import_files/import_service.md +26 -0
  266. endoreg_db/import_files/processing/__init__.py +11 -0
  267. endoreg_db/import_files/processing/report_processing/report_anonymization.py +99 -0
  268. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  269. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  270. endoreg_db/import_files/pseudonymization/__init__.py +0 -0
  271. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  272. endoreg_db/import_files/pseudonymization/k_anonymity.py +181 -0
  273. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +139 -0
  274. endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
  275. endoreg_db/import_files/report_import_service.py +147 -0
  276. endoreg_db/import_files/video_import_service.py +154 -0
  277. endoreg_db/logger_conf.py +156 -0
  278. endoreg_db/management/__init__.py +1 -0
  279. endoreg_db/management/commands/__init__.py +1 -0
  280. endoreg_db/management/commands/anonymize_video.py +0 -0
  281. endoreg_db/management/commands/check_auth.py +132 -0
  282. endoreg_db/management/commands/create_model_meta_from_huggingface.py +177 -0
  283. endoreg_db/management/commands/create_multilabel_model_meta.py +419 -0
  284. endoreg_db/management/commands/export_frame_annot.py +196 -0
  285. endoreg_db/management/commands/fix_missing_patient_data.py +206 -0
  286. endoreg_db/management/commands/fix_video_paths.py +186 -0
  287. endoreg_db/management/commands/import_report.py +361 -0
  288. endoreg_db/management/commands/list_routes.py +20 -0
  289. endoreg_db/management/commands/load_ai_model_data.py +83 -0
  290. endoreg_db/management/commands/load_ai_model_label_data.py +60 -0
  291. endoreg_db/management/commands/load_base_db_data.py +63 -0
  292. endoreg_db/management/commands/load_center_data.py +68 -0
  293. endoreg_db/management/commands/load_contraindication_data.py +39 -0
  294. endoreg_db/management/commands/load_disease_classification_choices_data.py +38 -0
  295. endoreg_db/management/commands/load_disease_classification_data.py +38 -0
  296. endoreg_db/management/commands/load_disease_data.py +59 -0
  297. endoreg_db/management/commands/load_distribution_data.py +63 -0
  298. endoreg_db/management/commands/load_endoscope_data.py +58 -0
  299. endoreg_db/management/commands/load_event_data.py +39 -0
  300. endoreg_db/management/commands/load_examination_data.py +78 -0
  301. endoreg_db/management/commands/load_examination_indication_data.py +85 -0
  302. endoreg_db/management/commands/load_finding_data.py +115 -0
  303. endoreg_db/management/commands/load_gender_data.py +37 -0
  304. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +142 -0
  305. endoreg_db/management/commands/load_information_source.py +46 -0
  306. endoreg_db/management/commands/load_lab_value_data.py +52 -0
  307. endoreg_db/management/commands/load_legacy_data.py +303 -0
  308. endoreg_db/management/commands/load_medication_data.py +104 -0
  309. endoreg_db/management/commands/load_name_data.py +36 -0
  310. endoreg_db/management/commands/load_organ_data.py +39 -0
  311. endoreg_db/management/commands/load_pdf_type_data.py +58 -0
  312. endoreg_db/management/commands/load_profession_data.py +40 -0
  313. endoreg_db/management/commands/load_qualification_data.py +56 -0
  314. endoreg_db/management/commands/load_report_reader_flag_data.py +40 -0
  315. endoreg_db/management/commands/load_requirement_data.py +207 -0
  316. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  317. endoreg_db/management/commands/load_risk_data.py +57 -0
  318. endoreg_db/management/commands/load_shift_data.py +57 -0
  319. endoreg_db/management/commands/load_tag_data.py +54 -0
  320. endoreg_db/management/commands/load_unit_data.py +40 -0
  321. endoreg_db/management/commands/load_user_groups.py +26 -0
  322. endoreg_db/management/commands/model_input.py +169 -0
  323. endoreg_db/management/commands/register_ai_model.py +70 -0
  324. endoreg_db/management/commands/setup_endoreg_db.py +459 -0
  325. endoreg_db/management/commands/start_filewatcher.py +115 -0
  326. endoreg_db/management/commands/storage_management.py +622 -0
  327. endoreg_db/management/commands/summarize_db_content.py +280 -0
  328. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  329. endoreg_db/management/commands/validate_video_files.py +189 -0
  330. endoreg_db/management/commands/video_validation.py +20 -0
  331. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  332. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  333. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  334. endoreg_db/mermaid/classification.md +8 -0
  335. endoreg_db/mermaid/examination.md +8 -0
  336. endoreg_db/mermaid/findings.md +7 -0
  337. endoreg_db/mermaid/image_classification.md +28 -0
  338. endoreg_db/mermaid/interventions.md +8 -0
  339. endoreg_db/mermaid/morphology.md +8 -0
  340. endoreg_db/mermaid/patient_creation.md +14 -0
  341. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  342. endoreg_db/migrations/0001_initial.py +1953 -0
  343. endoreg_db/migrations/__init__.py +0 -0
  344. endoreg_db/models/__init__.py +322 -0
  345. endoreg_db/models/administration/__init__.py +95 -0
  346. endoreg_db/models/administration/ai/__init__.py +9 -0
  347. endoreg_db/models/administration/ai/active_model.py +35 -0
  348. endoreg_db/models/administration/ai/ai_model.py +180 -0
  349. endoreg_db/models/administration/ai/model_type.py +42 -0
  350. endoreg_db/models/administration/case/__init__.py +5 -0
  351. endoreg_db/models/administration/case/case.py +114 -0
  352. endoreg_db/models/administration/case/case_template/__init__.py +3 -0
  353. endoreg_db/models/administration/case/case_template/case_template.py +3 -0
  354. endoreg_db/models/administration/case/case_template/case_template_rule.py +3 -0
  355. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +3 -0
  356. endoreg_db/models/administration/case/case_template/case_template_type.py +3 -0
  357. endoreg_db/models/administration/center/__init__.py +13 -0
  358. endoreg_db/models/administration/center/center.py +85 -0
  359. endoreg_db/models/administration/center/center_product.py +67 -0
  360. endoreg_db/models/administration/center/center_resource.py +69 -0
  361. endoreg_db/models/administration/center/center_shift.py +94 -0
  362. endoreg_db/models/administration/center/center_waste.py +42 -0
  363. endoreg_db/models/administration/person/__init__.py +26 -0
  364. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  365. endoreg_db/models/administration/person/employee/employee.py +40 -0
  366. endoreg_db/models/administration/person/employee/employee_qualification.py +44 -0
  367. endoreg_db/models/administration/person/employee/employee_type.py +50 -0
  368. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  369. endoreg_db/models/administration/person/examiner/examiner.py +64 -0
  370. endoreg_db/models/administration/person/names/__init__.py +0 -0
  371. endoreg_db/models/administration/person/names/first_name.py +20 -0
  372. endoreg_db/models/administration/person/names/last_name.py +20 -0
  373. endoreg_db/models/administration/person/patient/__init__.py +7 -0
  374. endoreg_db/models/administration/person/patient/patient.py +488 -0
  375. endoreg_db/models/administration/person/patient/patient_external_id.py +36 -0
  376. endoreg_db/models/administration/person/person.py +35 -0
  377. endoreg_db/models/administration/person/profession/__init__.py +28 -0
  378. endoreg_db/models/administration/person/user/__init__.py +5 -0
  379. endoreg_db/models/administration/person/user/portal_user_information.py +41 -0
  380. endoreg_db/models/administration/product/__init__.py +15 -0
  381. endoreg_db/models/administration/product/product.py +106 -0
  382. endoreg_db/models/administration/product/product_group.py +41 -0
  383. endoreg_db/models/administration/product/product_material.py +60 -0
  384. endoreg_db/models/administration/product/product_weight.py +51 -0
  385. endoreg_db/models/administration/product/reference_product.py +147 -0
  386. endoreg_db/models/administration/qualification/__init__.py +7 -0
  387. endoreg_db/models/administration/qualification/qualification.py +43 -0
  388. endoreg_db/models/administration/qualification/qualification_type.py +39 -0
  389. endoreg_db/models/administration/shift/__init__.py +9 -0
  390. endoreg_db/models/administration/shift/scheduled_days.py +72 -0
  391. endoreg_db/models/administration/shift/shift.py +57 -0
  392. endoreg_db/models/administration/shift/shift_type.py +108 -0
  393. endoreg_db/models/aidataset/__init__.py +5 -0
  394. endoreg_db/models/aidataset/aidataset.py +193 -0
  395. endoreg_db/models/label/__init__.py +23 -0
  396. endoreg_db/models/label/annotation/__init__.py +12 -0
  397. endoreg_db/models/label/annotation/image_classification.py +85 -0
  398. endoreg_db/models/label/annotation/video_segmentation_annotation.py +61 -0
  399. endoreg_db/models/label/label.py +91 -0
  400. endoreg_db/models/label/label_set.py +68 -0
  401. endoreg_db/models/label/label_type.py +29 -0
  402. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  403. endoreg_db/models/label/label_video_segment/_create_from_video.py +42 -0
  404. endoreg_db/models/label/label_video_segment/label_video_segment.py +611 -0
  405. endoreg_db/models/label/video_segmentation_label.py +35 -0
  406. endoreg_db/models/label/video_segmentation_labelset.py +28 -0
  407. endoreg_db/models/media/__init__.py +23 -0
  408. endoreg_db/models/media/frame/__init__.py +3 -0
  409. endoreg_db/models/media/frame/frame.py +137 -0
  410. endoreg_db/models/media/pdf/__init__.py +12 -0
  411. endoreg_db/models/media/pdf/raw_pdf.py +764 -0
  412. endoreg_db/models/media/pdf/report_file.py +162 -0
  413. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  414. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +85 -0
  415. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +46 -0
  416. endoreg_db/models/media/video/__init__.py +9 -0
  417. endoreg_db/models/media/video/create_from_file.py +402 -0
  418. endoreg_db/models/media/video/pipe_1.py +258 -0
  419. endoreg_db/models/media/video/pipe_2.py +129 -0
  420. endoreg_db/models/media/video/video_file.py +907 -0
  421. endoreg_db/models/media/video/video_file_ai.py +828 -0
  422. endoreg_db/models/media/video/video_file_anonymize.py +524 -0
  423. endoreg_db/models/media/video/video_file_frames/__init__.py +49 -0
  424. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +25 -0
  425. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  426. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +126 -0
  427. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +233 -0
  428. endoreg_db/models/media/video/video_file_frames/_get_frame.py +36 -0
  429. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +13 -0
  430. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +24 -0
  431. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +40 -0
  432. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +44 -0
  433. endoreg_db/models/media/video/video_file_frames/_get_frames.py +30 -0
  434. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +205 -0
  435. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +228 -0
  436. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +107 -0
  437. endoreg_db/models/media/video/video_file_io.py +272 -0
  438. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  439. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +58 -0
  440. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +62 -0
  441. endoreg_db/models/media/video/video_file_meta/get_fps.py +183 -0
  442. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +198 -0
  443. endoreg_db/models/media/video/video_file_meta/text_meta.py +178 -0
  444. endoreg_db/models/media/video/video_file_meta/video_meta.py +105 -0
  445. endoreg_db/models/media/video/video_file_segments.py +317 -0
  446. endoreg_db/models/media/video/video_metadata.py +67 -0
  447. endoreg_db/models/media/video/video_processing.py +192 -0
  448. endoreg_db/models/medical/__init__.py +136 -0
  449. endoreg_db/models/medical/contraindication/README.md +1 -0
  450. endoreg_db/models/medical/contraindication/__init__.py +29 -0
  451. endoreg_db/models/medical/disease.py +174 -0
  452. endoreg_db/models/medical/event.py +154 -0
  453. endoreg_db/models/medical/examination/__init__.py +20 -0
  454. endoreg_db/models/medical/examination/examination.py +183 -0
  455. endoreg_db/models/medical/examination/examination_indication.py +229 -0
  456. endoreg_db/models/medical/examination/examination_time.py +68 -0
  457. endoreg_db/models/medical/examination/examination_time_type.py +44 -0
  458. endoreg_db/models/medical/examination/examination_type.py +47 -0
  459. endoreg_db/models/medical/finding/__init__.py +20 -0
  460. endoreg_db/models/medical/finding/finding.py +113 -0
  461. endoreg_db/models/medical/finding/finding_classification.py +131 -0
  462. endoreg_db/models/medical/finding/finding_intervention.py +68 -0
  463. endoreg_db/models/medical/finding/finding_type.py +38 -0
  464. endoreg_db/models/medical/hardware/__init__.py +8 -0
  465. endoreg_db/models/medical/hardware/endoscope.py +77 -0
  466. endoreg_db/models/medical/hardware/endoscopy_processor.py +182 -0
  467. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  468. endoreg_db/models/medical/laboratory/lab_value.py +490 -0
  469. endoreg_db/models/medical/medication/__init__.py +23 -0
  470. endoreg_db/models/medical/medication/medication.py +45 -0
  471. endoreg_db/models/medical/medication/medication_indication.py +78 -0
  472. endoreg_db/models/medical/medication/medication_indication_type.py +58 -0
  473. endoreg_db/models/medical/medication/medication_intake_time.py +58 -0
  474. endoreg_db/models/medical/medication/medication_schedule.py +58 -0
  475. endoreg_db/models/medical/organ/__init__.py +38 -0
  476. endoreg_db/models/medical/patient/__init__.py +48 -0
  477. endoreg_db/models/medical/patient/medication_examples.py +56 -0
  478. endoreg_db/models/medical/patient/patient_disease.py +72 -0
  479. endoreg_db/models/medical/patient/patient_event.py +80 -0
  480. endoreg_db/models/medical/patient/patient_examination.py +280 -0
  481. endoreg_db/models/medical/patient/patient_examination_indication.py +57 -0
  482. endoreg_db/models/medical/patient/patient_finding.py +416 -0
  483. endoreg_db/models/medical/patient/patient_finding_classification.py +231 -0
  484. endoreg_db/models/medical/patient/patient_finding_intervention.py +37 -0
  485. endoreg_db/models/medical/patient/patient_lab_sample.py +157 -0
  486. endoreg_db/models/medical/patient/patient_lab_value.py +247 -0
  487. endoreg_db/models/medical/patient/patient_medication.py +111 -0
  488. endoreg_db/models/medical/patient/patient_medication_schedule.py +152 -0
  489. endoreg_db/models/medical/risk/__init__.py +7 -0
  490. endoreg_db/models/medical/risk/risk.py +73 -0
  491. endoreg_db/models/medical/risk/risk_type.py +54 -0
  492. endoreg_db/models/metadata/__init__.py +19 -0
  493. endoreg_db/models/metadata/model_meta.py +266 -0
  494. endoreg_db/models/metadata/model_meta_logic.py +485 -0
  495. endoreg_db/models/metadata/pdf_meta.py +96 -0
  496. endoreg_db/models/metadata/sensitive_meta.py +345 -0
  497. endoreg_db/models/metadata/sensitive_meta_logic.py +1161 -0
  498. endoreg_db/models/metadata/video_meta.py +459 -0
  499. endoreg_db/models/metadata/video_prediction_logic.py +232 -0
  500. endoreg_db/models/metadata/video_prediction_meta.py +319 -0
  501. endoreg_db/models/operation_log.py +63 -0
  502. endoreg_db/models/other/__init__.py +40 -0
  503. endoreg_db/models/other/distribution/__init__.py +46 -0
  504. endoreg_db/models/other/distribution/base_value_distribution.py +22 -0
  505. endoreg_db/models/other/distribution/date_value_distribution.py +163 -0
  506. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +50 -0
  507. endoreg_db/models/other/distribution/numeric_value_distribution.py +211 -0
  508. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +23 -0
  509. endoreg_db/models/other/emission/__init__.py +5 -0
  510. endoreg_db/models/other/emission/emission_factor.py +110 -0
  511. endoreg_db/models/other/gender.py +32 -0
  512. endoreg_db/models/other/information_source.py +190 -0
  513. endoreg_db/models/other/material.py +34 -0
  514. endoreg_db/models/other/resource.py +24 -0
  515. endoreg_db/models/other/tag.py +32 -0
  516. endoreg_db/models/other/transport_route.py +40 -0
  517. endoreg_db/models/other/unit.py +40 -0
  518. endoreg_db/models/other/waste.py +28 -0
  519. endoreg_db/models/report/__init__.py +0 -0
  520. endoreg_db/models/report/images.py +0 -0
  521. endoreg_db/models/report/report.py +5 -0
  522. endoreg_db/models/requirement/__init__.py +11 -0
  523. endoreg_db/models/requirement/requirement.py +792 -0
  524. endoreg_db/models/requirement/requirement_error.py +84 -0
  525. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  526. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  527. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  528. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +6 -0
  529. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +137 -0
  530. endoreg_db/models/requirement/requirement_operator.py +187 -0
  531. endoreg_db/models/requirement/requirement_set.py +327 -0
  532. endoreg_db/models/state/__init__.py +13 -0
  533. endoreg_db/models/state/abstract.py +11 -0
  534. endoreg_db/models/state/anonymization.py +30 -0
  535. endoreg_db/models/state/audit_ledger.py +155 -0
  536. endoreg_db/models/state/label_video_segment.py +31 -0
  537. endoreg_db/models/state/processing_history/__init__.py +3 -0
  538. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  539. endoreg_db/models/state/raw_pdf.py +219 -0
  540. endoreg_db/models/state/sensitive_meta.py +50 -0
  541. endoreg_db/models/state/video.py +251 -0
  542. endoreg_db/models/upload_job.py +100 -0
  543. endoreg_db/models/utils.py +138 -0
  544. endoreg_db/queries/__init__.py +3 -0
  545. endoreg_db/queries/annotations/__init__.py +1 -0
  546. endoreg_db/queries/annotations/legacy.py +169 -0
  547. endoreg_db/queries/sanity/__init_.py +0 -0
  548. endoreg_db/root_urls.py +27 -0
  549. endoreg_db/schemas/__init__.py +0 -0
  550. endoreg_db/schemas/examination_evaluation.py +30 -0
  551. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +861 -0
  552. endoreg_db/serializers/__init__.py +104 -0
  553. endoreg_db/serializers/administration/__init__.py +13 -0
  554. endoreg_db/serializers/administration/ai/__init__.py +9 -0
  555. endoreg_db/serializers/administration/ai/active_model.py +12 -0
  556. endoreg_db/serializers/administration/ai/ai_model.py +20 -0
  557. endoreg_db/serializers/administration/ai/model_type.py +12 -0
  558. endoreg_db/serializers/administration/center.py +14 -0
  559. endoreg_db/serializers/administration/gender.py +11 -0
  560. endoreg_db/serializers/anonymization.py +77 -0
  561. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -0
  562. endoreg_db/serializers/examination/__init__.py +10 -0
  563. endoreg_db/serializers/examination/base.py +45 -0
  564. endoreg_db/serializers/examination/dropdown.py +20 -0
  565. endoreg_db/serializers/examination_serializer.py +9 -0
  566. endoreg_db/serializers/finding/__init__.py +5 -0
  567. endoreg_db/serializers/finding/finding.py +61 -0
  568. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  569. endoreg_db/serializers/finding_classification/choice.py +19 -0
  570. endoreg_db/serializers/finding_classification/classification.py +11 -0
  571. endoreg_db/serializers/label_video_segment/__init__.py +9 -0
  572. endoreg_db/serializers/label_video_segment/image_classification_annotation.py +62 -0
  573. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  574. endoreg_db/serializers/label_video_segment/label/label.py +15 -0
  575. endoreg_db/serializers/label_video_segment/label_video_segment.py +427 -0
  576. endoreg_db/serializers/meta/__init__.py +13 -0
  577. endoreg_db/serializers/meta/sensitive_meta_detail.py +122 -0
  578. endoreg_db/serializers/meta/sensitive_meta_update.py +153 -0
  579. endoreg_db/serializers/meta/sensitive_meta_verification.py +62 -0
  580. endoreg_db/serializers/meta/video_meta.py +39 -0
  581. endoreg_db/serializers/misc/__init__.py +14 -0
  582. endoreg_db/serializers/misc/file_overview.py +72 -0
  583. endoreg_db/serializers/misc/sensitive_patient_data.py +144 -0
  584. endoreg_db/serializers/misc/stats.py +35 -0
  585. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  586. endoreg_db/serializers/misc/upload_job.py +74 -0
  587. endoreg_db/serializers/patient/__init__.py +12 -0
  588. endoreg_db/serializers/patient/patient.py +103 -0
  589. endoreg_db/serializers/patient/patient_dropdown.py +35 -0
  590. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  591. endoreg_db/serializers/patient_examination/patient_examination.py +168 -0
  592. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  593. endoreg_db/serializers/patient_finding/patient_finding.py +32 -0
  594. endoreg_db/serializers/patient_finding/patient_finding_classification.py +47 -0
  595. endoreg_db/serializers/patient_finding/patient_finding_detail.py +62 -0
  596. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +28 -0
  597. endoreg_db/serializers/patient_finding/patient_finding_list.py +40 -0
  598. endoreg_db/serializers/patient_finding/patient_finding_write.py +135 -0
  599. endoreg_db/serializers/pdf/__init__.py +3 -0
  600. endoreg_db/serializers/pdf/anony_text_validation.py +101 -0
  601. endoreg_db/serializers/requirements/requirement_schema.py +20 -0
  602. endoreg_db/serializers/requirements/requirement_sets.py +99 -0
  603. endoreg_db/serializers/sensitive_meta_serializer.py +301 -0
  604. endoreg_db/serializers/video/__init__.py +7 -0
  605. endoreg_db/serializers/video/video_file.py +283 -0
  606. endoreg_db/serializers/video/video_file_brief.py +14 -0
  607. endoreg_db/serializers/video/video_file_detail.py +96 -0
  608. endoreg_db/serializers/video/video_file_list.py +100 -0
  609. endoreg_db/serializers/video/video_processing_history.py +172 -0
  610. endoreg_db/serializers/video_examination.py +198 -0
  611. endoreg_db/services/__init__.py +5 -0
  612. endoreg_db/services/anonymization.py +274 -0
  613. endoreg_db/services/examination_evaluation.py +172 -0
  614. endoreg_db/services/finding_description_service.py +0 -0
  615. endoreg_db/services/lookup_service.py +424 -0
  616. endoreg_db/services/lookup_store.py +266 -0
  617. endoreg_db/services/model_meta_from_hf.py +76 -0
  618. endoreg_db/services/pdf_import.py +0 -0
  619. endoreg_db/services/polling_coordinator.py +319 -0
  620. endoreg_db/services/pseudonym_service.py +94 -0
  621. endoreg_db/services/report_import.py +13 -0
  622. endoreg_db/services/segment_sync.py +171 -0
  623. endoreg_db/services/video_import.py +9 -0
  624. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  625. endoreg_db/templates/admin/start_examination.html +12 -0
  626. endoreg_db/templates/timeline.html +176 -0
  627. endoreg_db/urls/__init__.py +56 -0
  628. endoreg_db/urls/ai.py +14 -0
  629. endoreg_db/urls/anonymization.py +78 -0
  630. endoreg_db/urls/auth.py +16 -0
  631. endoreg_db/urls/classification.py +34 -0
  632. endoreg_db/urls/examination.py +63 -0
  633. endoreg_db/urls/media.py +251 -0
  634. endoreg_db/urls/patient.py +23 -0
  635. endoreg_db/urls/requirements.py +15 -0
  636. endoreg_db/urls/root_urls.py +28 -0
  637. endoreg_db/urls/stats.py +54 -0
  638. endoreg_db/urls/upload.py +12 -0
  639. endoreg_db/urls.py +9 -0
  640. endoreg_db/utils/__init__.py +97 -0
  641. endoreg_db/utils/ai/__init__.py +9 -0
  642. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  643. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  644. endoreg_db/utils/ai/get.py +6 -0
  645. endoreg_db/utils/ai/inference_dataset.py +51 -0
  646. endoreg_db/utils/ai/model_training/config.py +117 -0
  647. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  648. endoreg_db/utils/ai/model_training/losses.py +68 -0
  649. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  650. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  651. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  652. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  653. endoreg_db/utils/ai/multilabel_classification_net.py +270 -0
  654. endoreg_db/utils/ai/postprocess.py +63 -0
  655. endoreg_db/utils/ai/predict.py +293 -0
  656. endoreg_db/utils/ai/preprocess.py +76 -0
  657. endoreg_db/utils/calc_duration_seconds.py +24 -0
  658. endoreg_db/utils/case_generator/__init__.py +3 -0
  659. endoreg_db/utils/case_generator/lab_sample_factory.py +32 -0
  660. endoreg_db/utils/check_video_files.py +175 -0
  661. endoreg_db/utils/cropping.py +30 -0
  662. endoreg_db/utils/dataloader.py +285 -0
  663. endoreg_db/utils/dates.py +59 -0
  664. endoreg_db/utils/defaults/set_default_center.py +33 -0
  665. endoreg_db/utils/env.py +37 -0
  666. endoreg_db/utils/extract_specific_frames.py +87 -0
  667. endoreg_db/utils/file_operations.py +70 -0
  668. endoreg_db/utils/fix_video_path_direct.py +157 -0
  669. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  670. endoreg_db/utils/hashs.py +138 -0
  671. endoreg_db/utils/links/__init__.py +0 -0
  672. endoreg_db/utils/links/requirement_link.py +237 -0
  673. endoreg_db/utils/mime_types.py +0 -0
  674. endoreg_db/utils/names.py +82 -0
  675. endoreg_db/utils/ocr.py +195 -0
  676. endoreg_db/utils/operation_log.py +87 -0
  677. endoreg_db/utils/parse_and_generate_yaml.py +45 -0
  678. endoreg_db/utils/paths.py +159 -0
  679. endoreg_db/utils/permissions.py +160 -0
  680. endoreg_db/utils/pipelines/Readme.md +235 -0
  681. endoreg_db/utils/pipelines/__init__.py +0 -0
  682. endoreg_db/utils/pipelines/process_video_dir.py +144 -0
  683. endoreg_db/utils/product/__init__.py +0 -0
  684. endoreg_db/utils/product/sum_emissions.py +22 -0
  685. endoreg_db/utils/product/sum_weights.py +20 -0
  686. endoreg_db/utils/pydantic_models/__init__.py +5 -0
  687. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  688. endoreg_db/utils/requirement_helpers.py +0 -0
  689. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  690. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +678 -0
  691. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +842 -0
  692. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +114 -0
  693. endoreg_db/utils/setup_config.py +196 -0
  694. endoreg_db/utils/storage.py +117 -0
  695. endoreg_db/utils/translation.py +31 -0
  696. endoreg_db/utils/uuid.py +5 -0
  697. endoreg_db/utils/validate_endo_roi.py +33 -0
  698. endoreg_db/utils/validate_subcategory_dict.py +93 -0
  699. endoreg_db/utils/validate_video_detailed.py +415 -0
  700. endoreg_db/utils/video/__init__.py +30 -0
  701. endoreg_db/utils/video/extract_frames.py +100 -0
  702. endoreg_db/utils/video/ffmpeg_wrapper.py +996 -0
  703. endoreg_db/utils/video/names.py +47 -0
  704. endoreg_db/utils/video/streaming_processor.py +386 -0
  705. endoreg_db/utils/video/video_splitter.py +105 -0
  706. endoreg_db/versioning.md +79 -0
  707. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +247 -0
  708. endoreg_db/views/__init__.py +157 -0
  709. endoreg_db/views/anonymization/__init__.py +31 -0
  710. endoreg_db/views/anonymization/media_management.py +486 -0
  711. endoreg_db/views/anonymization/overview.py +307 -0
  712. endoreg_db/views/anonymization/validate.py +310 -0
  713. endoreg_db/views/auth/__init__.py +13 -0
  714. endoreg_db/views/auth/keycloak.py +146 -0
  715. endoreg_db/views/examination/__init__.py +30 -0
  716. endoreg_db/views/examination/examination.py +37 -0
  717. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  718. endoreg_db/views/examination/get_finding_classification_choices.py +62 -0
  719. endoreg_db/views/examination/get_finding_classifications.py +38 -0
  720. endoreg_db/views/examination/get_findings.py +39 -0
  721. endoreg_db/views/examination/get_instruments.py +19 -0
  722. endoreg_db/views/examination/get_interventions.py +14 -0
  723. endoreg_db/views/finding/__init__.py +9 -0
  724. endoreg_db/views/finding/finding.py +116 -0
  725. endoreg_db/views/finding/get_classifications.py +14 -0
  726. endoreg_db/views/finding/get_interventions.py +17 -0
  727. endoreg_db/views/finding_classification/__init__.py +13 -0
  728. endoreg_db/views/finding_classification/base.py +0 -0
  729. endoreg_db/views/finding_classification/finding_classification.py +41 -0
  730. endoreg_db/views/finding_classification/get_classification_choices.py +54 -0
  731. endoreg_db/views/media/__init__.py +32 -0
  732. endoreg_db/views/media/pdf_media.py +411 -0
  733. endoreg_db/views/media/sensitive_metadata.py +372 -0
  734. endoreg_db/views/media/video_media.py +275 -0
  735. endoreg_db/views/meta/__init__.py +7 -0
  736. endoreg_db/views/meta/sensitive_meta_list.py +102 -0
  737. endoreg_db/views/meta/sensitive_meta_verification.py +74 -0
  738. endoreg_db/views/misc/__init__.py +29 -0
  739. endoreg_db/views/misc/center.py +14 -0
  740. endoreg_db/views/misc/csrf.py +8 -0
  741. endoreg_db/views/misc/gender.py +15 -0
  742. endoreg_db/views/misc/stats.py +255 -0
  743. endoreg_db/views/misc/upload_views.py +241 -0
  744. endoreg_db/views/patient/__init__.py +3 -0
  745. endoreg_db/views/patient/patient.py +253 -0
  746. endoreg_db/views/patient_examination/__init__.py +11 -0
  747. endoreg_db/views/patient_examination/patient_examination.py +141 -0
  748. endoreg_db/views/patient_examination/patient_examination_create.py +58 -0
  749. endoreg_db/views/patient_examination/patient_examination_detail.py +63 -0
  750. endoreg_db/views/patient_examination/patient_examination_list.py +72 -0
  751. endoreg_db/views/patient_examination/video.py +228 -0
  752. endoreg_db/views/patient_finding/__init__.py +7 -0
  753. endoreg_db/views/patient_finding/base.py +0 -0
  754. endoreg_db/views/patient_finding/patient_finding.py +71 -0
  755. endoreg_db/views/patient_finding/patient_finding_optimized.py +291 -0
  756. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  757. endoreg_db/views/patient_finding_classification/pfc_create.py +75 -0
  758. endoreg_db/views/report/__init__.py +7 -0
  759. endoreg_db/views/report/reimport.py +177 -0
  760. endoreg_db/views/report/report_stream.py +191 -0
  761. endoreg_db/views/requirement/__init__.py +11 -0
  762. endoreg_db/views/requirement/evaluate.py +278 -0
  763. endoreg_db/views/requirement/lookup.py +380 -0
  764. endoreg_db/views/requirement/lookup_store.py +183 -0
  765. endoreg_db/views/requirement/requirement_utils.py +87 -0
  766. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  767. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  768. endoreg_db/views/stats/__init__.py +13 -0
  769. endoreg_db/views/stats/stats_views.py +266 -0
  770. endoreg_db/views/video/__init__.py +49 -0
  771. endoreg_db/views/video/ai/__init__.py +8 -0
  772. endoreg_db/views/video/ai/label.py +159 -0
  773. endoreg_db/views/video/correction.py +529 -0
  774. endoreg_db/views/video/reimport.py +230 -0
  775. endoreg_db/views/video/segments_crud.py +709 -0
  776. endoreg_db/views/video/video_apply_mask.py +49 -0
  777. endoreg_db/views/video/video_correction.py +22 -0
  778. endoreg_db/views/video/video_download_processed.py +58 -0
  779. endoreg_db/views/video/video_examination_viewset.py +242 -0
  780. endoreg_db/views/video/video_metadata.py +101 -0
  781. endoreg_db/views/video/video_processing_history.py +25 -0
  782. endoreg_db/views/video/video_remove_frames.py +49 -0
  783. endoreg_db/views/video/video_stream.py +334 -0
  784. endoreg_db-0.8.9.32.dist-info/METADATA +404 -0
  785. endoreg_db-0.8.9.32.dist-info/RECORD +787 -0
  786. endoreg_db-0.8.9.32.dist-info/WHEEL +4 -0
  787. endoreg_db-0.8.9.32.dist-info/licenses/LICENSE +674 -0
@@ -0,0 +1,828 @@
1
+ import logging
2
+ from collections import Counter, defaultdict
3
+ from pathlib import Path
4
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
5
+ import os
6
+ import numpy as np
7
+ from safetensors import safe_open
8
+
9
+ from ...metadata import ModelMeta, VideoPredictionMeta
10
+ from ...utils import TEST_RUN as GLOBAL_TEST_RUN
11
+
12
+ #TODO is this needed?
13
+ n = os.getenv("GLOBAL_N_TEST_FRAMES", 100)
14
+ if not isinstance(n, int):
15
+ GLOBAL_N_TEST_FRAMES = 100
16
+ else:
17
+ GLOBAL_N_TEST_FRAMES = n
18
+
19
+ if TYPE_CHECKING:
20
+ from ...medical.hardware import EndoscopyProcessor
21
+ from .video_file import VideoFile
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+
26
+ def _is_stub_weights_file(weights_path: Path) -> bool:
27
+ """Return True if the provided weights file is a known test stub."""
28
+
29
+ name_hint = weights_path.name.lower()
30
+ if "stub" in name_hint:
31
+ return True
32
+
33
+ try:
34
+ size_bytes = weights_path.stat().st_size
35
+ except OSError:
36
+ return False
37
+
38
+ if size_bytes < 4096:
39
+ try:
40
+ with weights_path.open("rb") as fh:
41
+ header = fh.read(32)
42
+ except OSError:
43
+ return False
44
+ return header.startswith(b"stub-weights") or not header
45
+
46
+ return False
47
+
48
+
49
+ def _resolve_label_names(model_meta: "ModelMeta") -> List[str]:
50
+ """Return deterministic label ordering for the associated label set."""
51
+
52
+ labelset = model_meta.labelset
53
+ if not labelset:
54
+ return []
55
+
56
+ try:
57
+ return [label.name for label in labelset.get_labels_in_order()]
58
+ except AttributeError:
59
+ # Fallback in case legacy labelsets provide only the raw manager interface.
60
+ return [label.name for label in labelset.labels.all().order_by("name")]
61
+
62
+
63
+ def _infer_model_type(model_meta: "ModelMeta", weights_path: Path) -> str:
64
+ """Best-effort detection of the backbone expected by the safetensors weights."""
65
+
66
+ candidates: List[Any] = [
67
+ getattr(model_meta.model, "model_subtype", None) if model_meta.model else None,
68
+ getattr(model_meta.model, "name", None) if model_meta.model else None,
69
+ model_meta.name,
70
+ model_meta.description,
71
+ weights_path.stem,
72
+ ]
73
+
74
+ for value in candidates:
75
+ if not value:
76
+ continue
77
+ text = str(value).lower()
78
+ if "regnet" in text:
79
+ return "RegNetX800MF"
80
+ if "efficientnet" in text and "b4" in text:
81
+ return "EfficientNetB4"
82
+ if "efficientnet" in text:
83
+ return "EfficientNetB4"
84
+
85
+ logger.warning(
86
+ "Unable to infer model backbone for %s; defaulting to EfficientNetB4.",
87
+ weights_path,
88
+ )
89
+ return "EfficientNetB4"
90
+
91
+
92
+ LEGACY_CLASS_LABELS = [
93
+ "appendix",
94
+ "blood",
95
+ "diverticule",
96
+ "grasper",
97
+ "ileocaecalvalve",
98
+ "ileum",
99
+ "low_quality",
100
+ "nbi",
101
+ "needle",
102
+ "outside",
103
+ "polyp",
104
+ "snare",
105
+ "water_jet",
106
+ "wound",
107
+ ]
108
+
109
+ LEGACY_LABEL_ALIASES = {
110
+ "nbi": "digital_chromo_endoscopy",
111
+ "grasper": "instrument",
112
+ "needle": "instrument",
113
+ "snare": "instrument",
114
+ }
115
+
116
+ LEGACY_IGNORED_LABELS = {"diverticule"}
117
+
118
+
119
+ def _infer_output_classes(weights_path: Path) -> Optional[int]:
120
+ if weights_path.suffix.lower() != ".safetensors":
121
+ return None
122
+
123
+ try:
124
+ with safe_open(weights_path, framework="pt", device="cpu") as handle:
125
+ return int(handle.get_tensor("model.fc.weight").shape[0])
126
+ except Exception as exc: # pragma: no cover - defensive logging only
127
+ logger.debug("Unable to infer output classes from %s: %s", weights_path, exc)
128
+ return None
129
+
130
+
131
+ def _build_label_mapping(
132
+ source_labels: List[str], target_labels: List[str]
133
+ ) -> Dict[str, List[str]]:
134
+ if source_labels == target_labels:
135
+ return {label: [label] for label in target_labels}
136
+
137
+ mapping: Dict[str, List[str]] = {label: [] for label in target_labels}
138
+
139
+ for label in source_labels:
140
+ alias = LEGACY_LABEL_ALIASES.get(label, label)
141
+ if alias in mapping:
142
+ mapping[alias].append(label)
143
+ elif label not in LEGACY_IGNORED_LABELS:
144
+ logger.debug("Label '%s' from source set has no mapping; dropping.", label)
145
+
146
+ for label in target_labels:
147
+ if not mapping[label]:
148
+ mapping[label] = [label]
149
+
150
+ return mapping
151
+
152
+
153
+ def _remap_prediction_dict(
154
+ predictions: Dict[str, Any], mapping: Dict[str, List[str]]
155
+ ) -> Dict[str, Any]:
156
+ remapped: Dict[str, Any] = {}
157
+ for target, sources in mapping.items():
158
+ values: List[Any] = []
159
+ for source in sources:
160
+ value = predictions.get(source)
161
+ if value is not None:
162
+ values.append(value)
163
+ if not values:
164
+ remapped[target] = 0.0
165
+ continue
166
+
167
+ first = values[0]
168
+ if isinstance(first, np.ndarray):
169
+ stacked = np.stack(values, axis=0)
170
+ remapped[target] = stacked.max(axis=0)
171
+ elif hasattr(first, "__iter__") and not isinstance(first, (float, int)):
172
+ stacked = np.stack([np.asarray(v) for v in values], axis=0)
173
+ remapped[target] = stacked.max(axis=0)
174
+ else:
175
+ remapped[target] = max(float(v) for v in values)
176
+
177
+ return remapped
178
+
179
+
180
+ def _extract_text_from_video_frames(
181
+ video: "VideoFile", frame_fraction: float = 0.001, cap: int = 15
182
+ ) -> Optional[Dict[str, str]]:
183
+ """
184
+ Extracts text from a sample of video frames using OCR based on processor ROIs.
185
+ Requires frames to be extracted. Raises ValueError on pre-condition failure.
186
+ Returns dictionary of extracted text or None if no text found.
187
+
188
+ State Transitions:
189
+ - Pre-condition: Requires state.frames_extracted=True.
190
+ - Post-condition: No state changes.
191
+ """
192
+ from endoreg_db.utils.ocr import (
193
+ extract_text_from_rois, # Local import for dependency isolation
194
+ )
195
+
196
+ state = video.get_or_create_state() # Use State helper
197
+ # --- Pre-condition Check ---
198
+ if not state.frames_extracted:
199
+ # Raise exception
200
+ raise ValueError(
201
+ f"Frames not extracted for video {video.video_hash}. Cannot extract text."
202
+ )
203
+ # --- End Pre-condition Check ---
204
+
205
+ processor: Optional["EndoscopyProcessor"] = video.processor
206
+ if not processor:
207
+ # Raise exception
208
+ raise ValueError(
209
+ f"Processor not set for video {video.video_hash}. Cannot extract text."
210
+ )
211
+
212
+ try:
213
+ frame_paths = video.get_frame_paths() # Use Frame helper
214
+ except Exception as e:
215
+ logger.error(
216
+ "Error getting frame paths for video %s: %s",
217
+ video.video_hash,
218
+ e,
219
+ exc_info=True,
220
+ )
221
+ raise RuntimeError(
222
+ f"Could not get frame paths for video {video.video_hash}"
223
+ ) from e
224
+
225
+ n_frames = len(frame_paths)
226
+ if n_frames == 0:
227
+ logger.warning(
228
+ "No frame paths found for video %s during text extraction.",
229
+ video.video_hash,
230
+ )
231
+ return (
232
+ None # Return None if no frames, not an error condition for this function
233
+ )
234
+
235
+ # Determine number of frames to process
236
+ n_frames_to_process = max(1, int(frame_fraction * n_frames))
237
+ n_frames_to_process = min(n_frames_to_process, cap, n_frames)
238
+
239
+ logger.info(
240
+ "Processing %d frames (out of %d) for text extraction from video %s.",
241
+ n_frames_to_process,
242
+ n_frames,
243
+ video.video_hash,
244
+ )
245
+
246
+ # Select evenly spaced frames
247
+ step = max(1, n_frames // n_frames_to_process)
248
+ selected_frame_paths = frame_paths[::step][:n_frames_to_process]
249
+
250
+ # Extract text from ROIs for selected frames
251
+ rois_texts = defaultdict(list)
252
+ errors_encountered = False
253
+ for frame_path in selected_frame_paths:
254
+ try:
255
+ extracted_texts = extract_text_from_rois(frame_path, processor)
256
+ for roi, text in extracted_texts.items():
257
+ if text: # Only append non-empty text
258
+ rois_texts[roi].append(text)
259
+ except Exception as e:
260
+ # Log error but continue processing other frames
261
+ logger.error(
262
+ "Error extracting text from frame %s for video %s: %s",
263
+ frame_path,
264
+ video.video_hash,
265
+ e,
266
+ exc_info=True,
267
+ )
268
+ errors_encountered = True # Flag that an error occurred
269
+
270
+ # Determine the most frequent text for each ROI
271
+ most_frequent_texts = {}
272
+ for roi, texts in rois_texts.items():
273
+ if not texts:
274
+ most_frequent_texts[roi] = None
275
+ continue
276
+ try:
277
+ counter = Counter(texts)
278
+ most_common = counter.most_common(1)
279
+ if most_common:
280
+ most_frequent_texts[roi] = most_common[0][0]
281
+ else:
282
+ most_frequent_texts[roi] = None
283
+ except Exception as e:
284
+ logger.error(
285
+ "Error finding most common text for ROI %s: %s", roi, e, exc_info=True
286
+ )
287
+ most_frequent_texts[roi] = None
288
+
289
+ if errors_encountered:
290
+ logger.warning(
291
+ "Errors occurred during text extraction for some frames of video %s. Results may be incomplete.",
292
+ video.video_hash,
293
+ )
294
+
295
+ if not most_frequent_texts:
296
+ logger.info("No text extracted for any ROI for video %s.", video.video_hash)
297
+ return None # Return None if no text found
298
+
299
+ logger.info(
300
+ "Extracted text for video %s: %s", video.video_hash, most_frequent_texts
301
+ )
302
+ return most_frequent_texts
303
+
304
+
305
+ def _predict_video_pipeline(
306
+ video: "VideoFile",
307
+ model_meta: "ModelMeta",
308
+ dataset_name: str = "inference_dataset",
309
+ smooth_window_size_s: int = 1,
310
+ binarize_threshold: float = 0.5,
311
+ test_run: bool = False,
312
+ n_test_frames: int = 10,
313
+ ) -> Dict[str, List[Tuple[int, int]]]: # Changed return type to non-optional
314
+ """
315
+ Executes the video prediction pipeline using an AI model.
316
+ Requires frames to be extracted. Raises exceptions on failure.
317
+
318
+ State Transitions:
319
+ - Pre-condition: Requires state.frames_extracted=True.
320
+ - Post-condition: No state changes directly. (Calling pipeline sets flags).
321
+ """
322
+ # Import heavy dependencies locally
323
+ from ...administration.ai import AiModel
324
+
325
+ try:
326
+ from ....utils.ai import (
327
+ Classifier,
328
+ InferenceDataset,
329
+ MultiLabelClassificationNet,
330
+ )
331
+ from ....utils.ai.postprocess import (
332
+ concat_pred_dicts,
333
+ find_true_pred_sequences,
334
+ make_smooth_preds,
335
+ )
336
+ except ImportError as e:
337
+ logger.error(
338
+ "Failed to import endo_ai components: %s. Prediction unavailable.",
339
+ e,
340
+ exc_info=True,
341
+ )
342
+ # Raise exception
343
+ raise ImportError(
344
+ "Failed to import required AI components for prediction."
345
+ ) from e
346
+
347
+ if not test_run and GLOBAL_TEST_RUN:
348
+ test_run = True
349
+ n_test_frames = GLOBAL_N_TEST_FRAMES
350
+ logger.info("Using global TEST_RUN settings for prediction pipeline.")
351
+
352
+ state = video.get_or_create_state() # Use State helper
353
+ # --- Pre-condition Check ---
354
+ if not state.frames_extracted:
355
+ # Raise exception
356
+ raise ValueError(
357
+ f"Frames not extracted for video {video.video_hash}. Prediction aborted."
358
+ )
359
+ # --- End Pre-condition Check ---
360
+
361
+ # Frame directory check
362
+ frame_dir = video.get_frame_dir_path() # Use IO helper
363
+ if not frame_dir or not frame_dir.exists() or not any(frame_dir.iterdir()):
364
+ # Raise exception
365
+ raise FileNotFoundError(
366
+ f"Frame directory {frame_dir} is empty or does not exist for video {video.video_hash}. Prediction aborted."
367
+ )
368
+
369
+ model: Optional[AiModel] = model_meta.model
370
+ if not model:
371
+ # Raise exception
372
+ raise ValueError(
373
+ f"Model not found in ModelMeta {model_meta.name} (Version: {model_meta.version}) for video {video.video_hash}. Prediction aborted."
374
+ )
375
+
376
+ # Ensure weights file exists
377
+ try:
378
+ weights_path = Path(model_meta.weights.path)
379
+ if not weights_path.exists():
380
+ # Raise exception
381
+ raise FileNotFoundError(
382
+ f"Model weights file {weights_path} not found for {model_meta.name} (Video: {video.video_hash}). Prediction aborted."
383
+ )
384
+ except Exception as e:
385
+ logger.error(
386
+ "Error accessing model weights path for %s (Video: %s): %s",
387
+ model_meta.name,
388
+ video.video_hash,
389
+ e,
390
+ exc_info=True,
391
+ )
392
+ raise RuntimeError(
393
+ f"Error accessing model weights for {model_meta.name}"
394
+ ) from e
395
+
396
+ # Get or create VideoPredictionMeta
397
+ try:
398
+ _video_prediction_meta, created = VideoPredictionMeta.objects.get_or_create(
399
+ video_file=video, model_meta=model_meta
400
+ )
401
+ if created:
402
+ logger.info(
403
+ "Created new VideoPredictionMeta for video %s, model %s.",
404
+ video.video_hash,
405
+ model_meta.name,
406
+ )
407
+ else:
408
+ logger.info(
409
+ "Found existing VideoPredictionMeta for video %s, model %s.",
410
+ video.video_hash,
411
+ model_meta.name,
412
+ )
413
+ # video_prediction_meta.save() # Save is handled by get_or_create
414
+ except Exception as e:
415
+ logger.error(
416
+ "Failed to get or create VideoPredictionMeta for video %s, model %s: %s",
417
+ video.video_hash,
418
+ model_meta.name,
419
+ e,
420
+ exc_info=True,
421
+ )
422
+ # Raise exception
423
+ raise RuntimeError("Failed to get or create VideoPredictionMeta") from e
424
+
425
+ if _is_stub_weights_file(weights_path):
426
+ logger.info(
427
+ "Detected stub weights at %s for video %s; skipping model inference and returning empty predictions.",
428
+ weights_path,
429
+ video.video_hash,
430
+ )
431
+ return {}
432
+
433
+ # --- Dataset Preparation ---
434
+ datasets = {
435
+ "inference_dataset": InferenceDataset,
436
+ # Add other dataset types here if needed
437
+ }
438
+ dataset_model_class = datasets.get(dataset_name)
439
+ if not dataset_model_class:
440
+ # Raise exception
441
+ raise ValueError(
442
+ f"Dataset class '{dataset_name}' not found for video {video.video_hash}. Prediction aborted."
443
+ )
444
+
445
+ try:
446
+ paths = video.get_frame_paths() # Use Frame helper
447
+ if not paths:
448
+ raise FileNotFoundError(
449
+ f"No frame paths returned by get_frame_paths for {frame_dir} (Video: {video.video_hash})"
450
+ )
451
+ except Exception as e:
452
+ logger.error(
453
+ "Error listing or getting frame files from %s for video %s: %s",
454
+ frame_dir,
455
+ video.video_hash,
456
+ e,
457
+ exc_info=True,
458
+ )
459
+ raise RuntimeError(f"Error getting frame paths from {frame_dir}") from e
460
+
461
+ logger.info(
462
+ "Found %d frame files in %s for video %s.",
463
+ len(paths),
464
+ frame_dir,
465
+ video.video_hash,
466
+ )
467
+
468
+ crop_template = video.get_crop_template() # Use Meta helper
469
+ string_paths = [p.as_posix() for p in paths]
470
+ crops = [crop_template] * len(paths) # Assuming same crop for all frames
471
+
472
+ if test_run:
473
+ logger.info(
474
+ "TEST RUN: Using first %d frames for video %s.",
475
+ n_test_frames,
476
+ video.video_hash,
477
+ )
478
+ string_paths = string_paths[:n_test_frames]
479
+ crops = crops[:n_test_frames]
480
+ if not string_paths:
481
+ # Raise exception
482
+ raise ValueError(
483
+ f"Not enough frames ({len(paths)}) for test run (required {n_test_frames}) for video {video.video_hash}."
484
+ )
485
+
486
+ label_names = _resolve_label_names(model_meta)
487
+ if not label_names:
488
+ raise ValueError(
489
+ f"Label set '{getattr(model_meta.labelset, 'name', 'unknown')}' has no labels configured."
490
+ )
491
+
492
+ outputs_hint = _infer_output_classes(weights_path)
493
+
494
+ network_labels = label_names
495
+ if outputs_hint and outputs_hint != len(label_names):
496
+ if outputs_hint == len(LEGACY_CLASS_LABELS):
497
+ network_labels = LEGACY_CLASS_LABELS
498
+ logger.info(
499
+ "Detected legacy multilabel checkpoint with %d classes; using legacy label ordering.",
500
+ outputs_hint,
501
+ )
502
+ else:
503
+ logger.warning(
504
+ "Weights %s expect %d outputs while label set '%s' defines %d labels.",
505
+ weights_path.name,
506
+ outputs_hint,
507
+ getattr(model_meta.labelset, "name", "unknown"),
508
+ len(label_names),
509
+ )
510
+
511
+ label_mapping = _build_label_mapping(network_labels, label_names)
512
+
513
+ load_kwargs: Dict[str, Any] = {}
514
+ if weights_path.suffix.lower() == ".safetensors":
515
+ load_kwargs.update(
516
+ {
517
+ "labels": network_labels,
518
+ "model_type": _infer_model_type(model_meta, weights_path),
519
+ "load_imagenet_weights": False,
520
+ "strict": False,
521
+ }
522
+ )
523
+
524
+ classifier_config: Optional[Dict[str, Any]] = None
525
+
526
+ try:
527
+ ds_config = model_meta.get_inference_dataset_config()
528
+ ds = dataset_model_class(string_paths, crops, config=ds_config)
529
+ logger.info(
530
+ "Created dataset '%s' with %d items for video %s.",
531
+ dataset_name,
532
+ len(ds),
533
+ video.video_hash,
534
+ )
535
+ if len(ds) > 0:
536
+ sample = ds[0] # Get a sample for debugging shape
537
+ logger.debug("Sample shape: %s", getattr(sample, "shape", None))
538
+
539
+ try:
540
+ activation = ModelMeta.get_activation_function(model_meta.activation)
541
+ except ValueError:
542
+ logger.warning(
543
+ "Unsupported activation '%s' for model %s; falling back to sigmoid.",
544
+ model_meta.activation,
545
+ model_meta.name,
546
+ )
547
+ activation = ModelMeta.get_activation_function("sigmoid")
548
+
549
+ classifier_config = {
550
+ **ds_config,
551
+ "batchsize": model_meta.batchsize or 16,
552
+ "num_workers": model_meta.num_workers or 0,
553
+ "activation": activation,
554
+ "labels": network_labels,
555
+ }
556
+ except Exception as e:
557
+ logger.error(
558
+ "Failed to create dataset '%s' for video %s: %s",
559
+ dataset_name,
560
+ video.video_hash,
561
+ e,
562
+ exc_info=True,
563
+ )
564
+ # Raise exception
565
+ raise RuntimeError(f"Failed to create dataset '{dataset_name}'") from e
566
+
567
+ # --- Model Loading ---
568
+ try:
569
+ # Check if CUDA is available
570
+ import torch
571
+
572
+ if torch.cuda.is_available():
573
+ try:
574
+ device = torch.device("cuda")
575
+ ai_model_instance = MultiLabelClassificationNet.load_from_checkpoint(
576
+ checkpoint_path=weights_path.as_posix(),
577
+ map_location=device,
578
+ **load_kwargs,
579
+ )
580
+ ai_model_instance = ai_model_instance.to(device)
581
+ logger.info("Loaded model on GPU for video %s.", video.video_hash)
582
+ except RuntimeError as cuda_err:
583
+ logger.warning(
584
+ "GPU loading failed for video %s: %s. Falling back to CPU.",
585
+ video.video_hash,
586
+ cuda_err,
587
+ )
588
+ device = torch.device("cpu")
589
+ ai_model_instance = MultiLabelClassificationNet.load_from_checkpoint(
590
+ checkpoint_path=weights_path.as_posix(),
591
+ map_location=device,
592
+ **load_kwargs,
593
+ )
594
+ ai_model_instance = ai_model_instance.to(device)
595
+ logger.info("Loaded model on CPU for video %s.", video.video_hash)
596
+ else:
597
+ # No CUDA available, load directly on CPU
598
+ logger.info(
599
+ "CUDA not available. Loading model on CPU for video %s.",
600
+ video.video_hash,
601
+ )
602
+ device = torch.device("cpu")
603
+ ai_model_instance = MultiLabelClassificationNet.load_from_checkpoint(
604
+ checkpoint_path=weights_path.as_posix(),
605
+ map_location=device,
606
+ **load_kwargs,
607
+ )
608
+ ai_model_instance = ai_model_instance.to(device)
609
+
610
+ _ = ai_model_instance.eval() # Set to evaluation mode
611
+ classifier = Classifier(
612
+ ai_model_instance, config=classifier_config or {}, verbose=True
613
+ )
614
+ logger.info(
615
+ "AI model loaded successfully for video %s from %s.",
616
+ video.video_hash,
617
+ weights_path,
618
+ )
619
+ except Exception as e:
620
+ logger.error(
621
+ "Failed to load AI model for video %s from %s: %s",
622
+ video.video_hash,
623
+ weights_path,
624
+ e,
625
+ exc_info=True,
626
+ )
627
+ # Raise exception
628
+ raise RuntimeError(f"Failed to load AI model from {weights_path}") from e
629
+
630
+ # --- Inference ---
631
+ try:
632
+ logger.info(
633
+ "Starting inference on %d frames for video %s...",
634
+ len(string_paths),
635
+ video.video_hash,
636
+ )
637
+ predictions = classifier.pipe(string_paths, crops)
638
+ logger.info("Inference completed for video %s.", video.video_hash)
639
+ except Exception as e:
640
+ logger.error(
641
+ "Inference failed for video %s: %s", video.video_hash, e, exc_info=True
642
+ )
643
+ # CUDA-OOM Fallback: Speicher freigeben und CPU versuchen
644
+ try:
645
+ import gc
646
+
647
+ import torch
648
+
649
+ is_oom = isinstance(
650
+ e, (getattr(torch.cuda, "OutOfMemoryError", RuntimeError), RuntimeError)
651
+ ) and (
652
+ "out of memory" in str(e).lower()
653
+ or "cuda out of memory" in str(e).lower()
654
+ )
655
+ except Exception:
656
+ is_oom = False
657
+ if "torch" in globals() or "torch" in locals():
658
+ try:
659
+ import torch # ensure available in this scope
660
+
661
+ if torch.cuda.is_available() and is_oom:
662
+ logger.warning(
663
+ "CUDA OOM detected. Freeing CUDA cache and retrying on CPU…"
664
+ )
665
+ try:
666
+ torch.cuda.empty_cache()
667
+ gc.collect()
668
+ except Exception:
669
+ pass
670
+ try:
671
+ # Move model to CPU and retry inference
672
+ _ = ai_model_instance.cpu()
673
+ classifier = Classifier(ai_model_instance, verbose=True)
674
+ predictions = classifier.pipe(string_paths, crops)
675
+ logger.info(
676
+ "Inference completed on CPU after CUDA OOM for video %s.",
677
+ video.video_hash,
678
+ )
679
+ except Exception as e2:
680
+ logger.error(
681
+ "CPU fallback inference failed for video %s: %s",
682
+ video.video_hash,
683
+ e2,
684
+ exc_info=True,
685
+ )
686
+ # Raise exception
687
+ raise RuntimeError("Inference failed") from e2
688
+ else:
689
+ # Raise exception
690
+ raise RuntimeError("Inference failed") from e
691
+ except Exception:
692
+ # Raise exception
693
+ raise RuntimeError("Inference failed") from e
694
+ else:
695
+ # Raise exception
696
+ raise RuntimeError("Inference failed") from e
697
+
698
+ # --- Post-processing ---
699
+ try:
700
+ logger.info("Post-processing predictions for video %s...", video.video_hash)
701
+ readable_predictions = [classifier.readable(p) for p in predictions]
702
+ if label_mapping:
703
+ readable_predictions = [
704
+ _remap_prediction_dict(prediction, label_mapping)
705
+ for prediction in readable_predictions
706
+ ]
707
+
708
+ merged_predictions = concat_pred_dicts(readable_predictions)
709
+
710
+ fps = video.get_fps() # Use Meta helper
711
+ if not fps:
712
+ logger.warning(
713
+ "Video FPS is unknown for %s. Smoothing/sequence calculations might be inaccurate. Using default 30 FPS.",
714
+ video.video_hash,
715
+ )
716
+ fps = 30 # Default FPS if unknown
717
+
718
+ fps = int(fps)
719
+ smooth_merged_predictions = {}
720
+ for key in merged_predictions.keys():
721
+ smooth_merged_predictions[key] = make_smooth_preds(
722
+ prediction_array=merged_predictions[key],
723
+ window_size_s=smooth_window_size_s,
724
+ fps=fps,
725
+ )
726
+
727
+ binary_smooth_merged_predictions = {}
728
+ for key in smooth_merged_predictions.keys():
729
+ binary_smooth_merged_predictions[key] = (
730
+ smooth_merged_predictions[key] > binarize_threshold
731
+ )
732
+
733
+ sequences = {}
734
+ for label, prediction_array in binary_smooth_merged_predictions.items():
735
+ sequences[label] = find_true_pred_sequences(prediction_array)
736
+
737
+ logger.info(
738
+ "Post-processing completed for video %s. Found sequences for labels: %s",
739
+ video.video_hash,
740
+ list(sequences.keys()),
741
+ )
742
+ return sequences if sequences is not None else {}
743
+
744
+ except Exception as e:
745
+ logger.error(
746
+ "Post-processing failed for video %s: %s",
747
+ video.video_hash,
748
+ e,
749
+ exc_info=True,
750
+ )
751
+ # Raise exception
752
+ raise RuntimeError("Post-processing failed") from e
753
+
754
+
755
+ def _predict_video_entry(
756
+ video: "VideoFile",
757
+ model_name: str,
758
+ model_meta_version: Optional[int] = None,
759
+ dataset_name: str = "inference_dataset",
760
+ smooth_window_size_s: int = 1,
761
+ binarize_threshold: float = 0.5,
762
+ test_run: bool = GLOBAL_TEST_RUN,
763
+ n_test_frames: int = GLOBAL_N_TEST_FRAMES,
764
+ save_results: bool = True, # Note: save_results is handled in video_file.py now
765
+ ):
766
+ """Entry point called from VideoFile.predict_video. Imports and calls the main prediction logic."""
767
+ from endoreg_db.models import AiModel, ModelMeta # Local import
768
+
769
+ try:
770
+ ai_model = AiModel.objects.get(name=model_name)
771
+ if not model_meta_version:
772
+ model_meta = ai_model.get_latest_version()
773
+ logger.info(
774
+ "Using latest ModelMeta version %s for model %s.",
775
+ model_meta_version,
776
+ model_name,
777
+ )
778
+ else:
779
+ model_meta = ai_model.get_version(model_meta_version)
780
+ logger.info(
781
+ "Using specified ModelMeta version %s for model %s.",
782
+ model_meta_version,
783
+ model_name,
784
+ )
785
+
786
+ logger.info(
787
+ "Using ModelMeta: %s (Version: %s)", model_meta.name, model_meta.version
788
+ )
789
+ except ModelMeta.DoesNotExist:
790
+ logger.error(
791
+ "ModelMeta '%s' (Version: %s) not found.", model_name, model_meta_version
792
+ )
793
+ raise
794
+
795
+ # --- Explicitly pass only the arguments expected by _predict_video_pipeline ---
796
+ predicted_sequences = _predict_video_pipeline(
797
+ video=video,
798
+ model_meta=model_meta, # Pass the fetched ModelMeta object
799
+ dataset_name=dataset_name,
800
+ smooth_window_size_s=smooth_window_size_s,
801
+ binarize_threshold=binarize_threshold,
802
+ test_run=test_run,
803
+ n_test_frames=n_test_frames,
804
+ )
805
+ # --- End Explicit Arguments ---
806
+
807
+ # Return the sequences and the ModelMeta object used
808
+ return predicted_sequences, model_meta
809
+
810
+
811
+ def _extract_text_information(
812
+ video: "VideoFile", frame_fraction: float = 0.001, cap: int = 15
813
+ ) -> Optional[Dict[str, str]]:
814
+ """Facade function to call the text extraction logic."""
815
+ logger.info("Attempting text extraction for video %s.", video.video_hash)
816
+
817
+ extracted_data = _extract_text_from_video_frames(
818
+ video=video, frame_fraction=frame_fraction, cap=cap
819
+ )
820
+
821
+ if extracted_data is not None:
822
+ logger.info("Text extraction successful for video %s.", video.video_hash)
823
+ else:
824
+ logger.warning(
825
+ "Text extraction returned no data for video %s.", video.video_hash
826
+ )
827
+
828
+ return extracted_data