endoreg-db 0.8.9.32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (787) hide show
  1. endoreg_db/__init__.py +0 -0
  2. endoreg_db/_version.py +34 -0
  3. endoreg_db/admin.py +97 -0
  4. endoreg_db/api/serializers/finding_descriptions.py +0 -0
  5. endoreg_db/api/views/finding_descriptions.py +0 -0
  6. endoreg_db/api_urls.py +4 -0
  7. endoreg_db/apps.py +17 -0
  8. endoreg_db/assets/dummy_model.ckpt +1 -0
  9. endoreg_db/authz/auth.py +78 -0
  10. endoreg_db/authz/backends.py +168 -0
  11. endoreg_db/authz/management/commands/list_routes.py +20 -0
  12. endoreg_db/authz/middleware.py +84 -0
  13. endoreg_db/authz/permissions.py +138 -0
  14. endoreg_db/authz/policy.py +224 -0
  15. endoreg_db/authz/settings.py +64 -0
  16. endoreg_db/authz/views_auth.py +70 -0
  17. endoreg_db/codemods/readme.md +88 -0
  18. endoreg_db/codemods/rename_datetime_fields.py +99 -0
  19. endoreg_db/config/__init__.py +0 -0
  20. endoreg_db/config/env.py +106 -0
  21. endoreg_db/config/settings/__init__.py +6 -0
  22. endoreg_db/config/settings/base.py +148 -0
  23. endoreg_db/config/settings/case_gen.py +32 -0
  24. endoreg_db/config/settings/dev.py +108 -0
  25. endoreg_db/config/settings/keycloak.py +177 -0
  26. endoreg_db/config/settings/prod.py +66 -0
  27. endoreg_db/config/settings/test.py +72 -0
  28. endoreg_db/data/__init__.py +135 -0
  29. endoreg_db/data/ai_model/data.yaml +7 -0
  30. endoreg_db/data/ai_model_label/label/data.yaml +88 -0
  31. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  32. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  33. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  34. endoreg_db/data/ai_model_label/label-type/data.yaml +7 -0
  35. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  36. endoreg_db/data/ai_model_type/data.yaml +7 -0
  37. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  38. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  39. endoreg_db/data/case_template/rule/00_patient_lab_sample_add_default_value.yaml +167 -0
  40. endoreg_db/data/case_template/rule/01_patient-set-age.yaml +8 -0
  41. endoreg_db/data/case_template/rule/01_patient-set-gender.yaml +9 -0
  42. endoreg_db/data/case_template/rule/11_create_patient_lab_sample.yaml +23 -0
  43. endoreg_db/data/case_template/rule/12_create-patient_medication-anticoagulation.yaml +19 -0
  44. endoreg_db/data/case_template/rule/13_create-patient_medication_schedule-anticoagulation.yaml +19 -0
  45. endoreg_db/data/case_template/rule/19_create_patient.yaml +17 -0
  46. endoreg_db/data/case_template/rule_type/base_types.yaml +35 -0
  47. endoreg_db/data/case_template/rule_value/.init +0 -0
  48. endoreg_db/data/case_template/rule_value_type/base_types.yaml +59 -0
  49. endoreg_db/data/case_template/template/base.yaml +8 -0
  50. endoreg_db/data/case_template/template_type/pre_endoscopy.yaml +3 -0
  51. endoreg_db/data/case_template/tmp/_rule_value +13 -0
  52. endoreg_db/data/case_template/tmp/rule/01_atrial_fibrillation.yaml +21 -0
  53. endoreg_db/data/case_template/tmp/rule/02_create_object.yaml +10 -0
  54. endoreg_db/data/case_template/tmp/template/atrial_fibrillation_low_risk.yaml +7 -0
  55. endoreg_db/data/center/data.yaml +99 -0
  56. endoreg_db/data/center_resource/green_endoscopy_dashboard_CenterResource.yaml +144 -0
  57. endoreg_db/data/center_shift/ukw.yaml +9 -0
  58. endoreg_db/data/center_waste/green_endoscopy_dashboard_CenterWaste.yaml +48 -0
  59. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  60. endoreg_db/data/db_summary.csv +58 -0
  61. endoreg_db/data/db_summary.xlsx +0 -0
  62. endoreg_db/data/disease/cardiovascular.yaml +37 -0
  63. endoreg_db/data/disease/hepatology.yaml +5 -0
  64. endoreg_db/data/disease/misc.yaml +5 -0
  65. endoreg_db/data/disease/renal.yaml +5 -0
  66. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +6 -0
  67. endoreg_db/data/disease_classification/coronary_vessel_disease.yaml +6 -0
  68. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +41 -0
  69. endoreg_db/data/disease_classification_choice/coronary_vessel_disease.yaml +20 -0
  70. endoreg_db/data/distribution/date/patient.yaml +7 -0
  71. endoreg_db/data/distribution/multiple_categorical/.init +0 -0
  72. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  73. endoreg_db/data/distribution/single_categorical/patient.yaml +7 -0
  74. endoreg_db/data/emission_factor/green_endoscopy_dashboard_EmissionFactor.yaml +132 -0
  75. endoreg_db/data/endoscope/data.yaml +93 -0
  76. endoreg_db/data/endoscope_type/data.yaml +11 -0
  77. endoreg_db/data/endoscopy_processor/data.yaml +50 -0
  78. endoreg_db/data/event/cardiology.yaml +15 -0
  79. endoreg_db/data/event/neurology.yaml +14 -0
  80. endoreg_db/data/event/surgery.yaml +13 -0
  81. endoreg_db/data/event/thrombembolism.yaml +20 -0
  82. endoreg_db/data/event_classification/data.yaml +4 -0
  83. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  84. endoreg_db/data/examination/examinations/data.yaml +172 -0
  85. endoreg_db/data/examination/time/data.yaml +48 -0
  86. endoreg_db/data/examination/time-type/data.yaml +5 -0
  87. endoreg_db/data/examination/type/data.yaml +17 -0
  88. endoreg_db/data/examination_indication/endoscopy.yaml +359 -0
  89. endoreg_db/data/examination_indication_classification/endoscopy.yaml +90 -0
  90. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +97 -0
  91. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  92. endoreg_db/data/finding/00_generic.yaml +35 -0
  93. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  94. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  95. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  96. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  97. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  98. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  99. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  100. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  101. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  102. endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
  103. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  104. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  105. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  106. endoreg_db/data/finding_classification_choice/00_generic.yaml +15 -0
  107. endoreg_db/data/finding_classification_choice/00_generic_baseline.yaml +23 -0
  108. endoreg_db/data/finding_classification_choice/00_generic_complication.yaml +15 -0
  109. endoreg_db/data/finding_classification_choice/00_generic_histology.yaml +21 -0
  110. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  111. endoreg_db/data/finding_classification_choice/02_colonoscopy_bowel_preparation.yaml +49 -0
  112. endoreg_db/data/finding_classification_choice/02_colonoscopy_generic.yaml +19 -0
  113. endoreg_db/data/finding_classification_choice/02_colonoscopy_histology.yaml +20 -0
  114. endoreg_db/data/finding_classification_choice/02_colonoscopy_location.yaml +248 -0
  115. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  116. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  117. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_morphology.yaml +75 -0
  118. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  119. endoreg_db/data/finding_classification_type/00_generic.yaml +53 -0
  120. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  121. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  122. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  123. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  124. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  125. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  126. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  127. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  128. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  129. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  130. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  131. endoreg_db/data/finding_type/data.yaml +39 -0
  132. endoreg_db/data/gender/data.yaml +42 -0
  133. endoreg_db/data/information_source/annotation.yaml +6 -0
  134. endoreg_db/data/information_source/data.yaml +30 -0
  135. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  136. endoreg_db/data/information_source/medication.yaml +6 -0
  137. endoreg_db/data/information_source/prediction.yaml +7 -0
  138. endoreg_db/data/information_source_type/data.yaml +8 -0
  139. endoreg_db/data/lab_value/cardiac_enzymes.yaml +37 -0
  140. endoreg_db/data/lab_value/coagulation.yaml +54 -0
  141. endoreg_db/data/lab_value/electrolytes.yaml +228 -0
  142. endoreg_db/data/lab_value/gastrointestinal_function.yaml +133 -0
  143. endoreg_db/data/lab_value/hematology.yaml +184 -0
  144. endoreg_db/data/lab_value/hormones.yaml +59 -0
  145. endoreg_db/data/lab_value/lipids.yaml +53 -0
  146. endoreg_db/data/lab_value/misc.yaml +76 -0
  147. endoreg_db/data/lab_value/renal_function.yaml +12 -0
  148. endoreg_db/data/log_type/data.yaml +57 -0
  149. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  150. endoreg_db/data/lx_client_type/base.yaml +30 -0
  151. endoreg_db/data/lx_permission/base.yaml +24 -0
  152. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  153. endoreg_db/data/material/material.yaml +91 -0
  154. endoreg_db/data/medication/anticoagulation.yaml +65 -0
  155. endoreg_db/data/medication/tah.yaml +70 -0
  156. endoreg_db/data/medication_indication/anticoagulation.yaml +115 -0
  157. endoreg_db/data/medication_indication_type/data.yaml +11 -0
  158. endoreg_db/data/medication_indication_type/thrombembolism.yaml +41 -0
  159. endoreg_db/data/medication_intake_time/base.yaml +31 -0
  160. endoreg_db/data/medication_schedule/apixaban.yaml +95 -0
  161. endoreg_db/data/medication_schedule/ass.yaml +12 -0
  162. endoreg_db/data/medication_schedule/enoxaparin.yaml +26 -0
  163. endoreg_db/data/names_first/first_names.yaml +54 -0
  164. endoreg_db/data/names_last/last_names.yaml +51 -0
  165. endoreg_db/data/network_device/data.yaml +59 -0
  166. endoreg_db/data/network_device_type/data.yaml +12 -0
  167. endoreg_db/data/organ/data.yaml +29 -0
  168. endoreg_db/data/patient_lab_sample_type/generic.yaml +6 -0
  169. endoreg_db/data/pdf_type/data.yaml +46 -0
  170. endoreg_db/data/product/green_endoscopy_dashboard_Product.yaml +66 -0
  171. endoreg_db/data/product_group/green_endoscopy_dashboard_ProductGroup.yaml +33 -0
  172. endoreg_db/data/product_material/green_endoscopy_dashboard_ProductMaterial.yaml +308 -0
  173. endoreg_db/data/product_weight/green_endoscopy_dashboard_ProductWeight.yaml +88 -0
  174. endoreg_db/data/profession/data.yaml +70 -0
  175. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  176. endoreg_db/data/qualification/m2.yaml +39 -0
  177. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  178. endoreg_db/data/qualification/sonography.yaml +36 -0
  179. endoreg_db/data/qualification_type/base.yaml +29 -0
  180. endoreg_db/data/reference_product/green_endoscopy_dashboard_ReferenceProduct.yaml +55 -0
  181. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  182. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +30 -0
  183. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +24 -0
  184. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  185. endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
  186. endoreg_db/data/requirement/old/colonoscopy_baseline_austria.yaml +45 -0
  187. endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
  188. endoreg_db/data/requirement/old/disease_cardiovascular.yaml +79 -0
  189. endoreg_db/data/requirement/old/disease_classification_choice_cardiovascular.yaml +41 -0
  190. endoreg_db/data/requirement/old/disease_hepatology.yaml +12 -0
  191. endoreg_db/data/requirement/old/disease_misc.yaml +12 -0
  192. endoreg_db/data/requirement/old/disease_renal.yaml +96 -0
  193. endoreg_db/data/requirement/old/endoscopy_bleeding_risk.yaml +59 -0
  194. endoreg_db/data/requirement/old/event_cardiology.yaml +251 -0
  195. endoreg_db/data/requirement/old/event_requirements.yaml +145 -0
  196. endoreg_db/data/requirement/old/finding_colon_polyp.yaml +50 -0
  197. endoreg_db/data/requirement/old/gender.yaml +0 -0
  198. endoreg_db/data/requirement/old/lab_value.yaml +441 -0
  199. endoreg_db/data/requirement/old/medication.yaml +93 -0
  200. endoreg_db/data/requirement_operator/_old/age.yaml +13 -0
  201. endoreg_db/data/requirement_operator/_old/lab_operators.yaml +129 -0
  202. endoreg_db/data/requirement_operator/_old/model_operators.yaml +96 -0
  203. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  204. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +65 -0
  205. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  206. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  207. endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
  208. endoreg_db/data/requirement_set/_old_ +109 -0
  209. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  210. endoreg_db/data/requirement_set_type/data.yaml +41 -0
  211. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  212. endoreg_db/data/resource/green_endoscopy_dashboard_Resource.yaml +15 -0
  213. endoreg_db/data/risk/bleeding.yaml +26 -0
  214. endoreg_db/data/risk/thrombosis.yaml +37 -0
  215. endoreg_db/data/risk_type/data.yaml +27 -0
  216. endoreg_db/data/setup_config.yaml +38 -0
  217. endoreg_db/data/shift/endoscopy.yaml +21 -0
  218. endoreg_db/data/shift/m2.yaml +0 -0
  219. endoreg_db/data/shift_type/base.yaml +35 -0
  220. endoreg_db/data/tag/requirement_set_tags.yaml +32 -0
  221. endoreg_db/data/tmp/chronic_kidney_disease.yaml +0 -0
  222. endoreg_db/data/tmp/congestive_heart_failure.yaml +0 -0
  223. endoreg_db/data/transport_route/green_endoscopy_dashboard_TransportRoute.yaml +12 -0
  224. endoreg_db/data/unit/concentration.yaml +115 -0
  225. endoreg_db/data/unit/data.yaml +17 -0
  226. endoreg_db/data/unit/length.yaml +31 -0
  227. endoreg_db/data/unit/misc.yaml +20 -0
  228. endoreg_db/data/unit/rate.yaml +6 -0
  229. endoreg_db/data/unit/time.yaml +48 -0
  230. endoreg_db/data/unit/volume.yaml +35 -0
  231. endoreg_db/data/unit/weight.yaml +38 -0
  232. endoreg_db/data/waste/data.yaml +12 -0
  233. endoreg_db/exceptions.py +24 -0
  234. endoreg_db/export/frames/export.py +6 -0
  235. endoreg_db/export/frames/export_frames_with_labels.py +616 -0
  236. endoreg_db/factories/__init__.py +0 -0
  237. endoreg_db/forms/__init__.py +4 -0
  238. endoreg_db/forms/examination_form.py +12 -0
  239. endoreg_db/forms/patient_finding_intervention_form.py +40 -0
  240. endoreg_db/forms/patient_form.py +23 -0
  241. endoreg_db/forms/questionnaires/__init__.py +1 -0
  242. endoreg_db/forms/questionnaires/tto_questionnaire.py +23 -0
  243. endoreg_db/forms/settings/__init__.py +11 -0
  244. endoreg_db/forms/unit.py +7 -0
  245. endoreg_db/helpers/__init__.py +0 -0
  246. endoreg_db/helpers/count_db.py +48 -0
  247. endoreg_db/helpers/data_loader.py +280 -0
  248. endoreg_db/helpers/default_objects.py +414 -0
  249. endoreg_db/helpers/download_segmentation_model.py +32 -0
  250. endoreg_db/helpers/interact.py +1 -0
  251. endoreg_db/helpers/test_video_helper.py +127 -0
  252. endoreg_db/import_files/__init__.py +27 -0
  253. endoreg_db/import_files/context/__init__.py +7 -0
  254. endoreg_db/import_files/context/default_sensitive_meta.py +83 -0
  255. endoreg_db/import_files/context/ensure_center.py +17 -0
  256. endoreg_db/import_files/context/file_lock.py +66 -0
  257. endoreg_db/import_files/context/import_context.py +42 -0
  258. endoreg_db/import_files/context/validate_directories.py +57 -0
  259. endoreg_db/import_files/file_storage/__init__.py +15 -0
  260. endoreg_db/import_files/file_storage/create_report_file.py +99 -0
  261. endoreg_db/import_files/file_storage/create_video_file.py +104 -0
  262. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +42 -0
  263. endoreg_db/import_files/file_storage/state_management.py +463 -0
  264. endoreg_db/import_files/file_storage/storage.py +42 -0
  265. endoreg_db/import_files/import_service.md +26 -0
  266. endoreg_db/import_files/processing/__init__.py +11 -0
  267. endoreg_db/import_files/processing/report_processing/report_anonymization.py +99 -0
  268. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  269. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  270. endoreg_db/import_files/pseudonymization/__init__.py +0 -0
  271. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  272. endoreg_db/import_files/pseudonymization/k_anonymity.py +181 -0
  273. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +139 -0
  274. endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
  275. endoreg_db/import_files/report_import_service.py +147 -0
  276. endoreg_db/import_files/video_import_service.py +154 -0
  277. endoreg_db/logger_conf.py +156 -0
  278. endoreg_db/management/__init__.py +1 -0
  279. endoreg_db/management/commands/__init__.py +1 -0
  280. endoreg_db/management/commands/anonymize_video.py +0 -0
  281. endoreg_db/management/commands/check_auth.py +132 -0
  282. endoreg_db/management/commands/create_model_meta_from_huggingface.py +177 -0
  283. endoreg_db/management/commands/create_multilabel_model_meta.py +419 -0
  284. endoreg_db/management/commands/export_frame_annot.py +196 -0
  285. endoreg_db/management/commands/fix_missing_patient_data.py +206 -0
  286. endoreg_db/management/commands/fix_video_paths.py +186 -0
  287. endoreg_db/management/commands/import_report.py +361 -0
  288. endoreg_db/management/commands/list_routes.py +20 -0
  289. endoreg_db/management/commands/load_ai_model_data.py +83 -0
  290. endoreg_db/management/commands/load_ai_model_label_data.py +60 -0
  291. endoreg_db/management/commands/load_base_db_data.py +63 -0
  292. endoreg_db/management/commands/load_center_data.py +68 -0
  293. endoreg_db/management/commands/load_contraindication_data.py +39 -0
  294. endoreg_db/management/commands/load_disease_classification_choices_data.py +38 -0
  295. endoreg_db/management/commands/load_disease_classification_data.py +38 -0
  296. endoreg_db/management/commands/load_disease_data.py +59 -0
  297. endoreg_db/management/commands/load_distribution_data.py +63 -0
  298. endoreg_db/management/commands/load_endoscope_data.py +58 -0
  299. endoreg_db/management/commands/load_event_data.py +39 -0
  300. endoreg_db/management/commands/load_examination_data.py +78 -0
  301. endoreg_db/management/commands/load_examination_indication_data.py +85 -0
  302. endoreg_db/management/commands/load_finding_data.py +115 -0
  303. endoreg_db/management/commands/load_gender_data.py +37 -0
  304. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +142 -0
  305. endoreg_db/management/commands/load_information_source.py +46 -0
  306. endoreg_db/management/commands/load_lab_value_data.py +52 -0
  307. endoreg_db/management/commands/load_legacy_data.py +303 -0
  308. endoreg_db/management/commands/load_medication_data.py +104 -0
  309. endoreg_db/management/commands/load_name_data.py +36 -0
  310. endoreg_db/management/commands/load_organ_data.py +39 -0
  311. endoreg_db/management/commands/load_pdf_type_data.py +58 -0
  312. endoreg_db/management/commands/load_profession_data.py +40 -0
  313. endoreg_db/management/commands/load_qualification_data.py +56 -0
  314. endoreg_db/management/commands/load_report_reader_flag_data.py +40 -0
  315. endoreg_db/management/commands/load_requirement_data.py +207 -0
  316. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  317. endoreg_db/management/commands/load_risk_data.py +57 -0
  318. endoreg_db/management/commands/load_shift_data.py +57 -0
  319. endoreg_db/management/commands/load_tag_data.py +54 -0
  320. endoreg_db/management/commands/load_unit_data.py +40 -0
  321. endoreg_db/management/commands/load_user_groups.py +26 -0
  322. endoreg_db/management/commands/model_input.py +169 -0
  323. endoreg_db/management/commands/register_ai_model.py +70 -0
  324. endoreg_db/management/commands/setup_endoreg_db.py +459 -0
  325. endoreg_db/management/commands/start_filewatcher.py +115 -0
  326. endoreg_db/management/commands/storage_management.py +622 -0
  327. endoreg_db/management/commands/summarize_db_content.py +280 -0
  328. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  329. endoreg_db/management/commands/validate_video_files.py +189 -0
  330. endoreg_db/management/commands/video_validation.py +20 -0
  331. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  332. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  333. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  334. endoreg_db/mermaid/classification.md +8 -0
  335. endoreg_db/mermaid/examination.md +8 -0
  336. endoreg_db/mermaid/findings.md +7 -0
  337. endoreg_db/mermaid/image_classification.md +28 -0
  338. endoreg_db/mermaid/interventions.md +8 -0
  339. endoreg_db/mermaid/morphology.md +8 -0
  340. endoreg_db/mermaid/patient_creation.md +14 -0
  341. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  342. endoreg_db/migrations/0001_initial.py +1953 -0
  343. endoreg_db/migrations/__init__.py +0 -0
  344. endoreg_db/models/__init__.py +322 -0
  345. endoreg_db/models/administration/__init__.py +95 -0
  346. endoreg_db/models/administration/ai/__init__.py +9 -0
  347. endoreg_db/models/administration/ai/active_model.py +35 -0
  348. endoreg_db/models/administration/ai/ai_model.py +180 -0
  349. endoreg_db/models/administration/ai/model_type.py +42 -0
  350. endoreg_db/models/administration/case/__init__.py +5 -0
  351. endoreg_db/models/administration/case/case.py +114 -0
  352. endoreg_db/models/administration/case/case_template/__init__.py +3 -0
  353. endoreg_db/models/administration/case/case_template/case_template.py +3 -0
  354. endoreg_db/models/administration/case/case_template/case_template_rule.py +3 -0
  355. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +3 -0
  356. endoreg_db/models/administration/case/case_template/case_template_type.py +3 -0
  357. endoreg_db/models/administration/center/__init__.py +13 -0
  358. endoreg_db/models/administration/center/center.py +85 -0
  359. endoreg_db/models/administration/center/center_product.py +67 -0
  360. endoreg_db/models/administration/center/center_resource.py +69 -0
  361. endoreg_db/models/administration/center/center_shift.py +94 -0
  362. endoreg_db/models/administration/center/center_waste.py +42 -0
  363. endoreg_db/models/administration/person/__init__.py +26 -0
  364. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  365. endoreg_db/models/administration/person/employee/employee.py +40 -0
  366. endoreg_db/models/administration/person/employee/employee_qualification.py +44 -0
  367. endoreg_db/models/administration/person/employee/employee_type.py +50 -0
  368. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  369. endoreg_db/models/administration/person/examiner/examiner.py +64 -0
  370. endoreg_db/models/administration/person/names/__init__.py +0 -0
  371. endoreg_db/models/administration/person/names/first_name.py +20 -0
  372. endoreg_db/models/administration/person/names/last_name.py +20 -0
  373. endoreg_db/models/administration/person/patient/__init__.py +7 -0
  374. endoreg_db/models/administration/person/patient/patient.py +488 -0
  375. endoreg_db/models/administration/person/patient/patient_external_id.py +36 -0
  376. endoreg_db/models/administration/person/person.py +35 -0
  377. endoreg_db/models/administration/person/profession/__init__.py +28 -0
  378. endoreg_db/models/administration/person/user/__init__.py +5 -0
  379. endoreg_db/models/administration/person/user/portal_user_information.py +41 -0
  380. endoreg_db/models/administration/product/__init__.py +15 -0
  381. endoreg_db/models/administration/product/product.py +106 -0
  382. endoreg_db/models/administration/product/product_group.py +41 -0
  383. endoreg_db/models/administration/product/product_material.py +60 -0
  384. endoreg_db/models/administration/product/product_weight.py +51 -0
  385. endoreg_db/models/administration/product/reference_product.py +147 -0
  386. endoreg_db/models/administration/qualification/__init__.py +7 -0
  387. endoreg_db/models/administration/qualification/qualification.py +43 -0
  388. endoreg_db/models/administration/qualification/qualification_type.py +39 -0
  389. endoreg_db/models/administration/shift/__init__.py +9 -0
  390. endoreg_db/models/administration/shift/scheduled_days.py +72 -0
  391. endoreg_db/models/administration/shift/shift.py +57 -0
  392. endoreg_db/models/administration/shift/shift_type.py +108 -0
  393. endoreg_db/models/aidataset/__init__.py +5 -0
  394. endoreg_db/models/aidataset/aidataset.py +193 -0
  395. endoreg_db/models/label/__init__.py +23 -0
  396. endoreg_db/models/label/annotation/__init__.py +12 -0
  397. endoreg_db/models/label/annotation/image_classification.py +85 -0
  398. endoreg_db/models/label/annotation/video_segmentation_annotation.py +61 -0
  399. endoreg_db/models/label/label.py +91 -0
  400. endoreg_db/models/label/label_set.py +68 -0
  401. endoreg_db/models/label/label_type.py +29 -0
  402. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  403. endoreg_db/models/label/label_video_segment/_create_from_video.py +42 -0
  404. endoreg_db/models/label/label_video_segment/label_video_segment.py +611 -0
  405. endoreg_db/models/label/video_segmentation_label.py +35 -0
  406. endoreg_db/models/label/video_segmentation_labelset.py +28 -0
  407. endoreg_db/models/media/__init__.py +23 -0
  408. endoreg_db/models/media/frame/__init__.py +3 -0
  409. endoreg_db/models/media/frame/frame.py +137 -0
  410. endoreg_db/models/media/pdf/__init__.py +12 -0
  411. endoreg_db/models/media/pdf/raw_pdf.py +764 -0
  412. endoreg_db/models/media/pdf/report_file.py +162 -0
  413. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  414. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +85 -0
  415. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +46 -0
  416. endoreg_db/models/media/video/__init__.py +9 -0
  417. endoreg_db/models/media/video/create_from_file.py +402 -0
  418. endoreg_db/models/media/video/pipe_1.py +258 -0
  419. endoreg_db/models/media/video/pipe_2.py +129 -0
  420. endoreg_db/models/media/video/video_file.py +907 -0
  421. endoreg_db/models/media/video/video_file_ai.py +828 -0
  422. endoreg_db/models/media/video/video_file_anonymize.py +524 -0
  423. endoreg_db/models/media/video/video_file_frames/__init__.py +49 -0
  424. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +25 -0
  425. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  426. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +126 -0
  427. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +233 -0
  428. endoreg_db/models/media/video/video_file_frames/_get_frame.py +36 -0
  429. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +13 -0
  430. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +24 -0
  431. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +40 -0
  432. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +44 -0
  433. endoreg_db/models/media/video/video_file_frames/_get_frames.py +30 -0
  434. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +205 -0
  435. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +228 -0
  436. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +107 -0
  437. endoreg_db/models/media/video/video_file_io.py +272 -0
  438. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  439. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +58 -0
  440. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +62 -0
  441. endoreg_db/models/media/video/video_file_meta/get_fps.py +183 -0
  442. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +198 -0
  443. endoreg_db/models/media/video/video_file_meta/text_meta.py +178 -0
  444. endoreg_db/models/media/video/video_file_meta/video_meta.py +105 -0
  445. endoreg_db/models/media/video/video_file_segments.py +317 -0
  446. endoreg_db/models/media/video/video_metadata.py +67 -0
  447. endoreg_db/models/media/video/video_processing.py +192 -0
  448. endoreg_db/models/medical/__init__.py +136 -0
  449. endoreg_db/models/medical/contraindication/README.md +1 -0
  450. endoreg_db/models/medical/contraindication/__init__.py +29 -0
  451. endoreg_db/models/medical/disease.py +174 -0
  452. endoreg_db/models/medical/event.py +154 -0
  453. endoreg_db/models/medical/examination/__init__.py +20 -0
  454. endoreg_db/models/medical/examination/examination.py +183 -0
  455. endoreg_db/models/medical/examination/examination_indication.py +229 -0
  456. endoreg_db/models/medical/examination/examination_time.py +68 -0
  457. endoreg_db/models/medical/examination/examination_time_type.py +44 -0
  458. endoreg_db/models/medical/examination/examination_type.py +47 -0
  459. endoreg_db/models/medical/finding/__init__.py +20 -0
  460. endoreg_db/models/medical/finding/finding.py +113 -0
  461. endoreg_db/models/medical/finding/finding_classification.py +131 -0
  462. endoreg_db/models/medical/finding/finding_intervention.py +68 -0
  463. endoreg_db/models/medical/finding/finding_type.py +38 -0
  464. endoreg_db/models/medical/hardware/__init__.py +8 -0
  465. endoreg_db/models/medical/hardware/endoscope.py +77 -0
  466. endoreg_db/models/medical/hardware/endoscopy_processor.py +182 -0
  467. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  468. endoreg_db/models/medical/laboratory/lab_value.py +490 -0
  469. endoreg_db/models/medical/medication/__init__.py +23 -0
  470. endoreg_db/models/medical/medication/medication.py +45 -0
  471. endoreg_db/models/medical/medication/medication_indication.py +78 -0
  472. endoreg_db/models/medical/medication/medication_indication_type.py +58 -0
  473. endoreg_db/models/medical/medication/medication_intake_time.py +58 -0
  474. endoreg_db/models/medical/medication/medication_schedule.py +58 -0
  475. endoreg_db/models/medical/organ/__init__.py +38 -0
  476. endoreg_db/models/medical/patient/__init__.py +48 -0
  477. endoreg_db/models/medical/patient/medication_examples.py +56 -0
  478. endoreg_db/models/medical/patient/patient_disease.py +72 -0
  479. endoreg_db/models/medical/patient/patient_event.py +80 -0
  480. endoreg_db/models/medical/patient/patient_examination.py +280 -0
  481. endoreg_db/models/medical/patient/patient_examination_indication.py +57 -0
  482. endoreg_db/models/medical/patient/patient_finding.py +416 -0
  483. endoreg_db/models/medical/patient/patient_finding_classification.py +231 -0
  484. endoreg_db/models/medical/patient/patient_finding_intervention.py +37 -0
  485. endoreg_db/models/medical/patient/patient_lab_sample.py +157 -0
  486. endoreg_db/models/medical/patient/patient_lab_value.py +247 -0
  487. endoreg_db/models/medical/patient/patient_medication.py +111 -0
  488. endoreg_db/models/medical/patient/patient_medication_schedule.py +152 -0
  489. endoreg_db/models/medical/risk/__init__.py +7 -0
  490. endoreg_db/models/medical/risk/risk.py +73 -0
  491. endoreg_db/models/medical/risk/risk_type.py +54 -0
  492. endoreg_db/models/metadata/__init__.py +19 -0
  493. endoreg_db/models/metadata/model_meta.py +266 -0
  494. endoreg_db/models/metadata/model_meta_logic.py +485 -0
  495. endoreg_db/models/metadata/pdf_meta.py +96 -0
  496. endoreg_db/models/metadata/sensitive_meta.py +345 -0
  497. endoreg_db/models/metadata/sensitive_meta_logic.py +1161 -0
  498. endoreg_db/models/metadata/video_meta.py +459 -0
  499. endoreg_db/models/metadata/video_prediction_logic.py +232 -0
  500. endoreg_db/models/metadata/video_prediction_meta.py +319 -0
  501. endoreg_db/models/operation_log.py +63 -0
  502. endoreg_db/models/other/__init__.py +40 -0
  503. endoreg_db/models/other/distribution/__init__.py +46 -0
  504. endoreg_db/models/other/distribution/base_value_distribution.py +22 -0
  505. endoreg_db/models/other/distribution/date_value_distribution.py +163 -0
  506. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +50 -0
  507. endoreg_db/models/other/distribution/numeric_value_distribution.py +211 -0
  508. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +23 -0
  509. endoreg_db/models/other/emission/__init__.py +5 -0
  510. endoreg_db/models/other/emission/emission_factor.py +110 -0
  511. endoreg_db/models/other/gender.py +32 -0
  512. endoreg_db/models/other/information_source.py +190 -0
  513. endoreg_db/models/other/material.py +34 -0
  514. endoreg_db/models/other/resource.py +24 -0
  515. endoreg_db/models/other/tag.py +32 -0
  516. endoreg_db/models/other/transport_route.py +40 -0
  517. endoreg_db/models/other/unit.py +40 -0
  518. endoreg_db/models/other/waste.py +28 -0
  519. endoreg_db/models/report/__init__.py +0 -0
  520. endoreg_db/models/report/images.py +0 -0
  521. endoreg_db/models/report/report.py +5 -0
  522. endoreg_db/models/requirement/__init__.py +11 -0
  523. endoreg_db/models/requirement/requirement.py +792 -0
  524. endoreg_db/models/requirement/requirement_error.py +84 -0
  525. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  526. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  527. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  528. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +6 -0
  529. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +137 -0
  530. endoreg_db/models/requirement/requirement_operator.py +187 -0
  531. endoreg_db/models/requirement/requirement_set.py +327 -0
  532. endoreg_db/models/state/__init__.py +13 -0
  533. endoreg_db/models/state/abstract.py +11 -0
  534. endoreg_db/models/state/anonymization.py +30 -0
  535. endoreg_db/models/state/audit_ledger.py +155 -0
  536. endoreg_db/models/state/label_video_segment.py +31 -0
  537. endoreg_db/models/state/processing_history/__init__.py +3 -0
  538. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  539. endoreg_db/models/state/raw_pdf.py +219 -0
  540. endoreg_db/models/state/sensitive_meta.py +50 -0
  541. endoreg_db/models/state/video.py +251 -0
  542. endoreg_db/models/upload_job.py +100 -0
  543. endoreg_db/models/utils.py +138 -0
  544. endoreg_db/queries/__init__.py +3 -0
  545. endoreg_db/queries/annotations/__init__.py +1 -0
  546. endoreg_db/queries/annotations/legacy.py +169 -0
  547. endoreg_db/queries/sanity/__init_.py +0 -0
  548. endoreg_db/root_urls.py +27 -0
  549. endoreg_db/schemas/__init__.py +0 -0
  550. endoreg_db/schemas/examination_evaluation.py +30 -0
  551. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +861 -0
  552. endoreg_db/serializers/__init__.py +104 -0
  553. endoreg_db/serializers/administration/__init__.py +13 -0
  554. endoreg_db/serializers/administration/ai/__init__.py +9 -0
  555. endoreg_db/serializers/administration/ai/active_model.py +12 -0
  556. endoreg_db/serializers/administration/ai/ai_model.py +20 -0
  557. endoreg_db/serializers/administration/ai/model_type.py +12 -0
  558. endoreg_db/serializers/administration/center.py +14 -0
  559. endoreg_db/serializers/administration/gender.py +11 -0
  560. endoreg_db/serializers/anonymization.py +77 -0
  561. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -0
  562. endoreg_db/serializers/examination/__init__.py +10 -0
  563. endoreg_db/serializers/examination/base.py +45 -0
  564. endoreg_db/serializers/examination/dropdown.py +20 -0
  565. endoreg_db/serializers/examination_serializer.py +9 -0
  566. endoreg_db/serializers/finding/__init__.py +5 -0
  567. endoreg_db/serializers/finding/finding.py +61 -0
  568. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  569. endoreg_db/serializers/finding_classification/choice.py +19 -0
  570. endoreg_db/serializers/finding_classification/classification.py +11 -0
  571. endoreg_db/serializers/label_video_segment/__init__.py +9 -0
  572. endoreg_db/serializers/label_video_segment/image_classification_annotation.py +62 -0
  573. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  574. endoreg_db/serializers/label_video_segment/label/label.py +15 -0
  575. endoreg_db/serializers/label_video_segment/label_video_segment.py +427 -0
  576. endoreg_db/serializers/meta/__init__.py +13 -0
  577. endoreg_db/serializers/meta/sensitive_meta_detail.py +122 -0
  578. endoreg_db/serializers/meta/sensitive_meta_update.py +153 -0
  579. endoreg_db/serializers/meta/sensitive_meta_verification.py +62 -0
  580. endoreg_db/serializers/meta/video_meta.py +39 -0
  581. endoreg_db/serializers/misc/__init__.py +14 -0
  582. endoreg_db/serializers/misc/file_overview.py +72 -0
  583. endoreg_db/serializers/misc/sensitive_patient_data.py +144 -0
  584. endoreg_db/serializers/misc/stats.py +35 -0
  585. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  586. endoreg_db/serializers/misc/upload_job.py +74 -0
  587. endoreg_db/serializers/patient/__init__.py +12 -0
  588. endoreg_db/serializers/patient/patient.py +103 -0
  589. endoreg_db/serializers/patient/patient_dropdown.py +35 -0
  590. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  591. endoreg_db/serializers/patient_examination/patient_examination.py +168 -0
  592. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  593. endoreg_db/serializers/patient_finding/patient_finding.py +32 -0
  594. endoreg_db/serializers/patient_finding/patient_finding_classification.py +47 -0
  595. endoreg_db/serializers/patient_finding/patient_finding_detail.py +62 -0
  596. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +28 -0
  597. endoreg_db/serializers/patient_finding/patient_finding_list.py +40 -0
  598. endoreg_db/serializers/patient_finding/patient_finding_write.py +135 -0
  599. endoreg_db/serializers/pdf/__init__.py +3 -0
  600. endoreg_db/serializers/pdf/anony_text_validation.py +101 -0
  601. endoreg_db/serializers/requirements/requirement_schema.py +20 -0
  602. endoreg_db/serializers/requirements/requirement_sets.py +99 -0
  603. endoreg_db/serializers/sensitive_meta_serializer.py +301 -0
  604. endoreg_db/serializers/video/__init__.py +7 -0
  605. endoreg_db/serializers/video/video_file.py +283 -0
  606. endoreg_db/serializers/video/video_file_brief.py +14 -0
  607. endoreg_db/serializers/video/video_file_detail.py +96 -0
  608. endoreg_db/serializers/video/video_file_list.py +100 -0
  609. endoreg_db/serializers/video/video_processing_history.py +172 -0
  610. endoreg_db/serializers/video_examination.py +198 -0
  611. endoreg_db/services/__init__.py +5 -0
  612. endoreg_db/services/anonymization.py +274 -0
  613. endoreg_db/services/examination_evaluation.py +172 -0
  614. endoreg_db/services/finding_description_service.py +0 -0
  615. endoreg_db/services/lookup_service.py +424 -0
  616. endoreg_db/services/lookup_store.py +266 -0
  617. endoreg_db/services/model_meta_from_hf.py +76 -0
  618. endoreg_db/services/pdf_import.py +0 -0
  619. endoreg_db/services/polling_coordinator.py +319 -0
  620. endoreg_db/services/pseudonym_service.py +94 -0
  621. endoreg_db/services/report_import.py +13 -0
  622. endoreg_db/services/segment_sync.py +171 -0
  623. endoreg_db/services/video_import.py +9 -0
  624. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  625. endoreg_db/templates/admin/start_examination.html +12 -0
  626. endoreg_db/templates/timeline.html +176 -0
  627. endoreg_db/urls/__init__.py +56 -0
  628. endoreg_db/urls/ai.py +14 -0
  629. endoreg_db/urls/anonymization.py +78 -0
  630. endoreg_db/urls/auth.py +16 -0
  631. endoreg_db/urls/classification.py +34 -0
  632. endoreg_db/urls/examination.py +63 -0
  633. endoreg_db/urls/media.py +251 -0
  634. endoreg_db/urls/patient.py +23 -0
  635. endoreg_db/urls/requirements.py +15 -0
  636. endoreg_db/urls/root_urls.py +28 -0
  637. endoreg_db/urls/stats.py +54 -0
  638. endoreg_db/urls/upload.py +12 -0
  639. endoreg_db/urls.py +9 -0
  640. endoreg_db/utils/__init__.py +97 -0
  641. endoreg_db/utils/ai/__init__.py +9 -0
  642. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  643. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  644. endoreg_db/utils/ai/get.py +6 -0
  645. endoreg_db/utils/ai/inference_dataset.py +51 -0
  646. endoreg_db/utils/ai/model_training/config.py +117 -0
  647. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  648. endoreg_db/utils/ai/model_training/losses.py +68 -0
  649. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  650. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  651. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  652. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  653. endoreg_db/utils/ai/multilabel_classification_net.py +270 -0
  654. endoreg_db/utils/ai/postprocess.py +63 -0
  655. endoreg_db/utils/ai/predict.py +293 -0
  656. endoreg_db/utils/ai/preprocess.py +76 -0
  657. endoreg_db/utils/calc_duration_seconds.py +24 -0
  658. endoreg_db/utils/case_generator/__init__.py +3 -0
  659. endoreg_db/utils/case_generator/lab_sample_factory.py +32 -0
  660. endoreg_db/utils/check_video_files.py +175 -0
  661. endoreg_db/utils/cropping.py +30 -0
  662. endoreg_db/utils/dataloader.py +285 -0
  663. endoreg_db/utils/dates.py +59 -0
  664. endoreg_db/utils/defaults/set_default_center.py +33 -0
  665. endoreg_db/utils/env.py +37 -0
  666. endoreg_db/utils/extract_specific_frames.py +87 -0
  667. endoreg_db/utils/file_operations.py +70 -0
  668. endoreg_db/utils/fix_video_path_direct.py +157 -0
  669. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  670. endoreg_db/utils/hashs.py +138 -0
  671. endoreg_db/utils/links/__init__.py +0 -0
  672. endoreg_db/utils/links/requirement_link.py +237 -0
  673. endoreg_db/utils/mime_types.py +0 -0
  674. endoreg_db/utils/names.py +82 -0
  675. endoreg_db/utils/ocr.py +195 -0
  676. endoreg_db/utils/operation_log.py +87 -0
  677. endoreg_db/utils/parse_and_generate_yaml.py +45 -0
  678. endoreg_db/utils/paths.py +159 -0
  679. endoreg_db/utils/permissions.py +160 -0
  680. endoreg_db/utils/pipelines/Readme.md +235 -0
  681. endoreg_db/utils/pipelines/__init__.py +0 -0
  682. endoreg_db/utils/pipelines/process_video_dir.py +144 -0
  683. endoreg_db/utils/product/__init__.py +0 -0
  684. endoreg_db/utils/product/sum_emissions.py +22 -0
  685. endoreg_db/utils/product/sum_weights.py +20 -0
  686. endoreg_db/utils/pydantic_models/__init__.py +5 -0
  687. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  688. endoreg_db/utils/requirement_helpers.py +0 -0
  689. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  690. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +678 -0
  691. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +842 -0
  692. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +114 -0
  693. endoreg_db/utils/setup_config.py +196 -0
  694. endoreg_db/utils/storage.py +117 -0
  695. endoreg_db/utils/translation.py +31 -0
  696. endoreg_db/utils/uuid.py +5 -0
  697. endoreg_db/utils/validate_endo_roi.py +33 -0
  698. endoreg_db/utils/validate_subcategory_dict.py +93 -0
  699. endoreg_db/utils/validate_video_detailed.py +415 -0
  700. endoreg_db/utils/video/__init__.py +30 -0
  701. endoreg_db/utils/video/extract_frames.py +100 -0
  702. endoreg_db/utils/video/ffmpeg_wrapper.py +996 -0
  703. endoreg_db/utils/video/names.py +47 -0
  704. endoreg_db/utils/video/streaming_processor.py +386 -0
  705. endoreg_db/utils/video/video_splitter.py +105 -0
  706. endoreg_db/versioning.md +79 -0
  707. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +247 -0
  708. endoreg_db/views/__init__.py +157 -0
  709. endoreg_db/views/anonymization/__init__.py +31 -0
  710. endoreg_db/views/anonymization/media_management.py +486 -0
  711. endoreg_db/views/anonymization/overview.py +307 -0
  712. endoreg_db/views/anonymization/validate.py +310 -0
  713. endoreg_db/views/auth/__init__.py +13 -0
  714. endoreg_db/views/auth/keycloak.py +146 -0
  715. endoreg_db/views/examination/__init__.py +30 -0
  716. endoreg_db/views/examination/examination.py +37 -0
  717. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  718. endoreg_db/views/examination/get_finding_classification_choices.py +62 -0
  719. endoreg_db/views/examination/get_finding_classifications.py +38 -0
  720. endoreg_db/views/examination/get_findings.py +39 -0
  721. endoreg_db/views/examination/get_instruments.py +19 -0
  722. endoreg_db/views/examination/get_interventions.py +14 -0
  723. endoreg_db/views/finding/__init__.py +9 -0
  724. endoreg_db/views/finding/finding.py +116 -0
  725. endoreg_db/views/finding/get_classifications.py +14 -0
  726. endoreg_db/views/finding/get_interventions.py +17 -0
  727. endoreg_db/views/finding_classification/__init__.py +13 -0
  728. endoreg_db/views/finding_classification/base.py +0 -0
  729. endoreg_db/views/finding_classification/finding_classification.py +41 -0
  730. endoreg_db/views/finding_classification/get_classification_choices.py +54 -0
  731. endoreg_db/views/media/__init__.py +32 -0
  732. endoreg_db/views/media/pdf_media.py +411 -0
  733. endoreg_db/views/media/sensitive_metadata.py +372 -0
  734. endoreg_db/views/media/video_media.py +275 -0
  735. endoreg_db/views/meta/__init__.py +7 -0
  736. endoreg_db/views/meta/sensitive_meta_list.py +102 -0
  737. endoreg_db/views/meta/sensitive_meta_verification.py +74 -0
  738. endoreg_db/views/misc/__init__.py +29 -0
  739. endoreg_db/views/misc/center.py +14 -0
  740. endoreg_db/views/misc/csrf.py +8 -0
  741. endoreg_db/views/misc/gender.py +15 -0
  742. endoreg_db/views/misc/stats.py +255 -0
  743. endoreg_db/views/misc/upload_views.py +241 -0
  744. endoreg_db/views/patient/__init__.py +3 -0
  745. endoreg_db/views/patient/patient.py +253 -0
  746. endoreg_db/views/patient_examination/__init__.py +11 -0
  747. endoreg_db/views/patient_examination/patient_examination.py +141 -0
  748. endoreg_db/views/patient_examination/patient_examination_create.py +58 -0
  749. endoreg_db/views/patient_examination/patient_examination_detail.py +63 -0
  750. endoreg_db/views/patient_examination/patient_examination_list.py +72 -0
  751. endoreg_db/views/patient_examination/video.py +228 -0
  752. endoreg_db/views/patient_finding/__init__.py +7 -0
  753. endoreg_db/views/patient_finding/base.py +0 -0
  754. endoreg_db/views/patient_finding/patient_finding.py +71 -0
  755. endoreg_db/views/patient_finding/patient_finding_optimized.py +291 -0
  756. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  757. endoreg_db/views/patient_finding_classification/pfc_create.py +75 -0
  758. endoreg_db/views/report/__init__.py +7 -0
  759. endoreg_db/views/report/reimport.py +177 -0
  760. endoreg_db/views/report/report_stream.py +191 -0
  761. endoreg_db/views/requirement/__init__.py +11 -0
  762. endoreg_db/views/requirement/evaluate.py +278 -0
  763. endoreg_db/views/requirement/lookup.py +380 -0
  764. endoreg_db/views/requirement/lookup_store.py +183 -0
  765. endoreg_db/views/requirement/requirement_utils.py +87 -0
  766. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  767. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  768. endoreg_db/views/stats/__init__.py +13 -0
  769. endoreg_db/views/stats/stats_views.py +266 -0
  770. endoreg_db/views/video/__init__.py +49 -0
  771. endoreg_db/views/video/ai/__init__.py +8 -0
  772. endoreg_db/views/video/ai/label.py +159 -0
  773. endoreg_db/views/video/correction.py +529 -0
  774. endoreg_db/views/video/reimport.py +230 -0
  775. endoreg_db/views/video/segments_crud.py +709 -0
  776. endoreg_db/views/video/video_apply_mask.py +49 -0
  777. endoreg_db/views/video/video_correction.py +22 -0
  778. endoreg_db/views/video/video_download_processed.py +58 -0
  779. endoreg_db/views/video/video_examination_viewset.py +242 -0
  780. endoreg_db/views/video/video_metadata.py +101 -0
  781. endoreg_db/views/video/video_processing_history.py +25 -0
  782. endoreg_db/views/video/video_remove_frames.py +49 -0
  783. endoreg_db/views/video/video_stream.py +334 -0
  784. endoreg_db-0.8.9.32.dist-info/METADATA +404 -0
  785. endoreg_db-0.8.9.32.dist-info/RECORD +787 -0
  786. endoreg_db-0.8.9.32.dist-info/WHEEL +4 -0
  787. endoreg_db-0.8.9.32.dist-info/licenses/LICENSE +674 -0
@@ -0,0 +1,907 @@
1
+ """Concrete model for video files, handling both raw and processed states."""
2
+
3
+ import logging
4
+ import os
5
+ from pathlib import Path
6
+ from typing import TYPE_CHECKING, Optional, Union, cast
7
+
8
+ from django.core.files import File
9
+ from django.core.validators import FileExtensionValidator
10
+ from django.db import models
11
+ from django.db.models import F
12
+ from django.db.models.fields.files import FieldFile
13
+
14
+ from endoreg_db.utils.calc_duration_seconds import _calc_duration_vf
15
+ from endoreg_db.utils.paths import ANONYM_VIDEO_DIR, SENSITIVE_VIDEO_DIR
16
+ from endoreg_db.utils.video.ffmpeg_wrapper import assemble_video_from_frames
17
+
18
+ from ...label import Label, LabelVideoSegment
19
+ from ...state import VideoState
20
+
21
+ # --- Import model-specific function modules ---
22
+ from .create_from_file import _create_from_file
23
+ from .pipe_1 import _pipe_1, _test_after_pipe_1
24
+ from .pipe_2 import _pipe_2
25
+ from .video_file_ai import _extract_text_from_video_frames, _predict_video_pipeline
26
+ from .video_file_anonymize import (
27
+ _anonymize,
28
+ _censor_outside_frames,
29
+ _cleanup_raw_assets,
30
+ _create_anonymized_frame_files,
31
+ )
32
+ from .video_file_frames import (
33
+ _bulk_create_frames,
34
+ _create_frame_object,
35
+ _delete_frames,
36
+ _extract_frames,
37
+ _get_frame,
38
+ _get_frame_number,
39
+ _get_frame_path,
40
+ _get_frame_paths,
41
+ _get_frame_range,
42
+ _get_frames,
43
+ _initialize_frames,
44
+ )
45
+
46
+ # Update import aliases for clarity and to use as helpers
47
+ from .video_file_frames._manage_frame_range import (
48
+ _delete_frame_range as _delete_frame_range_helper,
49
+ )
50
+ from .video_file_frames._manage_frame_range import (
51
+ _extract_frame_range as _extract_frame_range_helper,
52
+ )
53
+ from .video_file_io import (
54
+ _delete_with_file,
55
+ _get_base_frame_dir,
56
+ _get_frame_dir_path,
57
+ _get_processed_file_path,
58
+ _get_raw_file_path,
59
+ _get_target_anonymized_video_path,
60
+ _get_temp_anonymized_frame_dir,
61
+ _set_frame_dir,
62
+ )
63
+ from .video_file_meta import (
64
+ _get_crop_template,
65
+ _get_endo_roi,
66
+ _get_fps,
67
+ _initialize_video_specs,
68
+ _update_text_metadata,
69
+ _update_video_meta,
70
+ )
71
+
72
+ # Configure logging
73
+ logger = logging.getLogger(__name__) # Changed from "video_file"
74
+
75
+ if TYPE_CHECKING:
76
+ from django.db.models.fields.files import FieldFile
77
+
78
+ from endoreg_db.models import (
79
+ Center,
80
+ EndoscopyProcessor,
81
+ FFMpegMeta,
82
+ Frame,
83
+ ModelMeta,
84
+ Patient,
85
+ PatientExamination,
86
+ SensitiveMeta,
87
+ VideoImportMeta,
88
+ VideoMeta,
89
+ VideoState,
90
+ )
91
+
92
+
93
+ class VideoQuerySet(models.QuerySet):
94
+ def next_after(self, last_id=None):
95
+ """
96
+ Return the next VideoFile instance with a primary key greater than the given last_id.
97
+
98
+ Parameters:
99
+ last_id (int or None): The primary key to start after. If None or invalid, returns the first instance.
100
+
101
+ Returns:
102
+ VideoFile or None: The next VideoFile instance, or None if not found.
103
+ """
104
+ if last_id is not None:
105
+ try:
106
+ last_id = int(last_id)
107
+ except (ValueError, TypeError):
108
+ return None
109
+ q = self if last_id is None else self.filter(pk__gt=last_id)
110
+ return q.order_by("pk").first()
111
+
112
+
113
+ class VideoFile(models.Model):
114
+ objects = VideoQuerySet.as_manager()
115
+
116
+ raw_file = models.FileField(
117
+ upload_to=SENSITIVE_VIDEO_DIR.name, # Use .name for relative path
118
+ validators=[FileExtensionValidator(allowed_extensions=["mp4"])],
119
+ null=True,
120
+ blank=True,
121
+ )
122
+ processed_file = models.FileField(
123
+ upload_to=ANONYM_VIDEO_DIR.name, # Use .name for relative path
124
+ validators=[FileExtensionValidator(allowed_extensions=["mp4"])],
125
+ null=True,
126
+ blank=True,
127
+ )
128
+
129
+ video_hash = models.CharField(
130
+ max_length=255, unique=True, help_text="Hash of the raw video file."
131
+ )
132
+ processed_video_hash = models.CharField(
133
+ max_length=255,
134
+ unique=True,
135
+ null=True,
136
+ blank=True,
137
+ help_text="Hash of the processed video file, unique if not null.",
138
+ )
139
+
140
+ sensitive_meta = models.OneToOneField(
141
+ "SensitiveMeta",
142
+ on_delete=models.SET_NULL,
143
+ null=True,
144
+ blank=True,
145
+ related_name="video_file",
146
+ )
147
+ center = models.ForeignKey("Center", on_delete=models.PROTECT)
148
+ processor = models.ForeignKey(
149
+ "EndoscopyProcessor", on_delete=models.PROTECT, blank=True, null=True
150
+ )
151
+ video_meta = models.OneToOneField(
152
+ "VideoMeta",
153
+ on_delete=models.SET_NULL,
154
+ null=True,
155
+ blank=True,
156
+ related_name="video_file",
157
+ )
158
+ examination = models.ForeignKey(
159
+ "PatientExamination",
160
+ on_delete=models.SET_NULL,
161
+ blank=True,
162
+ null=True,
163
+ related_name="video_files",
164
+ )
165
+ patient = models.ForeignKey(
166
+ "Patient",
167
+ on_delete=models.SET_NULL,
168
+ blank=True,
169
+ null=True,
170
+ related_name="video_files",
171
+ )
172
+ ai_model_meta = models.ForeignKey(
173
+ "ModelMeta", on_delete=models.SET_NULL, blank=True, null=True
174
+ )
175
+ state = models.OneToOneField(
176
+ "VideoState",
177
+ on_delete=models.SET_NULL,
178
+ null=True,
179
+ blank=True,
180
+ related_name="video_file",
181
+ )
182
+ import_meta = models.OneToOneField(
183
+ "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
184
+ )
185
+
186
+ original_file_name = models.CharField(max_length=255, blank=True, null=True)
187
+ uploaded_at = models.DateTimeField(auto_now_add=True)
188
+ frame_dir = models.CharField(
189
+ max_length=512,
190
+ blank=True,
191
+ help_text="Path to frames extracted from the raw video.",
192
+ )
193
+ fps = models.FloatField(blank=True, null=True)
194
+ duration = models.FloatField(blank=True, null=True)
195
+ frame_count = models.IntegerField(blank=True, null=True)
196
+ width = models.IntegerField(blank=True, null=True)
197
+ height = models.IntegerField(blank=True, null=True)
198
+ suffix = models.CharField(max_length=10, blank=True, null=True)
199
+ sequences = models.JSONField(
200
+ default=dict,
201
+ blank=True,
202
+ help_text="AI prediction sequences based on raw frames.",
203
+ )
204
+ date = models.DateField(blank=True, null=True)
205
+ meta = models.JSONField(blank=True, null=True)
206
+ date_created = models.DateTimeField(auto_now_add=True)
207
+ date_modified = models.DateTimeField(auto_now=True)
208
+
209
+ if TYPE_CHECKING:
210
+ from django.db.models.manager import RelatedManager
211
+
212
+ @property
213
+ def label_video_segments(self) -> RelatedManager[LabelVideoSegment]: ...
214
+
215
+ @property
216
+ def frames(self) -> RelatedManager[Frame]: ...
217
+
218
+ center: models.ForeignKey["Center"]
219
+ processor: models.ForeignKey["EndoscopyProcessor | None"]
220
+ video_meta: models.OneToOneField["VideoMeta | None"]
221
+ examination: models.ForeignKey["PatientExamination | None"]
222
+ patient: models.ForeignKey["Patient | None"]
223
+ sensitive_meta: models.OneToOneField["SensitiveMeta | None"]
224
+ state: models.OneToOneField["VideoState | None"]
225
+ ai_model_meta: models.ForeignKey["ModelMeta | None"]
226
+ import_meta: models.OneToOneField["VideoImportMeta | None"]
227
+ raw_file = cast(FieldFile, raw_file)
228
+ processed_file = cast(FieldFile, processed_file)
229
+
230
+ @property
231
+ def ffmpeg_meta(self) -> "FFMpegMeta":
232
+ """
233
+ Return the associated FFMpegMeta instance for this video, initializing video specs if necessary.
234
+
235
+ Returns:
236
+ FFMpegMeta: The FFMpegMeta object containing metadata for this video.
237
+ """
238
+ from endoreg_db.models import FFMpegMeta
239
+
240
+ if self.video_meta is not None:
241
+ if self.video_meta.ffmpeg_meta is not None:
242
+ return self.video_meta.ffmpeg_meta
243
+ raise AssertionError("Expected FFMpegMeta instance.")
244
+ else:
245
+ self.initialize_video_specs()
246
+ ffmpeg_meta = self.video_meta.ffmpeg_meta if self.video_meta else None
247
+ assert isinstance(ffmpeg_meta, FFMpegMeta), "Expected FFMpegMeta instance."
248
+ return ffmpeg_meta
249
+
250
+ # Exception message constants
251
+
252
+ NO_ACTIVE_FILE = "Has no raw file"
253
+ NO_FILE_ASSOCIATED = "Active file has no associated file."
254
+
255
+ @property
256
+ def active_raw_file(self) -> File:
257
+ """Return the raw file if available, otherwise raise ValueError."""
258
+ if self.has_raw:
259
+ return self.raw_file
260
+ raise ValueError(self.NO_ACTIVE_FILE)
261
+
262
+ @property
263
+ def active_raw_file_url(self) -> str:
264
+ """Return the URL of the active raw file, or raise ValueError if unavailable."""
265
+ _file = self.active_raw_file
266
+ assert _file is not None, self.NO_ACTIVE_FILE
267
+ if not _file or not _file.name:
268
+ raise ValueError(self.NO_FILE_ASSOCIATED)
269
+ url = getattr(_file, "url", None)
270
+ if not url:
271
+ raise ValueError("Active raw file URL could not be resolved.")
272
+ return str(url)
273
+
274
+ # Pipeline Functions
275
+ pipe_1 = _pipe_1
276
+ test_after_pipe_1 = _test_after_pipe_1
277
+ pipe_2 = _pipe_2
278
+
279
+ # Metadata Funtions
280
+ update_video_meta = _update_video_meta
281
+ initialize_video_specs = _initialize_video_specs
282
+ get_fps = _get_fps
283
+ get_endo_roi = _get_endo_roi
284
+ get_crop_template = _get_crop_template
285
+ update_text_metadata = _update_text_metadata
286
+
287
+ extract_frames = _extract_frames
288
+ initialize_frames = _initialize_frames
289
+ delete_frames = _delete_frames
290
+ get_frame_path = _get_frame_path
291
+ get_frame_paths = _get_frame_paths
292
+ get_frame_number = _get_frame_number
293
+ get_frames = _get_frames
294
+ get_frame = _get_frame
295
+ get_frame_range = _get_frame_range
296
+ get_duration = _calc_duration_vf
297
+ create_frame_object = _create_frame_object
298
+ bulk_create_frames = _bulk_create_frames
299
+
300
+ # Define new methods that call the helper functions
301
+ def extract_specific_frame_range(
302
+ self, start_frame: int, end_frame: int, overwrite: bool = False, **kwargs
303
+ ) -> bool:
304
+ """
305
+ Extract frames from the video within the specified frame range.
306
+
307
+ Parameters:
308
+ start_frame (int): The starting frame number (inclusive).
309
+ end_frame (int): The ending frame number (exclusive).
310
+ overwrite (bool): Whether to overwrite existing frames in the range.
311
+
312
+ Returns:
313
+ bool: True if frame extraction was successful, False otherwise.
314
+
315
+ Additional keyword arguments:
316
+ quality (int, optional): Quality setting for extracted frames.
317
+ ext (str, optional): File extension for extracted frames.
318
+ verbose (bool, optional): Whether to enable verbose output.
319
+ """
320
+ quality = kwargs.get("quality", 2)
321
+ ext = kwargs.get("ext", "jpg")
322
+ verbose = kwargs.get("verbose", False)
323
+
324
+ # Log if unexpected kwargs are passed, beyond those used by the helper
325
+ expected_helper_kwargs = {"quality", "ext", "verbose"}
326
+ unexpected_kwargs = {
327
+ k: v for k, v in kwargs.items() if k not in expected_helper_kwargs
328
+ }
329
+ if unexpected_kwargs:
330
+ logger.warning(
331
+ f"Unexpected keyword arguments for extract_specific_frame_range, will be ignored by helper: {unexpected_kwargs}"
332
+ )
333
+
334
+ return _extract_frame_range_helper(
335
+ video=self,
336
+ start_frame=start_frame,
337
+ end_frame=end_frame,
338
+ quality=quality,
339
+ overwrite=overwrite,
340
+ ext=ext,
341
+ verbose=verbose,
342
+ )
343
+
344
+ def delete_specific_frame_range(self, start_frame: int, end_frame: int) -> None:
345
+ """
346
+ Deletes frame files for a specific range [start_frame, end_frame).
347
+ """
348
+ _delete_frame_range_helper(
349
+ video=self, start_frame=start_frame, end_frame=end_frame
350
+ )
351
+
352
+ delete_with_file = _delete_with_file
353
+ get_base_frame_dir = _get_base_frame_dir
354
+ set_frame_dir = _set_frame_dir
355
+ get_frame_dir_path = _get_frame_dir_path
356
+ get_temp_anonymized_frame_dir = _get_temp_anonymized_frame_dir
357
+ get_target_anonymized_video_path = _get_target_anonymized_video_path
358
+ get_raw_file_path = _get_raw_file_path
359
+ get_processed_file_path = _get_processed_file_path
360
+
361
+ anonymize = _anonymize
362
+ _create_anonymized_frame_files = _create_anonymized_frame_files
363
+ _cleanup_raw_assets = _cleanup_raw_assets
364
+
365
+ predict_video = _predict_video_pipeline
366
+ extract_text_from_frames = _extract_text_from_video_frames
367
+
368
+ @classmethod
369
+ def check_hash_exists(cls, video_hash: str) -> bool:
370
+ """
371
+ Checks if a VideoFile with the given raw video hash already exists.
372
+ """
373
+ return cls.objects.filter(video_hash=video_hash).exists()
374
+
375
+ @property
376
+ def is_processed(self) -> bool:
377
+ return bool(self.processed_file and self.processed_file.name)
378
+
379
+ @property
380
+ def has_raw(self) -> bool:
381
+ """
382
+ Return True if a raw video file is associated with this instance.
383
+ """
384
+ return bool(self.raw_file and self.raw_file.name)
385
+
386
+ @property
387
+ def active_file(self) -> FieldFile:
388
+ """
389
+ Return the active video file, preferring the processed file if available.
390
+
391
+ Returns:
392
+ File: The processed file if present; otherwise, the raw file.
393
+
394
+ Raises:
395
+ ValueError: If neither a processed nor a raw file is available.
396
+ """
397
+ processed = self.processed_file
398
+ if isinstance(processed, FieldFile) and processed.name:
399
+ return processed
400
+
401
+ raw = self.raw_file
402
+ if isinstance(raw, FieldFile) and raw.name:
403
+ return raw
404
+
405
+ raise ValueError(
406
+ "No active file available. VideoFile has neither raw nor processed file."
407
+ )
408
+
409
+ @property
410
+ def active_file_path(self) -> Path:
411
+ """
412
+ Return the filesystem path of the active video file.
413
+
414
+ Returns:
415
+ Path: The path to the processed file if available, otherwise the raw file.
416
+
417
+ Raises:
418
+ ValueError: If neither a processed nor raw file is present.
419
+ """
420
+ active = self.active_file
421
+ if active is self.processed_file:
422
+ path = _get_processed_file_path(self)
423
+ elif active is self.raw_file:
424
+ path = _get_raw_file_path(self)
425
+ else:
426
+ raise ValueError(
427
+ "No active file path available. VideoFile has neither raw nor processed file."
428
+ )
429
+
430
+ if path is None:
431
+ raise ValueError(
432
+ "Active file path could not be resolved. VideoFile raw file is missing."
433
+ )
434
+ return path
435
+
436
+ @property
437
+ def active_file_url(self) -> str:
438
+ """Return the URL of the active video file, if available."""
439
+ file_obj = self.active_file
440
+ if not isinstance(file_obj, FieldFile):
441
+ raise ValueError("Active file is not a valid Django FieldFile instance.")
442
+ try:
443
+ url = getattr(file_obj, "url", None)
444
+ except Exception as exc: # storage backends may raise when missing
445
+ logger.warning(
446
+ "Active file URL unavailable for video %s: %s",
447
+ self.video_hash,
448
+ exc,
449
+ )
450
+ raise ValueError(
451
+ "Active file URL could not be resolved for this VideoFile."
452
+ ) from exc
453
+
454
+ if not url:
455
+ raise ValueError("Active file URL is empty for this VideoFile.")
456
+
457
+ return str(url)
458
+
459
+ @classmethod
460
+ def create_from_file(
461
+ cls, file_path: Union[str, Path], center_name: str, **kwargs
462
+ ) -> Optional["VideoFile"]:
463
+ # Ensure file_path is a Path object
464
+ if isinstance(file_path, str):
465
+ file_path = Path(file_path)
466
+ # Pass center_name and other kwargs to the helper function
467
+ if not center_name:
468
+ try:
469
+ center_name = os.environ["CENTER_NAME"]
470
+ except KeyError:
471
+ logger.error(
472
+ "Center name must be provided to create VideoFile from file. You can set CENTER_NAME in environment variables."
473
+ )
474
+ return None
475
+ return _create_from_file(cls, file_path, center_name=center_name, **kwargs)
476
+
477
+ @classmethod
478
+ def create_from_file_initialized(
479
+ cls,
480
+ file_path: Union[str, Path],
481
+ center_name: str,
482
+ processor_name: Optional[str],
483
+ video_hash: str,
484
+ delete_source: bool = False,
485
+ save_video_file: bool = True,
486
+ ):
487
+ """
488
+ Creates a VideoFile instance from a given video file path.
489
+ Handles transcoding (if necessary), hashing, file storage, and database record creation.
490
+ Raises exceptions on failure.
491
+ """
492
+ # Ensure file_path is a Path object
493
+ if isinstance(file_path, str):
494
+ file_path = Path(file_path)
495
+
496
+ # Call the helper function to create the VideoFile instance
497
+ video_file = _create_from_file(
498
+ cls_model=VideoFile,
499
+ file_path=file_path,
500
+ center_name=center_name,
501
+ processor_name=processor_name,
502
+ video_hash=video_hash,
503
+ delete_source=delete_source,
504
+ save=save_video_file, # Add this line
505
+ )
506
+
507
+ video_file = video_file.initialize()
508
+ return video_file
509
+
510
+ def delete(self, using=None, keep_parents=False) -> tuple[int, dict[str, int]]:
511
+ """
512
+ Delete the VideoFile instance, including associated files and frames.
513
+
514
+ Overrides the default delete method to ensure proper cleanup of related resources.
515
+ """
516
+ # Ensure frames are deleted before the main instance
517
+ _delete_frames(self)
518
+
519
+ # Call the original delete method to remove the instance from the database
520
+ try:
521
+ active_path = self.active_file_path
522
+ logger.info(f"Deleting VideoFile: {self.video_hash} - {active_path}")
523
+
524
+ except ValueError:
525
+ logger.info(
526
+ f"Deleting VideoFile: {self.video_hash} - No active file path found."
527
+ )
528
+ active_path = None
529
+
530
+ # Delete associated files if they exist
531
+ if active_path and active_path.exists():
532
+ active_path.unlink(missing_ok=True)
533
+
534
+ # Delete file storage
535
+ if self.raw_file and self.raw_file.storage.exists(self.raw_file.name):
536
+ self.raw_file.storage.delete(self.raw_file.name)
537
+ if self.processed_file and self.processed_file.storage.exists(
538
+ self.processed_file.name
539
+ ):
540
+ self.processed_file.storage.delete(self.processed_file.name)
541
+
542
+ # Use proper database connection
543
+ if using is None:
544
+ using = "default"
545
+
546
+ raw_file_path = self.get_raw_file_path()
547
+ if raw_file_path:
548
+ raw_file_path = Path(raw_file_path)
549
+ lock_path = raw_file_path.with_suffix(raw_file_path.suffix + ".lock")
550
+ if lock_path.exists():
551
+ try:
552
+ lock_path.unlink()
553
+ logger.info(f"Removed processing lock: {lock_path}")
554
+ except Exception as e:
555
+ logger.warning(f"Could not remove processing lock {lock_path}: {e}")
556
+
557
+ try:
558
+ # Call parent delete with proper parameters
559
+ result = super().delete(using=using, keep_parents=keep_parents)
560
+ logger.info(f"VideoFile {self.video_hash} deleted successfully.")
561
+ return result
562
+ except Exception as e:
563
+ logger.error(f"Error deleting VideoFile {self.video_hash}: {e}")
564
+ raise
565
+
566
+ def validate_metadata_annotation(
567
+ self, extracted_data_dict: Optional[dict] = None
568
+ ) -> bool:
569
+ """
570
+ Validate the metadata of the VideoFile instance.
571
+
572
+ Called after annotation in the frontend, this method:
573
+ 1. Updates sensitive metadata with user-annotated data
574
+ 2. Deletes the RAW video file (keeping only the anonymized version)
575
+ 3. Marks the video as validated
576
+
577
+ **IMPORTANT:** Only the raw video is deleted. The processed (anonymized)
578
+ video is preserved as the final validated output.
579
+ """
580
+
581
+ # CRITICAL FIX: Delete RAW video file, not the processed (anonymized) one
582
+ # CRITICAL: Update metadata BEFORE deleting raw video
583
+ if extracted_data_dict and self.sensitive_meta:
584
+ self.sensitive_meta.update_from_dict(extracted_data_dict)
585
+ else:
586
+ return False
587
+
588
+ # After validation and metadata update, only the anonymized video should remain
589
+ from .video_file_io import _get_raw_file_path
590
+
591
+ raw_path = _get_raw_file_path(self)
592
+
593
+ if raw_path and raw_path.exists():
594
+ logger.info(f"Deleting raw video file after validation: {raw_path}")
595
+ raw_path.unlink(missing_ok=True)
596
+ # Clear the raw_file field in database (use delete() to avoid save issues)
597
+ if self.raw_file:
598
+ self.raw_file.delete(save=False)
599
+ logger.info(
600
+ f"Raw video deleted for {self.video_hash}. Anonymized video preserved."
601
+ )
602
+ else:
603
+ logger.warning(
604
+ "Raw video file not found for deletion during validation %s.",
605
+ self.video_hash,
606
+ )
607
+
608
+ if self.sensitive_meta:
609
+ # Mark as processed after validation
610
+ self.get_or_create_state().mark_anonymization_validated(save=True)
611
+ # Save the VideoFile instance to persist changes
612
+ self.save()
613
+ logger.info(
614
+ f"Metadata annotation validated and saved for video {self.video_hash}."
615
+ )
616
+ return True
617
+ else:
618
+ logger.error(
619
+ f"Failed to validate metadata annotation for video {self.video_hash}."
620
+ )
621
+ return False
622
+
623
+ def initialize(self):
624
+ """
625
+ Initialize the VideoFile instance by updating metadata, setting up video specs, assigning frame directory, ensuring related state and sensitive metadata exist, saving the instance, and initializing frames.
626
+
627
+ Returns:
628
+ VideoFile: The initialized VideoFile instance.
629
+ """
630
+
631
+ self.update_video_meta()
632
+ try:
633
+ # We explicitly check this because initialize_video_specs raises RuntimeError
634
+ # if the file is missing, which kills the import pipeline.
635
+ if self.get_raw_file_path():
636
+ self.initialize_video_specs(use_raw=True)
637
+ else:
638
+ logger.error(f"Skipping video specs init for {self.video_hash}: Raw file path not found.")
639
+ except Exception as e:
640
+ # Log the specific error but allow the function to continue to state creation
641
+ logger.error(f"Failed to initialize video specs for {self.video_hash}: {e}")
642
+
643
+ # Set the frame directory
644
+ self.set_frame_dir()
645
+
646
+ # Create a new state if it doesn't exist
647
+ self.state = self.get_or_create_state()
648
+
649
+ self.save()
650
+ # Initialize frames based on the video specs
651
+ self.initialize_frames()
652
+
653
+ return self
654
+
655
+ def __str__(self):
656
+ """
657
+ Return a human-readable string summarizing the video's state, active file name, and UUID.
658
+ """
659
+ active_path = self.active_file_path
660
+ file_name = active_path.name if active_path else "No file"
661
+ state = (
662
+ "Processed" if self.is_processed else ("Raw" if self.has_raw else "No File")
663
+ )
664
+ return f"VideoFile ({state}): {file_name} (UUID: {self.video_hash})"
665
+
666
+ # --- Convenience state/meta helpers used in tests and admin workflows ---
667
+ def mark_sensitive_meta_processed(self, *, save: bool = True) -> "VideoFile":
668
+ """
669
+ Mark this video's processing state as having its sensitive meta fully processed.
670
+ This proxies to the related VideoState and persists by default.
671
+ """
672
+ sm = self.sensitive_meta
673
+ from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
674
+
675
+ if not isinstance(sm, SensitiveMeta):
676
+ raise AttributeError()
677
+ state = self.get_or_create_state()
678
+ state.mark_sensitive_meta_processed(save=save)
679
+ return self
680
+
681
+ def mark_sensitive_meta_verified(self) -> "VideoFile":
682
+ """
683
+ Mark the associated SensitiveMeta as verified by setting both DOB and names as verified.
684
+ Ensures the SensitiveMeta and its state exist.
685
+ """
686
+ sm = self.sensitive_meta
687
+ # Use SensitiveMeta methods to update underlying SensitiveMetaState
688
+ from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
689
+
690
+ if not isinstance(sm, SensitiveMeta):
691
+ raise AttributeError()
692
+
693
+ sm.mark_dob_verified()
694
+ sm.mark_names_verified()
695
+ return self
696
+
697
+ def save(self, *args, **kwargs):
698
+ # Ensure state exists or is created before the main save operation
699
+ # Now call the original save method
700
+ """
701
+ Saves the VideoFile instance to the database.
702
+
703
+ Overrides the default save method to persist changes to the VideoFile model.
704
+ """
705
+ super().save(*args, **kwargs)
706
+
707
+ def get_or_create_state(self) -> "VideoState":
708
+ """Ensure this video has a persisted ``VideoState`` and return it."""
709
+
710
+ state = self.state
711
+
712
+ # When tests reuse cached instances across database flushes, ``state`` may reference
713
+ # a row that no longer exists. Guard against that by validating persistence.
714
+ state_pk = getattr(state, "pk", None)
715
+ if state is not None and state_pk is not None:
716
+ if not VideoState.objects.filter(pk=state_pk).exists():
717
+ state = None
718
+
719
+ if state is None:
720
+ # Create a fresh state to avoid refresh_from_db() failures on unsaved instances.
721
+ state = VideoState.objects.create()
722
+ self.state = state
723
+
724
+ # Persist the relation immediately if the VideoFile already exists in the DB so
725
+ # later refreshes see the association without requiring additional saves.
726
+ if self.pk:
727
+ self.save(update_fields=["state"])
728
+
729
+ return state
730
+
731
+ def get_outside_segments(
732
+ self, only_validated: bool = False
733
+ ) -> models.QuerySet["LabelVideoSegment"]:
734
+ """
735
+ Return all video segments labeled as "outside" for this video.
736
+
737
+ Parameters:
738
+ only_validated (bool): If True, only segments with a validated state are included.
739
+
740
+ Returns:
741
+ QuerySet: A queryset of LabelVideoSegment instances labeled as "outside". Returns an empty queryset if the label does not exist or an error occurs.
742
+ """
743
+ try:
744
+ outside_label = Label.objects.get(name__iexact="outside")
745
+ segments = self.label_video_segments.filter(label=outside_label)
746
+
747
+ if only_validated:
748
+ # Filter based on the is_validated field in the related state object
749
+ segments = segments.filter(state__is_validated=True)
750
+
751
+ return segments
752
+ except Label.DoesNotExist:
753
+ logger.warning("Outside label not found in the database.")
754
+ return self.label_video_segments.none()
755
+ except Exception as e:
756
+ logger.error(
757
+ "Error getting outside segments for video %s: %s",
758
+ self.video_hash,
759
+ e,
760
+ exc_info=True,
761
+ )
762
+ return self.label_video_segments.none()
763
+
764
+ @classmethod
765
+ def create_video_without_outside_frames(
766
+ cls, instance: "VideoFile", only_validated: bool = False
767
+ ) -> bool:
768
+ """
769
+ Creates a new video by excluding frames that belong to 'outside' segments.
770
+
771
+ Parameters:
772
+ only_validated (bool): If True, only validated segments are considered for frame exclusion.
773
+
774
+ Returns:
775
+ VideoFile: A new VideoFile instance with the frames excluding those labeled as 'outside'.
776
+ """
777
+ video = instance
778
+
779
+ if not video:
780
+ logger.warning(
781
+ "No processed video file available for VideoFile %s.",
782
+ instance.video_hash,
783
+ )
784
+ return False
785
+ try:
786
+ extracted = video.extract_frames(
787
+ quality=2,
788
+ overwrite=False,
789
+ ext="jpg",
790
+ verbose=False,
791
+ from_processed=True,
792
+ )
793
+ assert extracted is True
794
+ except AssertionError:
795
+ extracted = video.extract_frames(
796
+ quality=2,
797
+ overwrite=False,
798
+ ext="jpg",
799
+ verbose=False,
800
+ from_processed=True,
801
+ )
802
+ assert extracted is True
803
+ try:
804
+ # Step 1: Get the "outside" labeled frames
805
+ censored = _censor_outside_frames(video)
806
+ frames = [instance.get_frame_dir_path()]
807
+ assert len(frames) != 0
808
+ fps = (
809
+ video.fps if video.fps else 120.0
810
+ ) # Default to 30 FPS if fps is not set
811
+ assert censored is True
812
+ assert fps is not None
813
+ assert video.width is not None
814
+ assert video.height is not None
815
+ # assert isinstance(frames, list[Path]) #TODO improve TypeCheck
816
+
817
+ # Step 2: Reassemble the video with frames excluding the 'outside' labeled frames
818
+ output_video_path = Path(f"/path/to/output/{video.video_hash}_filtered.mp4")
819
+ fps = (
820
+ video.fps if video.fps else 30.0
821
+ ) # Default to 30 FPS if fps is not set
822
+ new_video_file = assemble_video_from_frames(
823
+ frames, output_video_path, fps, width=video.width, height=video.height
824
+ )
825
+ video.processed_file = new_video_file
826
+ return True
827
+ except AssertionError as ae:
828
+ logger.error(
829
+ f"Assertion error while creating video without 'outside' frames for VideoFile {video.video_hash}: {ae}",
830
+ exc_info=True,
831
+ )
832
+ return False
833
+ except Label.DoesNotExist:
834
+ logger.warning("Outside label not found in the database.")
835
+ return False
836
+ except Exception as e:
837
+ logger.error(
838
+ f"Error creating video without 'outside' frames for VideoFile {video.video_hash}: {e}",
839
+ exc_info=True,
840
+ )
841
+ return False
842
+
843
+ @classmethod
844
+ def get_all_videos(cls) -> models.QuerySet["VideoFile"]:
845
+ """
846
+ Returns a queryset containing all VideoFile records.
847
+
848
+ This class method retrieves every VideoFile instance in the database without filtering.
849
+ """
850
+ return cast(models.QuerySet["VideoFile"], cls.objects.all())
851
+
852
+ def count_unmodified_others(self) -> int:
853
+ """
854
+ Count the number of other VideoFile instances that have not been modified since creation.
855
+
856
+ Returns:
857
+ int: The count of VideoFile records, excluding this instance, where the modification timestamp matches the creation timestamp.
858
+ """
859
+ return (
860
+ VideoFile.objects.filter(
861
+ date_modified=F("date_created")
862
+ ) # compare the two fields in SQL
863
+ .exclude(pk=self.pk) # exclude this instance
864
+ .count() # run a fast COUNT(*) on the filtered set
865
+ )
866
+
867
+ def frame_number_to_s(self, frame_number: int) -> float:
868
+ """
869
+ Convert a frame number to its corresponding time in seconds based on the video's frames per second (FPS).
870
+
871
+ Parameters:
872
+ frame_number (int): The frame number to convert.
873
+
874
+ Returns:
875
+ float: The time in seconds corresponding to the given frame number.
876
+
877
+ Raises:
878
+ ValueError: If the video's FPS is not set or is less than or equal to zero.
879
+ """
880
+ fps = self.get_fps()
881
+ if fps is None or fps <= 0:
882
+ raise ValueError("FPS must be set and greater than zero.")
883
+ return frame_number / fps
884
+
885
+ @staticmethod
886
+ def get_video_by_pk(pk: int) -> "VideoFile":
887
+ """
888
+ Retrieve a VideoFile instance by its primary key (ID).
889
+
890
+ Parameters:
891
+ video_id (int): The primary key of the VideoFile to retrieve.
892
+
893
+ Returns:
894
+ VideoFile: The VideoFile instance with the specified ID.
895
+
896
+ Raises:
897
+ VideoFile.DoesNotExist: If no VideoFile with the given ID exists.
898
+ """
899
+ return VideoFile.objects.get(pk=pk)
900
+
901
+ @staticmethod
902
+ def get_video_by_content_hash(hash: str) -> "VideoFile":
903
+ try:
904
+ return VideoFile.objects.get(video_hash=hash)
905
+ except Exception as e:
906
+ logger.error(f"Video cant be returned for known hash. {e}")
907
+ raise