endoreg-db 0.4.5__py3-none-any.whl → 0.8.6.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (846) hide show
  1. endoreg_db/admin.py +90 -1
  2. endoreg_db/api_urls.py +4 -0
  3. endoreg_db/apps.py +12 -0
  4. endoreg_db/assets/dummy_model.ckpt +1 -0
  5. endoreg_db/codemods/readme.md +88 -0
  6. endoreg_db/codemods/rename_datetime_fields.py +92 -0
  7. endoreg_db/config/env.py +101 -0
  8. endoreg_db/data/__init__.py +76 -4
  9. endoreg_db/data/ai_model/data.yaml +7 -0
  10. endoreg_db/data/{label → ai_model_label}/label/data.yaml +27 -1
  11. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  12. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  13. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  14. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  15. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  16. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  17. endoreg_db/data/center/data.yaml +40 -9
  18. endoreg_db/data/center_shift/ukw.yaml +9 -0
  19. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  20. endoreg_db/data/db_summary.csv +58 -0
  21. endoreg_db/data/db_summary.xlsx +0 -0
  22. endoreg_db/data/disease/misc.yaml +1 -2
  23. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +2 -2
  24. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +6 -6
  25. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  26. endoreg_db/data/endoscope/data.yaml +93 -0
  27. endoreg_db/data/endoscopy_processor/data.yaml +3 -0
  28. endoreg_db/data/event/cardiology.yaml +0 -13
  29. endoreg_db/data/examination/examinations/data.yaml +34 -28
  30. endoreg_db/data/examination/type/data.yaml +12 -0
  31. endoreg_db/data/examination_indication/endoscopy.yaml +424 -0
  32. endoreg_db/data/examination_indication_classification/endoscopy.yaml +160 -0
  33. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +101 -0
  34. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  35. endoreg_db/data/finding/anatomy_colon.yaml +128 -0
  36. endoreg_db/data/finding/colonoscopy.yaml +40 -0
  37. endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +56 -0
  38. endoreg_db/data/finding/complication.yaml +16 -0
  39. endoreg_db/data/finding/data.yaml +105 -0
  40. endoreg_db/data/finding/examination_setting.yaml +16 -0
  41. endoreg_db/data/finding/medication_related.yaml +18 -0
  42. endoreg_db/data/finding/outcome.yaml +12 -0
  43. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +95 -0
  44. endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +22 -0
  45. endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +25 -0
  46. endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  47. endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  48. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  49. endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  50. endoreg_db/data/finding_classification/colonoscopy_location.yaml +80 -0
  51. endoreg_db/data/finding_classification/colonoscopy_lst.yaml +21 -0
  52. endoreg_db/data/finding_classification/colonoscopy_nice.yaml +20 -0
  53. endoreg_db/data/finding_classification/colonoscopy_paris.yaml +26 -0
  54. endoreg_db/data/finding_classification/colonoscopy_sano.yaml +22 -0
  55. endoreg_db/data/finding_classification/colonoscopy_summary.yaml +53 -0
  56. endoreg_db/data/finding_classification/complication_generic.yaml +25 -0
  57. endoreg_db/data/finding_classification/examination_setting_generic.yaml +40 -0
  58. endoreg_db/data/finding_classification/histology_colo.yaml +51 -0
  59. endoreg_db/data/finding_classification/intervention_required.yaml +26 -0
  60. endoreg_db/data/finding_classification/medication_related.yaml +23 -0
  61. endoreg_db/data/finding_classification/visualized.yaml +33 -0
  62. endoreg_db/data/finding_classification_choice/bowel_preparation.yaml +78 -0
  63. endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
  64. endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  65. endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  66. endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  67. endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +17 -0
  68. endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +57 -0
  69. endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
  70. endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  71. endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
  72. endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +229 -0
  73. endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  74. endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +82 -0
  75. endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  76. endoreg_db/data/finding_classification_choice/complication_generic_types.yaml +15 -0
  77. endoreg_db/data/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  78. endoreg_db/data/finding_classification_choice/histology.yaml +24 -0
  79. endoreg_db/data/finding_classification_choice/histology_polyp.yaml +20 -0
  80. endoreg_db/data/finding_classification_choice/outcome.yaml +19 -0
  81. endoreg_db/data/finding_classification_choice/yes_no_na.yaml +11 -0
  82. endoreg_db/data/finding_classification_type/colonoscopy_basic.yaml +48 -0
  83. endoreg_db/data/finding_intervention/endoscopy.yaml +43 -0
  84. endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
  85. endoreg_db/data/finding_intervention/endoscopy_egd.yaml +128 -0
  86. endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +32 -0
  87. endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +9 -0
  88. endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +36 -0
  89. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  90. endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +79 -0
  91. endoreg_db/data/finding_type/data.yaml +43 -0
  92. endoreg_db/data/gender/data.yaml +24 -0
  93. endoreg_db/data/information_source/annotation.yaml +6 -0
  94. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  95. endoreg_db/data/information_source/prediction.yaml +7 -0
  96. endoreg_db/data/information_source_type/data.yaml +8 -0
  97. endoreg_db/data/lab_value/cardiac_enzymes.yaml +7 -1
  98. endoreg_db/data/lab_value/coagulation.yaml +6 -1
  99. endoreg_db/data/lab_value/electrolytes.yaml +39 -1
  100. endoreg_db/data/lab_value/gastrointestinal_function.yaml +12 -0
  101. endoreg_db/data/lab_value/hematology.yaml +17 -2
  102. endoreg_db/data/lab_value/hormones.yaml +6 -0
  103. endoreg_db/data/lab_value/lipids.yaml +12 -3
  104. endoreg_db/data/lab_value/misc.yaml +48 -2
  105. endoreg_db/data/lab_value/renal_function.yaml +2 -1
  106. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  107. endoreg_db/data/lx_client_type/base.yaml +30 -0
  108. endoreg_db/data/lx_permission/base.yaml +24 -0
  109. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  110. endoreg_db/data/medication/anticoagulation.yaml +5 -5
  111. endoreg_db/data/medication/tah.yaml +5 -5
  112. endoreg_db/data/medication_indication/anticoagulation.yaml +48 -53
  113. endoreg_db/data/medication_intake_time/base.yaml +4 -4
  114. endoreg_db/data/names_first/first_names.yaml +54 -0
  115. endoreg_db/data/names_last/last_names.yaml +51 -0
  116. endoreg_db/data/network_device/data.yaml +30 -0
  117. endoreg_db/data/organ/data.yaml +29 -0
  118. endoreg_db/data/pdf_type/data.yaml +27 -9
  119. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  120. endoreg_db/data/qualification/m2.yaml +39 -0
  121. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  122. endoreg_db/data/qualification/sonography.yaml +36 -0
  123. endoreg_db/data/qualification_type/base.yaml +29 -0
  124. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  125. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +4 -0
  126. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +5 -0
  127. endoreg_db/data/requirement/age.yaml +26 -0
  128. endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +45 -0
  129. endoreg_db/data/requirement/disease_cardiovascular.yaml +79 -0
  130. endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
  131. endoreg_db/data/requirement/disease_hepatology.yaml +12 -0
  132. endoreg_db/data/requirement/disease_misc.yaml +12 -0
  133. endoreg_db/data/requirement/disease_renal.yaml +96 -0
  134. endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +59 -0
  135. endoreg_db/data/requirement/event_cardiology.yaml +251 -0
  136. endoreg_db/data/requirement/event_requirements.yaml +145 -0
  137. endoreg_db/data/requirement/finding_colon_polyp.yaml +50 -0
  138. endoreg_db/data/requirement/gender.yaml +25 -0
  139. endoreg_db/data/requirement/lab_value.yaml +441 -0
  140. endoreg_db/data/requirement/medication.yaml +93 -0
  141. endoreg_db/data/requirement_operator/age.yaml +13 -0
  142. endoreg_db/data/requirement_operator/lab_operators.yaml +129 -0
  143. endoreg_db/data/requirement_operator/model_operators.yaml +96 -0
  144. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +48 -0
  145. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  146. endoreg_db/data/requirement_set/endoscopy_bleeding_risk.yaml +52 -0
  147. endoreg_db/data/requirement_set_type/data.yaml +20 -0
  148. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  149. endoreg_db/data/risk/bleeding.yaml +26 -0
  150. endoreg_db/data/risk/thrombosis.yaml +37 -0
  151. endoreg_db/data/risk_type/data.yaml +27 -0
  152. endoreg_db/data/setup_config.yaml +38 -0
  153. endoreg_db/data/shift/endoscopy.yaml +21 -0
  154. endoreg_db/data/shift_type/base.yaml +35 -0
  155. endoreg_db/data/tag/requirement_set_tags.yaml +11 -0
  156. endoreg_db/data/unit/concentration.yaml +23 -0
  157. endoreg_db/data/unit/time.yaml +36 -1
  158. endoreg_db/exceptions.py +19 -0
  159. endoreg_db/forms/__init__.py +3 -1
  160. endoreg_db/forms/examination_form.py +11 -0
  161. endoreg_db/forms/patient_finding_intervention_form.py +18 -0
  162. endoreg_db/forms/patient_form.py +27 -0
  163. endoreg_db/forms/questionnaires/__init__.py +1 -1
  164. endoreg_db/forms/questionnaires/tto_questionnaire.py +19 -19
  165. endoreg_db/helpers/count_db.py +45 -0
  166. endoreg_db/helpers/data_loader.py +208 -0
  167. endoreg_db/helpers/default_objects.py +378 -0
  168. endoreg_db/helpers/download_segmentation_model.py +31 -0
  169. endoreg_db/helpers/interact.py +6 -0
  170. endoreg_db/helpers/test_video_helper.py +119 -0
  171. endoreg_db/logger_conf.py +140 -0
  172. endoreg_db/management/__init__.py +1 -0
  173. endoreg_db/management/commands/__init__.py +1 -0
  174. endoreg_db/management/commands/anonymize_video.py +0 -0
  175. endoreg_db/management/commands/check_auth.py +125 -0
  176. endoreg_db/management/commands/create_model_meta_from_huggingface.py +115 -0
  177. endoreg_db/management/commands/create_multilabel_model_meta.py +214 -0
  178. endoreg_db/management/commands/fix_missing_patient_data.py +172 -0
  179. endoreg_db/management/commands/fix_video_paths.py +165 -0
  180. endoreg_db/management/commands/import_fallback_video.py +203 -0
  181. endoreg_db/management/commands/import_report.py +298 -0
  182. endoreg_db/management/commands/import_video.py +423 -0
  183. endoreg_db/management/commands/import_video_with_classification.py +367 -0
  184. endoreg_db/management/commands/init_default_ai_model.py +112 -0
  185. endoreg_db/management/commands/load_ai_model_data.py +58 -26
  186. endoreg_db/management/commands/load_ai_model_label_data.py +59 -0
  187. endoreg_db/management/commands/load_base_db_data.py +174 -118
  188. endoreg_db/management/commands/load_center_data.py +46 -21
  189. endoreg_db/management/commands/{load_logging_data.py → load_contraindication_data.py} +4 -2
  190. endoreg_db/management/commands/load_disease_data.py +29 -7
  191. endoreg_db/management/commands/{load_endoscope_type_data.py → load_endoscope_data.py} +30 -7
  192. endoreg_db/management/commands/load_examination_indication_data.py +86 -0
  193. endoreg_db/management/commands/load_finding_data.py +128 -0
  194. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +0 -1
  195. endoreg_db/management/commands/load_information_source.py +13 -7
  196. endoreg_db/management/commands/load_lab_value_data.py +3 -3
  197. endoreg_db/management/commands/load_medication_data.py +83 -21
  198. endoreg_db/management/commands/load_name_data.py +37 -0
  199. endoreg_db/management/commands/{load_medication_intake_time_data.py → load_organ_data.py} +7 -5
  200. endoreg_db/management/commands/load_qualification_data.py +59 -0
  201. endoreg_db/management/commands/load_requirement_data.py +180 -0
  202. endoreg_db/management/commands/load_risk_data.py +56 -0
  203. endoreg_db/management/commands/load_shift_data.py +60 -0
  204. endoreg_db/management/commands/load_tag_data.py +57 -0
  205. endoreg_db/management/commands/register_ai_model.py +1 -1
  206. endoreg_db/management/commands/setup_endoreg_db.py +381 -0
  207. endoreg_db/management/commands/start_filewatcher.py +106 -0
  208. endoreg_db/management/commands/storage_management.py +548 -0
  209. endoreg_db/management/commands/summarize_db_content.py +189 -0
  210. endoreg_db/management/commands/validate_video.py +204 -0
  211. endoreg_db/management/commands/validate_video_files.py +161 -0
  212. endoreg_db/management/commands/video_validation.py +22 -0
  213. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  214. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  215. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  216. endoreg_db/mermaid/classification.md +8 -0
  217. endoreg_db/mermaid/examination.md +8 -0
  218. endoreg_db/mermaid/findings.md +7 -0
  219. endoreg_db/mermaid/image_classification.md +28 -0
  220. endoreg_db/mermaid/interventions.md +8 -0
  221. endoreg_db/mermaid/morphology.md +8 -0
  222. endoreg_db/mermaid/patient_creation.md +14 -0
  223. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  224. endoreg_db/migrations/0001_initial.py +1234 -944
  225. endoreg_db/migrations/0002_add_video_correction_models.py +52 -0
  226. endoreg_db/migrations/0003_add_center_display_name.py +30 -0
  227. endoreg_db/models/__init__.py +339 -53
  228. endoreg_db/models/administration/__init__.py +116 -0
  229. endoreg_db/models/administration/ai/__init__.py +9 -0
  230. endoreg_db/models/administration/ai/active_model.py +35 -0
  231. endoreg_db/models/administration/ai/ai_model.py +156 -0
  232. endoreg_db/models/{ai_model → administration/ai}/model_type.py +19 -4
  233. endoreg_db/models/administration/case/__init__.py +19 -0
  234. endoreg_db/models/administration/case/case.py +114 -0
  235. endoreg_db/models/{case_template → administration/case/case_template}/__init__.py +10 -1
  236. endoreg_db/models/{case_template → administration/case/case_template}/case_template.py +60 -16
  237. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule.py +6 -13
  238. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule_value.py +21 -8
  239. endoreg_db/models/{case_template → administration/case/case_template}/case_template_type.py +1 -3
  240. endoreg_db/models/{center → administration/center}/__init__.py +9 -0
  241. endoreg_db/models/administration/center/center.py +67 -0
  242. endoreg_db/models/administration/center/center_product.py +64 -0
  243. endoreg_db/models/administration/center/center_resource.py +49 -0
  244. endoreg_db/models/administration/center/center_shift.py +88 -0
  245. endoreg_db/models/administration/center/center_waste.py +30 -0
  246. endoreg_db/models/administration/permissions/__init__.py +44 -0
  247. endoreg_db/models/administration/person/__init__.py +24 -0
  248. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  249. endoreg_db/models/administration/person/employee/employee.py +35 -0
  250. endoreg_db/models/administration/person/employee/employee_qualification.py +39 -0
  251. endoreg_db/models/administration/person/employee/employee_type.py +42 -0
  252. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  253. endoreg_db/models/administration/person/examiner/examiner.py +54 -0
  254. endoreg_db/models/administration/person/names/__init__.py +0 -0
  255. endoreg_db/models/{persons → administration/person/names}/first_name.py +1 -1
  256. endoreg_db/models/{persons → administration/person/names}/last_name.py +2 -3
  257. endoreg_db/models/administration/person/patient/__init__.py +5 -0
  258. endoreg_db/models/administration/person/patient/patient.py +460 -0
  259. endoreg_db/models/administration/person/profession/__init__.py +24 -0
  260. endoreg_db/models/administration/person/user/__init__.py +5 -0
  261. endoreg_db/models/administration/person/user/portal_user_information.py +37 -0
  262. endoreg_db/models/administration/product/__init__.py +14 -0
  263. endoreg_db/models/administration/product/product.py +97 -0
  264. endoreg_db/models/administration/product/product_group.py +39 -0
  265. endoreg_db/models/administration/product/product_material.py +54 -0
  266. endoreg_db/models/{product → administration/product}/product_weight.py +21 -0
  267. endoreg_db/models/{product → administration/product}/reference_product.py +44 -13
  268. endoreg_db/models/administration/qualification/__init__.py +7 -0
  269. endoreg_db/models/administration/qualification/qualification.py +37 -0
  270. endoreg_db/models/administration/qualification/qualification_type.py +35 -0
  271. endoreg_db/models/administration/shift/__init__.py +9 -0
  272. endoreg_db/models/administration/shift/scheduled_days.py +69 -0
  273. endoreg_db/models/administration/shift/shift.py +51 -0
  274. endoreg_db/models/administration/shift/shift_type.py +108 -0
  275. endoreg_db/models/label/__init__.py +24 -1
  276. endoreg_db/models/label/annotation/__init__.py +12 -0
  277. endoreg_db/models/label/annotation/image_classification.py +84 -0
  278. endoreg_db/models/label/annotation/video_segmentation_annotation.py +66 -0
  279. endoreg_db/models/label/label.py +53 -54
  280. endoreg_db/models/label/label_set.py +53 -0
  281. endoreg_db/models/label/label_type.py +29 -0
  282. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  283. endoreg_db/models/label/label_video_segment/_create_from_video.py +41 -0
  284. endoreg_db/models/label/label_video_segment/label_video_segment.py +511 -0
  285. endoreg_db/models/label/video_segmentation_label.py +31 -0
  286. endoreg_db/models/label/video_segmentation_labelset.py +27 -0
  287. endoreg_db/models/media/__init__.py +16 -0
  288. endoreg_db/models/media/frame/__init__.py +3 -0
  289. endoreg_db/models/media/frame/frame.py +111 -0
  290. endoreg_db/models/media/pdf/__init__.py +11 -0
  291. endoreg_db/models/media/pdf/raw_pdf.py +757 -0
  292. endoreg_db/models/media/pdf/report_file.py +162 -0
  293. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  294. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +77 -0
  295. endoreg_db/models/media/video/__init__.py +8 -0
  296. endoreg_db/models/media/video/create_from_file.py +358 -0
  297. endoreg_db/models/media/video/pipe_1.py +213 -0
  298. endoreg_db/models/media/video/pipe_2.py +105 -0
  299. endoreg_db/models/media/video/refactor_plan.md +0 -0
  300. endoreg_db/models/media/video/video_file.py +825 -0
  301. endoreg_db/models/media/video/video_file_ai.py +443 -0
  302. endoreg_db/models/media/video/video_file_anonymize.py +349 -0
  303. endoreg_db/models/media/video/video_file_frames/__init__.py +47 -0
  304. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +22 -0
  305. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  306. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +104 -0
  307. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +174 -0
  308. endoreg_db/models/media/video/video_file_frames/_get_frame.py +28 -0
  309. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +27 -0
  310. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +20 -0
  311. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +27 -0
  312. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +34 -0
  313. endoreg_db/models/media/video/video_file_frames/_get_frames.py +27 -0
  314. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +129 -0
  315. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +141 -0
  316. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +65 -0
  317. endoreg_db/models/media/video/video_file_frames.py +0 -0
  318. endoreg_db/models/media/video/video_file_io.py +168 -0
  319. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  320. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +45 -0
  321. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +39 -0
  322. endoreg_db/models/media/video/video_file_meta/get_fps.py +147 -0
  323. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +143 -0
  324. endoreg_db/models/media/video/video_file_meta/text_meta.py +134 -0
  325. endoreg_db/models/media/video/video_file_meta/video_meta.py +70 -0
  326. endoreg_db/models/media/video/video_file_segments.py +209 -0
  327. endoreg_db/models/media/video/video_metadata.py +65 -0
  328. endoreg_db/models/media/video/video_processing.py +152 -0
  329. endoreg_db/models/medical/__init__.py +146 -0
  330. endoreg_db/models/medical/contraindication/__init__.py +17 -0
  331. endoreg_db/models/medical/disease.py +156 -0
  332. endoreg_db/models/medical/event.py +137 -0
  333. endoreg_db/models/medical/examination/__init__.py +9 -0
  334. endoreg_db/models/medical/examination/examination.py +148 -0
  335. endoreg_db/models/medical/examination/examination_indication.py +278 -0
  336. endoreg_db/models/medical/examination/examination_time.py +49 -0
  337. endoreg_db/models/medical/examination/examination_time_type.py +41 -0
  338. endoreg_db/models/medical/examination/examination_type.py +48 -0
  339. endoreg_db/models/medical/finding/__init__.py +18 -0
  340. endoreg_db/models/medical/finding/finding.py +96 -0
  341. endoreg_db/models/medical/finding/finding_classification.py +142 -0
  342. endoreg_db/models/medical/finding/finding_intervention.py +52 -0
  343. endoreg_db/models/medical/finding/finding_type.py +35 -0
  344. endoreg_db/models/medical/hardware/__init__.py +8 -0
  345. endoreg_db/models/medical/hardware/endoscope.py +65 -0
  346. endoreg_db/models/{hardware → medical/hardware}/endoscopy_processor.py +68 -29
  347. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  348. endoreg_db/models/medical/laboratory/lab_value.py +419 -0
  349. endoreg_db/models/medical/medication/__init__.py +19 -0
  350. endoreg_db/models/medical/medication/medication.py +31 -0
  351. endoreg_db/models/medical/medication/medication_indication.py +50 -0
  352. endoreg_db/models/medical/medication/medication_indication_type.py +39 -0
  353. endoreg_db/models/medical/medication/medication_intake_time.py +44 -0
  354. endoreg_db/models/medical/medication/medication_schedule.py +45 -0
  355. endoreg_db/models/medical/organ/__init__.py +35 -0
  356. endoreg_db/models/medical/patient/__init__.py +56 -0
  357. endoreg_db/models/medical/patient/medication_examples.py +38 -0
  358. endoreg_db/models/medical/patient/patient_disease.py +63 -0
  359. endoreg_db/models/medical/patient/patient_event.py +75 -0
  360. endoreg_db/models/medical/patient/patient_examination.py +249 -0
  361. endoreg_db/models/medical/patient/patient_examination_indication.py +44 -0
  362. endoreg_db/models/medical/patient/patient_finding.py +357 -0
  363. endoreg_db/models/medical/patient/patient_finding_classification.py +207 -0
  364. endoreg_db/models/medical/patient/patient_finding_intervention.py +40 -0
  365. endoreg_db/models/medical/patient/patient_lab_sample.py +148 -0
  366. endoreg_db/models/{persons → medical}/patient/patient_lab_value.py +68 -22
  367. endoreg_db/models/medical/patient/patient_medication.py +104 -0
  368. endoreg_db/models/medical/patient/patient_medication_schedule.py +136 -0
  369. endoreg_db/models/medical/risk/__init__.py +7 -0
  370. endoreg_db/models/medical/risk/risk.py +72 -0
  371. endoreg_db/models/medical/risk/risk_type.py +51 -0
  372. endoreg_db/models/metadata/__init__.py +19 -0
  373. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  374. endoreg_db/models/metadata/model_meta.py +206 -0
  375. endoreg_db/models/metadata/model_meta_logic.py +343 -0
  376. endoreg_db/models/{data_file/metadata → metadata}/pdf_meta.py +32 -13
  377. endoreg_db/models/metadata/sensitive_meta.py +288 -0
  378. endoreg_db/models/metadata/sensitive_meta_logic.py +1048 -0
  379. endoreg_db/models/metadata/video_meta.py +332 -0
  380. endoreg_db/models/metadata/video_prediction_logic.py +190 -0
  381. endoreg_db/models/metadata/video_prediction_meta.py +270 -0
  382. endoreg_db/models/other/__init__.py +36 -1
  383. endoreg_db/models/other/distribution/__init__.py +44 -0
  384. endoreg_db/models/other/distribution/base_value_distribution.py +20 -0
  385. endoreg_db/models/other/distribution/date_value_distribution.py +89 -0
  386. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +32 -0
  387. endoreg_db/models/other/distribution/numeric_value_distribution.py +125 -0
  388. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +22 -0
  389. endoreg_db/models/other/emission/__init__.py +5 -0
  390. endoreg_db/models/other/emission/emission_factor.py +94 -0
  391. endoreg_db/models/{persons → other}/gender.py +8 -3
  392. endoreg_db/models/other/information_source.py +159 -0
  393. endoreg_db/models/other/material.py +14 -2
  394. endoreg_db/models/other/resource.py +6 -2
  395. endoreg_db/models/other/tag.py +27 -0
  396. endoreg_db/models/other/transport_route.py +15 -3
  397. endoreg_db/models/{unit.py → other/unit.py} +16 -6
  398. endoreg_db/models/other/waste.py +10 -3
  399. endoreg_db/models/requirement/__init__.py +11 -0
  400. endoreg_db/models/requirement/requirement.py +767 -0
  401. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  402. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  403. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +9 -0
  404. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +95 -0
  405. endoreg_db/models/requirement/requirement_operator.py +176 -0
  406. endoreg_db/models/requirement/requirement_set.py +287 -0
  407. endoreg_db/models/rule/__init__.py +13 -0
  408. endoreg_db/models/{rules → rule}/rule.py +6 -3
  409. endoreg_db/models/{rules → rule}/rule_attribute_dtype.py +0 -2
  410. endoreg_db/models/{rules → rule}/rule_type.py +0 -2
  411. endoreg_db/models/{rules → rule}/ruleset.py +0 -2
  412. endoreg_db/models/state/__init__.py +12 -0
  413. endoreg_db/models/state/abstract.py +11 -0
  414. endoreg_db/models/state/audit_ledger.py +150 -0
  415. endoreg_db/models/state/label_video_segment.py +22 -0
  416. endoreg_db/models/state/raw_pdf.py +187 -0
  417. endoreg_db/models/state/sensitive_meta.py +46 -0
  418. endoreg_db/models/state/video.py +232 -0
  419. endoreg_db/models/upload_job.py +99 -0
  420. endoreg_db/models/utils.py +135 -0
  421. endoreg_db/renames.yml +8 -0
  422. endoreg_db/root_urls.py +9 -0
  423. endoreg_db/schemas/__init__.py +0 -0
  424. endoreg_db/schemas/examination_evaluation.py +27 -0
  425. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +775 -0
  426. endoreg_db/serializers/__init__.py +118 -10
  427. endoreg_db/serializers/_old/raw_pdf_meta_validation.py +223 -0
  428. endoreg_db/serializers/_old/raw_video_meta_validation.py +179 -0
  429. endoreg_db/serializers/_old/video.py +71 -0
  430. endoreg_db/serializers/administration/__init__.py +14 -0
  431. endoreg_db/serializers/administration/ai/__init__.py +10 -0
  432. endoreg_db/serializers/administration/ai/active_model.py +10 -0
  433. endoreg_db/serializers/administration/ai/ai_model.py +18 -0
  434. endoreg_db/serializers/administration/ai/model_type.py +10 -0
  435. endoreg_db/serializers/administration/center.py +9 -0
  436. endoreg_db/serializers/administration/gender.py +9 -0
  437. endoreg_db/serializers/anonymization.py +69 -0
  438. endoreg_db/serializers/evaluation/examination_evaluation.py +1 -0
  439. endoreg_db/serializers/examination/__init__.py +10 -0
  440. endoreg_db/serializers/examination/base.py +46 -0
  441. endoreg_db/serializers/examination/dropdown.py +21 -0
  442. endoreg_db/serializers/examination_serializer.py +12 -0
  443. endoreg_db/serializers/finding/__init__.py +5 -0
  444. endoreg_db/serializers/finding/finding.py +54 -0
  445. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  446. endoreg_db/serializers/finding_classification/choice.py +19 -0
  447. endoreg_db/serializers/finding_classification/classification.py +13 -0
  448. endoreg_db/serializers/label/__init__.py +7 -0
  449. endoreg_db/serializers/label/image_classification_annotation.py +62 -0
  450. endoreg_db/serializers/label/label.py +15 -0
  451. endoreg_db/serializers/label_video_segment/__init__.py +7 -0
  452. endoreg_db/serializers/label_video_segment/_lvs_create.py +149 -0
  453. endoreg_db/serializers/label_video_segment/_lvs_update.py +138 -0
  454. endoreg_db/serializers/label_video_segment/_lvs_validate.py +149 -0
  455. endoreg_db/serializers/label_video_segment/label_video_segment.py +344 -0
  456. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +99 -0
  457. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +163 -0
  458. endoreg_db/serializers/meta/__init__.py +19 -0
  459. endoreg_db/serializers/meta/pdf_file_meta_extraction.py +115 -0
  460. endoreg_db/serializers/meta/report_meta.py +53 -0
  461. endoreg_db/serializers/meta/sensitive_meta_detail.py +162 -0
  462. endoreg_db/serializers/meta/sensitive_meta_update.py +148 -0
  463. endoreg_db/serializers/meta/sensitive_meta_verification.py +59 -0
  464. endoreg_db/serializers/meta/video_meta.py +39 -0
  465. endoreg_db/serializers/misc/__init__.py +14 -0
  466. endoreg_db/serializers/misc/file_overview.py +182 -0
  467. endoreg_db/serializers/misc/sensitive_patient_data.py +120 -0
  468. endoreg_db/serializers/misc/stats.py +33 -0
  469. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  470. endoreg_db/serializers/misc/upload_job.py +71 -0
  471. endoreg_db/serializers/patient/__init__.py +11 -0
  472. endoreg_db/serializers/patient/patient.py +86 -0
  473. endoreg_db/serializers/patient/patient_dropdown.py +27 -0
  474. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  475. endoreg_db/serializers/patient_examination/patient_examination.py +141 -0
  476. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  477. endoreg_db/serializers/patient_finding/patient_finding.py +31 -0
  478. endoreg_db/serializers/patient_finding/patient_finding_classification.py +39 -0
  479. endoreg_db/serializers/patient_finding/patient_finding_detail.py +53 -0
  480. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +26 -0
  481. endoreg_db/serializers/patient_finding/patient_finding_list.py +41 -0
  482. endoreg_db/serializers/patient_finding/patient_finding_write.py +126 -0
  483. endoreg_db/serializers/pdf/__init__.py +5 -0
  484. endoreg_db/serializers/pdf/anony_text_validation.py +85 -0
  485. endoreg_db/serializers/report/__init__.py +9 -0
  486. endoreg_db/serializers/report/mixins.py +45 -0
  487. endoreg_db/serializers/report/report.py +105 -0
  488. endoreg_db/serializers/report/report_list.py +22 -0
  489. endoreg_db/serializers/report/secure_file_url.py +26 -0
  490. endoreg_db/serializers/requirements/requirement_schema.py +25 -0
  491. endoreg_db/serializers/requirements/requirement_sets.py +29 -0
  492. endoreg_db/serializers/sensitive_meta_serializer.py +282 -0
  493. endoreg_db/serializers/video/__init__.py +7 -0
  494. endoreg_db/serializers/video/segmentation.py +263 -0
  495. endoreg_db/serializers/video/video_file_brief.py +10 -0
  496. endoreg_db/serializers/video/video_file_detail.py +83 -0
  497. endoreg_db/serializers/video/video_file_list.py +67 -0
  498. endoreg_db/serializers/video/video_metadata.py +105 -0
  499. endoreg_db/serializers/video/video_processing_history.py +153 -0
  500. endoreg_db/serializers/video_examination.py +198 -0
  501. endoreg_db/services/__init__.py +5 -0
  502. endoreg_db/services/anonymization.py +223 -0
  503. endoreg_db/services/examination_evaluation.py +149 -0
  504. endoreg_db/services/finding_description_service.py +0 -0
  505. endoreg_db/services/lookup_service.py +411 -0
  506. endoreg_db/services/lookup_store.py +266 -0
  507. endoreg_db/services/pdf_import.py +1382 -0
  508. endoreg_db/services/polling_coordinator.py +288 -0
  509. endoreg_db/services/pseudonym_service.py +89 -0
  510. endoreg_db/services/requirements_object.py +147 -0
  511. endoreg_db/services/segment_sync.py +155 -0
  512. endoreg_db/services/storage_aware_video_processor.py +344 -0
  513. endoreg_db/services/video_import.py +1259 -0
  514. endoreg_db/tasks/upload_tasks.py +207 -0
  515. endoreg_db/tasks/video_ingest.py +157 -0
  516. endoreg_db/tasks/video_processing_tasks.py +327 -0
  517. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  518. endoreg_db/templates/admin/start_examination.html +12 -0
  519. endoreg_db/templates/timeline.html +176 -0
  520. endoreg_db/urls/__init__.py +83 -0
  521. endoreg_db/urls/anonymization.py +32 -0
  522. endoreg_db/urls/auth.py +16 -0
  523. endoreg_db/urls/classification.py +39 -0
  524. endoreg_db/urls/examination.py +54 -0
  525. endoreg_db/urls/files.py +6 -0
  526. endoreg_db/urls/label_video_segment_validate.py +33 -0
  527. endoreg_db/urls/label_video_segments.py +46 -0
  528. endoreg_db/urls/media.py +227 -0
  529. endoreg_db/urls/patient.py +19 -0
  530. endoreg_db/urls/report.py +48 -0
  531. endoreg_db/urls/requirements.py +13 -0
  532. endoreg_db/urls/sensitive_meta.py +0 -0
  533. endoreg_db/urls/stats.py +46 -0
  534. endoreg_db/urls/upload.py +20 -0
  535. endoreg_db/urls/video.py +61 -0
  536. endoreg_db/urls.py +9 -0
  537. endoreg_db/utils/__init__.py +88 -1
  538. endoreg_db/utils/ai/__init__.py +9 -0
  539. endoreg_db/{models/ai_model/utils.py → utils/ai/get.py} +1 -4
  540. endoreg_db/utils/ai/inference_dataset.py +52 -0
  541. endoreg_db/utils/ai/multilabel_classification_net.py +159 -0
  542. endoreg_db/utils/ai/postprocess.py +63 -0
  543. endoreg_db/utils/ai/predict.py +291 -0
  544. endoreg_db/utils/ai/preprocess.py +68 -0
  545. endoreg_db/utils/calc_duration_seconds.py +24 -0
  546. endoreg_db/utils/case_generator/__init__.py +0 -0
  547. endoreg_db/utils/case_generator/case_generator.py +159 -0
  548. endoreg_db/utils/case_generator/lab_sample_factory.py +33 -0
  549. endoreg_db/utils/case_generator/utils.py +30 -0
  550. endoreg_db/utils/check_video_files.py +148 -0
  551. endoreg_db/utils/dataloader.py +118 -35
  552. endoreg_db/utils/dates.py +60 -0
  553. endoreg_db/utils/env.py +33 -0
  554. endoreg_db/utils/extract_specific_frames.py +72 -0
  555. endoreg_db/utils/file_operations.py +29 -1
  556. endoreg_db/utils/fix_video_path_direct.py +141 -0
  557. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  558. endoreg_db/utils/hashs.py +123 -4
  559. endoreg_db/utils/links/__init__.py +0 -0
  560. endoreg_db/utils/links/requirement_link.py +193 -0
  561. endoreg_db/utils/mime_types.py +0 -0
  562. endoreg_db/utils/names.py +76 -0
  563. endoreg_db/utils/parse_and_generate_yaml.py +46 -0
  564. endoreg_db/utils/paths.py +95 -0
  565. endoreg_db/utils/permissions.py +143 -0
  566. endoreg_db/utils/pipelines/Readme.md +235 -0
  567. endoreg_db/utils/pipelines/__init__.py +0 -0
  568. endoreg_db/utils/pipelines/process_video_dir.py +120 -0
  569. endoreg_db/utils/product/__init__.py +0 -0
  570. endoreg_db/utils/product/sum_emissions.py +20 -0
  571. endoreg_db/utils/product/sum_weights.py +18 -0
  572. endoreg_db/utils/pydantic_models/__init__.py +6 -0
  573. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  574. endoreg_db/utils/requirement_helpers.py +0 -0
  575. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  576. endoreg_db/utils/requirement_operator_logic/lab_value_operators.py +578 -0
  577. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +368 -0
  578. endoreg_db/utils/setup_config.py +177 -0
  579. endoreg_db/utils/translation.py +27 -0
  580. endoreg_db/utils/validate_endo_roi.py +19 -0
  581. endoreg_db/utils/validate_subcategory_dict.py +91 -0
  582. endoreg_db/utils/validate_video_detailed.py +357 -0
  583. endoreg_db/utils/video/__init__.py +26 -0
  584. endoreg_db/utils/video/extract_frames.py +88 -0
  585. endoreg_db/utils/video/ffmpeg_wrapper.py +835 -0
  586. endoreg_db/utils/video/names.py +42 -0
  587. endoreg_db/utils/video/streaming_processor.py +312 -0
  588. endoreg_db/utils/video/video_splitter.py +94 -0
  589. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +238 -0
  590. endoreg_db/views/__init__.py +274 -0
  591. endoreg_db/views/anonymization/__init__.py +27 -0
  592. endoreg_db/views/anonymization/media_management.py +454 -0
  593. endoreg_db/views/anonymization/overview.py +216 -0
  594. endoreg_db/views/anonymization/validate.py +107 -0
  595. endoreg_db/views/auth/__init__.py +13 -0
  596. endoreg_db/views/auth/keycloak.py +113 -0
  597. endoreg_db/views/examination/__init__.py +33 -0
  598. endoreg_db/views/examination/examination.py +37 -0
  599. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  600. endoreg_db/views/examination/get_finding_classification_choices.py +59 -0
  601. endoreg_db/views/examination/get_finding_classifications.py +36 -0
  602. endoreg_db/views/examination/get_findings.py +41 -0
  603. endoreg_db/views/examination/get_instruments.py +18 -0
  604. endoreg_db/views/examination/get_interventions.py +14 -0
  605. endoreg_db/views/finding/__init__.py +9 -0
  606. endoreg_db/views/finding/finding.py +112 -0
  607. endoreg_db/views/finding/get_classifications.py +14 -0
  608. endoreg_db/views/finding/get_interventions.py +17 -0
  609. endoreg_db/views/finding_classification/__init__.py +13 -0
  610. endoreg_db/views/finding_classification/base.py +0 -0
  611. endoreg_db/views/finding_classification/finding_classification.py +42 -0
  612. endoreg_db/views/finding_classification/get_classification_choices.py +55 -0
  613. endoreg_db/views/label/__init__.py +5 -0
  614. endoreg_db/views/label/label.py +15 -0
  615. endoreg_db/views/label_video_segment/__init__.py +16 -0
  616. endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +44 -0
  617. endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +50 -0
  618. endoreg_db/views/label_video_segment/label_video_segment.py +77 -0
  619. endoreg_db/views/label_video_segment/label_video_segment_by_label.py +174 -0
  620. endoreg_db/views/label_video_segment/label_video_segment_detail.py +73 -0
  621. endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +46 -0
  622. endoreg_db/views/label_video_segment/validate.py +226 -0
  623. endoreg_db/views/media/__init__.py +45 -0
  624. endoreg_db/views/media/pdf_media.py +388 -0
  625. endoreg_db/views/media/segments.py +71 -0
  626. endoreg_db/views/media/sensitive_metadata.py +314 -0
  627. endoreg_db/views/media/video_media.py +272 -0
  628. endoreg_db/views/media/video_segments.py +524 -0
  629. endoreg_db/views/meta/__init__.py +15 -0
  630. endoreg_db/views/meta/available_files_list.py +146 -0
  631. endoreg_db/views/meta/report_meta.py +53 -0
  632. endoreg_db/views/meta/sensitive_meta_detail.py +148 -0
  633. endoreg_db/views/meta/sensitive_meta_list.py +104 -0
  634. endoreg_db/views/meta/sensitive_meta_verification.py +71 -0
  635. endoreg_db/views/misc/__init__.py +63 -0
  636. endoreg_db/views/misc/center.py +13 -0
  637. endoreg_db/views/misc/csrf.py +7 -0
  638. endoreg_db/views/misc/gender.py +14 -0
  639. endoreg_db/views/misc/secure_file_serving_view.py +80 -0
  640. endoreg_db/views/misc/secure_file_url_view.py +84 -0
  641. endoreg_db/views/misc/secure_url_validate.py +79 -0
  642. endoreg_db/views/misc/stats.py +220 -0
  643. endoreg_db/views/misc/translation.py +182 -0
  644. endoreg_db/views/misc/upload_views.py +240 -0
  645. endoreg_db/views/patient/__init__.py +5 -0
  646. endoreg_db/views/patient/patient.py +210 -0
  647. endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +164 -0
  648. endoreg_db/views/patient_examination/__init__.py +11 -0
  649. endoreg_db/views/patient_examination/patient_examination.py +140 -0
  650. endoreg_db/views/patient_examination/patient_examination_create.py +63 -0
  651. endoreg_db/views/patient_examination/patient_examination_detail.py +66 -0
  652. endoreg_db/views/patient_examination/patient_examination_list.py +68 -0
  653. endoreg_db/views/patient_examination/video.py +194 -0
  654. endoreg_db/views/patient_finding/__init__.py +7 -0
  655. endoreg_db/views/patient_finding/base.py +0 -0
  656. endoreg_db/views/patient_finding/patient_finding.py +64 -0
  657. endoreg_db/views/patient_finding/patient_finding_optimized.py +259 -0
  658. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  659. endoreg_db/views/patient_finding_classification/pfc_create.py +67 -0
  660. endoreg_db/views/patient_finding_location/__init__.py +5 -0
  661. endoreg_db/views/patient_finding_location/pfl_create.py +70 -0
  662. endoreg_db/views/patient_finding_morphology/__init__.py +5 -0
  663. endoreg_db/views/patient_finding_morphology/pfm_create.py +70 -0
  664. endoreg_db/views/pdf/__init__.py +8 -0
  665. endoreg_db/views/pdf/pdf_stream.py +187 -0
  666. endoreg_db/views/pdf/reimport.py +177 -0
  667. endoreg_db/views/report/__init__.py +9 -0
  668. endoreg_db/views/report/report_list.py +112 -0
  669. endoreg_db/views/report/report_with_secure_url.py +28 -0
  670. endoreg_db/views/report/start_examination.py +7 -0
  671. endoreg_db/views/requirement/__init__.py +10 -0
  672. endoreg_db/views/requirement/evaluate.py +279 -0
  673. endoreg_db/views/requirement/lookup.py +367 -0
  674. endoreg_db/views/requirement/lookup_store.py +252 -0
  675. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  676. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  677. endoreg_db/views/stats/__init__.py +13 -0
  678. endoreg_db/views/stats/stats_views.py +229 -0
  679. endoreg_db/views/video/__init__.py +59 -0
  680. endoreg_db/views/video/correction.py +530 -0
  681. endoreg_db/views/video/reimport.py +195 -0
  682. endoreg_db/views/video/segmentation.py +274 -0
  683. endoreg_db/views/video/task_status.py +49 -0
  684. endoreg_db/views/video/timeline.py +46 -0
  685. endoreg_db/views/video/video_analyze.py +52 -0
  686. endoreg_db/views/video/video_apply_mask.py +48 -0
  687. endoreg_db/views/video/video_correction.py +21 -0
  688. endoreg_db/views/video/video_download_processed.py +58 -0
  689. endoreg_db/views/video/video_examination_viewset.py +242 -0
  690. endoreg_db/views/video/video_meta.py +29 -0
  691. endoreg_db/views/video/video_processing_history.py +24 -0
  692. endoreg_db/views/video/video_remove_frames.py +48 -0
  693. endoreg_db/views/video/video_stream.py +306 -0
  694. endoreg_db/views.py +0 -3
  695. endoreg_db-0.8.6.3.dist-info/METADATA +383 -0
  696. endoreg_db-0.8.6.3.dist-info/RECORD +793 -0
  697. {endoreg_db-0.4.5.dist-info → endoreg_db-0.8.6.3.dist-info}/WHEEL +1 -1
  698. endoreg_db/data/active_model/data.yaml +0 -3
  699. endoreg_db/data/agl_service/data.yaml +0 -19
  700. endoreg_db/data/label/label-set/data.yaml +0 -18
  701. endoreg_db/management/commands/_load_model_template.py +0 -41
  702. endoreg_db/management/commands/delete_all.py +0 -18
  703. endoreg_db/management/commands/delete_legacy_images.py +0 -19
  704. endoreg_db/management/commands/delete_legacy_videos.py +0 -17
  705. endoreg_db/management/commands/extract_legacy_video_frames.py +0 -18
  706. endoreg_db/management/commands/fetch_legacy_image_dataset.py +0 -32
  707. endoreg_db/management/commands/fix_auth_permission.py +0 -20
  708. endoreg_db/management/commands/import_legacy_images.py +0 -94
  709. endoreg_db/management/commands/import_legacy_videos.py +0 -76
  710. endoreg_db/management/commands/load_active_model_data.py +0 -45
  711. endoreg_db/management/commands/load_endoscopy_processor_data.py +0 -45
  712. endoreg_db/management/commands/load_g_play_data.py +0 -113
  713. endoreg_db/management/commands/load_label_data.py +0 -67
  714. endoreg_db/management/commands/load_medication_indication_data.py +0 -63
  715. endoreg_db/management/commands/load_medication_indication_type_data.py +0 -41
  716. endoreg_db/management/commands/load_medication_schedule_data.py +0 -55
  717. endoreg_db/management/commands/load_network_data.py +0 -57
  718. endoreg_db/migrations/0002_anonymizedimagelabel_anonymousimageannotation_and_more.py +0 -55
  719. endoreg_db/migrations/0003_anonymousimageannotation_original_image_url_and_more.py +0 -39
  720. endoreg_db/migrations/0004_alter_rawpdffile_file.py +0 -20
  721. endoreg_db/migrations/0005_uploadedfile_alter_rawpdffile_file_anonymizedfile.py +0 -40
  722. endoreg_db/migrations/0006_alter_rawpdffile_file.py +0 -20
  723. endoreg_db/migrations/0007_networkdevicelogentry_datetime_and_more.py +0 -43
  724. endoreg_db/models/ai_model/__init__.py +0 -3
  725. endoreg_db/models/ai_model/active_model.py +0 -9
  726. endoreg_db/models/ai_model/model_meta.py +0 -24
  727. endoreg_db/models/annotation/__init__.py +0 -3
  728. endoreg_db/models/annotation/anonymized_image_annotation.py +0 -60
  729. endoreg_db/models/annotation/binary_classification_annotation_task.py +0 -80
  730. endoreg_db/models/annotation/image_classification.py +0 -27
  731. endoreg_db/models/center/center.py +0 -25
  732. endoreg_db/models/center/center_product.py +0 -34
  733. endoreg_db/models/center/center_resource.py +0 -19
  734. endoreg_db/models/center/center_waste.py +0 -11
  735. endoreg_db/models/data_file/__init__.py +0 -6
  736. endoreg_db/models/data_file/base_classes/__init__.py +0 -2
  737. endoreg_db/models/data_file/base_classes/abstract_frame.py +0 -51
  738. endoreg_db/models/data_file/base_classes/abstract_video.py +0 -201
  739. endoreg_db/models/data_file/frame.py +0 -45
  740. endoreg_db/models/data_file/import_classes/__init__.py +0 -32
  741. endoreg_db/models/data_file/import_classes/processing_functions/__init__.py +0 -35
  742. endoreg_db/models/data_file/import_classes/processing_functions/pdf.py +0 -28
  743. endoreg_db/models/data_file/import_classes/processing_functions/video.py +0 -260
  744. endoreg_db/models/data_file/import_classes/raw_pdf.py +0 -188
  745. endoreg_db/models/data_file/import_classes/raw_video.py +0 -343
  746. endoreg_db/models/data_file/metadata/__init__.py +0 -3
  747. endoreg_db/models/data_file/metadata/sensitive_meta.py +0 -31
  748. endoreg_db/models/data_file/metadata/video_meta.py +0 -133
  749. endoreg_db/models/data_file/report_file.py +0 -89
  750. endoreg_db/models/data_file/video/__init__.py +0 -7
  751. endoreg_db/models/data_file/video/import_meta.py +0 -25
  752. endoreg_db/models/data_file/video/video.py +0 -25
  753. endoreg_db/models/data_file/video_segment.py +0 -107
  754. endoreg_db/models/disease.py +0 -56
  755. endoreg_db/models/emission/__init__.py +0 -1
  756. endoreg_db/models/emission/emission_factor.py +0 -20
  757. endoreg_db/models/event.py +0 -22
  758. endoreg_db/models/examination/__init__.py +0 -4
  759. endoreg_db/models/examination/examination.py +0 -26
  760. endoreg_db/models/examination/examination_time.py +0 -27
  761. endoreg_db/models/examination/examination_time_type.py +0 -24
  762. endoreg_db/models/examination/examination_type.py +0 -18
  763. endoreg_db/models/hardware/__init__.py +0 -2
  764. endoreg_db/models/hardware/endoscope.py +0 -44
  765. endoreg_db/models/information_source.py +0 -29
  766. endoreg_db/models/laboratory/__init__.py +0 -1
  767. endoreg_db/models/laboratory/lab_value.py +0 -102
  768. endoreg_db/models/legacy_data/__init__.py +0 -3
  769. endoreg_db/models/legacy_data/image.py +0 -34
  770. endoreg_db/models/logging/__init__.py +0 -4
  771. endoreg_db/models/logging/agl_service.py +0 -19
  772. endoreg_db/models/logging/base.py +0 -22
  773. endoreg_db/models/logging/log_type.py +0 -23
  774. endoreg_db/models/logging/network_device.py +0 -24
  775. endoreg_db/models/medication/__init__.py +0 -1
  776. endoreg_db/models/medication/medication.py +0 -148
  777. endoreg_db/models/network/__init__.py +0 -3
  778. endoreg_db/models/network/agl_service.py +0 -38
  779. endoreg_db/models/network/network_device.py +0 -53
  780. endoreg_db/models/network/network_device_type.py +0 -23
  781. endoreg_db/models/other/distribution.py +0 -215
  782. endoreg_db/models/patient_examination/__init__.py +0 -35
  783. endoreg_db/models/permissions/__init__.py +0 -44
  784. endoreg_db/models/persons/__init__.py +0 -7
  785. endoreg_db/models/persons/examiner/__init__.py +0 -2
  786. endoreg_db/models/persons/examiner/examiner.py +0 -16
  787. endoreg_db/models/persons/examiner/examiner_type.py +0 -2
  788. endoreg_db/models/persons/patient/__init__.py +0 -8
  789. endoreg_db/models/persons/patient/case/case.py +0 -30
  790. endoreg_db/models/persons/patient/patient.py +0 -216
  791. endoreg_db/models/persons/patient/patient_disease.py +0 -16
  792. endoreg_db/models/persons/patient/patient_event.py +0 -22
  793. endoreg_db/models/persons/patient/patient_lab_sample.py +0 -106
  794. endoreg_db/models/persons/patient/patient_medication.py +0 -44
  795. endoreg_db/models/persons/patient/patient_medication_schedule.py +0 -28
  796. endoreg_db/models/persons/portal_user_information.py +0 -27
  797. endoreg_db/models/prediction/__init__.py +0 -2
  798. endoreg_db/models/prediction/image_classification.py +0 -37
  799. endoreg_db/models/prediction/video_prediction_meta.py +0 -244
  800. endoreg_db/models/product/__init__.py +0 -5
  801. endoreg_db/models/product/product.py +0 -97
  802. endoreg_db/models/product/product_group.py +0 -19
  803. endoreg_db/models/product/product_material.py +0 -24
  804. endoreg_db/models/questionnaires/__init__.py +0 -114
  805. endoreg_db/models/quiz/__init__.py +0 -2
  806. endoreg_db/models/quiz/quiz_answer.py +0 -41
  807. endoreg_db/models/quiz/quiz_question.py +0 -54
  808. endoreg_db/models/report_reader/__init__.py +0 -2
  809. endoreg_db/models/report_reader/report_reader_config.py +0 -53
  810. endoreg_db/models/rules/__init__.py +0 -5
  811. endoreg_db/queries/get/__init__.py +0 -6
  812. endoreg_db/queries/get/center.py +0 -42
  813. endoreg_db/queries/get/model.py +0 -13
  814. endoreg_db/queries/get/patient.py +0 -14
  815. endoreg_db/queries/get/patient_examination.py +0 -20
  816. endoreg_db/queries/get/report_file.py +0 -33
  817. endoreg_db/queries/get/video.py +0 -31
  818. endoreg_db/serializers/ai_model.py +0 -19
  819. endoreg_db/serializers/annotation.py +0 -17
  820. endoreg_db/serializers/center.py +0 -11
  821. endoreg_db/serializers/examination.py +0 -33
  822. endoreg_db/serializers/frame.py +0 -13
  823. endoreg_db/serializers/hardware.py +0 -21
  824. endoreg_db/serializers/label.py +0 -22
  825. endoreg_db/serializers/patient.py +0 -10
  826. endoreg_db/serializers/prediction.py +0 -15
  827. endoreg_db/serializers/report_file.py +0 -7
  828. endoreg_db/serializers/video.py +0 -27
  829. endoreg_db/tests.py +0 -3
  830. endoreg_db/utils/legacy_ocr.py +0 -201
  831. endoreg_db/utils/video_metadata.py +0 -87
  832. endoreg_db-0.4.5.dist-info/METADATA +0 -34
  833. endoreg_db-0.4.5.dist-info/RECORD +0 -316
  834. /endoreg_db/{data/distribution/numeric/.init → api/serializers/finding_descriptions.py} +0 -0
  835. /endoreg_db/{models/persons/patient/case/__init__.py → api/views/finding_descriptions.py} +0 -0
  836. /endoreg_db/{queries/get/annotation.py → config/__init__.py} +0 -0
  837. /endoreg_db/data/{label → ai_model_label}/label-type/data.yaml +0 -0
  838. /endoreg_db/data/{model_type → ai_model_type}/data.yaml +0 -0
  839. /endoreg_db/{queries/get/prediction.py → data/shift/m2.yaml} +0 -0
  840. /endoreg_db/{queries/get/video_import_meta.py → factories/__init__.py} +0 -0
  841. /endoreg_db/{queries/get/video_prediction_meta.py → helpers/__init__.py} +0 -0
  842. /endoreg_db/management/commands/{load_report_reader_flag.py → load_report_reader_flag_data.py} +0 -0
  843. /endoreg_db/models/{persons → administration/person}/person.py +0 -0
  844. /endoreg_db/models/{report_reader → media/pdf/report_reader}/report_reader_flag.py +0 -0
  845. /endoreg_db/models/{rules → rule}/rule_applicator.py +0 -0
  846. {endoreg_db-0.4.5.dist-info → endoreg_db-0.8.6.3.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,42 @@
1
+ from typing import Optional
2
+
3
+ def get_video_key(examination_alias:str, content:str, is_anonymous:bool=False) -> str:
4
+ """
5
+ Generates a video key based on the examination alias, content, and anonymity status.
6
+ """
7
+ if is_anonymous:
8
+ return f"{examination_alias}-{content}-anonymous"
9
+ else:
10
+ return f"{examination_alias}-{content}-non_anonymous"
11
+
12
+ def identify_video_key(video_key:str) -> str:
13
+ """
14
+ Identifies the video key based on the provided string.
15
+ """
16
+ split_key = video_key.split("-")
17
+ assert len(split_key) == 3, f"Invalid video key format: {video_key}"
18
+ examination_alias = split_key[0]
19
+ content = split_key[1]
20
+ is_anonymous = split_key[2] == "anonymous"
21
+ return examination_alias, content, is_anonymous
22
+
23
+
24
+ def get_video_key_regex_by_examination_alias(
25
+ examination_alias:Optional[str]=None, content:Optional[str]=None, is_anonymous:Optional[bool]=None
26
+ ):
27
+ """
28
+ Generates a regex pattern to match video keys based on examination alias, content, and anonymity status.
29
+ If any of the parameters are None, they will be ignored in the regex.
30
+ """
31
+ pattern = ""
32
+ if examination_alias:
33
+ pattern += f"{examination_alias}-"
34
+ if content:
35
+ pattern += f"{content}-"
36
+ if is_anonymous is not None:
37
+ pattern += "anonymous" if is_anonymous else "non_anonymous"
38
+ else:
39
+ pattern += "(anonymous|non_anonymous)"
40
+
41
+ return f"^{pattern}$"
42
+
@@ -0,0 +1,312 @@
1
+ import logging
2
+ import subprocess
3
+ from pathlib import Path
4
+ from typing import Optional, Iterator, Tuple
5
+ import tempfile
6
+ import shutil
7
+
8
+ from ...exceptions import InsufficientStorageError, VideoProcessingError
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+ class StreamingVideoProcessor:
13
+ """
14
+ Streaming video processor for memory-efficient video anonymization.
15
+ Processes videos in chunks to reduce memory usage and improve performance.
16
+ """
17
+
18
+ def __init__(self, chunk_duration: int = 30, temp_dir: Optional[Path] = None):
19
+ """
20
+ Initialize the streaming processor.
21
+
22
+ Args:
23
+ chunk_duration: Duration of each chunk in seconds
24
+ temp_dir: Temporary directory for processing chunks
25
+ """
26
+ self.chunk_duration = chunk_duration
27
+ self.temp_dir = Path(temp_dir) if temp_dir else Path(tempfile.gettempdir()) / 'video_streaming'
28
+ self.temp_dir.mkdir(parents=True, exist_ok=True)
29
+
30
+ def check_ffmpeg_available(self) -> bool:
31
+ """Check if FFmpeg is available in the system."""
32
+ try:
33
+ subprocess.run(['ffmpeg', '-version'],
34
+ capture_output=True, check=True, timeout=10)
35
+ return True
36
+ except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
37
+ return False
38
+
39
+ def get_video_duration(self, video_path: Path) -> float:
40
+ """Get video duration in seconds using FFprobe."""
41
+ try:
42
+ cmd = [
43
+ 'ffprobe', '-v', 'quiet', '-show_entries', 'format=duration',
44
+ '-of', 'csv=p=0', str(video_path)
45
+ ]
46
+ result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=30)
47
+ return float(result.stdout.strip())
48
+ except (subprocess.CalledProcessError, ValueError, subprocess.TimeoutExpired) as e:
49
+ logger.error(f"Failed to get video duration for {video_path}: {e}")
50
+ raise VideoProcessingError(f"Could not determine video duration: {e}")
51
+
52
+ def split_video_chunks(self, video_path: Path) -> Iterator[Tuple[Path, float, float]]:
53
+ """
54
+ Split video into chunks for streaming processing.
55
+
56
+ Args:
57
+ video_path: Path to the input video
58
+
59
+ Yields:
60
+ Tuple of (chunk_path, start_time, end_time)
61
+ """
62
+ if not self.check_ffmpeg_available():
63
+ raise VideoProcessingError("FFmpeg not available for video processing")
64
+
65
+ try:
66
+ total_duration = self.get_video_duration(video_path)
67
+ logger.info(f"Video duration: {total_duration:.2f}s, splitting into {self.chunk_duration}s chunks")
68
+
69
+ chunk_count = 0
70
+ for start_time in range(0, int(total_duration), self.chunk_duration):
71
+ end_time = min(start_time + self.chunk_duration, total_duration)
72
+
73
+ # Create chunk filename
74
+ chunk_filename = f"chunk_{chunk_count:04d}_{start_time}_{int(end_time)}.mp4"
75
+ chunk_path = self.temp_dir / chunk_filename
76
+
77
+ # Extract chunk using FFmpeg
78
+ cmd = [
79
+ 'ffmpeg', '-y', # Overwrite output files
80
+ '-ss', str(start_time), # Start time
81
+ '-i', str(video_path), # Input file
82
+ '-t', str(end_time - start_time), # Duration
83
+ '-c', 'copy', # Copy streams without re-encoding for speed
84
+ '-avoid_negative_ts', 'make_zero', # Handle timestamp issues
85
+ str(chunk_path)
86
+ ]
87
+
88
+ try:
89
+ logger.debug(f"Creating chunk {chunk_count}: {start_time}s-{end_time}s")
90
+ result = subprocess.run(cmd, capture_output=True, text=True,
91
+ check=True, timeout=300) # 5 minute timeout per chunk
92
+
93
+ if chunk_path.exists() and chunk_path.stat().st_size > 0:
94
+ yield chunk_path, start_time, end_time
95
+ chunk_count += 1
96
+ else:
97
+ logger.warning(f"Chunk {chunk_count} was not created or is empty")
98
+
99
+ except subprocess.CalledProcessError as e:
100
+ logger.error(f"FFmpeg failed for chunk {chunk_count}: {e.stderr}")
101
+ # Skip this chunk but continue with others
102
+ continue
103
+ except subprocess.TimeoutExpired:
104
+ logger.error(f"FFmpeg timeout for chunk {chunk_count}")
105
+ continue
106
+
107
+ except Exception as e:
108
+ logger.error(f"Error in video chunking: {e}")
109
+ raise VideoProcessingError(f"Video chunking failed: {e}")
110
+
111
+ def process_chunk_anonymization(self, chunk_path: Path, anonymizer_func, **kwargs) -> Path:
112
+ """
113
+ Process a single chunk with the anonymization function.
114
+
115
+ Args:
116
+ chunk_path: Path to the video chunk
117
+ anonymizer_func: Function to anonymize the chunk
118
+ **kwargs: Additional arguments for the anonymizer
119
+
120
+ Returns:
121
+ Path to the anonymized chunk
122
+ """
123
+ try:
124
+ output_path = chunk_path.with_suffix('.anonymized.mp4')
125
+
126
+ # Call the anonymization function
127
+ result = anonymizer_func(chunk_path, output_path, **kwargs)
128
+
129
+ if isinstance(result, Path):
130
+ return result
131
+ elif result is True and output_path.exists():
132
+ return output_path
133
+ else:
134
+ raise VideoProcessingError(f"Anonymization failed for chunk {chunk_path}")
135
+
136
+ except Exception as e:
137
+ logger.error(f"Chunk anonymization failed for {chunk_path}: {e}")
138
+ raise VideoProcessingError(f"Chunk processing failed: {e}")
139
+
140
+ def merge_chunks(self, chunk_paths: list[Path], output_path: Path) -> Path:
141
+ """
142
+ Merge anonymized chunks back into a single video.
143
+
144
+ Args:
145
+ chunk_paths: List of paths to anonymized chunks
146
+ output_path: Path for the final merged video
147
+
148
+ Returns:
149
+ Path to the merged video
150
+ """
151
+ if not chunk_paths:
152
+ raise VideoProcessingError("No chunks to merge")
153
+
154
+ try:
155
+ # Check storage space before merging
156
+ total_chunk_size = sum(chunk.stat().st_size for chunk in chunk_paths if chunk.exists())
157
+ free_space = shutil.disk_usage(output_path.parent).free
158
+
159
+ if free_space < total_chunk_size * 1.2: # 20% safety margin
160
+ raise InsufficientStorageError(
161
+ f"Insufficient space for merging. Required: {total_chunk_size * 1.2 / 1e9:.1f} GB, "
162
+ f"Available: {free_space / 1e9:.1f} GB",
163
+ required_space=int(total_chunk_size * 1.2),
164
+ available_space=free_space
165
+ )
166
+
167
+ # Create concat file for FFmpeg
168
+ concat_file = self.temp_dir / f"concat_{output_path.stem}.txt"
169
+
170
+ with open(concat_file, 'w') as f:
171
+ for chunk_path in chunk_paths:
172
+ if chunk_path.exists():
173
+ # Use relative paths in concat file for better portability
174
+ f.write(f"file '{chunk_path.name}'\n")
175
+
176
+ # Merge using FFmpeg concat demuxer
177
+ cmd = [
178
+ 'ffmpeg', '-y', # Overwrite output
179
+ '-f', 'concat', # Use concat demuxer
180
+ '-safe', '0', # Allow unsafe file paths
181
+ '-i', str(concat_file), # Input concat file
182
+ '-c', 'copy', # Copy streams without re-encoding
183
+ str(output_path)
184
+ ]
185
+
186
+ logger.info(f"Merging {len(chunk_paths)} chunks into {output_path}")
187
+
188
+ # Change working directory to temp_dir for relative paths
189
+ result = subprocess.run(cmd, cwd=str(self.temp_dir),
190
+ capture_output=True, text=True, check=True, timeout=600)
191
+
192
+ if not output_path.exists() or output_path.stat().st_size == 0:
193
+ raise VideoProcessingError("Merged video was not created or is empty")
194
+
195
+ logger.info(f"Successfully merged video: {output_path}")
196
+
197
+ # Clean up concat file
198
+ concat_file.unlink(missing_ok=True)
199
+
200
+ return output_path
201
+
202
+ except subprocess.CalledProcessError as e:
203
+ logger.error(f"FFmpeg merge failed: {e.stderr}")
204
+ raise VideoProcessingError(f"Video merging failed: {e.stderr}")
205
+ except subprocess.TimeoutExpired:
206
+ logger.error("FFmpeg merge timeout")
207
+ raise VideoProcessingError("Video merging timed out")
208
+ except Exception as e:
209
+ logger.error(f"Unexpected error during merge: {e}")
210
+ raise VideoProcessingError(f"Video merging failed: {e}")
211
+
212
+ def process_video_streaming(self, input_path: Path, output_path: Path,
213
+ anonymizer_func, progress_callback=None, **kwargs) -> Path:
214
+ """
215
+ Process a video using streaming approach for memory efficiency.
216
+
217
+ Args:
218
+ input_path: Path to input video
219
+ output_path: Path for output video
220
+ anonymizer_func: Function to anonymize video chunks
221
+ progress_callback: Optional callback for progress updates
222
+ **kwargs: Additional arguments for anonymizer
223
+
224
+ Returns:
225
+ Path to the processed video
226
+ """
227
+ processed_chunks = []
228
+ total_chunks = 0
229
+
230
+ try:
231
+ logger.info(f"Starting streaming video processing: {input_path} -> {output_path}")
232
+
233
+ # First pass: count total chunks for progress tracking
234
+ chunk_list = list(self.split_video_chunks(input_path))
235
+ total_chunks = len(chunk_list)
236
+
237
+ if total_chunks == 0:
238
+ raise VideoProcessingError("No chunks were created from the input video")
239
+
240
+ logger.info(f"Processing {total_chunks} chunks")
241
+
242
+ # Process each chunk
243
+ for i, (chunk_path, start_time, end_time) in enumerate(chunk_list):
244
+ try:
245
+ logger.debug(f"Processing chunk {i+1}/{total_chunks}: {chunk_path}")
246
+
247
+ # Process the chunk
248
+ processed_chunk = self.process_chunk_anonymization(
249
+ chunk_path, anonymizer_func, **kwargs
250
+ )
251
+ processed_chunks.append(processed_chunk)
252
+
253
+ # Update progress
254
+ if progress_callback:
255
+ progress = int((i + 1) / total_chunks * 80) # Reserve 20% for merging
256
+ progress_callback(progress, f"Processed chunk {i+1}/{total_chunks}")
257
+
258
+ # Clean up original chunk to save space
259
+ chunk_path.unlink(missing_ok=True)
260
+
261
+ except Exception as e:
262
+ logger.error(f"Failed to process chunk {i}: {e}")
263
+ # Clean up failed chunk
264
+ chunk_path.unlink(missing_ok=True)
265
+ # Continue with other chunks
266
+ continue
267
+
268
+ if not processed_chunks:
269
+ raise VideoProcessingError("No chunks were successfully processed")
270
+
271
+ # Update progress for merging phase
272
+ if progress_callback:
273
+ progress_callback(80, f"Merging {len(processed_chunks)} processed chunks...")
274
+
275
+ # Merge processed chunks
276
+ final_output = self.merge_chunks(processed_chunks, output_path)
277
+
278
+ # Final progress update
279
+ if progress_callback:
280
+ progress_callback(100, "Video processing completed")
281
+
282
+ logger.info(f"Streaming video processing completed: {final_output}")
283
+ return final_output
284
+
285
+ except (InsufficientStorageError, VideoProcessingError):
286
+ # Re-raise these specific errors as-is
287
+ raise
288
+ except Exception as e:
289
+ logger.error(f"Streaming video processing failed: {e}")
290
+ raise VideoProcessingError(f"Streaming processing failed: {e}")
291
+ finally:
292
+ # Clean up all temporary chunks
293
+ self.cleanup_chunks(processed_chunks)
294
+
295
+ def cleanup_chunks(self, chunk_paths: list[Path]) -> None:
296
+ """Clean up temporary chunk files."""
297
+ for chunk_path in chunk_paths:
298
+ if chunk_path and chunk_path.exists():
299
+ try:
300
+ chunk_path.unlink()
301
+ logger.debug(f"Cleaned up chunk: {chunk_path}")
302
+ except Exception as e:
303
+ logger.warning(f"Failed to clean up chunk {chunk_path}: {e}")
304
+
305
+ def cleanup_temp_dir(self) -> None:
306
+ """Clean up the entire temporary directory."""
307
+ try:
308
+ if self.temp_dir.exists():
309
+ shutil.rmtree(self.temp_dir)
310
+ logger.debug(f"Cleaned up temp directory: {self.temp_dir}")
311
+ except Exception as e:
312
+ logger.warning(f"Failed to clean up temp directory {self.temp_dir}: {e}")
@@ -0,0 +1,94 @@
1
+ import os
2
+ import subprocess
3
+ import pathlib
4
+ import math
5
+ import logging
6
+ import json
7
+
8
+ # Setup basic logging
9
+ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
10
+
11
+ def get_video_duration(video_path: pathlib.Path) -> float:
12
+ """Gets the duration of a video file using ffprobe."""
13
+ cmd = [
14
+ "ffprobe",
15
+ "-v", "error",
16
+ "-show_entries", "format=duration",
17
+ "-of", "default=noprint_wrappers=1:nokey=1",
18
+ str(video_path)
19
+ ]
20
+ try:
21
+ result = subprocess.run(cmd, capture_output=True, text=True, check=True)
22
+ duration = float(result.stdout.strip())
23
+ return duration
24
+ except subprocess.CalledProcessError as e:
25
+ logging.error(f"Error getting duration for {video_path}: {e.stderr}")
26
+ raise
27
+ except ValueError:
28
+ logging.error(f"Could not parse duration from ffprobe output: {result.stdout}")
29
+ raise
30
+
31
+ def split_video(input_path: str, interval: int):
32
+ """
33
+ Splits a video into segments of a specified interval using ffmpeg.
34
+
35
+ Args:
36
+ input_path: Path to the input MP4 video file.
37
+ interval: The desired duration of each segment in seconds.
38
+ """
39
+ input_file = pathlib.Path(input_path)
40
+ if not input_file.is_file():
41
+ logging.error(f"Input file not found: {input_path}")
42
+ return
43
+
44
+ video_name = input_file.stem
45
+ output_dir = pathlib.Path("data/video_splitter") / video_name
46
+ output_dir.mkdir(parents=True, exist_ok=True)
47
+ logging.info(f"Created output directory: {output_dir}")
48
+
49
+ try:
50
+ duration = get_video_duration(input_file)
51
+ logging.info(f"Video duration: {duration:.2f} seconds")
52
+ except Exception as e:
53
+ logging.error(f"Could not get video duration. Aborting split. Error: {e}")
54
+ return
55
+
56
+ num_segments = math.ceil(duration / interval)
57
+ logging.info(f"Splitting into {num_segments} segments of approximately {interval} seconds each.")
58
+
59
+ for i in range(num_segments):
60
+ start_time = i * interval
61
+ output_filename = output_dir / f"segment_{i+1:03d}{input_file.suffix}"
62
+
63
+ # Use -t for interval duration. For the last segment, ffmpeg with -c copy
64
+ # might automatically stop at the end, or we could calculate exact duration.
65
+ # Using -t interval is simpler and usually works well with -c copy.
66
+ # If the last segment needs precise duration without relying on -c copy behavior:
67
+ # segment_duration = min(interval, duration - start_time)
68
+ # However, -t interval with -c copy is generally robust.
69
+
70
+ cmd = [
71
+ "ffmpeg",
72
+ "-i", str(input_file),
73
+ "-ss", str(start_time),
74
+ "-t", str(interval),
75
+ "-c", "copy", # Fast, lossless splitting
76
+ "-avoid_negative_ts", "make_zero", # Avoids issues with negative timestamps
77
+ str(output_filename)
78
+ ]
79
+
80
+ logging.info(f"Running command: {' '.join(cmd)}")
81
+ try:
82
+ result = subprocess.run(cmd, capture_output=True, text=True, check=True)
83
+ logging.info(f"Successfully created segment: {output_filename}")
84
+ if result.stderr: # ffmpeg often outputs info to stderr
85
+ logging.debug(f"ffmpeg output for segment {i+1}:\n{result.stderr}")
86
+ except subprocess.CalledProcessError as e:
87
+ logging.error(f"Error creating segment {i+1}: {output_filename}")
88
+ logging.error(f"Command failed: {' '.join(cmd)}")
89
+ logging.error(f"ffmpeg stderr:\n{e.stderr}")
90
+ # Decide if you want to stop on error or continue
91
+ # return # Uncomment to stop on first error
92
+
93
+ logging.info("Video splitting completed.")
94
+
@@ -0,0 +1,238 @@
1
+ from rest_framework.views import APIView
2
+ from rest_framework.response import Response
3
+ from rest_framework import status
4
+ from endoreg_db.models import VideoFile
5
+ from ..serializers.Frames_NICE_and_PARIS_classifications import (
6
+ ForNiceClassificationSerializer,
7
+ ForParisClassificationSerializer
8
+ )
9
+
10
+ import logging
11
+
12
+
13
+ class ForNiceClassificationView(APIView):
14
+ """
15
+ NICE Classification API View
16
+
17
+ GET: Führt NICE-Klassifikation für alle Videos durch
18
+ POST: Führt NICE-Klassifikation für spezifizierte Videos durch
19
+
20
+ POST Body: {"video_ids": [1, 2, 3]} oder leerer Body für alle Videos
21
+ """
22
+
23
+ def get(self, request):
24
+ """Legacy GET endpoint - processes all videos"""
25
+ print("[DEBUG] NICE Classification View hit (GET)")
26
+ return self._process_classification(request)
27
+
28
+ def post(self, request):
29
+ """New POST endpoint - processes specified videos"""
30
+ print("[DEBUG] NICE Classification View hit (POST)")
31
+ return self._process_classification(request)
32
+
33
+ def _process_classification(self, request):
34
+ try:
35
+ # Handle POST data for specific video IDs
36
+ video_ids = None
37
+ if request.method == 'POST' and hasattr(request, 'data'):
38
+ video_ids = request.data.get('video_ids', None)
39
+
40
+ if video_ids:
41
+ videos = VideoFile.objects.filter(id__in=video_ids)
42
+ print(f"[DEBUG] Processing NICE classification for specific videos: {video_ids}")
43
+ else:
44
+ videos = VideoFile.objects.all()
45
+ print("[DEBUG] Processing NICE classification for all videos")
46
+
47
+ print(f"[DEBUG] Total videos found: {videos.count()}")
48
+ for v in videos:
49
+ print(f"[DEBUG] Video ID: {v.id}")
50
+
51
+ if not videos.exists():
52
+ return Response(
53
+ {"error": "No videos found in the database."},
54
+ status=status.HTTP_404_NOT_FOUND
55
+ )
56
+
57
+ serializer = ForNiceClassificationSerializer()
58
+ response_data = serializer.to_representation(videos)
59
+
60
+ if not response_data:
61
+ return Response(
62
+ {"error": "No valid segments for NICE classification."},
63
+ status=status.HTTP_404_NOT_FOUND
64
+ )
65
+
66
+ return Response(response_data, status=status.HTTP_200_OK)
67
+
68
+ except Exception as e:
69
+ return Response(
70
+ {"error": f"Internal server error: {str(e)}"},
71
+ status=status.HTTP_500_INTERNAL_SERVER_ERROR
72
+ )
73
+
74
+
75
+ class ForParisClassificationView(APIView):
76
+ """
77
+ PARIS Classification API View
78
+
79
+ GET: Führt PARIS-Klassifikation für alle Videos durch
80
+ POST: Führt PARIS-Klassifikation für spezifizierte Videos durch
81
+
82
+ POST Body: {"video_ids": [1, 2, 3]} oder leerer Body für alle Videos
83
+ """
84
+
85
+ def get(self, request):
86
+ """Legacy GET endpoint - processes all videos"""
87
+ print("[DEBUG] PARIS Classification View hit (GET)")
88
+ return self._process_classification(request)
89
+
90
+ def post(self, request):
91
+ """New POST endpoint - processes specified videos"""
92
+ print("[DEBUG] PARIS Classification View hit (POST)")
93
+ return self._process_classification(request)
94
+
95
+ def _process_classification(self, request):
96
+ try:
97
+ # Handle POST data for specific video IDs
98
+ video_ids = None
99
+ if request.method == 'POST' and hasattr(request, 'data'):
100
+ video_ids = request.data.get('video_ids', None)
101
+
102
+ if video_ids:
103
+ videos = VideoFile.objects.filter(id__in=video_ids)
104
+ print(f"[DEBUG] Processing PARIS classification for specific videos: {video_ids}")
105
+ else:
106
+ videos = VideoFile.objects.all()
107
+ print(f"[DEBUG] Processing PARIS classification for all videos")
108
+
109
+ print(f"[DEBUG] Total videos found: {videos.count()}")
110
+
111
+ filtered_videos = [
112
+ video for video in videos
113
+ if getattr(video, "frame_dir", None) # no more readable_predictions check
114
+ ]
115
+
116
+ if not filtered_videos:
117
+ return Response(
118
+ {"error": "No videos with valid frame_dir found."},
119
+ status=status.HTTP_404_NOT_FOUND
120
+ )
121
+
122
+ serializer = ForParisClassificationSerializer()
123
+ response_data = serializer.to_representation(filtered_videos)
124
+
125
+ if not response_data:
126
+ return Response(
127
+ {"error": "No valid PARIS segments found."},
128
+ status=status.HTTP_404_NOT_FOUND
129
+ )
130
+
131
+ return Response(response_data, status=status.HTTP_200_OK)
132
+
133
+ except Exception as e:
134
+ return Response(
135
+ {"error": f"Internal server error: {str(e)}"},
136
+ status=status.HTTP_500_INTERNAL_SERVER_ERROR
137
+ )
138
+
139
+
140
+ class BatchClassificationView(APIView):
141
+ """
142
+ Batch Classification API View
143
+
144
+ POST: Führt beide Klassifikationstypen (NICE und PARIS) für spezifizierte Videos durch
145
+
146
+ POST Body: {
147
+ "video_ids": [1, 2, 3],
148
+ "types": ["nice", "paris"] # Optional, default beide
149
+ }
150
+ """
151
+
152
+ def post(self, request):
153
+ try:
154
+ video_ids = request.data.get('video_ids', None)
155
+ classification_types = request.data.get('types', ['nice', 'paris'])
156
+
157
+ if video_ids:
158
+ videos = VideoFile.objects.filter(id__in=video_ids)
159
+ else:
160
+ videos = VideoFile.objects.all()
161
+
162
+ if not videos.exists():
163
+ return Response(
164
+ {"error": "No videos found."},
165
+ status=status.HTTP_404_NOT_FOUND
166
+ )
167
+
168
+ results = {}
169
+
170
+ if 'nice' in classification_types:
171
+ nice_serializer = ForNiceClassificationSerializer()
172
+ results['nice'] = nice_serializer.to_representation(videos)
173
+
174
+ if 'paris' in classification_types:
175
+ # Filter videos for PARIS (need frame_dir)
176
+ filtered_videos = [
177
+ video for video in videos
178
+ if getattr(video, "frame_dir", None)
179
+ ]
180
+
181
+ if filtered_videos:
182
+ paris_serializer = ForParisClassificationSerializer()
183
+ results['paris'] = paris_serializer.to_representation(filtered_videos)
184
+ else:
185
+ results['paris'] = {"error": "No videos with valid frame_dir found for PARIS classification."}
186
+
187
+ return Response({
188
+ "message": "Batch classification completed.",
189
+ "results": results
190
+ }, status=status.HTTP_200_OK)
191
+
192
+ except Exception as e:
193
+ return Response(
194
+ {"error": f"Internal server error: {str(e)}"},
195
+ status=status.HTTP_500_INTERNAL_SERVER_ERROR
196
+ )
197
+
198
+
199
+ class ClassificationStatusView(APIView):
200
+ """
201
+ Classification Status API View
202
+
203
+ GET: Gibt den Status der Klassifikationen für ein Video zurück
204
+ """
205
+
206
+ def get(self, request, video_id):
207
+ try:
208
+ video = VideoFile.objects.get(id=video_id)
209
+
210
+ # Check if classifications exist for this video
211
+ # This would typically check for saved classification results in the database
212
+ # For now, we'll return basic status information
213
+
214
+ status_info = {
215
+ "video_id": video_id,
216
+ "video_name": getattr(video, 'original_file_name', 'Unknown'),
217
+ "has_frame_dir": bool(getattr(video, "frame_dir", None)),
218
+ "nice_classification_available": True, # Always available for NICE
219
+ "paris_classification_available": bool(getattr(video, "frame_dir", None)),
220
+ "last_processed": None, # Would come from classification results table
221
+ "classification_results": {
222
+ "nice": None, # Would contain saved NICE results
223
+ "paris": None # Would contain saved PARIS results
224
+ }
225
+ }
226
+
227
+ return Response(status_info, status=status.HTTP_200_OK)
228
+
229
+ except VideoFile.DoesNotExist:
230
+ return Response(
231
+ {"error": f"Video with ID {video_id} not found."},
232
+ status=status.HTTP_404_NOT_FOUND
233
+ )
234
+ except Exception as e:
235
+ return Response(
236
+ {"error": f"Internal server error: {str(e)}"},
237
+ status=status.HTTP_500_INTERNAL_SERVER_ERROR
238
+ )