endoreg-db 0.3.6__py3-none-any.whl → 0.8.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (970) hide show
  1. endoreg_db/admin.py +92 -3
  2. endoreg_db/api_urls.py +4 -0
  3. endoreg_db/apps.py +18 -6
  4. endoreg_db/assets/dummy_model.ckpt +1 -0
  5. endoreg_db/codemods/readme.md +88 -0
  6. endoreg_db/codemods/rename_datetime_fields.py +92 -0
  7. endoreg_db/config/env.py +101 -0
  8. endoreg_db/data/__init__.py +144 -65
  9. endoreg_db/data/ai_model/data.yaml +7 -0
  10. endoreg_db/data/{label → ai_model_label}/label/data.yaml +88 -62
  11. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  12. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  13. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  14. endoreg_db/data/{label → ai_model_label}/label-type/data.yaml +6 -6
  15. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  16. endoreg_db/data/{model_type → ai_model_type}/data.yaml +6 -6
  17. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  18. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  19. endoreg_db/data/case_template/rule/00_patient_lab_sample_add_default_value.yaml +167 -167
  20. endoreg_db/data/case_template/rule/01_patient-set-age.yaml +7 -7
  21. endoreg_db/data/case_template/rule/01_patient-set-gender.yaml +8 -8
  22. endoreg_db/data/case_template/rule/11_create_patient_lab_sample.yaml +22 -22
  23. endoreg_db/data/case_template/rule/12_create-patient_medication-anticoagulation.yaml +18 -18
  24. endoreg_db/data/case_template/rule/13_create-patient_medication_schedule-anticoagulation.yaml +18 -18
  25. endoreg_db/data/case_template/rule/19_create_patient.yaml +16 -16
  26. endoreg_db/data/case_template/rule_type/base_types.yaml +35 -35
  27. endoreg_db/data/case_template/rule_value_type/base_types.yaml +58 -58
  28. endoreg_db/data/case_template/template/base.yaml +7 -7
  29. endoreg_db/data/case_template/template_type/pre_endoscopy.yaml +2 -2
  30. endoreg_db/data/case_template/tmp/_rule_value +13 -13
  31. endoreg_db/data/case_template/tmp/rule/01_atrial_fibrillation.yaml +21 -21
  32. endoreg_db/data/case_template/tmp/rule/02_create_object.yaml +9 -9
  33. endoreg_db/data/case_template/tmp/template/atrial_fibrillation_low_risk.yaml +6 -6
  34. endoreg_db/data/center/data.yaml +90 -59
  35. endoreg_db/data/center_resource/green_endoscopy_dashboard_CenterResource.yaml +144 -144
  36. endoreg_db/data/center_shift/ukw.yaml +9 -0
  37. endoreg_db/data/center_waste/green_endoscopy_dashboard_CenterWaste.yaml +48 -48
  38. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  39. endoreg_db/data/db_summary.csv +58 -0
  40. endoreg_db/data/db_summary.xlsx +0 -0
  41. endoreg_db/data/disease/cardiovascular.yaml +37 -37
  42. endoreg_db/data/disease/hepatology.yaml +4 -4
  43. endoreg_db/data/disease/misc.yaml +5 -6
  44. endoreg_db/data/disease/renal.yaml +4 -4
  45. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +6 -6
  46. endoreg_db/data/disease_classification/coronary_vessel_disease.yaml +5 -5
  47. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +41 -41
  48. endoreg_db/data/disease_classification_choice/coronary_vessel_disease.yaml +19 -19
  49. endoreg_db/data/distribution/date/patient.yaml +6 -6
  50. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  51. endoreg_db/data/distribution/single_categorical/patient.yaml +6 -6
  52. endoreg_db/data/emission_factor/green_endoscopy_dashboard_EmissionFactor.yaml +132 -132
  53. endoreg_db/data/endoscope/data.yaml +93 -0
  54. endoreg_db/data/endoscope_type/data.yaml +10 -10
  55. endoreg_db/data/endoscopy_processor/data.yaml +50 -45
  56. endoreg_db/data/event/cardiology.yaml +15 -28
  57. endoreg_db/data/event/neurology.yaml +13 -13
  58. endoreg_db/data/event/surgery.yaml +12 -12
  59. endoreg_db/data/event/thrombembolism.yaml +19 -19
  60. endoreg_db/data/examination/examinations/data.yaml +72 -66
  61. endoreg_db/data/examination/time/data.yaml +47 -47
  62. endoreg_db/data/examination/time-type/data.yaml +7 -7
  63. endoreg_db/data/examination/type/data.yaml +17 -5
  64. endoreg_db/data/examination_indication/endoscopy.yaml +424 -0
  65. endoreg_db/data/examination_indication_classification/endoscopy.yaml +160 -0
  66. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +101 -0
  67. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  68. endoreg_db/data/finding/anatomy_colon.yaml +128 -0
  69. endoreg_db/data/finding/colonoscopy.yaml +40 -0
  70. endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +56 -0
  71. endoreg_db/data/finding/complication.yaml +16 -0
  72. endoreg_db/data/finding/data.yaml +105 -0
  73. endoreg_db/data/finding/examination_setting.yaml +16 -0
  74. endoreg_db/data/finding/medication_related.yaml +18 -0
  75. endoreg_db/data/finding/outcome.yaml +12 -0
  76. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +95 -0
  77. endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +22 -0
  78. endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +25 -0
  79. endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  80. endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  81. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  82. endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  83. endoreg_db/data/finding_classification/colonoscopy_location.yaml +80 -0
  84. endoreg_db/data/finding_classification/colonoscopy_lst.yaml +21 -0
  85. endoreg_db/data/finding_classification/colonoscopy_nice.yaml +20 -0
  86. endoreg_db/data/finding_classification/colonoscopy_paris.yaml +26 -0
  87. endoreg_db/data/finding_classification/colonoscopy_sano.yaml +22 -0
  88. endoreg_db/data/finding_classification/colonoscopy_summary.yaml +53 -0
  89. endoreg_db/data/finding_classification/complication_generic.yaml +25 -0
  90. endoreg_db/data/finding_classification/examination_setting_generic.yaml +40 -0
  91. endoreg_db/data/finding_classification/histology_colo.yaml +51 -0
  92. endoreg_db/data/finding_classification/intervention_required.yaml +26 -0
  93. endoreg_db/data/finding_classification/medication_related.yaml +23 -0
  94. endoreg_db/data/finding_classification/visualized.yaml +33 -0
  95. endoreg_db/data/finding_classification_choice/bowel_preparation.yaml +78 -0
  96. endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
  97. endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  98. endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  99. endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  100. endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +17 -0
  101. endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +57 -0
  102. endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
  103. endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  104. endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
  105. endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +229 -0
  106. endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  107. endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +82 -0
  108. endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  109. endoreg_db/data/finding_classification_choice/complication_generic_types.yaml +15 -0
  110. endoreg_db/data/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  111. endoreg_db/data/finding_classification_choice/histology.yaml +24 -0
  112. endoreg_db/data/finding_classification_choice/histology_polyp.yaml +20 -0
  113. endoreg_db/data/finding_classification_choice/outcome.yaml +19 -0
  114. endoreg_db/data/finding_classification_choice/yes_no_na.yaml +11 -0
  115. endoreg_db/data/finding_classification_type/colonoscopy_basic.yaml +48 -0
  116. endoreg_db/data/finding_intervention/endoscopy.yaml +43 -0
  117. endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
  118. endoreg_db/data/finding_intervention/endoscopy_egd.yaml +128 -0
  119. endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +32 -0
  120. endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +9 -0
  121. endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +36 -0
  122. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  123. endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +79 -0
  124. endoreg_db/data/finding_type/data.yaml +43 -0
  125. endoreg_db/data/gender/data.yaml +42 -18
  126. endoreg_db/data/information_source/annotation.yaml +6 -0
  127. endoreg_db/data/information_source/data.yaml +30 -30
  128. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  129. endoreg_db/data/information_source/medication.yaml +5 -5
  130. endoreg_db/data/information_source/prediction.yaml +7 -0
  131. endoreg_db/data/information_source_type/data.yaml +8 -0
  132. endoreg_db/data/lab_value/cardiac_enzymes.yaml +37 -31
  133. endoreg_db/data/lab_value/coagulation.yaml +54 -49
  134. endoreg_db/data/lab_value/electrolytes.yaml +228 -190
  135. endoreg_db/data/lab_value/gastrointestinal_function.yaml +133 -121
  136. endoreg_db/data/lab_value/hematology.yaml +184 -169
  137. endoreg_db/data/lab_value/hormones.yaml +59 -53
  138. endoreg_db/data/lab_value/lipids.yaml +53 -44
  139. endoreg_db/data/lab_value/misc.yaml +76 -30
  140. endoreg_db/data/lab_value/renal_function.yaml +12 -11
  141. endoreg_db/data/log_type/data.yaml +57 -0
  142. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  143. endoreg_db/data/lx_client_type/base.yaml +30 -0
  144. endoreg_db/data/lx_permission/base.yaml +24 -0
  145. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  146. endoreg_db/data/medication/anticoagulation.yaml +64 -64
  147. endoreg_db/data/medication/tah.yaml +69 -69
  148. endoreg_db/data/medication_indication/anticoagulation.yaml +115 -120
  149. endoreg_db/data/medication_indication_type/data.yaml +10 -10
  150. endoreg_db/data/medication_indication_type/thrombembolism.yaml +40 -40
  151. endoreg_db/data/medication_intake_time/base.yaml +30 -30
  152. endoreg_db/data/medication_schedule/apixaban.yaml +94 -94
  153. endoreg_db/data/medication_schedule/ass.yaml +12 -12
  154. endoreg_db/data/medication_schedule/enoxaparin.yaml +26 -26
  155. endoreg_db/data/names_first/first_names.yaml +54 -0
  156. endoreg_db/data/names_last/last_names.yaml +51 -0
  157. endoreg_db/data/network_device/data.yaml +59 -0
  158. endoreg_db/data/network_device_type/data.yaml +12 -0
  159. endoreg_db/data/organ/data.yaml +29 -0
  160. endoreg_db/data/patient_lab_sample_type/generic.yaml +5 -5
  161. endoreg_db/data/pdf_type/data.yaml +46 -28
  162. endoreg_db/data/product/green_endoscopy_dashboard_Product.yaml +66 -66
  163. endoreg_db/data/product_group/green_endoscopy_dashboard_ProductGroup.yaml +33 -33
  164. endoreg_db/data/product_material/green_endoscopy_dashboard_ProductMaterial.yaml +308 -308
  165. endoreg_db/data/product_weight/green_endoscopy_dashboard_ProductWeight.yaml +88 -88
  166. endoreg_db/data/profession/data.yaml +70 -70
  167. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  168. endoreg_db/data/qualification/m2.yaml +39 -0
  169. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  170. endoreg_db/data/qualification/sonography.yaml +36 -0
  171. endoreg_db/data/qualification_type/base.yaml +29 -0
  172. endoreg_db/data/reference_product/green_endoscopy_dashboard_ReferenceProduct.yaml +55 -55
  173. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  174. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +30 -26
  175. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +24 -19
  176. endoreg_db/data/requirement/age.yaml +26 -0
  177. endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +45 -0
  178. endoreg_db/data/requirement/disease_cardiovascular.yaml +79 -0
  179. endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
  180. endoreg_db/data/requirement/disease_hepatology.yaml +12 -0
  181. endoreg_db/data/requirement/disease_misc.yaml +12 -0
  182. endoreg_db/data/requirement/disease_renal.yaml +96 -0
  183. endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +59 -0
  184. endoreg_db/data/requirement/event_cardiology.yaml +251 -0
  185. endoreg_db/data/requirement/event_requirements.yaml +145 -0
  186. endoreg_db/data/requirement/finding_colon_polyp.yaml +50 -0
  187. endoreg_db/data/requirement/gender.yaml +25 -0
  188. endoreg_db/data/requirement/lab_value.yaml +441 -0
  189. endoreg_db/data/requirement/medication.yaml +93 -0
  190. endoreg_db/data/requirement_operator/age.yaml +13 -0
  191. endoreg_db/data/requirement_operator/lab_operators.yaml +129 -0
  192. endoreg_db/data/requirement_operator/model_operators.yaml +96 -0
  193. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +48 -0
  194. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  195. endoreg_db/data/requirement_set/endoscopy_bleeding_risk.yaml +52 -0
  196. endoreg_db/data/requirement_set_type/data.yaml +20 -0
  197. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  198. endoreg_db/data/resource/green_endoscopy_dashboard_Resource.yaml +15 -15
  199. endoreg_db/data/risk/bleeding.yaml +26 -0
  200. endoreg_db/data/risk/thrombosis.yaml +37 -0
  201. endoreg_db/data/risk_type/data.yaml +27 -0
  202. endoreg_db/data/setup_config.yaml +38 -0
  203. endoreg_db/data/shift/endoscopy.yaml +21 -0
  204. endoreg_db/data/shift/m2.yaml +0 -0
  205. endoreg_db/data/shift_type/base.yaml +35 -0
  206. endoreg_db/data/tag/requirement_set_tags.yaml +11 -0
  207. endoreg_db/data/transport_route/green_endoscopy_dashboard_TransportRoute.yaml +12 -12
  208. endoreg_db/data/unit/concentration.yaml +115 -92
  209. endoreg_db/data/unit/data.yaml +17 -17
  210. endoreg_db/data/unit/length.yaml +30 -30
  211. endoreg_db/data/unit/misc.yaml +19 -19
  212. endoreg_db/data/unit/rate.yaml +5 -5
  213. endoreg_db/data/unit/time.yaml +48 -13
  214. endoreg_db/data/unit/volume.yaml +35 -35
  215. endoreg_db/data/unit/weight.yaml +37 -37
  216. endoreg_db/data/waste/data.yaml +11 -11
  217. endoreg_db/exceptions.py +19 -0
  218. endoreg_db/factories/__init__.py +0 -0
  219. endoreg_db/forms/__init__.py +5 -3
  220. endoreg_db/forms/examination_form.py +11 -0
  221. endoreg_db/forms/patient_finding_intervention_form.py +18 -0
  222. endoreg_db/forms/patient_form.py +27 -0
  223. endoreg_db/forms/questionnaires/__init__.py +1 -1
  224. endoreg_db/forms/questionnaires/tto_questionnaire.py +23 -23
  225. endoreg_db/forms/settings/__init__.py +8 -8
  226. endoreg_db/forms/unit.py +5 -5
  227. endoreg_db/helpers/__init__.py +0 -0
  228. endoreg_db/helpers/count_db.py +45 -0
  229. endoreg_db/helpers/data_loader.py +208 -0
  230. endoreg_db/helpers/default_objects.py +378 -0
  231. endoreg_db/helpers/download_segmentation_model.py +31 -0
  232. endoreg_db/helpers/interact.py +6 -0
  233. endoreg_db/helpers/test_video_helper.py +119 -0
  234. endoreg_db/logger_conf.py +140 -0
  235. endoreg_db/management/__init__.py +1 -0
  236. endoreg_db/management/commands/__init__.py +1 -0
  237. endoreg_db/management/commands/anonymize_video.py +0 -0
  238. endoreg_db/management/commands/check_auth.py +125 -0
  239. endoreg_db/management/commands/create_model_meta_from_huggingface.py +115 -0
  240. endoreg_db/management/commands/create_multilabel_model_meta.py +214 -0
  241. endoreg_db/management/commands/fix_missing_patient_data.py +172 -0
  242. endoreg_db/management/commands/fix_video_paths.py +165 -0
  243. endoreg_db/management/commands/import_fallback_video.py +203 -0
  244. endoreg_db/management/commands/import_report.py +298 -0
  245. endoreg_db/management/commands/import_video.py +423 -0
  246. endoreg_db/management/commands/import_video_with_classification.py +367 -0
  247. endoreg_db/management/commands/init_default_ai_model.py +112 -0
  248. endoreg_db/management/commands/load_ai_model_data.py +77 -45
  249. endoreg_db/management/commands/load_ai_model_label_data.py +59 -0
  250. endoreg_db/management/commands/load_base_db_data.py +192 -128
  251. endoreg_db/management/commands/load_center_data.py +68 -43
  252. endoreg_db/management/commands/{load_medication_intake_time_data.py → load_contraindication_data.py} +40 -40
  253. endoreg_db/management/commands/load_disease_classification_choices_data.py +40 -40
  254. endoreg_db/management/commands/load_disease_classification_data.py +40 -40
  255. endoreg_db/management/commands/load_disease_data.py +61 -39
  256. endoreg_db/management/commands/load_distribution_data.py +65 -65
  257. endoreg_db/management/commands/{load_endoscope_type_data.py → load_endoscope_data.py} +67 -44
  258. endoreg_db/management/commands/load_event_data.py +40 -40
  259. endoreg_db/management/commands/load_examination_data.py +74 -74
  260. endoreg_db/management/commands/load_examination_indication_data.py +86 -0
  261. endoreg_db/management/commands/load_finding_data.py +128 -0
  262. endoreg_db/management/commands/load_gender_data.py +43 -43
  263. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +131 -132
  264. endoreg_db/management/commands/load_information_source.py +50 -44
  265. endoreg_db/management/commands/load_lab_value_data.py +49 -49
  266. endoreg_db/management/commands/load_medication_data.py +103 -41
  267. endoreg_db/management/commands/load_name_data.py +37 -0
  268. endoreg_db/management/commands/{load_medication_indication_type_data.py → load_organ_data.py} +42 -40
  269. endoreg_db/management/commands/load_pdf_type_data.py +60 -60
  270. endoreg_db/management/commands/load_profession_data.py +43 -43
  271. endoreg_db/management/commands/load_qualification_data.py +59 -0
  272. endoreg_db/management/commands/{load_report_reader_flag.py → load_report_reader_flag_data.py} +45 -45
  273. endoreg_db/management/commands/load_requirement_data.py +180 -0
  274. endoreg_db/management/commands/load_risk_data.py +56 -0
  275. endoreg_db/management/commands/load_shift_data.py +60 -0
  276. endoreg_db/management/commands/load_tag_data.py +57 -0
  277. endoreg_db/management/commands/load_unit_data.py +45 -45
  278. endoreg_db/management/commands/load_user_groups.py +28 -28
  279. endoreg_db/management/commands/register_ai_model.py +64 -65
  280. endoreg_db/management/commands/reset_celery_schedule.py +9 -9
  281. endoreg_db/management/commands/setup_endoreg_db.py +381 -0
  282. endoreg_db/management/commands/start_filewatcher.py +106 -0
  283. endoreg_db/management/commands/storage_management.py +548 -0
  284. endoreg_db/management/commands/summarize_db_content.py +189 -0
  285. endoreg_db/management/commands/validate_video.py +204 -0
  286. endoreg_db/management/commands/validate_video_files.py +161 -0
  287. endoreg_db/management/commands/video_validation.py +22 -0
  288. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  289. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  290. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  291. endoreg_db/mermaid/classification.md +8 -0
  292. endoreg_db/mermaid/examination.md +8 -0
  293. endoreg_db/mermaid/findings.md +7 -0
  294. endoreg_db/mermaid/image_classification.md +28 -0
  295. endoreg_db/mermaid/interventions.md +8 -0
  296. endoreg_db/mermaid/morphology.md +8 -0
  297. endoreg_db/mermaid/patient_creation.md +14 -0
  298. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  299. endoreg_db/migrations/0001_initial.py +1857 -582
  300. endoreg_db/migrations/0002_add_video_correction_models.py +52 -0
  301. endoreg_db/migrations/0003_add_center_display_name.py +30 -0
  302. endoreg_db/models/__init__.py +359 -74
  303. endoreg_db/models/administration/__init__.py +116 -0
  304. endoreg_db/models/administration/ai/__init__.py +9 -0
  305. endoreg_db/models/administration/ai/active_model.py +35 -0
  306. endoreg_db/models/administration/ai/ai_model.py +156 -0
  307. endoreg_db/models/{ai_model → administration/ai}/model_type.py +41 -26
  308. endoreg_db/models/administration/case/__init__.py +19 -0
  309. endoreg_db/models/administration/case/case.py +114 -0
  310. endoreg_db/models/{case_template → administration/case/case_template}/__init__.py +15 -6
  311. endoreg_db/models/{case_template → administration/case/case_template}/case_template.py +125 -81
  312. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule.py +269 -276
  313. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule_value.py +86 -73
  314. endoreg_db/models/{case_template → administration/case/case_template}/case_template_type.py +26 -28
  315. endoreg_db/models/{center → administration/center}/__init__.py +13 -4
  316. endoreg_db/models/administration/center/center.py +67 -0
  317. endoreg_db/models/administration/center/center_product.py +64 -0
  318. endoreg_db/models/administration/center/center_resource.py +49 -0
  319. endoreg_db/models/administration/center/center_shift.py +88 -0
  320. endoreg_db/models/administration/center/center_waste.py +30 -0
  321. endoreg_db/models/administration/permissions/__init__.py +44 -0
  322. endoreg_db/models/administration/person/__init__.py +24 -0
  323. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  324. endoreg_db/models/administration/person/employee/employee.py +35 -0
  325. endoreg_db/models/administration/person/employee/employee_qualification.py +39 -0
  326. endoreg_db/models/administration/person/employee/employee_type.py +42 -0
  327. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  328. endoreg_db/models/administration/person/examiner/examiner.py +54 -0
  329. endoreg_db/models/administration/person/names/__init__.py +0 -0
  330. endoreg_db/models/{persons → administration/person/names}/first_name.py +18 -18
  331. endoreg_db/models/{persons → administration/person/names}/last_name.py +18 -19
  332. endoreg_db/models/administration/person/patient/__init__.py +5 -0
  333. endoreg_db/models/administration/person/patient/patient.py +460 -0
  334. endoreg_db/models/{persons → administration/person}/person.py +31 -31
  335. endoreg_db/models/administration/person/profession/__init__.py +24 -0
  336. endoreg_db/models/administration/person/user/__init__.py +5 -0
  337. endoreg_db/models/administration/person/user/portal_user_information.py +37 -0
  338. endoreg_db/models/administration/product/__init__.py +14 -0
  339. endoreg_db/models/administration/product/product.py +97 -0
  340. endoreg_db/models/administration/product/product_group.py +39 -0
  341. endoreg_db/models/administration/product/product_material.py +54 -0
  342. endoreg_db/models/{product → administration/product}/product_weight.py +47 -26
  343. endoreg_db/models/{product → administration/product}/reference_product.py +130 -99
  344. endoreg_db/models/administration/qualification/__init__.py +7 -0
  345. endoreg_db/models/administration/qualification/qualification.py +37 -0
  346. endoreg_db/models/administration/qualification/qualification_type.py +35 -0
  347. endoreg_db/models/administration/shift/__init__.py +9 -0
  348. endoreg_db/models/administration/shift/scheduled_days.py +69 -0
  349. endoreg_db/models/administration/shift/shift.py +51 -0
  350. endoreg_db/models/administration/shift/shift_type.py +108 -0
  351. endoreg_db/models/label/__init__.py +24 -1
  352. endoreg_db/models/label/annotation/__init__.py +12 -0
  353. endoreg_db/models/label/annotation/image_classification.py +84 -0
  354. endoreg_db/models/label/annotation/video_segmentation_annotation.py +66 -0
  355. endoreg_db/models/label/label.py +83 -84
  356. endoreg_db/models/label/label_set.py +53 -0
  357. endoreg_db/models/label/label_type.py +29 -0
  358. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  359. endoreg_db/models/label/label_video_segment/_create_from_video.py +41 -0
  360. endoreg_db/models/label/label_video_segment/label_video_segment.py +511 -0
  361. endoreg_db/models/label/video_segmentation_label.py +31 -0
  362. endoreg_db/models/label/video_segmentation_labelset.py +27 -0
  363. endoreg_db/models/media/__init__.py +16 -0
  364. endoreg_db/models/media/frame/__init__.py +3 -0
  365. endoreg_db/models/media/frame/frame.py +111 -0
  366. endoreg_db/models/media/pdf/__init__.py +11 -0
  367. endoreg_db/models/media/pdf/raw_pdf.py +757 -0
  368. endoreg_db/models/media/pdf/report_file.py +162 -0
  369. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  370. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +77 -0
  371. endoreg_db/models/{report_reader → media/pdf/report_reader}/report_reader_flag.py +19 -19
  372. endoreg_db/models/media/video/__init__.py +8 -0
  373. endoreg_db/models/media/video/create_from_file.py +358 -0
  374. endoreg_db/models/media/video/pipe_1.py +213 -0
  375. endoreg_db/models/media/video/pipe_2.py +105 -0
  376. endoreg_db/models/media/video/refactor_plan.md +0 -0
  377. endoreg_db/models/media/video/video_file.py +825 -0
  378. endoreg_db/models/media/video/video_file_ai.py +443 -0
  379. endoreg_db/models/media/video/video_file_anonymize.py +349 -0
  380. endoreg_db/models/media/video/video_file_frames/__init__.py +47 -0
  381. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +22 -0
  382. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  383. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +104 -0
  384. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +174 -0
  385. endoreg_db/models/media/video/video_file_frames/_get_frame.py +28 -0
  386. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +27 -0
  387. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +20 -0
  388. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +27 -0
  389. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +34 -0
  390. endoreg_db/models/media/video/video_file_frames/_get_frames.py +27 -0
  391. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +129 -0
  392. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +141 -0
  393. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +65 -0
  394. endoreg_db/models/media/video/video_file_frames.py +0 -0
  395. endoreg_db/models/media/video/video_file_io.py +168 -0
  396. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  397. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +45 -0
  398. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +39 -0
  399. endoreg_db/models/media/video/video_file_meta/get_fps.py +147 -0
  400. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +143 -0
  401. endoreg_db/models/media/video/video_file_meta/text_meta.py +134 -0
  402. endoreg_db/models/media/video/video_file_meta/video_meta.py +70 -0
  403. endoreg_db/models/media/video/video_file_segments.py +209 -0
  404. endoreg_db/models/media/video/video_metadata.py +65 -0
  405. endoreg_db/models/media/video/video_processing.py +152 -0
  406. endoreg_db/models/medical/__init__.py +146 -0
  407. endoreg_db/models/medical/contraindication/__init__.py +17 -0
  408. endoreg_db/models/medical/disease.py +156 -0
  409. endoreg_db/models/medical/event.py +137 -0
  410. endoreg_db/models/medical/examination/__init__.py +9 -0
  411. endoreg_db/models/medical/examination/examination.py +148 -0
  412. endoreg_db/models/medical/examination/examination_indication.py +278 -0
  413. endoreg_db/models/medical/examination/examination_time.py +49 -0
  414. endoreg_db/models/medical/examination/examination_time_type.py +41 -0
  415. endoreg_db/models/medical/examination/examination_type.py +48 -0
  416. endoreg_db/models/medical/finding/__init__.py +18 -0
  417. endoreg_db/models/medical/finding/finding.py +96 -0
  418. endoreg_db/models/medical/finding/finding_classification.py +142 -0
  419. endoreg_db/models/medical/finding/finding_intervention.py +52 -0
  420. endoreg_db/models/medical/finding/finding_type.py +35 -0
  421. endoreg_db/models/medical/hardware/__init__.py +8 -0
  422. endoreg_db/models/medical/hardware/endoscope.py +65 -0
  423. endoreg_db/models/{hardware → medical/hardware}/endoscopy_processor.py +182 -143
  424. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  425. endoreg_db/models/medical/laboratory/lab_value.py +419 -0
  426. endoreg_db/models/medical/medication/__init__.py +19 -0
  427. endoreg_db/models/medical/medication/medication.py +31 -0
  428. endoreg_db/models/medical/medication/medication_indication.py +50 -0
  429. endoreg_db/models/medical/medication/medication_indication_type.py +39 -0
  430. endoreg_db/models/medical/medication/medication_intake_time.py +44 -0
  431. endoreg_db/models/medical/medication/medication_schedule.py +45 -0
  432. endoreg_db/models/medical/organ/__init__.py +35 -0
  433. endoreg_db/models/medical/patient/__init__.py +56 -0
  434. endoreg_db/models/medical/patient/medication_examples.py +38 -0
  435. endoreg_db/models/medical/patient/patient_disease.py +63 -0
  436. endoreg_db/models/medical/patient/patient_event.py +75 -0
  437. endoreg_db/models/medical/patient/patient_examination.py +249 -0
  438. endoreg_db/models/medical/patient/patient_examination_indication.py +44 -0
  439. endoreg_db/models/medical/patient/patient_finding.py +357 -0
  440. endoreg_db/models/medical/patient/patient_finding_classification.py +207 -0
  441. endoreg_db/models/medical/patient/patient_finding_intervention.py +40 -0
  442. endoreg_db/models/medical/patient/patient_lab_sample.py +148 -0
  443. endoreg_db/models/{persons → medical}/patient/patient_lab_value.py +222 -176
  444. endoreg_db/models/medical/patient/patient_medication.py +104 -0
  445. endoreg_db/models/medical/patient/patient_medication_schedule.py +136 -0
  446. endoreg_db/models/medical/risk/__init__.py +7 -0
  447. endoreg_db/models/medical/risk/risk.py +72 -0
  448. endoreg_db/models/medical/risk/risk_type.py +51 -0
  449. endoreg_db/models/metadata/__init__.py +19 -0
  450. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  451. endoreg_db/models/metadata/model_meta.py +206 -0
  452. endoreg_db/models/metadata/model_meta_logic.py +343 -0
  453. endoreg_db/models/{data_file/metadata → metadata}/pdf_meta.py +89 -70
  454. endoreg_db/models/metadata/sensitive_meta.py +288 -0
  455. endoreg_db/models/metadata/sensitive_meta_logic.py +1048 -0
  456. endoreg_db/models/metadata/video_meta.py +332 -0
  457. endoreg_db/models/metadata/video_prediction_logic.py +190 -0
  458. endoreg_db/models/metadata/video_prediction_meta.py +270 -0
  459. endoreg_db/models/other/__init__.py +40 -5
  460. endoreg_db/models/other/distribution/__init__.py +44 -0
  461. endoreg_db/models/other/distribution/base_value_distribution.py +20 -0
  462. endoreg_db/models/other/distribution/date_value_distribution.py +89 -0
  463. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +32 -0
  464. endoreg_db/models/other/distribution/numeric_value_distribution.py +125 -0
  465. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +22 -0
  466. endoreg_db/models/other/emission/__init__.py +5 -0
  467. endoreg_db/models/other/emission/emission_factor.py +94 -0
  468. endoreg_db/models/{persons → other}/gender.py +27 -22
  469. endoreg_db/models/other/information_source.py +159 -0
  470. endoreg_db/models/other/material.py +28 -16
  471. endoreg_db/models/other/resource.py +21 -17
  472. endoreg_db/models/other/tag.py +27 -0
  473. endoreg_db/models/other/transport_route.py +33 -21
  474. endoreg_db/models/{unit.py → other/unit.py} +32 -22
  475. endoreg_db/models/other/waste.py +27 -20
  476. endoreg_db/models/requirement/__init__.py +11 -0
  477. endoreg_db/models/requirement/requirement.py +767 -0
  478. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  479. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  480. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +9 -0
  481. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +95 -0
  482. endoreg_db/models/requirement/requirement_operator.py +176 -0
  483. endoreg_db/models/requirement/requirement_set.py +287 -0
  484. endoreg_db/models/rule/__init__.py +13 -0
  485. endoreg_db/models/{rules → rule}/rule.py +27 -24
  486. endoreg_db/models/{rules → rule}/rule_applicator.py +224 -224
  487. endoreg_db/models/{rules → rule}/rule_attribute_dtype.py +16 -18
  488. endoreg_db/models/{rules → rule}/rule_type.py +19 -21
  489. endoreg_db/models/{rules → rule}/ruleset.py +17 -19
  490. endoreg_db/models/state/__init__.py +12 -0
  491. endoreg_db/models/state/abstract.py +11 -0
  492. endoreg_db/models/state/audit_ledger.py +150 -0
  493. endoreg_db/models/state/label_video_segment.py +22 -0
  494. endoreg_db/models/state/raw_pdf.py +187 -0
  495. endoreg_db/models/state/sensitive_meta.py +46 -0
  496. endoreg_db/models/state/video.py +232 -0
  497. endoreg_db/models/upload_job.py +99 -0
  498. endoreg_db/models/utils.py +135 -0
  499. endoreg_db/queries/__init__.py +4 -4
  500. endoreg_db/queries/annotations/__init__.py +2 -2
  501. endoreg_db/queries/annotations/legacy.py +158 -159
  502. endoreg_db/renames.yml +8 -0
  503. endoreg_db/root_urls.py +9 -0
  504. endoreg_db/schemas/__init__.py +0 -0
  505. endoreg_db/schemas/examination_evaluation.py +27 -0
  506. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +775 -0
  507. endoreg_db/serializers/__init__.py +118 -10
  508. endoreg_db/serializers/_old/raw_pdf_meta_validation.py +223 -0
  509. endoreg_db/serializers/_old/raw_video_meta_validation.py +179 -0
  510. endoreg_db/serializers/_old/video.py +71 -0
  511. endoreg_db/serializers/administration/__init__.py +14 -0
  512. endoreg_db/serializers/administration/ai/__init__.py +10 -0
  513. endoreg_db/serializers/administration/ai/active_model.py +10 -0
  514. endoreg_db/serializers/administration/ai/ai_model.py +18 -0
  515. endoreg_db/serializers/administration/ai/model_type.py +10 -0
  516. endoreg_db/serializers/administration/center.py +9 -0
  517. endoreg_db/serializers/administration/gender.py +9 -0
  518. endoreg_db/serializers/anonymization.py +69 -0
  519. endoreg_db/serializers/evaluation/examination_evaluation.py +1 -0
  520. endoreg_db/serializers/examination/__init__.py +10 -0
  521. endoreg_db/serializers/examination/base.py +46 -0
  522. endoreg_db/serializers/examination/dropdown.py +21 -0
  523. endoreg_db/serializers/examination_serializer.py +12 -0
  524. endoreg_db/serializers/finding/__init__.py +5 -0
  525. endoreg_db/serializers/finding/finding.py +54 -0
  526. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  527. endoreg_db/serializers/finding_classification/choice.py +19 -0
  528. endoreg_db/serializers/finding_classification/classification.py +13 -0
  529. endoreg_db/serializers/label/__init__.py +7 -0
  530. endoreg_db/serializers/label/image_classification_annotation.py +62 -0
  531. endoreg_db/serializers/label/label.py +15 -0
  532. endoreg_db/serializers/label_video_segment/__init__.py +7 -0
  533. endoreg_db/serializers/label_video_segment/_lvs_create.py +149 -0
  534. endoreg_db/serializers/label_video_segment/_lvs_update.py +138 -0
  535. endoreg_db/serializers/label_video_segment/_lvs_validate.py +149 -0
  536. endoreg_db/serializers/label_video_segment/label_video_segment.py +344 -0
  537. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +99 -0
  538. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +163 -0
  539. endoreg_db/serializers/meta/__init__.py +19 -0
  540. endoreg_db/serializers/meta/pdf_file_meta_extraction.py +115 -0
  541. endoreg_db/serializers/meta/report_meta.py +53 -0
  542. endoreg_db/serializers/meta/sensitive_meta_detail.py +162 -0
  543. endoreg_db/serializers/meta/sensitive_meta_update.py +148 -0
  544. endoreg_db/serializers/meta/sensitive_meta_verification.py +59 -0
  545. endoreg_db/serializers/meta/video_meta.py +39 -0
  546. endoreg_db/serializers/misc/__init__.py +14 -0
  547. endoreg_db/serializers/misc/file_overview.py +182 -0
  548. endoreg_db/serializers/misc/sensitive_patient_data.py +120 -0
  549. endoreg_db/serializers/misc/stats.py +33 -0
  550. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  551. endoreg_db/serializers/misc/upload_job.py +71 -0
  552. endoreg_db/serializers/patient/__init__.py +11 -0
  553. endoreg_db/serializers/patient/patient.py +86 -0
  554. endoreg_db/serializers/patient/patient_dropdown.py +27 -0
  555. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  556. endoreg_db/serializers/patient_examination/patient_examination.py +141 -0
  557. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  558. endoreg_db/serializers/patient_finding/patient_finding.py +31 -0
  559. endoreg_db/serializers/patient_finding/patient_finding_classification.py +39 -0
  560. endoreg_db/serializers/patient_finding/patient_finding_detail.py +53 -0
  561. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +26 -0
  562. endoreg_db/serializers/patient_finding/patient_finding_list.py +41 -0
  563. endoreg_db/serializers/patient_finding/patient_finding_write.py +126 -0
  564. endoreg_db/serializers/pdf/__init__.py +5 -0
  565. endoreg_db/serializers/pdf/anony_text_validation.py +85 -0
  566. endoreg_db/serializers/report/__init__.py +9 -0
  567. endoreg_db/serializers/report/mixins.py +45 -0
  568. endoreg_db/serializers/report/report.py +105 -0
  569. endoreg_db/serializers/report/report_list.py +22 -0
  570. endoreg_db/serializers/report/secure_file_url.py +26 -0
  571. endoreg_db/serializers/requirements/requirement_schema.py +25 -0
  572. endoreg_db/serializers/requirements/requirement_sets.py +29 -0
  573. endoreg_db/serializers/sensitive_meta_serializer.py +282 -0
  574. endoreg_db/serializers/video/__init__.py +7 -0
  575. endoreg_db/serializers/video/segmentation.py +263 -0
  576. endoreg_db/serializers/video/video_file_brief.py +10 -0
  577. endoreg_db/serializers/video/video_file_detail.py +83 -0
  578. endoreg_db/serializers/video/video_file_list.py +67 -0
  579. endoreg_db/serializers/video/video_metadata.py +105 -0
  580. endoreg_db/serializers/video/video_processing_history.py +153 -0
  581. endoreg_db/serializers/video_examination.py +198 -0
  582. endoreg_db/services/__init__.py +5 -0
  583. endoreg_db/services/anonymization.py +223 -0
  584. endoreg_db/services/examination_evaluation.py +149 -0
  585. endoreg_db/services/finding_description_service.py +0 -0
  586. endoreg_db/services/lookup_service.py +411 -0
  587. endoreg_db/services/lookup_store.py +266 -0
  588. endoreg_db/services/pdf_import.py +1382 -0
  589. endoreg_db/services/polling_coordinator.py +288 -0
  590. endoreg_db/services/pseudonym_service.py +89 -0
  591. endoreg_db/services/requirements_object.py +147 -0
  592. endoreg_db/services/segment_sync.py +155 -0
  593. endoreg_db/services/storage_aware_video_processor.py +344 -0
  594. endoreg_db/services/video_import.py +1259 -0
  595. endoreg_db/tasks/upload_tasks.py +207 -0
  596. endoreg_db/tasks/video_ingest.py +157 -0
  597. endoreg_db/tasks/video_processing_tasks.py +327 -0
  598. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  599. endoreg_db/templates/admin/start_examination.html +12 -0
  600. endoreg_db/templates/timeline.html +176 -0
  601. endoreg_db/urls/__init__.py +83 -0
  602. endoreg_db/urls/anonymization.py +32 -0
  603. endoreg_db/urls/auth.py +16 -0
  604. endoreg_db/urls/classification.py +39 -0
  605. endoreg_db/urls/examination.py +54 -0
  606. endoreg_db/urls/files.py +6 -0
  607. endoreg_db/urls/label_video_segment_validate.py +33 -0
  608. endoreg_db/urls/label_video_segments.py +46 -0
  609. endoreg_db/urls/media.py +227 -0
  610. endoreg_db/urls/patient.py +19 -0
  611. endoreg_db/urls/report.py +48 -0
  612. endoreg_db/urls/requirements.py +13 -0
  613. endoreg_db/urls/sensitive_meta.py +0 -0
  614. endoreg_db/urls/stats.py +46 -0
  615. endoreg_db/urls/upload.py +20 -0
  616. endoreg_db/urls/video.py +61 -0
  617. endoreg_db/urls.py +9 -0
  618. endoreg_db/utils/__init__.py +88 -1
  619. endoreg_db/utils/ai/__init__.py +9 -0
  620. endoreg_db/{models/ai_model/utils.py → utils/ai/get.py} +5 -8
  621. endoreg_db/utils/ai/inference_dataset.py +52 -0
  622. endoreg_db/utils/ai/multilabel_classification_net.py +159 -0
  623. endoreg_db/utils/ai/postprocess.py +63 -0
  624. endoreg_db/utils/ai/predict.py +291 -0
  625. endoreg_db/utils/ai/preprocess.py +68 -0
  626. endoreg_db/utils/calc_duration_seconds.py +24 -0
  627. endoreg_db/utils/case_generator/__init__.py +0 -0
  628. endoreg_db/utils/case_generator/case_generator.py +159 -0
  629. endoreg_db/utils/case_generator/lab_sample_factory.py +33 -0
  630. endoreg_db/utils/case_generator/utils.py +30 -0
  631. endoreg_db/utils/check_video_files.py +148 -0
  632. endoreg_db/utils/cropping.py +28 -28
  633. endoreg_db/utils/dataloader.py +175 -92
  634. endoreg_db/utils/dates.py +60 -0
  635. endoreg_db/utils/env.py +33 -0
  636. endoreg_db/utils/extract_specific_frames.py +72 -0
  637. endoreg_db/utils/file_operations.py +58 -30
  638. endoreg_db/utils/fix_video_path_direct.py +141 -0
  639. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  640. endoreg_db/utils/hashs.py +153 -34
  641. endoreg_db/utils/links/__init__.py +0 -0
  642. endoreg_db/utils/links/requirement_link.py +193 -0
  643. endoreg_db/utils/mime_types.py +0 -0
  644. endoreg_db/utils/names.py +76 -0
  645. endoreg_db/utils/ocr.py +190 -197
  646. endoreg_db/utils/parse_and_generate_yaml.py +46 -0
  647. endoreg_db/utils/paths.py +95 -0
  648. endoreg_db/utils/permissions.py +143 -0
  649. endoreg_db/utils/pipelines/Readme.md +235 -0
  650. endoreg_db/utils/pipelines/__init__.py +0 -0
  651. endoreg_db/utils/pipelines/process_video_dir.py +120 -0
  652. endoreg_db/utils/product/__init__.py +0 -0
  653. endoreg_db/utils/product/sum_emissions.py +20 -0
  654. endoreg_db/utils/product/sum_weights.py +18 -0
  655. endoreg_db/utils/pydantic_models/__init__.py +6 -0
  656. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  657. endoreg_db/utils/requirement_helpers.py +0 -0
  658. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  659. endoreg_db/utils/requirement_operator_logic/lab_value_operators.py +578 -0
  660. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +368 -0
  661. endoreg_db/utils/setup_config.py +177 -0
  662. endoreg_db/utils/translation.py +27 -0
  663. endoreg_db/utils/uuid.py +4 -4
  664. endoreg_db/utils/validate_endo_roi.py +19 -0
  665. endoreg_db/utils/validate_subcategory_dict.py +91 -0
  666. endoreg_db/utils/validate_video_detailed.py +357 -0
  667. endoreg_db/utils/video/__init__.py +26 -0
  668. endoreg_db/utils/video/extract_frames.py +88 -0
  669. endoreg_db/utils/video/ffmpeg_wrapper.py +835 -0
  670. endoreg_db/utils/video/names.py +42 -0
  671. endoreg_db/utils/video/streaming_processor.py +312 -0
  672. endoreg_db/utils/video/video_splitter.py +94 -0
  673. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +238 -0
  674. endoreg_db/views/__init__.py +275 -0
  675. endoreg_db/views/anonymization/__init__.py +27 -0
  676. endoreg_db/views/anonymization/media_management.py +454 -0
  677. endoreg_db/views/anonymization/overview.py +216 -0
  678. endoreg_db/views/anonymization/validate.py +107 -0
  679. endoreg_db/views/auth/__init__.py +13 -0
  680. endoreg_db/views/auth/keycloak.py +113 -0
  681. endoreg_db/views/examination/__init__.py +33 -0
  682. endoreg_db/views/examination/examination.py +37 -0
  683. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  684. endoreg_db/views/examination/get_finding_classification_choices.py +59 -0
  685. endoreg_db/views/examination/get_finding_classifications.py +36 -0
  686. endoreg_db/views/examination/get_findings.py +41 -0
  687. endoreg_db/views/examination/get_instruments.py +18 -0
  688. endoreg_db/views/examination/get_interventions.py +14 -0
  689. endoreg_db/views/finding/__init__.py +9 -0
  690. endoreg_db/views/finding/finding.py +112 -0
  691. endoreg_db/views/finding/get_classifications.py +14 -0
  692. endoreg_db/views/finding/get_interventions.py +17 -0
  693. endoreg_db/views/finding_classification/__init__.py +13 -0
  694. endoreg_db/views/finding_classification/base.py +0 -0
  695. endoreg_db/views/finding_classification/finding_classification.py +42 -0
  696. endoreg_db/views/finding_classification/get_classification_choices.py +55 -0
  697. endoreg_db/views/label/__init__.py +5 -0
  698. endoreg_db/views/label/label.py +15 -0
  699. endoreg_db/views/label_video_segment/__init__.py +16 -0
  700. endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +44 -0
  701. endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +50 -0
  702. endoreg_db/views/label_video_segment/label_video_segment.py +77 -0
  703. endoreg_db/views/label_video_segment/label_video_segment_by_label.py +174 -0
  704. endoreg_db/views/label_video_segment/label_video_segment_detail.py +73 -0
  705. endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +46 -0
  706. endoreg_db/views/label_video_segment/validate.py +226 -0
  707. endoreg_db/views/media/__init__.py +45 -0
  708. endoreg_db/views/media/pdf_media.py +388 -0
  709. endoreg_db/views/media/segments.py +71 -0
  710. endoreg_db/views/media/sensitive_metadata.py +314 -0
  711. endoreg_db/views/media/video_media.py +272 -0
  712. endoreg_db/views/media/video_segments.py +524 -0
  713. endoreg_db/views/meta/__init__.py +15 -0
  714. endoreg_db/views/meta/available_files_list.py +146 -0
  715. endoreg_db/views/meta/report_meta.py +53 -0
  716. endoreg_db/views/meta/sensitive_meta_detail.py +148 -0
  717. endoreg_db/views/meta/sensitive_meta_list.py +104 -0
  718. endoreg_db/views/meta/sensitive_meta_verification.py +71 -0
  719. endoreg_db/views/misc/__init__.py +63 -0
  720. endoreg_db/views/misc/center.py +13 -0
  721. endoreg_db/views/misc/csrf.py +7 -0
  722. endoreg_db/views/misc/gender.py +14 -0
  723. endoreg_db/views/misc/secure_file_serving_view.py +80 -0
  724. endoreg_db/views/misc/secure_file_url_view.py +84 -0
  725. endoreg_db/views/misc/secure_url_validate.py +79 -0
  726. endoreg_db/views/misc/stats.py +220 -0
  727. endoreg_db/views/misc/translation.py +182 -0
  728. endoreg_db/views/misc/upload_views.py +240 -0
  729. endoreg_db/views/patient/__init__.py +5 -0
  730. endoreg_db/views/patient/patient.py +210 -0
  731. endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +164 -0
  732. endoreg_db/views/patient_examination/__init__.py +11 -0
  733. endoreg_db/views/patient_examination/patient_examination.py +140 -0
  734. endoreg_db/views/patient_examination/patient_examination_create.py +63 -0
  735. endoreg_db/views/patient_examination/patient_examination_detail.py +66 -0
  736. endoreg_db/views/patient_examination/patient_examination_list.py +68 -0
  737. endoreg_db/views/patient_examination/video.py +194 -0
  738. endoreg_db/views/patient_finding/__init__.py +7 -0
  739. endoreg_db/views/patient_finding/base.py +0 -0
  740. endoreg_db/views/patient_finding/patient_finding.py +64 -0
  741. endoreg_db/views/patient_finding/patient_finding_optimized.py +259 -0
  742. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  743. endoreg_db/views/patient_finding_classification/pfc_create.py +67 -0
  744. endoreg_db/views/patient_finding_location/__init__.py +5 -0
  745. endoreg_db/views/patient_finding_location/pfl_create.py +70 -0
  746. endoreg_db/views/patient_finding_morphology/__init__.py +5 -0
  747. endoreg_db/views/patient_finding_morphology/pfm_create.py +70 -0
  748. endoreg_db/views/pdf/__init__.py +8 -0
  749. endoreg_db/views/pdf/pdf_stream.py +187 -0
  750. endoreg_db/views/pdf/reimport.py +177 -0
  751. endoreg_db/views/report/__init__.py +9 -0
  752. endoreg_db/views/report/report_list.py +112 -0
  753. endoreg_db/views/report/report_with_secure_url.py +28 -0
  754. endoreg_db/views/report/start_examination.py +7 -0
  755. endoreg_db/views/requirement/__init__.py +10 -0
  756. endoreg_db/views/requirement/evaluate.py +279 -0
  757. endoreg_db/views/requirement/lookup.py +367 -0
  758. endoreg_db/views/requirement/lookup_store.py +252 -0
  759. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  760. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  761. endoreg_db/views/stats/__init__.py +13 -0
  762. endoreg_db/views/stats/stats_views.py +229 -0
  763. endoreg_db/views/video/__init__.py +59 -0
  764. endoreg_db/views/video/correction.py +530 -0
  765. endoreg_db/views/video/reimport.py +195 -0
  766. endoreg_db/views/video/segmentation.py +274 -0
  767. endoreg_db/views/video/task_status.py +49 -0
  768. endoreg_db/views/video/timeline.py +46 -0
  769. endoreg_db/views/video/video_analyze.py +52 -0
  770. endoreg_db/views/video/video_apply_mask.py +48 -0
  771. endoreg_db/views/video/video_correction.py +21 -0
  772. endoreg_db/views/video/video_download_processed.py +58 -0
  773. endoreg_db/views/video/video_examination_viewset.py +242 -0
  774. endoreg_db/views/video/video_meta.py +29 -0
  775. endoreg_db/views/video/video_processing_history.py +24 -0
  776. endoreg_db/views/video/video_remove_frames.py +48 -0
  777. endoreg_db/views/video/video_stream.py +306 -0
  778. endoreg_db/views.py +0 -3
  779. endoreg_db-0.8.6.1.dist-info/METADATA +383 -0
  780. endoreg_db-0.8.6.1.dist-info/RECORD +793 -0
  781. {endoreg_db-0.3.6.dist-info → endoreg_db-0.8.6.1.dist-info}/WHEEL +1 -1
  782. {endoreg_db-0.3.6.dist-info → endoreg_db-0.8.6.1.dist-info/licenses}/LICENSE +674 -674
  783. endoreg_db/data/active_model/data.yaml +0 -3
  784. endoreg_db/data/label/label-set/data.yaml +0 -18
  785. endoreg_db/management/commands/_load_model_template.py +0 -41
  786. endoreg_db/management/commands/delete_all.py +0 -18
  787. endoreg_db/management/commands/delete_legacy_images.py +0 -19
  788. endoreg_db/management/commands/delete_legacy_videos.py +0 -17
  789. endoreg_db/management/commands/extract_legacy_video_frames.py +0 -18
  790. endoreg_db/management/commands/fetch_legacy_image_dataset.py +0 -32
  791. endoreg_db/management/commands/fix_auth_permission.py +0 -20
  792. endoreg_db/management/commands/import_legacy_images.py +0 -94
  793. endoreg_db/management/commands/import_legacy_videos.py +0 -76
  794. endoreg_db/management/commands/load_active_model_data.py +0 -45
  795. endoreg_db/management/commands/load_endoscopy_processor_data.py +0 -45
  796. endoreg_db/management/commands/load_g_play_data.py +0 -113
  797. endoreg_db/management/commands/load_label_data.py +0 -67
  798. endoreg_db/management/commands/load_medication_indication_data.py +0 -63
  799. endoreg_db/management/commands/load_medication_schedule_data.py +0 -55
  800. endoreg_db/migrations/0002_rawvideofile.py +0 -26
  801. endoreg_db/migrations/0003_rawvideofile_frames_required.py +0 -18
  802. endoreg_db/migrations/0004_rename_hash_rawvideofile_video_hash.py +0 -18
  803. endoreg_db/migrations/0005_ffmpegmeta_remove_videoimportmeta_center_and_more.py +0 -56
  804. endoreg_db/migrations/0006_rawvideofile_center_alter_videometa_processor.py +0 -25
  805. endoreg_db/migrations/0007_rawvideofile_processor.py +0 -19
  806. endoreg_db/migrations/0008_rename_frames_required_rawvideofile_state_frames_required.py +0 -18
  807. endoreg_db/migrations/0009_sensitivemeta_rawvideofile_sensitive_meta.py +0 -31
  808. endoreg_db/migrations/0010_rename_endoscope_serial_number_sensitivemeta_endoscope_sn.py +0 -18
  809. endoreg_db/migrations/0011_rawvideofile_state_sensitive_data_retrieved.py +0 -18
  810. endoreg_db/migrations/0012_rawvideofile_prediction_dir_and_more.py +0 -109
  811. endoreg_db/migrations/0013_rawpdffile.py +0 -31
  812. endoreg_db/migrations/0014_pdftype_alter_rawpdffile_file_pdfmeta.py +0 -38
  813. endoreg_db/migrations/0015_rename_report_processed_rawpdffile_state_report_processed_and_more.py +0 -31
  814. endoreg_db/migrations/0016_rawpdffile_state_report_processing_required.py +0 -18
  815. endoreg_db/migrations/0017_firstname_lastname_center_first_names_and_more.py +0 -37
  816. endoreg_db/migrations/0018_reportreaderflag_reportreaderconfig.py +0 -37
  817. endoreg_db/migrations/0019_pdftype_cut_off_above_lines_and_more.py +0 -42
  818. endoreg_db/migrations/0020_rename_endoscopy_info_line_pdftype_endoscope_info_line.py +0 -18
  819. endoreg_db/migrations/0021_alter_pdftype_endoscope_info_line.py +0 -19
  820. endoreg_db/migrations/0022_alter_pdftype_endoscope_info_line.py +0 -19
  821. endoreg_db/migrations/0023_ttoquestionnaire_alter_pdftype_endoscope_info_line.py +0 -59
  822. endoreg_db/migrations/0024_remove_ttoquestionnaire_infections_and_more.py +0 -27
  823. endoreg_db/migrations/0025_event_alter_rawpdffile_file_patientevent.py +0 -42
  824. endoreg_db/migrations/0026_disease_diseaseclassification_and_more.py +0 -166
  825. endoreg_db/migrations/0027_labvalue_abbreviation_labvalue_default_normal_range_and_more.py +0 -38
  826. endoreg_db/migrations/0028_alter_unit_abbreviation.py +0 -18
  827. endoreg_db/migrations/0029_medicationintaketime_and_more.py +0 -75
  828. endoreg_db/migrations/0030_medicationindicationtype_and_more.py +0 -101
  829. endoreg_db/migrations/0031_rename_adapt_to_liver_function_medication_adapt_to_age_and_more.py +0 -38
  830. endoreg_db/migrations/0032_alter_medicationschedule_therapy_duration_d.py +0 -18
  831. endoreg_db/migrations/0033_medicationindication_sources.py +0 -18
  832. endoreg_db/migrations/0034_alter_rawpdffile_file.py +0 -20
  833. endoreg_db/migrations/0035_alter_medicationindication_sources.py +0 -18
  834. endoreg_db/migrations/0036_alter_rawpdffile_file.py +0 -20
  835. endoreg_db/migrations/0037_alter_medicationindication_sources.py +0 -18
  836. endoreg_db/migrations/0038_emissionfactor_material_product_productgroup_and_more.py +0 -164
  837. endoreg_db/migrations/0039_referenceproduct_name.py +0 -19
  838. endoreg_db/migrations/0040_quizanswertype_quizquestiontype_quizquestion_and_more.py +0 -50
  839. endoreg_db/migrations/0041_gender_patientmedication_medication_indication_and_more.py +0 -40
  840. endoreg_db/migrations/0042_casetemplateruletype_casetemplaterulevalue_and_more.py +0 -74
  841. endoreg_db/migrations/0043_casetemplatetype_name_de_casetemplatetype_name_en.py +0 -23
  842. endoreg_db/migrations/0044_casetemplateruletype_name_de_and_more.py +0 -23
  843. endoreg_db/migrations/0045_casetemplaterulevalue_value_type.py +0 -19
  844. endoreg_db/migrations/0046_casetemplaterulevalue_target_field.py +0 -18
  845. endoreg_db/migrations/0047_casetemplaterule_target_model.py +0 -18
  846. endoreg_db/migrations/0048_remove_casetemplaterule_chained_rules_and_more.py +0 -22
  847. endoreg_db/migrations/0049_remove_casetemplaterule_rule_values.py +0 -17
  848. endoreg_db/migrations/0050_casetemplaterule_rule_values.py +0 -18
  849. endoreg_db/migrations/0051_remove_casetemplaterule_calling_rules_and_more.py +0 -27
  850. endoreg_db/migrations/0052_rename_case_template_type_casetemplate_template_type.py +0 -18
  851. endoreg_db/migrations/0053_patientlabsampletype_patientlabsample_and_more.py +0 -38
  852. endoreg_db/migrations/0054_multiplecategoricalvaluedistribution_and_more.py +0 -69
  853. endoreg_db/migrations/0055_remove_casetemplaterule_rule_values_and_more.py +0 -59
  854. endoreg_db/migrations/0056_datevaluedistribution_and_more.py +0 -32
  855. endoreg_db/migrations/0057_remove_datevaluedistribution_max_date_and_more.py +0 -72
  856. endoreg_db/migrations/0058_datevaluedistribution_description_and_more.py +0 -28
  857. endoreg_db/migrations/0059_casetemplaterule_rule_values.py +0 -18
  858. endoreg_db/migrations/0060_labvalue__default_date_value_distribution_and_more.py +0 -44
  859. endoreg_db/migrations/0061_remove_patientlabvalue_date_patientlabvalue_datetime.py +0 -24
  860. endoreg_db/migrations/0062_labvalue_numeric_precision.py +0 -18
  861. endoreg_db/migrations/0063_alter_labvalue_numeric_precision.py +0 -18
  862. endoreg_db/migrations/0064_casetemplaterule_extra_parameters_and_more.py +0 -23
  863. endoreg_db/migrations/0065_rename__date_value_distribution_casetemplaterule_date_value_distribution_and_more.py +0 -58
  864. endoreg_db/migrations/0066_alter_patientlabvalue_patient_and_more.py +0 -29
  865. endoreg_db/migrations/0067_alter_medicationindication_indication_type.py +0 -19
  866. endoreg_db/models/ai_model/__init__.py +0 -3
  867. endoreg_db/models/ai_model/active_model.py +0 -9
  868. endoreg_db/models/ai_model/model_meta.py +0 -24
  869. endoreg_db/models/annotation/__init__.py +0 -2
  870. endoreg_db/models/annotation/binary_classification_annotation_task.py +0 -80
  871. endoreg_db/models/annotation/image_classification.py +0 -27
  872. endoreg_db/models/center/center.py +0 -25
  873. endoreg_db/models/center/center_product.py +0 -34
  874. endoreg_db/models/center/center_resource.py +0 -19
  875. endoreg_db/models/center/center_waste.py +0 -11
  876. endoreg_db/models/data_file/__init__.py +0 -6
  877. endoreg_db/models/data_file/base_classes/__init__.py +0 -2
  878. endoreg_db/models/data_file/base_classes/abstract_frame.py +0 -51
  879. endoreg_db/models/data_file/base_classes/abstract_video.py +0 -201
  880. endoreg_db/models/data_file/frame.py +0 -45
  881. endoreg_db/models/data_file/import_classes/__init__.py +0 -32
  882. endoreg_db/models/data_file/import_classes/processing_functions/__init__.py +0 -35
  883. endoreg_db/models/data_file/import_classes/processing_functions/pdf.py +0 -28
  884. endoreg_db/models/data_file/import_classes/processing_functions/video.py +0 -260
  885. endoreg_db/models/data_file/import_classes/raw_pdf.py +0 -188
  886. endoreg_db/models/data_file/import_classes/raw_video.py +0 -343
  887. endoreg_db/models/data_file/metadata/__init__.py +0 -3
  888. endoreg_db/models/data_file/metadata/sensitive_meta.py +0 -31
  889. endoreg_db/models/data_file/metadata/video_meta.py +0 -133
  890. endoreg_db/models/data_file/report_file.py +0 -89
  891. endoreg_db/models/data_file/video/__init__.py +0 -7
  892. endoreg_db/models/data_file/video/import_meta.py +0 -25
  893. endoreg_db/models/data_file/video/video.py +0 -25
  894. endoreg_db/models/data_file/video_segment.py +0 -107
  895. endoreg_db/models/disease.py +0 -56
  896. endoreg_db/models/emission/__init__.py +0 -1
  897. endoreg_db/models/emission/emission_factor.py +0 -20
  898. endoreg_db/models/event.py +0 -22
  899. endoreg_db/models/examination/__init__.py +0 -4
  900. endoreg_db/models/examination/examination.py +0 -26
  901. endoreg_db/models/examination/examination_time.py +0 -27
  902. endoreg_db/models/examination/examination_time_type.py +0 -24
  903. endoreg_db/models/examination/examination_type.py +0 -18
  904. endoreg_db/models/hardware/__init__.py +0 -2
  905. endoreg_db/models/hardware/endoscope.py +0 -44
  906. endoreg_db/models/information_source.py +0 -29
  907. endoreg_db/models/laboratory/__init__.py +0 -1
  908. endoreg_db/models/laboratory/lab_value.py +0 -102
  909. endoreg_db/models/legacy_data/__init__.py +0 -3
  910. endoreg_db/models/legacy_data/image.py +0 -34
  911. endoreg_db/models/medication/__init__.py +0 -1
  912. endoreg_db/models/medication/medication.py +0 -148
  913. endoreg_db/models/other/distribution.py +0 -215
  914. endoreg_db/models/patient_examination/__init__.py +0 -35
  915. endoreg_db/models/permissions/__init__.py +0 -44
  916. endoreg_db/models/persons/__init__.py +0 -7
  917. endoreg_db/models/persons/examiner/__init__.py +0 -2
  918. endoreg_db/models/persons/examiner/examiner.py +0 -16
  919. endoreg_db/models/persons/examiner/examiner_type.py +0 -2
  920. endoreg_db/models/persons/patient/__init__.py +0 -8
  921. endoreg_db/models/persons/patient/case/case.py +0 -30
  922. endoreg_db/models/persons/patient/patient.py +0 -216
  923. endoreg_db/models/persons/patient/patient_disease.py +0 -16
  924. endoreg_db/models/persons/patient/patient_event.py +0 -22
  925. endoreg_db/models/persons/patient/patient_lab_sample.py +0 -106
  926. endoreg_db/models/persons/patient/patient_medication.py +0 -44
  927. endoreg_db/models/persons/patient/patient_medication_schedule.py +0 -28
  928. endoreg_db/models/persons/portal_user_information.py +0 -27
  929. endoreg_db/models/prediction/__init__.py +0 -2
  930. endoreg_db/models/prediction/image_classification.py +0 -37
  931. endoreg_db/models/prediction/video_prediction_meta.py +0 -244
  932. endoreg_db/models/product/__init__.py +0 -5
  933. endoreg_db/models/product/product.py +0 -97
  934. endoreg_db/models/product/product_group.py +0 -19
  935. endoreg_db/models/product/product_material.py +0 -24
  936. endoreg_db/models/questionnaires/__init__.py +0 -114
  937. endoreg_db/models/quiz/__init__.py +0 -2
  938. endoreg_db/models/quiz/quiz_answer.py +0 -41
  939. endoreg_db/models/quiz/quiz_question.py +0 -54
  940. endoreg_db/models/report_reader/__init__.py +0 -2
  941. endoreg_db/models/report_reader/report_reader_config.py +0 -53
  942. endoreg_db/models/rules/__init__.py +0 -5
  943. endoreg_db/queries/get/__init__.py +0 -6
  944. endoreg_db/queries/get/center.py +0 -42
  945. endoreg_db/queries/get/model.py +0 -13
  946. endoreg_db/queries/get/patient.py +0 -14
  947. endoreg_db/queries/get/patient_examination.py +0 -20
  948. endoreg_db/queries/get/report_file.py +0 -33
  949. endoreg_db/queries/get/video.py +0 -31
  950. endoreg_db/serializers/ai_model.py +0 -19
  951. endoreg_db/serializers/annotation.py +0 -17
  952. endoreg_db/serializers/center.py +0 -11
  953. endoreg_db/serializers/examination.py +0 -33
  954. endoreg_db/serializers/frame.py +0 -13
  955. endoreg_db/serializers/hardware.py +0 -21
  956. endoreg_db/serializers/label.py +0 -22
  957. endoreg_db/serializers/patient.py +0 -10
  958. endoreg_db/serializers/prediction.py +0 -15
  959. endoreg_db/serializers/report_file.py +0 -7
  960. endoreg_db/serializers/video.py +0 -27
  961. endoreg_db/tests.py +0 -3
  962. endoreg_db/utils/legacy_ocr.py +0 -201
  963. endoreg_db/utils/video_metadata.py +0 -87
  964. endoreg_db-0.3.6.dist-info/METADATA +0 -33
  965. endoreg_db-0.3.6.dist-info/RECORD +0 -357
  966. /endoreg_db/{models/persons/patient/case/__init__.py → api/serializers/finding_descriptions.py} +0 -0
  967. /endoreg_db/{queries/get/annotation.py → api/views/finding_descriptions.py} +0 -0
  968. /endoreg_db/{queries/get/prediction.py → config/__init__.py} +0 -0
  969. /endoreg_db/{queries/get/video_import_meta.py → data/case_template/rule_value/.init} +0 -0
  970. /endoreg_db/{queries/get/video_prediction_meta.py → data/distribution/multiple_categorical/.init} +0 -0
@@ -0,0 +1,207 @@
1
+ import os
2
+ import logging
3
+ from typing import Optional
4
+ from celery_app import shared_task
5
+ from django.core.files.storage import default_storage
6
+
7
+ from endoreg_db.models.upload_job import UploadJob
8
+ from endoreg_db.models import SensitiveMeta, VideoFile
9
+
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ @shared_task(bind=True)
15
+ def process_upload_job(self, job_id: str) -> dict:
16
+ """
17
+ Process an uploaded file (PDF or video) asynchronously.
18
+
19
+ This task:
20
+ 1. Updates job status to 'processing'
21
+ 2. Determines file type and routes to appropriate processor
22
+ 3. For PDFs: Uses ReportReader to extract metadata and create SensitiveMeta
23
+ 4. For videos: Uses VideoFile.create_from_file_initialized() and processing pipeline
24
+ 5. Links the created SensitiveMeta to the job and marks as 'anonymized'
25
+ 6. On error: marks job as 'error' with details
26
+
27
+ Args:
28
+ job_id: UUID string of the UploadJob to process
29
+
30
+ Returns:
31
+ dict: Processing result with status and details
32
+ """
33
+ try:
34
+ # Get the upload job
35
+ try:
36
+ upload_job = UploadJob.objects.get(id=job_id)
37
+ except UploadJob.DoesNotExist:
38
+ error_msg = f"UploadJob with id {job_id} not found"
39
+ logger.error(error_msg)
40
+ return {'status': 'error', 'error': error_msg}
41
+
42
+ # Mark as processing
43
+ upload_job.mark_processing()
44
+ logger.info(f"Started processing upload job {job_id}")
45
+
46
+ # Get file path
47
+ file_path = upload_job.file.path
48
+ if not os.path.exists(file_path):
49
+ error_msg = f"Uploaded file not found at {file_path}"
50
+ upload_job.mark_error(error_msg)
51
+ logger.error(error_msg)
52
+ return {'status': 'error', 'error': error_msg}
53
+
54
+ # Route to appropriate processor based on content type
55
+ sensitive_meta = None
56
+
57
+ if upload_job.content_type == 'application/pdf':
58
+ sensitive_meta = _process_pdf_file(upload_job, file_path)
59
+ elif upload_job.content_type.startswith('video/'):
60
+ sensitive_meta = _process_video_file(upload_job, file_path)
61
+ else:
62
+ error_msg = f"Unsupported content type: {upload_job.content_type}"
63
+ upload_job.mark_error(error_msg)
64
+ logger.error(error_msg)
65
+ return {'status': 'error', 'error': error_msg}
66
+
67
+ if sensitive_meta:
68
+ # Mark job as completed with the created SensitiveMeta
69
+ upload_job.mark_completed(sensitive_meta)
70
+ logger.info(f"Successfully processed upload job {job_id}, created SensitiveMeta {sensitive_meta.id}")
71
+ return {
72
+ 'status': 'anonymized',
73
+ 'sensitive_meta_id': sensitive_meta.id,
74
+ 'job_id': job_id
75
+ }
76
+ else:
77
+ error_msg = "Processing completed but no SensitiveMeta was created"
78
+ upload_job.mark_error(error_msg)
79
+ logger.error(f"Upload job {job_id}: {error_msg}")
80
+ return {'status': 'error', 'error': error_msg}
81
+
82
+ except Exception as e:
83
+ error_msg = f"Unexpected error processing upload job {job_id}: {str(e)}"
84
+ logger.exception(error_msg)
85
+
86
+ try:
87
+ upload_job = UploadJob.objects.get(id=job_id)
88
+ upload_job.mark_error(error_msg)
89
+ except Exception as inner_e:
90
+ logger.error(f"Failed to update job status after error: {inner_e}")
91
+
92
+ return {'status': 'error', 'error': error_msg}
93
+
94
+
95
+ def _process_pdf_file(upload_job: UploadJob, file_path: str) -> Optional[SensitiveMeta]:
96
+ """
97
+ Process a PDF file using ReportReader.
98
+
99
+ Args:
100
+ upload_job: The UploadJob instance
101
+ file_path: Path to the uploaded PDF file
102
+
103
+ Returns:
104
+ SensitiveMeta instance if successful, None otherwise
105
+ """
106
+ try:
107
+ # Import ReportReader (lazy import to avoid startup issues)
108
+ try:
109
+ from lx_anonymizer.report_reader import ReportReader
110
+ except ImportError as e:
111
+ raise ImportError(f"Could not import ReportReader: {e}. Make sure lx-anonymizer is available.")
112
+
113
+ logger.info(f"Processing PDF file: {file_path}")
114
+
115
+ # Create ReportReader instance
116
+ reader = ReportReader(
117
+ locale="de_DE",
118
+ text_date_format="%d.%m.%Y"
119
+ )
120
+
121
+ # Process the report
122
+ original_text, anonymized_text, metadata = reader.process_report(
123
+ pdf_path=file_path,
124
+ use_ensemble=False, # Can be made configurable
125
+ verbose=False,
126
+ use_llm_extractor=None # Use default extractor
127
+ )
128
+
129
+ if not metadata:
130
+ raise ValueError("No metadata extracted from PDF")
131
+
132
+ # Create SensitiveMeta from extracted metadata
133
+ sensitive_meta = SensitiveMeta.create_from_dict(metadata)
134
+ sensitive_meta.save()
135
+
136
+ logger.info(f"Created SensitiveMeta {sensitive_meta.id} from PDF processing")
137
+ return sensitive_meta
138
+
139
+ except Exception as e:
140
+ logger.exception(f"Error processing PDF file {file_path}: {e}")
141
+ raise
142
+
143
+
144
+ def _process_video_file(upload_job: UploadJob, file_path: str) -> Optional[SensitiveMeta]:
145
+ """
146
+ Process a video file using VideoFile creation and processing pipeline.
147
+
148
+ Args:
149
+ upload_job: The UploadJob instance
150
+ file_path: Path to the uploaded video file
151
+
152
+ Returns:
153
+ SensitiveMeta instance if successful, None otherwise
154
+ """
155
+ try:
156
+ logger.info(f"Processing video file: {file_path}")
157
+
158
+ # Get the relative path for storage
159
+ relative_path = os.path.relpath(file_path, default_storage.location)
160
+
161
+ # Create VideoFile instance using the existing method
162
+ video_file = VideoFile.create_from_file_initialized(
163
+ file_path=relative_path,
164
+ skip_frame_extraction=False, # Extract frames for processing
165
+ skip_meta_extraction=False, # Extract metadata
166
+ )
167
+
168
+ if not video_file:
169
+ raise ValueError("Failed to create VideoFile instance")
170
+
171
+ # Run the video processing pipeline (pipe_1)
172
+ try:
173
+ # Import the processing function
174
+ from endoreg_db.management.commands.import_video import pipe_1
175
+
176
+ # Run processing pipeline
177
+ pipe_1(
178
+ video_file=video_file,
179
+ skip_frame_extraction=False,
180
+ skip_meta_extraction=False,
181
+ skip_anonymization=False
182
+ )
183
+
184
+ except Exception as e:
185
+ logger.warning(f"Video processing pipeline failed, but VideoFile was created: {e}")
186
+ # Continue anyway - we might still have usable metadata
187
+
188
+ # Get the associated SensitiveMeta
189
+ if hasattr(video_file, 'sensitive_meta') and video_file.sensitive_meta:
190
+ sensitive_meta = video_file.sensitive_meta
191
+ logger.info(f"Found SensitiveMeta {sensitive_meta.id} from video processing")
192
+ return sensitive_meta
193
+ else:
194
+ # Create a basic SensitiveMeta if none exists
195
+ sensitive_meta = SensitiveMeta.objects.create()
196
+
197
+ # Link it to the video if possible
198
+ if hasattr(video_file, 'sensitive_meta'):
199
+ video_file.sensitive_meta = sensitive_meta
200
+ video_file.save(update_fields=['sensitive_meta'])
201
+
202
+ logger.info(f"Created basic SensitiveMeta {sensitive_meta.id} for video file")
203
+ return sensitive_meta
204
+
205
+ except Exception as e:
206
+ logger.exception(f"Error processing video file {file_path}: {e}")
207
+ raise
@@ -0,0 +1,157 @@
1
+ """
2
+ Video ingestion tasks using RQ (Redis Queue).
3
+
4
+ This module provides background task processing for video file imports,
5
+ allowing asynchronous processing of video files dropped into the inbox directory.
6
+ """
7
+
8
+ import logging
9
+ import os
10
+ from pathlib import Path
11
+ from typing import Optional
12
+
13
+ import rq
14
+ import django
15
+ from django.conf import settings
16
+ from django.core.management import call_command
17
+ from redis import Redis
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+ # Redis connection configuration
22
+ REDIS_URL = os.getenv('RQ_REDIS_URL', 'redis://localhost:6379/0')
23
+
24
+ def get_redis_connection():
25
+ """Get Redis connection for RQ."""
26
+ try:
27
+ return Redis.from_url(REDIS_URL)
28
+ except Exception as e:
29
+ logger.error(f"Failed to connect to Redis at {REDIS_URL}: {e}")
30
+ raise
31
+
32
+ def get_queue():
33
+ """Get the 'anonymize' RQ queue."""
34
+ redis_conn = get_redis_connection()
35
+ return rq.Queue('anonymize', connection=redis_conn)
36
+
37
+ def run_import_video(file_path: str, center_name: Optional[str] = None,
38
+ processor_name: Optional[str] = None) -> dict:
39
+ """
40
+ RQ task to import a video file using the video import service.
41
+
42
+ Args:
43
+ file_path: Path to the video file to import
44
+ center_name: Optional center name (defaults to settings.DEFAULT_CENTER)
45
+ processor_name: Optional processor name (defaults to settings.DEFAULT_PROCESSOR)
46
+
47
+ Returns:
48
+ Dict containing task result status and metadata
49
+ """
50
+ try:
51
+ # Ensure Django is properly set up
52
+ if not django.apps.apps.ready:
53
+ django.setup()
54
+
55
+ file_path = Path(file_path)
56
+ if not file_path.exists():
57
+ error_msg = f"Video file not found: {file_path}"
58
+ logger.error(error_msg)
59
+ return {
60
+ 'status': 'error',
61
+ 'message': error_msg,
62
+ 'file_path': str(file_path)
63
+ }
64
+
65
+ # Use settings defaults if not provided
66
+ center_name = center_name or getattr(settings, 'DEFAULT_CENTER', 'university_hospital_wuerzburg')
67
+ processor_name = processor_name or getattr(settings, 'DEFAULT_PROCESSOR', 'olympus_cv_1500')
68
+
69
+ logger.info(f"Starting import for video: {file_path}")
70
+ logger.info(f"Using center: {center_name}, processor: {processor_name}")
71
+
72
+ # Try to use the service first
73
+ try:
74
+ from endoreg_db.services import video_import
75
+
76
+ video_file_obj = video_import.import_and_anonymize(
77
+ file_path=file_path,
78
+ center_name=center_name,
79
+ processor_name=processor_name,
80
+ save_video=True,
81
+ delete_source=True
82
+ )
83
+
84
+ success_msg = f"Successfully imported video: {file_path.name} (UUID: {video_file_obj.uuid})"
85
+ logger.info(success_msg)
86
+
87
+ return {
88
+ 'status': 'success',
89
+ 'message': success_msg,
90
+ 'file_path': str(file_path),
91
+ 'video_uuid': str(video_file_obj.uuid),
92
+ 'center_name': center_name,
93
+ 'processor_name': processor_name
94
+ }
95
+
96
+ except Exception as service_error:
97
+ logger.warning(f"Service import failed: {service_error}, falling back to management command")
98
+
99
+ # Fallback to Django management command
100
+ call_command(
101
+ "import_video",
102
+ str(file_path),
103
+ "--center_name", center_name,
104
+ "--processor_name", processor_name,
105
+ "--save_video_file",
106
+ "--delete_source"
107
+ )
108
+
109
+ success_msg = f"Successfully imported video via fallback: {file_path.name}"
110
+ logger.info(success_msg)
111
+
112
+ return {
113
+ 'status': 'success',
114
+ 'message': success_msg,
115
+ 'file_path': str(file_path),
116
+ 'center_name': center_name,
117
+ 'processor_name': processor_name,
118
+ 'method': 'fallback_command'
119
+ }
120
+
121
+ except Exception as e:
122
+ error_msg = f"Failed to import video {file_path}: {str(e)}"
123
+ logger.error(error_msg, exc_info=True)
124
+
125
+ return {
126
+ 'status': 'error',
127
+ 'message': error_msg,
128
+ 'file_path': str(file_path),
129
+ 'exception': str(e)
130
+ }
131
+
132
+ def enqueue_video_import(file_path: str, center_name: Optional[str] = None,
133
+ processor_name: Optional[str] = None) -> rq.Job:
134
+ """
135
+ Enqueue a video import task.
136
+
137
+ Args:
138
+ file_path: Path to the video file to import
139
+ center_name: Optional center name
140
+ processor_name: Optional processor name
141
+
142
+ Returns:
143
+ RQ Job instance
144
+ """
145
+ queue = get_queue()
146
+
147
+ job = queue.enqueue(
148
+ run_import_video,
149
+ file_path,
150
+ center_name,
151
+ processor_name,
152
+ job_timeout='30m', # 30 minute timeout for video processing
153
+ result_ttl=86400 # Keep results for 24 hours
154
+ )
155
+
156
+ logger.info(f"Enqueued video import job {job.id} for file: {file_path}")
157
+ return job
@@ -0,0 +1,327 @@
1
+ """
2
+ Video processing tasks for Celery-based asynchronous video processing.
3
+
4
+ This module provides Celery tasks for:
5
+ - Video masking with streaming FFmpeg processing
6
+ - Frame removal with streaming optimization
7
+ - Video reprocessing workflows
8
+ - Progress tracking and status updates
9
+ """
10
+
11
+ from pathlib import Path
12
+ from typing import Dict, Any, List, Optional
13
+ from celery import shared_task
14
+ from celery.utils.log import get_task_logger
15
+ from django.shortcuts import get_object_or_404
16
+
17
+ logger = get_task_logger(__name__)
18
+
19
+ @shared_task(bind=True)
20
+ def apply_video_mask_task(self, video_id: int, mask_type: str = 'device_default',
21
+ device_name: str = 'olympus_cv_1500', use_streaming: bool = True,
22
+ custom_mask: Optional[Dict[str, Any]] = None):
23
+ """
24
+ Apply mask to video using streaming processing.
25
+
26
+ Args:
27
+ video_id: ID of the VideoFile to process
28
+ mask_type: Type of mask to apply ('device_default', 'roi_based', 'custom')
29
+ device_name: Device name for default mask
30
+ use_streaming: Whether to use streaming processing (recommended)
31
+ custom_mask: Custom mask configuration (if mask_type='custom')
32
+ """
33
+ try:
34
+ # Update task progress
35
+ self.update_state(state='PROGRESS', meta={'progress': 0, 'message': 'Initializing mask application...'})
36
+
37
+ # Import models here to avoid circular imports
38
+ from endoreg_db.models import VideoFile
39
+ from lx_anonymizer.frame_cleaner import FrameCleaner
40
+
41
+ # Get video file
42
+ video = get_object_or_404(VideoFile, pk=video_id)
43
+ video_path = Path(video.file.path)
44
+
45
+ if not video_path.exists():
46
+ raise FileNotFoundError(f"Video file not found: {video_path}")
47
+
48
+ # Create output path
49
+ output_dir = video_path.parent / "processed"
50
+ output_dir.mkdir(exist_ok=True)
51
+ output_path = output_dir / f"{video_path.stem}_masked{video_path.suffix}"
52
+
53
+ self.update_state(state='PROGRESS', meta={'progress': 10, 'message': 'Setting up FrameCleaner...'})
54
+
55
+ # Initialize FrameCleaner
56
+ cleaner = FrameCleaner()
57
+
58
+ # Determine mask configuration
59
+ if mask_type == 'custom' and custom_mask:
60
+ mask_config = custom_mask
61
+ elif mask_type == 'roi_based':
62
+ # Would need ROI data from video metadata
63
+ mask_config = cleaner._load_mask(device_name) # Fallback for now
64
+ else: # device_default
65
+ mask_config = cleaner._load_mask(device_name)
66
+
67
+ self.update_state(state='PROGRESS', meta={'progress': 20, 'message': f'Applying {mask_type} mask...'})
68
+
69
+ # Apply mask using streaming
70
+ if use_streaming:
71
+ success = cleaner._mask_video_streaming(
72
+ video_path, mask_config, output_path, use_named_pipe=True
73
+ )
74
+ else:
75
+ success = cleaner._mask_video(video_path, mask_config, output_path)
76
+
77
+ if not success:
78
+ raise RuntimeError("Video masking failed")
79
+
80
+ self.update_state(state='PROGRESS', meta={'progress': 90, 'message': 'Finalizing...'})
81
+
82
+ # Verify output
83
+ if not output_path.exists() or output_path.stat().st_size == 0:
84
+ raise RuntimeError("Output video is empty or missing")
85
+
86
+ result = {
87
+ 'status': 'completed',
88
+ 'output_path': str(output_path),
89
+ 'mask_type': mask_type,
90
+ 'device_name': device_name,
91
+ 'use_streaming': use_streaming,
92
+ 'output_size': output_path.stat().st_size,
93
+ 'message': 'Video masking completed successfully'
94
+ }
95
+
96
+ logger.info(f"Video masking completed for video {video_id}: {output_path}")
97
+ return result
98
+
99
+ except Exception as e:
100
+ logger.error(f"Video masking failed for video {video_id}: {e}")
101
+ self.update_state(
102
+ state='FAILURE',
103
+ meta={'error': str(e), 'message': f'Video masking failed: {e}'}
104
+ )
105
+ raise
106
+
107
+
108
+ def _setup_frame_removal(video_id: int, detection_engine: str):
109
+ from endoreg_db.models import VideoFile
110
+ from lx_anonymizer.frame_cleaner import FrameCleaner
111
+ from django.shortcuts import get_object_or_404
112
+ video = get_object_or_404(VideoFile, pk=video_id)
113
+ video_path = Path(video.raw_file.path)
114
+ if not video_path.exists():
115
+ raise FileNotFoundError(f"Video file not found: {video_path}")
116
+ output_dir = video_path.parent / "processed"
117
+ output_dir.mkdir(exist_ok=True)
118
+ output_path = output_dir / f"{video_path.stem}_cleaned{video_path.suffix}"
119
+ use_minicpm = detection_engine == 'minicpm'
120
+ cleaner = FrameCleaner()
121
+ return video, video_path, output_path, cleaner
122
+
123
+ def _detect_sensitive_frames(self, cleaner, video_path, selection_method, manual_frames, total_frames):
124
+ frames_to_remove = []
125
+ if selection_method == 'manual' and manual_frames:
126
+ frames_to_remove = manual_frames
127
+ self.update_state(state='PROGRESS', meta={'progress': 50, 'message': f'Using manual frame selection: {len(frames_to_remove)} frames'})
128
+ elif selection_method == 'automatic':
129
+ self.update_state(state='PROGRESS', meta={'progress': 20, 'message': 'Detecting sensitive frames...'})
130
+ if total_frames > 10000:
131
+ self.update_state(state='PROGRESS', meta={'progress': 30, 'message': 'Using statistical sampling for large video...'})
132
+ sensitive_frames, estimated_ratio, early_stopped = cleaner._sample_frames_coroutine(
133
+ video_path, total_frames, max_samples=500
134
+ )
135
+ frames_to_remove = sensitive_frames
136
+ else:
137
+ self.update_state(state='PROGRESS', meta={'progress': 30, 'message': 'Analyzing all frames...'})
138
+ for abs_i, gray_frame, skip in cleaner._iter_video(video_path, total_frames):
139
+ progress = 30 + (abs_i / total_frames) * 40 # 30-70% for detection
140
+ if abs_i % 100 == 0:
141
+ self.update_state(state='PROGRESS', meta={
142
+ 'progress': int(progress),
143
+ 'message': f'Analyzing frame {abs_i}/{total_frames}...'
144
+ })
145
+ ocr_text, avg_conf, _ = cleaner.frame_ocr.extract_text_from_frame(
146
+ gray_frame, roi=None, high_quality=True
147
+ )
148
+ if ocr_text:
149
+ frame_metadata = cleaner.extract_metadata_deepseek(ocr_text)
150
+ if not frame_metadata:
151
+ frame_metadata = cleaner.frame_metadata_extractor.extract_metadata_from_frame_text(ocr_text)
152
+ has_sensitive = cleaner.frame_metadata_extractor.is_sensitive_content(frame_metadata)
153
+ if has_sensitive:
154
+ frames_to_remove.append(abs_i)
155
+ return frames_to_remove
156
+
157
+ def _remove_frames_from_video(cleaner, video_path, frames_to_remove, output_path, use_streaming):
158
+ if use_streaming:
159
+ return cleaner.remove_frames_from_video_streaming(
160
+ video_path, frames_to_remove, output_path, use_named_pipe=True
161
+ )
162
+ else:
163
+ return cleaner.remove_frames_from_video(
164
+ video_path, frames_to_remove, output_path
165
+ )
166
+
167
+ def _verify_frame_removal_output(output_path):
168
+ if not output_path.exists() or output_path.stat().st_size == 0:
169
+ raise RuntimeError("Output video is empty or missing")
170
+
171
+ @shared_task(bind=True)
172
+ def remove_video_frames_task(self, video_id: int, selection_method: str = 'automatic',
173
+ detection_engine: str = 'minicpm', use_streaming: bool = True,
174
+ manual_frames: Optional[List[int]] = None):
175
+ """
176
+ Remove frames from video using streaming processing.
177
+
178
+ Args:
179
+ video_id: ID of the VideoFile to process
180
+ selection_method: 'automatic' or 'manual'
181
+ detection_engine: 'minicpm' or 'traditional'
182
+ use_streaming: Whether to use streaming processing
183
+ manual_frames: List of frame indices to remove (for manual selection)
184
+ """
185
+ try:
186
+ # Update task progress
187
+ self.update_state(state='PROGRESS', meta={'progress': 0, 'message': 'Initializing frame removal...'})
188
+
189
+ video, video_path, output_path, cleaner = _setup_frame_removal(video_id, detection_engine)
190
+
191
+ self.update_state(state='PROGRESS', meta={'progress': 10, 'message': 'Setting up FrameCleaner...'})
192
+
193
+ # Get total frame count
194
+ total_frames = cleaner._get_total_frames()
195
+
196
+ frames_to_remove = _detect_sensitive_frames(
197
+ self, cleaner, video_path, selection_method, manual_frames, total_frames
198
+ )
199
+
200
+ self.update_state(state='PROGRESS', meta={'progress': 70, 'message': f'Removing {len(frames_to_remove)} frames...'})
201
+
202
+ # Remove frames using streaming
203
+ success = _remove_frames_from_video(cleaner, video_path, frames_to_remove, output_path, use_streaming)
204
+
205
+ if not success:
206
+ raise RuntimeError("Frame removal failed")
207
+
208
+ self.update_state(state='PROGRESS', meta={'progress': 90, 'message': 'Finalizing...'})
209
+
210
+ _verify_frame_removal_output(output_path)
211
+
212
+ result = {
213
+ 'status': 'completed',
214
+ 'output_path': str(output_path),
215
+ 'frames_removed': len(frames_to_remove),
216
+ 'selection_method': selection_method,
217
+ 'detection_engine': detection_engine,
218
+ 'use_streaming': use_streaming,
219
+ 'output_size': output_path.stat().st_size,
220
+ 'message': f'Successfully removed {len(frames_to_remove)} frames'
221
+ }
222
+
223
+ logger.info(f"Frame removal completed for video {video_id}: {len(frames_to_remove)} frames removed")
224
+ return result
225
+
226
+ except Exception as e:
227
+ logger.error(f"Frame removal failed for video {video_id}: {e}")
228
+ self.update_state(
229
+ state='FAILURE',
230
+ meta={'error': str(e), 'message': f'Frame removal failed: {e}'}
231
+ )
232
+ raise
233
+
234
+
235
+ @shared_task(bind=True)
236
+ def reprocess_video_task(self, video_id: int):
237
+ """
238
+ Reprocess video with updated settings.
239
+
240
+ Args:
241
+ video_id: ID of the VideoFile to reprocess
242
+ """
243
+ try:
244
+ self.update_state(state='PROGRESS', meta={'progress': 0, 'message': 'Starting video reprocessing...'})
245
+
246
+ # Import models here to avoid circular imports
247
+ from endoreg_db.models import VideoFile
248
+ from lx_anonymizer.frame_cleaner import FrameCleaner
249
+
250
+ # Get video file
251
+ video = get_object_or_404(VideoFile, pk=video_id)
252
+ video_path = Path(video.file.path)
253
+
254
+ if not video_path.exists():
255
+ raise FileNotFoundError(f"Video file not found: {video_path}")
256
+
257
+ self.update_state(state='PROGRESS', meta={'progress': 20, 'message': 'Initializing FrameCleaner...'})
258
+
259
+ # Initialize FrameCleaner with optimal settings
260
+ cleaner = FrameCleaner()
261
+
262
+ # Create output path
263
+ output_dir = video_path.parent / "processed"
264
+ output_dir.mkdir(exist_ok=True)
265
+ output_path = output_dir / f"{video_path.stem}_reprocessed{video_path.suffix}"
266
+
267
+ self.update_state(state='PROGRESS', meta={'progress': 30, 'message': 'Analyzing video...'})
268
+
269
+ # Perform full analysis
270
+ analysis_result = cleaner.analyze_video_sensitivity()
271
+
272
+ self.update_state(state='PROGRESS', meta={'progress': 60, 'message': f'Processing with {analysis_result["recommended_method"]}...'})
273
+
274
+ # Process based on analysis recommendation
275
+ if analysis_result["recommended_method"] == "masking":
276
+ # Apply device-specific masking
277
+ mask_config = cleaner._load_mask("olympus_cv_1500") # Default device
278
+ success = cleaner._mask_video_streaming(
279
+ video_path, mask_config, output_path, use_named_pipe=True
280
+ )
281
+ else:
282
+ # Remove sensitive frames
283
+ sensitive_frames = analysis_result.get("sensitive_frame_list", [])
284
+ # Filter out string elements like "...truncated"
285
+ sensitive_frames = [f for f in sensitive_frames if isinstance(f, int)]
286
+
287
+ success = cleaner.remove_frames_from_video_streaming(
288
+ video_path, sensitive_frames, output_path, use_named_pipe=True
289
+ )
290
+
291
+ if not success:
292
+ raise RuntimeError("Video reprocessing failed")
293
+
294
+ self.update_state(state='PROGRESS', meta={'progress': 90, 'message': 'Finalizing...'})
295
+
296
+ # Update video metadata with analysis results
297
+ if hasattr(video, 'sensitive_frame_count'):
298
+ video.sensitive_frame_count = analysis_result.get('sensitive_frames', 0)
299
+ if hasattr(video, 'total_frames'):
300
+ video.total_frames = analysis_result.get('total_frames', 0)
301
+ if hasattr(video, 'sensitive_ratio'):
302
+ video.sensitive_ratio = analysis_result.get('sensitivity_ratio', 0.0)
303
+
304
+ try:
305
+ video.save()
306
+ except Exception as e:
307
+ logger.warning(f"Could not save video metadata: {e}")
308
+
309
+ result = {
310
+ 'status': 'completed',
311
+ 'output_path': str(output_path),
312
+ 'analysis_result': analysis_result,
313
+ 'processing_method': analysis_result["recommended_method"],
314
+ 'output_size': output_path.stat().st_size if output_path.exists() else 0,
315
+ 'message': 'Video reprocessing completed successfully'
316
+ }
317
+
318
+ logger.info(f"Video reprocessing completed for video {video_id}")
319
+ return result
320
+
321
+ except Exception as e:
322
+ logger.error(f"Video reprocessing failed for video {video_id}: {e}")
323
+ self.update_state(
324
+ state='FAILURE',
325
+ meta={'error': str(e), 'message': f'Video reprocessing failed: {e}'}
326
+ )
327
+ raise