endoreg-db 0.6.4__py3-none-any.whl → 0.8.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (778) hide show
  1. endoreg_db/admin.py +26 -26
  2. endoreg_db/api_urls.py +4 -0
  3. endoreg_db/apps.py +12 -0
  4. endoreg_db/assets/dummy_model.ckpt +1 -0
  5. endoreg_db/codemods/readme.md +88 -0
  6. endoreg_db/codemods/rename_datetime_fields.py +92 -0
  7. endoreg_db/config/env.py +101 -0
  8. endoreg_db/data/__init__.py +12 -0
  9. endoreg_db/data/ai_model/data.yaml +1 -1
  10. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  11. endoreg_db/data/ai_model_label/label-set/data.yaml +20 -1
  12. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  13. endoreg_db/data/center/data.yaml +13 -12
  14. endoreg_db/data/center_shift/ukw.yaml +9 -0
  15. endoreg_db/data/db_summary.csv +58 -0
  16. endoreg_db/data/db_summary.xlsx +0 -0
  17. endoreg_db/data/disease/misc.yaml +1 -2
  18. endoreg_db/data/endoscopy_processor/data.yaml +3 -0
  19. endoreg_db/data/event/cardiology.yaml +0 -13
  20. endoreg_db/data/examination/examinations/data.yaml +14 -9
  21. endoreg_db/data/examination_indication/endoscopy.yaml +30 -30
  22. endoreg_db/data/examination_indication_classification/endoscopy.yaml +11 -11
  23. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  24. endoreg_db/data/finding/anatomy_colon.yaml +128 -0
  25. endoreg_db/data/finding/colonoscopy.yaml +40 -0
  26. endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +56 -0
  27. endoreg_db/data/finding/complication.yaml +16 -0
  28. endoreg_db/data/finding/data.yaml +3 -46
  29. endoreg_db/data/finding/examination_setting.yaml +16 -0
  30. endoreg_db/data/finding/medication_related.yaml +18 -0
  31. endoreg_db/data/finding/outcome.yaml +12 -0
  32. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +95 -0
  33. endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +22 -0
  34. endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +25 -0
  35. endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  36. endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  37. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  38. endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  39. endoreg_db/data/finding_classification/colonoscopy_location.yaml +80 -0
  40. endoreg_db/data/finding_classification/colonoscopy_lst.yaml +21 -0
  41. endoreg_db/data/finding_classification/colonoscopy_nice.yaml +20 -0
  42. endoreg_db/data/finding_classification/colonoscopy_paris.yaml +26 -0
  43. endoreg_db/data/finding_classification/colonoscopy_sano.yaml +22 -0
  44. endoreg_db/data/finding_classification/colonoscopy_summary.yaml +53 -0
  45. endoreg_db/data/finding_classification/complication_generic.yaml +25 -0
  46. endoreg_db/data/finding_classification/examination_setting_generic.yaml +40 -0
  47. endoreg_db/data/finding_classification/histology_colo.yaml +51 -0
  48. endoreg_db/data/finding_classification/intervention_required.yaml +26 -0
  49. endoreg_db/data/finding_classification/medication_related.yaml +23 -0
  50. endoreg_db/data/finding_classification/visualized.yaml +33 -0
  51. endoreg_db/data/finding_classification_choice/bowel_preparation.yaml +78 -0
  52. endoreg_db/data/{finding_morphology_classification_choice → finding_classification_choice}/colon_lesion_circularity_default.yaml +0 -2
  53. endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  54. endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  55. endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  56. endoreg_db/data/{finding_morphology_classification_choice → finding_classification_choice}/colon_lesion_nice.yaml +4 -7
  57. endoreg_db/data/{finding_morphology_classification_choice → finding_classification_choice}/colon_lesion_paris.yaml +0 -8
  58. endoreg_db/data/{finding_morphology_classification_choice → finding_classification_choice}/colon_lesion_planarity_default.yaml +6 -13
  59. endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  60. endoreg_db/data/{finding_morphology_classification_choice → finding_classification_choice}/colon_lesion_surface_intact_default.yaml +3 -6
  61. endoreg_db/data/{finding_location_classification_choice/colonoscopy.yaml → finding_classification_choice/colonoscopy_location.yaml} +11 -22
  62. endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  63. endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +82 -0
  64. endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  65. endoreg_db/data/finding_classification_choice/complication_generic_types.yaml +15 -0
  66. endoreg_db/data/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  67. endoreg_db/data/finding_classification_choice/histology.yaml +24 -0
  68. endoreg_db/data/finding_classification_choice/histology_polyp.yaml +20 -0
  69. endoreg_db/data/finding_classification_choice/outcome.yaml +19 -0
  70. endoreg_db/data/finding_classification_choice/yes_no_na.yaml +11 -0
  71. endoreg_db/data/finding_classification_type/colonoscopy_basic.yaml +48 -0
  72. endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +8 -3
  73. endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +6 -6
  74. endoreg_db/data/finding_type/data.yaml +23 -10
  75. endoreg_db/data/gender/data.yaml +8 -1
  76. endoreg_db/data/information_source/annotation.yaml +6 -0
  77. endoreg_db/data/information_source/prediction.yaml +7 -0
  78. endoreg_db/data/information_source_type/data.yaml +8 -0
  79. endoreg_db/data/lab_value/misc.yaml +43 -0
  80. endoreg_db/data/medication/anticoagulation.yaml +5 -5
  81. endoreg_db/data/medication/tah.yaml +5 -5
  82. endoreg_db/data/medication_intake_time/base.yaml +4 -4
  83. endoreg_db/data/names_first/first_names.yaml +3 -0
  84. endoreg_db/data/pdf_type/data.yaml +26 -2
  85. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  86. endoreg_db/data/qualification/m2.yaml +39 -0
  87. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  88. endoreg_db/data/qualification/sonography.yaml +36 -0
  89. endoreg_db/data/qualification_type/base.yaml +29 -0
  90. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  91. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +5 -0
  92. endoreg_db/data/requirement/age.yaml +26 -0
  93. endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +45 -0
  94. endoreg_db/data/requirement/disease_cardiovascular.yaml +6 -6
  95. endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +9 -6
  96. endoreg_db/data/requirement/disease_hepatology.yaml +1 -1
  97. endoreg_db/data/requirement/disease_misc.yaml +3 -3
  98. endoreg_db/data/requirement/disease_renal.yaml +18 -2
  99. endoreg_db/data/requirement/{colonoscopy_indications.yaml → endoscopy_bleeding_risk.yaml} +6 -3
  100. endoreg_db/data/requirement/event_cardiology.yaml +17 -17
  101. endoreg_db/data/requirement/event_requirements.yaml +145 -0
  102. endoreg_db/data/requirement/finding_colon_polyp.yaml +50 -0
  103. endoreg_db/data/requirement/gender.yaml +25 -0
  104. endoreg_db/data/requirement/lab_value.yaml +352 -31
  105. endoreg_db/data/requirement/medication.yaml +93 -0
  106. endoreg_db/data/requirement_operator/age.yaml +13 -0
  107. endoreg_db/data/requirement_operator/lab_operators.yaml +36 -35
  108. endoreg_db/data/requirement_operator/model_operators.yaml +13 -7
  109. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +48 -0
  110. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  111. endoreg_db/data/requirement_set/endoscopy_bleeding_risk.yaml +42 -2
  112. endoreg_db/data/requirement_type/requirement_types.yaml +82 -0
  113. endoreg_db/data/shift/endoscopy.yaml +21 -0
  114. endoreg_db/data/shift_type/base.yaml +35 -0
  115. endoreg_db/data/tag/requirement_set_tags.yaml +11 -0
  116. endoreg_db/data/unit/concentration.yaml +23 -0
  117. endoreg_db/exceptions.py +19 -0
  118. endoreg_db/forms/patient_finding_intervention_form.py +4 -5
  119. endoreg_db/forms/patient_form.py +7 -6
  120. endoreg_db/forms/questionnaires/__init__.py +1 -1
  121. endoreg_db/forms/questionnaires/tto_questionnaire.py +19 -19
  122. endoreg_db/helpers/count_db.py +45 -0
  123. endoreg_db/helpers/data_loader.py +208 -0
  124. endoreg_db/helpers/default_objects.py +359 -0
  125. endoreg_db/helpers/interact.py +6 -0
  126. endoreg_db/helpers/test_video_helper.py +119 -0
  127. endoreg_db/logger_conf.py +140 -0
  128. endoreg_db/management/__init__.py +1 -0
  129. endoreg_db/management/commands/__init__.py +1 -0
  130. endoreg_db/management/commands/anonymize_video.py +0 -0
  131. endoreg_db/management/commands/check_auth.py +125 -0
  132. endoreg_db/management/commands/create_multilabel_model_meta.py +214 -0
  133. endoreg_db/management/commands/fix_missing_patient_data.py +172 -0
  134. endoreg_db/management/commands/fix_video_paths.py +165 -0
  135. endoreg_db/management/commands/import_fallback_video.py +203 -0
  136. endoreg_db/management/commands/import_report.py +298 -0
  137. endoreg_db/management/commands/import_video.py +422 -0
  138. endoreg_db/management/commands/import_video_with_classification.py +367 -0
  139. endoreg_db/management/commands/init_default_ai_model.py +112 -0
  140. endoreg_db/management/commands/load_ai_model_data.py +2 -7
  141. endoreg_db/management/commands/load_base_db_data.py +1 -0
  142. endoreg_db/management/commands/load_endoscope_data.py +2 -2
  143. endoreg_db/management/commands/load_examination_indication_data.py +2 -3
  144. endoreg_db/management/commands/load_finding_data.py +49 -92
  145. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +0 -1
  146. endoreg_db/management/commands/load_information_source.py +13 -7
  147. endoreg_db/management/commands/load_name_data.py +37 -0
  148. endoreg_db/management/commands/load_qualification_data.py +59 -0
  149. endoreg_db/management/commands/load_requirement_data.py +30 -6
  150. endoreg_db/management/commands/load_shift_data.py +60 -0
  151. endoreg_db/management/commands/load_tag_data.py +57 -0
  152. endoreg_db/management/commands/register_ai_model.py +1 -1
  153. endoreg_db/management/commands/start_filewatcher.py +106 -0
  154. endoreg_db/management/commands/storage_management.py +548 -0
  155. endoreg_db/management/commands/summarize_db_content.py +189 -0
  156. endoreg_db/management/commands/validate_video.py +204 -0
  157. endoreg_db/management/commands/validate_video_files.py +161 -0
  158. endoreg_db/management/commands/video_validation.py +22 -0
  159. endoreg_db/migrations/0001_initial.py +625 -813
  160. endoreg_db/migrations/0002_add_video_correction_models.py +52 -0
  161. endoreg_db/models/__init__.py +270 -307
  162. endoreg_db/models/administration/__init__.py +116 -0
  163. endoreg_db/models/{ai_model → administration/ai}/__init__.py +6 -1
  164. endoreg_db/models/administration/ai/active_model.py +35 -0
  165. endoreg_db/models/administration/ai/ai_model.py +156 -0
  166. endoreg_db/models/{ai_model → administration/ai}/model_type.py +6 -1
  167. endoreg_db/models/administration/case/__init__.py +19 -0
  168. endoreg_db/models/administration/case/case.py +114 -0
  169. endoreg_db/models/{case_template → administration/case/case_template}/case_template.py +3 -3
  170. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule.py +3 -10
  171. endoreg_db/models/{case_template → administration/case/case_template}/case_template_rule_value.py +2 -4
  172. endoreg_db/models/{case_template → administration/case/case_template}/case_template_type.py +1 -3
  173. endoreg_db/models/{center → administration/center}/__init__.py +3 -1
  174. endoreg_db/models/administration/center/center.py +61 -0
  175. endoreg_db/models/administration/center/center_product.py +64 -0
  176. endoreg_db/models/{center → administration/center}/center_resource.py +19 -3
  177. endoreg_db/models/administration/center/center_shift.py +88 -0
  178. endoreg_db/models/administration/center/center_waste.py +30 -0
  179. endoreg_db/models/administration/permissions/__init__.py +44 -0
  180. endoreg_db/models/administration/person/__init__.py +24 -0
  181. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  182. endoreg_db/models/administration/person/employee/employee.py +35 -0
  183. endoreg_db/models/administration/person/employee/employee_qualification.py +39 -0
  184. endoreg_db/models/administration/person/employee/employee_type.py +42 -0
  185. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  186. endoreg_db/models/administration/person/examiner/examiner.py +54 -0
  187. endoreg_db/models/administration/person/names/__init__.py +0 -0
  188. endoreg_db/models/{persons → administration/person/names}/first_name.py +1 -1
  189. endoreg_db/models/{persons → administration/person/names}/last_name.py +2 -3
  190. endoreg_db/models/administration/person/patient/__init__.py +5 -0
  191. endoreg_db/models/administration/person/patient/patient.py +460 -0
  192. endoreg_db/models/administration/person/profession/__init__.py +24 -0
  193. endoreg_db/models/administration/person/user/__init__.py +5 -0
  194. endoreg_db/models/administration/person/user/portal_user_information.py +37 -0
  195. endoreg_db/models/administration/product/product.py +97 -0
  196. endoreg_db/models/administration/product/product_group.py +39 -0
  197. endoreg_db/models/administration/product/product_material.py +54 -0
  198. endoreg_db/models/{product → administration/product}/product_weight.py +9 -0
  199. endoreg_db/models/{product → administration/product}/reference_product.py +26 -11
  200. endoreg_db/models/administration/qualification/__init__.py +7 -0
  201. endoreg_db/models/administration/qualification/qualification.py +37 -0
  202. endoreg_db/models/administration/qualification/qualification_type.py +35 -0
  203. endoreg_db/models/administration/shift/__init__.py +9 -0
  204. endoreg_db/models/administration/shift/scheduled_days.py +69 -0
  205. endoreg_db/models/administration/shift/shift.py +51 -0
  206. endoreg_db/models/administration/shift/shift_type.py +108 -0
  207. endoreg_db/models/label/__init__.py +24 -1
  208. endoreg_db/models/label/annotation/__init__.py +12 -0
  209. endoreg_db/models/label/annotation/image_classification.py +84 -0
  210. endoreg_db/models/label/annotation/video_segmentation_annotation.py +66 -0
  211. endoreg_db/models/label/label.py +45 -74
  212. endoreg_db/models/label/label_set.py +53 -0
  213. endoreg_db/models/label/label_type.py +29 -0
  214. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  215. endoreg_db/models/label/label_video_segment/_create_from_video.py +41 -0
  216. endoreg_db/models/label/label_video_segment/label_video_segment.py +511 -0
  217. endoreg_db/models/label/video_segmentation_label.py +31 -0
  218. endoreg_db/models/{annotation → label}/video_segmentation_labelset.py +7 -0
  219. endoreg_db/models/media/__init__.py +14 -0
  220. endoreg_db/models/media/frame/__init__.py +3 -0
  221. endoreg_db/models/media/frame/frame.py +111 -0
  222. endoreg_db/models/media/pdf/__init__.py +11 -0
  223. endoreg_db/models/media/pdf/raw_pdf.py +608 -0
  224. endoreg_db/models/media/pdf/report_file.py +162 -0
  225. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +77 -0
  226. endoreg_db/models/media/video/__init__.py +4 -0
  227. endoreg_db/models/media/video/create_from_file.py +336 -0
  228. endoreg_db/models/media/video/pipe_1.py +195 -0
  229. endoreg_db/models/media/video/pipe_2.py +105 -0
  230. endoreg_db/models/media/video/refactor_plan.md +0 -0
  231. endoreg_db/models/media/video/video_file.py +680 -0
  232. endoreg_db/models/media/video/video_file_ai.py +443 -0
  233. endoreg_db/models/media/video/video_file_anonymize.py +348 -0
  234. endoreg_db/models/media/video/video_file_frames/__init__.py +47 -0
  235. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +22 -0
  236. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  237. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +104 -0
  238. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +174 -0
  239. endoreg_db/models/media/video/video_file_frames/_get_frame.py +28 -0
  240. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +27 -0
  241. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +20 -0
  242. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +27 -0
  243. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +34 -0
  244. endoreg_db/models/media/video/video_file_frames/_get_frames.py +27 -0
  245. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +129 -0
  246. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +129 -0
  247. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +65 -0
  248. endoreg_db/models/media/video/video_file_frames.py +0 -0
  249. endoreg_db/models/media/video/video_file_io.py +166 -0
  250. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  251. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +45 -0
  252. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +39 -0
  253. endoreg_db/models/media/video/video_file_meta/get_fps.py +147 -0
  254. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +143 -0
  255. endoreg_db/models/media/video/video_file_meta/text_meta.py +134 -0
  256. endoreg_db/models/media/video/video_file_meta/video_meta.py +70 -0
  257. endoreg_db/models/media/video/video_file_meta.py +11 -0
  258. endoreg_db/models/media/video/video_file_segments.py +209 -0
  259. endoreg_db/models/medical/__init__.py +146 -0
  260. endoreg_db/models/{contraindication → medical/contraindication}/__init__.py +1 -5
  261. endoreg_db/models/{disease.py → medical/disease.py} +60 -52
  262. endoreg_db/models/{event.py → medical/event.py} +31 -54
  263. endoreg_db/models/{examination → medical/examination}/__init__.py +1 -1
  264. endoreg_db/models/medical/examination/examination.py +148 -0
  265. endoreg_db/models/{examination → medical/examination}/examination_indication.py +64 -35
  266. endoreg_db/models/{examination → medical/examination}/examination_time.py +0 -4
  267. endoreg_db/models/{examination → medical/examination}/examination_time_type.py +1 -8
  268. endoreg_db/models/{examination → medical/examination}/examination_type.py +1 -7
  269. endoreg_db/models/medical/finding/__init__.py +18 -0
  270. endoreg_db/models/medical/finding/finding.py +96 -0
  271. endoreg_db/models/medical/finding/finding_classification.py +142 -0
  272. endoreg_db/models/{finding → medical/finding}/finding_intervention.py +2 -10
  273. endoreg_db/models/medical/finding/finding_type.py +35 -0
  274. endoreg_db/models/medical/hardware/__init__.py +8 -0
  275. endoreg_db/models/{hardware → medical/hardware}/endoscope.py +28 -23
  276. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  277. endoreg_db/models/medical/laboratory/lab_value.py +419 -0
  278. endoreg_db/models/{medication → medical/medication}/medication.py +1 -3
  279. endoreg_db/models/{medication → medical/medication}/medication_indication_type.py +8 -3
  280. endoreg_db/models/{medication → medical/medication}/medication_intake_time.py +21 -3
  281. endoreg_db/models/{medication → medical/medication}/medication_schedule.py +13 -5
  282. endoreg_db/models/{organ → medical/organ}/__init__.py +3 -6
  283. endoreg_db/models/medical/patient/__init__.py +56 -0
  284. endoreg_db/models/medical/patient/medication_examples.py +38 -0
  285. endoreg_db/models/medical/patient/patient_disease.py +63 -0
  286. endoreg_db/models/medical/patient/patient_event.py +75 -0
  287. endoreg_db/models/medical/patient/patient_examination.py +249 -0
  288. endoreg_db/models/{persons → medical}/patient/patient_examination_indication.py +21 -9
  289. endoreg_db/models/medical/patient/patient_finding.py +357 -0
  290. endoreg_db/models/medical/patient/patient_finding_classification.py +207 -0
  291. endoreg_db/models/{patient → medical/patient}/patient_finding_intervention.py +15 -1
  292. endoreg_db/models/medical/patient/patient_lab_sample.py +148 -0
  293. endoreg_db/models/{persons → medical}/patient/patient_lab_value.py +40 -15
  294. endoreg_db/models/medical/patient/patient_medication.py +104 -0
  295. endoreg_db/models/medical/patient/patient_medication_schedule.py +136 -0
  296. endoreg_db/models/{risk → medical/risk}/risk_type.py +0 -4
  297. endoreg_db/models/{data_file/metadata → metadata}/__init__.py +6 -0
  298. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  299. endoreg_db/models/metadata/model_meta.py +193 -0
  300. endoreg_db/models/metadata/model_meta_logic.py +236 -0
  301. endoreg_db/models/{data_file/metadata → metadata}/pdf_meta.py +28 -13
  302. endoreg_db/models/metadata/sensitive_meta.py +288 -0
  303. endoreg_db/models/metadata/sensitive_meta_logic.py +643 -0
  304. endoreg_db/models/metadata/video_meta.py +332 -0
  305. endoreg_db/models/metadata/video_prediction_logic.py +190 -0
  306. endoreg_db/models/metadata/video_prediction_meta.py +270 -0
  307. endoreg_db/models/other/__init__.py +17 -0
  308. endoreg_db/models/other/distribution/date_value_distribution.py +0 -2
  309. endoreg_db/models/other/distribution/numeric_value_distribution.py +30 -2
  310. endoreg_db/models/{emission → other/emission}/emission_factor.py +15 -6
  311. endoreg_db/models/{persons → other}/gender.py +8 -3
  312. endoreg_db/models/other/information_source.py +159 -0
  313. endoreg_db/models/other/material.py +10 -2
  314. endoreg_db/models/other/resource.py +6 -2
  315. endoreg_db/models/other/tag.py +27 -0
  316. endoreg_db/models/other/transport_route.py +13 -2
  317. endoreg_db/models/{unit.py → other/unit.py} +16 -6
  318. endoreg_db/models/other/waste.py +10 -3
  319. endoreg_db/models/requirement/requirement.py +556 -114
  320. endoreg_db/models/requirement/requirement_evaluation/__init__.py +4 -132
  321. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  322. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +9 -0
  323. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +80 -87
  324. endoreg_db/models/requirement/requirement_operator.py +132 -14
  325. endoreg_db/models/requirement/requirement_set.py +181 -21
  326. endoreg_db/models/rule/__init__.py +13 -0
  327. endoreg_db/models/{rules → rule}/rule.py +6 -3
  328. endoreg_db/models/{rules → rule}/rule_attribute_dtype.py +0 -2
  329. endoreg_db/models/{rules → rule}/rule_type.py +0 -2
  330. endoreg_db/models/{rules → rule}/ruleset.py +0 -2
  331. endoreg_db/models/state/__init__.py +12 -0
  332. endoreg_db/models/state/abstract.py +11 -0
  333. endoreg_db/models/state/audit_ledger.py +150 -0
  334. endoreg_db/models/state/label_video_segment.py +22 -0
  335. endoreg_db/models/state/raw_pdf.py +187 -0
  336. endoreg_db/models/state/sensitive_meta.py +46 -0
  337. endoreg_db/models/state/video.py +232 -0
  338. endoreg_db/models/upload_job.py +99 -0
  339. endoreg_db/models/utils.py +135 -0
  340. endoreg_db/models/video_metadata.py +66 -0
  341. endoreg_db/models/video_processing.py +153 -0
  342. endoreg_db/renames.yml +8 -0
  343. endoreg_db/root_urls.py +9 -0
  344. endoreg_db/schemas/__init__.py +0 -0
  345. endoreg_db/schemas/examination_evaluation.py +27 -0
  346. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +775 -0
  347. endoreg_db/serializers/__init__.py +147 -10
  348. endoreg_db/serializers/{raw_pdf_meta_validation.py → _old/raw_pdf_meta_validation.py} +3 -3
  349. endoreg_db/serializers/{raw_video_meta_validation.py → _old/raw_video_meta_validation.py} +18 -14
  350. endoreg_db/serializers/_old/video.py +71 -0
  351. endoreg_db/serializers/administration/__init__.py +14 -0
  352. endoreg_db/serializers/administration/ai/__init__.py +10 -0
  353. endoreg_db/serializers/administration/ai/active_model.py +10 -0
  354. endoreg_db/serializers/administration/ai/ai_model.py +18 -0
  355. endoreg_db/serializers/administration/ai/model_type.py +10 -0
  356. endoreg_db/serializers/administration/center.py +9 -0
  357. endoreg_db/serializers/administration/gender.py +9 -0
  358. endoreg_db/serializers/anonymization.py +66 -0
  359. endoreg_db/serializers/evaluation/examination_evaluation.py +1 -0
  360. endoreg_db/serializers/examination/__init__.py +10 -0
  361. endoreg_db/serializers/examination/base.py +46 -0
  362. endoreg_db/serializers/examination/dropdown.py +21 -0
  363. endoreg_db/serializers/examination_serializer.py +12 -0
  364. endoreg_db/serializers/finding/__init__.py +5 -0
  365. endoreg_db/serializers/finding/finding.py +54 -0
  366. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  367. endoreg_db/serializers/finding_classification/choice.py +19 -0
  368. endoreg_db/serializers/finding_classification/classification.py +13 -0
  369. endoreg_db/serializers/label/__init__.py +7 -0
  370. endoreg_db/serializers/label/image_classification_annotation.py +62 -0
  371. endoreg_db/serializers/label/label.py +15 -0
  372. endoreg_db/serializers/label_video_segment/__init__.py +7 -0
  373. endoreg_db/serializers/label_video_segment/_lvs_create.py +149 -0
  374. endoreg_db/serializers/label_video_segment/_lvs_update.py +138 -0
  375. endoreg_db/serializers/label_video_segment/_lvs_validate.py +149 -0
  376. endoreg_db/serializers/label_video_segment/label_video_segment.py +344 -0
  377. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +99 -0
  378. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +163 -0
  379. endoreg_db/serializers/meta/__init__.py +19 -0
  380. endoreg_db/serializers/meta/pdf_file_meta_extraction.py +115 -0
  381. endoreg_db/serializers/meta/report_meta.py +53 -0
  382. endoreg_db/serializers/meta/sensitive_meta_detail.py +162 -0
  383. endoreg_db/serializers/meta/sensitive_meta_update.py +148 -0
  384. endoreg_db/serializers/meta/sensitive_meta_verification.py +59 -0
  385. endoreg_db/serializers/meta/video_meta.py +39 -0
  386. endoreg_db/serializers/misc/__init__.py +14 -0
  387. endoreg_db/serializers/misc/file_overview.py +152 -0
  388. endoreg_db/serializers/misc/stats.py +33 -0
  389. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  390. endoreg_db/serializers/misc/upload_job.py +71 -0
  391. endoreg_db/serializers/misc/vop_patient_data.py +120 -0
  392. endoreg_db/serializers/patient/__init__.py +11 -0
  393. endoreg_db/serializers/patient/patient.py +86 -0
  394. endoreg_db/serializers/patient/patient_dropdown.py +27 -0
  395. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  396. endoreg_db/serializers/patient_examination/patient_examination.py +141 -0
  397. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  398. endoreg_db/serializers/patient_finding/patient_finding.py +31 -0
  399. endoreg_db/serializers/patient_finding/patient_finding_classification.py +39 -0
  400. endoreg_db/serializers/patient_finding/patient_finding_detail.py +53 -0
  401. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +26 -0
  402. endoreg_db/serializers/patient_finding/patient_finding_list.py +41 -0
  403. endoreg_db/serializers/patient_finding/patient_finding_write.py +126 -0
  404. endoreg_db/serializers/pdf/__init__.py +5 -0
  405. endoreg_db/serializers/pdf/anony_text_validation.py +85 -0
  406. endoreg_db/serializers/report/__init__.py +9 -0
  407. endoreg_db/serializers/report/mixins.py +45 -0
  408. endoreg_db/serializers/report/report.py +105 -0
  409. endoreg_db/serializers/report/report_list.py +22 -0
  410. endoreg_db/serializers/report/secure_file_url.py +26 -0
  411. endoreg_db/serializers/requirements/requirement_schema.py +25 -0
  412. endoreg_db/serializers/requirements/requirement_sets.py +29 -0
  413. endoreg_db/serializers/sensitive_meta_serializer.py +282 -0
  414. endoreg_db/serializers/video/__init__.py +7 -0
  415. endoreg_db/serializers/video/segmentation.py +263 -0
  416. endoreg_db/serializers/video/video_file_brief.py +10 -0
  417. endoreg_db/serializers/video/video_file_detail.py +83 -0
  418. endoreg_db/serializers/video/video_file_list.py +67 -0
  419. endoreg_db/serializers/video/video_metadata.py +105 -0
  420. endoreg_db/serializers/video/video_processing_history.py +153 -0
  421. endoreg_db/services/__init__.py +5 -0
  422. endoreg_db/services/anonymization.py +223 -0
  423. endoreg_db/services/examination_evaluation.py +149 -0
  424. endoreg_db/services/finding_description_service.py +0 -0
  425. endoreg_db/services/lookup_service.py +241 -0
  426. endoreg_db/services/lookup_store.py +122 -0
  427. endoreg_db/services/ollama_api_docs.py +1528 -0
  428. endoreg_db/services/pdf_import.py +993 -0
  429. endoreg_db/services/polling_coordinator.py +288 -0
  430. endoreg_db/services/pseudonym_service.py +89 -0
  431. endoreg_db/services/requirements_object.py +147 -0
  432. endoreg_db/services/segment_sync.py +155 -0
  433. endoreg_db/services/storage_aware_video_processor.py +344 -0
  434. endoreg_db/services/video_import.py +915 -0
  435. endoreg_db/tasks/upload_tasks.py +207 -0
  436. endoreg_db/tasks/video_ingest.py +157 -0
  437. endoreg_db/tasks/video_processing_tasks.py +327 -0
  438. endoreg_db/urls/__init__.py +72 -0
  439. endoreg_db/urls/anonymization.py +32 -0
  440. endoreg_db/urls/auth.py +16 -0
  441. endoreg_db/urls/classification.py +39 -0
  442. endoreg_db/urls/examination.py +54 -0
  443. endoreg_db/urls/files.py +6 -0
  444. endoreg_db/urls/label_video_segment_validate.py +33 -0
  445. endoreg_db/urls/label_video_segments.py +44 -0
  446. endoreg_db/urls/media.py +32 -0
  447. endoreg_db/urls/patient.py +19 -0
  448. endoreg_db/urls/pdf.py +0 -0
  449. endoreg_db/urls/report.py +78 -0
  450. endoreg_db/urls/requirements.py +13 -0
  451. endoreg_db/urls/sensitive_meta.py +36 -0
  452. endoreg_db/urls/stats.py +46 -0
  453. endoreg_db/urls/upload.py +20 -0
  454. endoreg_db/urls/video.py +119 -0
  455. endoreg_db/urls.py +6 -283
  456. endoreg_db/utils/__init__.py +66 -57
  457. endoreg_db/utils/ai/__init__.py +9 -0
  458. endoreg_db/{models/ai_model/utils.py → utils/ai/get.py} +1 -4
  459. endoreg_db/{models/ai_model/lightning → utils/ai}/inference_dataset.py +0 -1
  460. endoreg_db/{models/ai_model/lightning → utils/ai}/multilabel_classification_net.py +14 -10
  461. endoreg_db/{models/ai_model/lightning → utils/ai}/postprocess.py +15 -5
  462. endoreg_db/utils/ai/predict.py +291 -0
  463. endoreg_db/{models/ai_model/lightning → utils/ai}/preprocess.py +1 -1
  464. endoreg_db/utils/calc_duration_seconds.py +24 -0
  465. endoreg_db/utils/case_generator/__init__.py +0 -0
  466. endoreg_db/utils/check_video_files.py +148 -0
  467. endoreg_db/utils/dataloader.py +50 -12
  468. endoreg_db/utils/dates.py +21 -0
  469. endoreg_db/utils/env.py +33 -0
  470. endoreg_db/utils/extract_specific_frames.py +72 -0
  471. endoreg_db/utils/file_operations.py +29 -1
  472. endoreg_db/utils/fix_video_path_direct.py +141 -0
  473. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  474. endoreg_db/utils/links/__init__.py +0 -0
  475. endoreg_db/utils/links/requirement_link.py +193 -0
  476. endoreg_db/utils/mime_types.py +0 -0
  477. endoreg_db/utils/names.py +2 -0
  478. endoreg_db/utils/paths.py +100 -82
  479. endoreg_db/utils/permissions.py +143 -0
  480. endoreg_db/utils/pipelines/Readme.md +235 -0
  481. endoreg_db/utils/pipelines/__init__.py +0 -0
  482. endoreg_db/utils/pipelines/process_video_dir.py +120 -0
  483. endoreg_db/utils/product/__init__.py +0 -0
  484. endoreg_db/utils/product/sum_emissions.py +20 -0
  485. endoreg_db/utils/product/sum_weights.py +18 -0
  486. endoreg_db/utils/pydantic_models/db_config.py +1 -1
  487. endoreg_db/utils/requirement_helpers.py +0 -0
  488. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  489. endoreg_db/utils/requirement_operator_logic/lab_value_operators.py +578 -0
  490. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +368 -0
  491. endoreg_db/utils/translation.py +27 -0
  492. endoreg_db/utils/validate_video_detailed.py +357 -0
  493. endoreg_db/utils/video/__init__.py +19 -6
  494. endoreg_db/utils/video/extract_frames.py +37 -70
  495. endoreg_db/utils/video/ffmpeg_wrapper.py +772 -0
  496. endoreg_db/utils/video/names.py +42 -0
  497. endoreg_db/utils/video/streaming_processor.py +312 -0
  498. endoreg_db/utils/video/video_splitter.py +94 -0
  499. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +238 -0
  500. endoreg_db/views/__init__.py +282 -2
  501. endoreg_db/views/anonymization/__init__.py +27 -0
  502. endoreg_db/views/anonymization/media_management.py +454 -0
  503. endoreg_db/views/anonymization/overview.py +216 -0
  504. endoreg_db/views/anonymization/validate.py +63 -0
  505. endoreg_db/views/auth/__init__.py +13 -0
  506. endoreg_db/views/{views.py → auth/keycloak.py} +19 -13
  507. endoreg_db/views/examination/__init__.py +33 -0
  508. endoreg_db/views/examination/examination.py +37 -0
  509. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  510. endoreg_db/views/examination/get_finding_classification_choices.py +59 -0
  511. endoreg_db/views/examination/get_finding_classifications.py +36 -0
  512. endoreg_db/views/examination/get_findings.py +41 -0
  513. endoreg_db/views/examination/get_instruments.py +18 -0
  514. endoreg_db/views/examination/get_interventions.py +14 -0
  515. endoreg_db/views/finding/__init__.py +9 -0
  516. endoreg_db/views/finding/finding.py +112 -0
  517. endoreg_db/views/finding/get_classifications.py +14 -0
  518. endoreg_db/views/finding/get_interventions.py +17 -0
  519. endoreg_db/views/finding_classification/__init__.py +13 -0
  520. endoreg_db/views/finding_classification/base.py +0 -0
  521. endoreg_db/views/finding_classification/finding_classification.py +42 -0
  522. endoreg_db/views/finding_classification/get_classification_choices.py +55 -0
  523. endoreg_db/views/label/__init__.py +5 -0
  524. endoreg_db/views/label/label.py +15 -0
  525. endoreg_db/views/label_video_segment/__init__.py +16 -0
  526. endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +44 -0
  527. endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +50 -0
  528. endoreg_db/views/label_video_segment/label_video_segment.py +77 -0
  529. endoreg_db/views/label_video_segment/label_video_segment_by_label.py +174 -0
  530. endoreg_db/views/label_video_segment/label_video_segment_detail.py +73 -0
  531. endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +46 -0
  532. endoreg_db/views/label_video_segment/validate.py +226 -0
  533. endoreg_db/views/media/__init__.py +9 -0
  534. endoreg_db/views/media/pdf_media.py +386 -0
  535. endoreg_db/views/media/video_media.py +272 -0
  536. endoreg_db/views/meta/__init__.py +15 -0
  537. endoreg_db/views/meta/available_files_list.py +146 -0
  538. endoreg_db/views/meta/report_meta.py +53 -0
  539. endoreg_db/views/meta/sensitive_meta_detail.py +148 -0
  540. endoreg_db/views/meta/sensitive_meta_list.py +104 -0
  541. endoreg_db/views/meta/sensitive_meta_verification.py +71 -0
  542. endoreg_db/views/misc/__init__.py +63 -0
  543. endoreg_db/views/misc/center.py +13 -0
  544. endoreg_db/views/misc/gender.py +14 -0
  545. endoreg_db/views/misc/secure_file_serving_view.py +80 -0
  546. endoreg_db/views/misc/secure_file_url_view.py +84 -0
  547. endoreg_db/views/misc/secure_url_validate.py +79 -0
  548. endoreg_db/views/misc/stats.py +220 -0
  549. endoreg_db/views/misc/translation.py +182 -0
  550. endoreg_db/views/misc/upload_views.py +240 -0
  551. endoreg_db/views/patient/__init__.py +5 -0
  552. endoreg_db/views/patient/patient.py +210 -0
  553. endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +164 -0
  554. endoreg_db/views/patient_examination/__init__.py +11 -0
  555. endoreg_db/views/patient_examination/patient_examination.py +140 -0
  556. endoreg_db/views/patient_examination/patient_examination_create.py +63 -0
  557. endoreg_db/views/patient_examination/patient_examination_detail.py +66 -0
  558. endoreg_db/views/patient_examination/patient_examination_list.py +68 -0
  559. endoreg_db/views/patient_examination/video.py +194 -0
  560. endoreg_db/views/patient_finding/__init__.py +7 -0
  561. endoreg_db/views/patient_finding/base.py +0 -0
  562. endoreg_db/views/patient_finding/patient_finding.py +64 -0
  563. endoreg_db/views/patient_finding/patient_finding_optimized.py +259 -0
  564. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  565. endoreg_db/views/patient_finding_classification/pfc_create.py +67 -0
  566. endoreg_db/views/patient_finding_location/__init__.py +5 -0
  567. endoreg_db/views/patient_finding_location/pfl_create.py +70 -0
  568. endoreg_db/views/patient_finding_morphology/__init__.py +5 -0
  569. endoreg_db/views/patient_finding_morphology/pfm_create.py +70 -0
  570. endoreg_db/views/pdf/__init__.py +11 -0
  571. endoreg_db/views/pdf/pdf_media.py +239 -0
  572. endoreg_db/views/pdf/pdf_stream_views.py +127 -0
  573. endoreg_db/views/pdf/reimport.py +151 -0
  574. endoreg_db/views/report/__init__.py +9 -0
  575. endoreg_db/views/report/report_list.py +112 -0
  576. endoreg_db/views/report/report_with_secure_url.py +28 -0
  577. endoreg_db/views/report/start_examination.py +7 -0
  578. endoreg_db/views/requirement/__init__.py +10 -0
  579. endoreg_db/views/requirement/evaluate.py +279 -0
  580. endoreg_db/views/requirement/lookup.py +483 -0
  581. endoreg_db/views/requirement/lookup_store.py +252 -0
  582. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  583. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  584. endoreg_db/views/stats/__init__.py +13 -0
  585. endoreg_db/views/stats/stats_views.py +229 -0
  586. endoreg_db/views/video/__init__.py +72 -0
  587. endoreg_db/views/video/correction.py +672 -0
  588. endoreg_db/views/video/media/__init__.py +23 -0
  589. endoreg_db/views/video/media/task_status.py +49 -0
  590. endoreg_db/views/video/media/video_analyze.py +52 -0
  591. endoreg_db/views/video/media/video_apply_mask.py +48 -0
  592. endoreg_db/views/video/media/video_correction.py +21 -0
  593. endoreg_db/views/video/media/video_download_processed.py +58 -0
  594. endoreg_db/views/video/media/video_media.py +158 -0
  595. endoreg_db/views/video/media/video_meta.py +29 -0
  596. endoreg_db/views/video/media/video_processing_history.py +24 -0
  597. endoreg_db/views/video/media/video_remove_frames.py +48 -0
  598. endoreg_db/views/video/media/video_reprocess.py +40 -0
  599. endoreg_db/views/video/reimport.py +192 -0
  600. endoreg_db/views/video/segmentation.py +274 -0
  601. endoreg_db/views/{views_for_timeline.py → video/timeline.py} +3 -3
  602. endoreg_db/views/video/video_examination_viewset.py +329 -0
  603. endoreg_db/views/video/video_stream.py +188 -0
  604. endoreg_db-0.8.1.dist-info/METADATA +384 -0
  605. endoreg_db-0.8.1.dist-info/RECORD +789 -0
  606. endoreg_db/data/agl_service/data.yaml +0 -19
  607. endoreg_db/data/finding_location_classification/colonoscopy.yaml +0 -46
  608. endoreg_db/data/finding_morphology_classification/colonoscopy.yaml +0 -48
  609. endoreg_db/data/finding_morphology_classification_choice/colonoscopy_size.yaml +0 -57
  610. endoreg_db/management/commands/_load_model_template.py +0 -41
  611. endoreg_db/management/commands/delete_all.py +0 -18
  612. endoreg_db/management/commands/fetch_legacy_image_dataset.py +0 -32
  613. endoreg_db/management/commands/fix_auth_permission.py +0 -20
  614. endoreg_db/management/commands/load_active_model_data.py +0 -45
  615. endoreg_db/management/commands/load_g_play_data.py +0 -113
  616. endoreg_db/management/commands/load_logging_data.py +0 -39
  617. endoreg_db/management/commands/load_lx_data.py +0 -64
  618. endoreg_db/management/commands/load_medication_indication_data.py +0 -63
  619. endoreg_db/management/commands/load_medication_indication_type_data.py +0 -41
  620. endoreg_db/management/commands/load_medication_intake_time_data.py +0 -41
  621. endoreg_db/management/commands/load_medication_schedule_data.py +0 -55
  622. endoreg_db/management/commands/load_network_data.py +0 -57
  623. endoreg_db/migrations/0002_alter_frame_image_alter_rawframe_image.py +0 -23
  624. endoreg_db/migrations/0003_alter_frame_image_alter_rawframe_image.py +0 -23
  625. endoreg_db/migrations/0004_alter_rawvideofile_file_alter_video_file.py +0 -25
  626. endoreg_db/migrations/0005_rawvideofile_frame_count_and_more.py +0 -33
  627. endoreg_db/migrations/0006_frame_extracted_rawframe_extracted.py +0 -23
  628. endoreg_db/migrations/0007_rename_pseudo_patient_video_patient_and_more.py +0 -24
  629. endoreg_db/migrations/0008_remove_reportfile_patient_examination_and_more.py +0 -48
  630. endoreg_db/migrations/0009_requirementoperator_requirementsettype_and_more.py +0 -154
  631. endoreg_db/models/ai_model/active_model.py +0 -9
  632. endoreg_db/models/ai_model/ai_model.py +0 -90
  633. endoreg_db/models/ai_model/lightning/__init__.py +0 -3
  634. endoreg_db/models/ai_model/lightning/predict.py +0 -172
  635. endoreg_db/models/ai_model/lightning/prediction_visualizer.py +0 -55
  636. endoreg_db/models/ai_model/lightning/run_visualizer.py +0 -21
  637. endoreg_db/models/ai_model/model_meta.py +0 -240
  638. endoreg_db/models/annotation/__init__.py +0 -32
  639. endoreg_db/models/annotation/anonymized_image_annotation.py +0 -115
  640. endoreg_db/models/annotation/binary_classification_annotation_task.py +0 -117
  641. endoreg_db/models/annotation/image_classification.py +0 -86
  642. endoreg_db/models/annotation/video_segmentation_annotation.py +0 -52
  643. endoreg_db/models/case/__init__.py +0 -1
  644. endoreg_db/models/case/case.py +0 -34
  645. endoreg_db/models/center/center.py +0 -51
  646. endoreg_db/models/center/center_product.py +0 -33
  647. endoreg_db/models/center/center_waste.py +0 -16
  648. endoreg_db/models/data_file/__init__.py +0 -39
  649. endoreg_db/models/data_file/base_classes/__init__.py +0 -7
  650. endoreg_db/models/data_file/base_classes/abstract_frame.py +0 -98
  651. endoreg_db/models/data_file/base_classes/abstract_pdf.py +0 -127
  652. endoreg_db/models/data_file/base_classes/abstract_video.py +0 -806
  653. endoreg_db/models/data_file/base_classes/frame_helpers.py +0 -17
  654. endoreg_db/models/data_file/base_classes/prepare_bulk_frames.py +0 -19
  655. endoreg_db/models/data_file/base_classes/utils.py +0 -58
  656. endoreg_db/models/data_file/frame.py +0 -29
  657. endoreg_db/models/data_file/import_classes/__init__.py +0 -18
  658. endoreg_db/models/data_file/import_classes/processing_functions/__init__.py +0 -35
  659. endoreg_db/models/data_file/import_classes/processing_functions/pdf.py +0 -28
  660. endoreg_db/models/data_file/import_classes/processing_functions/video.py +0 -260
  661. endoreg_db/models/data_file/import_classes/raw_pdf.py +0 -254
  662. endoreg_db/models/data_file/import_classes/raw_video.py +0 -290
  663. endoreg_db/models/data_file/metadata/sensitive_meta.py +0 -290
  664. endoreg_db/models/data_file/metadata/video_meta.py +0 -199
  665. endoreg_db/models/data_file/report_file.py +0 -56
  666. endoreg_db/models/data_file/video/__init__.py +0 -11
  667. endoreg_db/models/data_file/video/import_meta.py +0 -25
  668. endoreg_db/models/data_file/video/video.py +0 -196
  669. endoreg_db/models/data_file/video_segment.py +0 -214
  670. endoreg_db/models/examination/examination.py +0 -67
  671. endoreg_db/models/finding/__init__.py +0 -11
  672. endoreg_db/models/finding/finding.py +0 -75
  673. endoreg_db/models/finding/finding_location_classification.py +0 -94
  674. endoreg_db/models/finding/finding_morphology_classification.py +0 -89
  675. endoreg_db/models/finding/finding_type.py +0 -22
  676. endoreg_db/models/hardware/__init__.py +0 -2
  677. endoreg_db/models/information_source.py +0 -65
  678. endoreg_db/models/laboratory/__init__.py +0 -1
  679. endoreg_db/models/laboratory/lab_value.py +0 -162
  680. endoreg_db/models/logging/__init__.py +0 -11
  681. endoreg_db/models/logging/agl_service.py +0 -19
  682. endoreg_db/models/logging/base.py +0 -22
  683. endoreg_db/models/logging/log_type.py +0 -23
  684. endoreg_db/models/logging/network_device.py +0 -27
  685. endoreg_db/models/lx/__init__.py +0 -4
  686. endoreg_db/models/lx/client.py +0 -57
  687. endoreg_db/models/lx/identity.py +0 -34
  688. endoreg_db/models/lx/permission.py +0 -18
  689. endoreg_db/models/lx/user.py +0 -16
  690. endoreg_db/models/network/__init__.py +0 -9
  691. endoreg_db/models/network/agl_service.py +0 -38
  692. endoreg_db/models/network/network_device.py +0 -58
  693. endoreg_db/models/network/network_device_type.py +0 -23
  694. endoreg_db/models/other/distribution.py +0 -5
  695. endoreg_db/models/patient/__init__.py +0 -24
  696. endoreg_db/models/patient/patient_examination.py +0 -182
  697. endoreg_db/models/patient/patient_finding.py +0 -143
  698. endoreg_db/models/patient/patient_finding_location.py +0 -120
  699. endoreg_db/models/patient/patient_finding_morphology.py +0 -166
  700. endoreg_db/models/permissions/__init__.py +0 -44
  701. endoreg_db/models/persons/__init__.py +0 -34
  702. endoreg_db/models/persons/examiner/__init__.py +0 -2
  703. endoreg_db/models/persons/examiner/examiner.py +0 -60
  704. endoreg_db/models/persons/examiner/examiner_type.py +0 -2
  705. endoreg_db/models/persons/patient/__init__.py +0 -8
  706. endoreg_db/models/persons/patient/patient.py +0 -389
  707. endoreg_db/models/persons/patient/patient_disease.py +0 -22
  708. endoreg_db/models/persons/patient/patient_event.py +0 -52
  709. endoreg_db/models/persons/patient/patient_lab_sample.py +0 -108
  710. endoreg_db/models/persons/patient/patient_medication.py +0 -59
  711. endoreg_db/models/persons/patient/patient_medication_schedule.py +0 -88
  712. endoreg_db/models/persons/portal_user_information.py +0 -27
  713. endoreg_db/models/prediction/__init__.py +0 -8
  714. endoreg_db/models/prediction/image_classification.py +0 -51
  715. endoreg_db/models/prediction/video_prediction_meta.py +0 -306
  716. endoreg_db/models/product/product.py +0 -110
  717. endoreg_db/models/product/product_group.py +0 -27
  718. endoreg_db/models/product/product_material.py +0 -28
  719. endoreg_db/models/questionnaires/__init__.py +0 -114
  720. endoreg_db/models/quiz/__init__.py +0 -9
  721. endoreg_db/models/quiz/quiz_answer.py +0 -41
  722. endoreg_db/models/quiz/quiz_question.py +0 -54
  723. endoreg_db/models/report_reader/report_reader_config.py +0 -53
  724. endoreg_db/models/rules/__init__.py +0 -5
  725. endoreg_db/queries/get/__init__.py +0 -6
  726. endoreg_db/queries/get/center.py +0 -42
  727. endoreg_db/queries/get/model.py +0 -13
  728. endoreg_db/queries/get/patient.py +0 -14
  729. endoreg_db/queries/get/patient_examination.py +0 -20
  730. endoreg_db/queries/get/report_file.py +0 -33
  731. endoreg_db/queries/get/video.py +0 -31
  732. endoreg_db/serializers/ai_model.py +0 -19
  733. endoreg_db/serializers/annotation.py +0 -14
  734. endoreg_db/serializers/center.py +0 -11
  735. endoreg_db/serializers/examination.py +0 -33
  736. endoreg_db/serializers/frame.py +0 -9
  737. endoreg_db/serializers/hardware.py +0 -21
  738. endoreg_db/serializers/label.py +0 -22
  739. endoreg_db/serializers/patient.py +0 -33
  740. endoreg_db/serializers/prediction.py +0 -10
  741. endoreg_db/serializers/raw_pdf_anony_text_validation.py +0 -137
  742. endoreg_db/serializers/report_file.py +0 -7
  743. endoreg_db/serializers/video.py +0 -20
  744. endoreg_db/serializers/video_segmentation.py +0 -574
  745. endoreg_db/tests.py +0 -3
  746. endoreg_db/utils/legacy_ocr.py +0 -201
  747. endoreg_db/utils/video/transcode_videofile.py +0 -111
  748. endoreg_db/views/patient_views.py +0 -90
  749. endoreg_db/views/raw_pdf_anony_text_validation_views.py +0 -95
  750. endoreg_db/views/raw_pdf_meta_validation_views.py +0 -111
  751. endoreg_db/views/raw_video_meta_validation_views.py +0 -148
  752. endoreg_db/views/report_views.py +0 -96
  753. endoreg_db/views/video_segmentation_views.py +0 -166
  754. endoreg_db-0.6.4.dist-info/METADATA +0 -161
  755. endoreg_db-0.6.4.dist-info/RECORD +0 -470
  756. /endoreg_db/{case_generator/__init__.py → api/serializers/finding_descriptions.py} +0 -0
  757. /endoreg_db/{queries/get/annotation.py → api/views/finding_descriptions.py} +0 -0
  758. /endoreg_db/{queries/get/prediction.py → data/shift/m2.yaml} +0 -0
  759. /endoreg_db/{queries/get/video_import_meta.py → factories/__init__.py} +0 -0
  760. /endoreg_db/{queries/get/video_prediction_meta.py → helpers/__init__.py} +0 -0
  761. /endoreg_db/models/{case_template → administration/case/case_template}/__init__.py +0 -0
  762. /endoreg_db/models/{persons → administration/person}/person.py +0 -0
  763. /endoreg_db/models/{product → administration/product}/__init__.py +0 -0
  764. /endoreg_db/models/{report_reader → media/pdf/report_reader}/__init__.py +0 -0
  765. /endoreg_db/models/{report_reader → media/pdf/report_reader}/report_reader_flag.py +0 -0
  766. /endoreg_db/models/{hardware → medical/hardware}/endoscopy_processor.py +0 -0
  767. /endoreg_db/models/{medication → medical/medication}/__init__.py +0 -0
  768. /endoreg_db/models/{medication → medical/medication}/medication_indication.py +0 -0
  769. /endoreg_db/models/{risk → medical/risk}/__init__.py +0 -0
  770. /endoreg_db/models/{risk → medical/risk}/risk.py +0 -0
  771. /endoreg_db/models/{emission → other/emission}/__init__.py +0 -0
  772. /endoreg_db/models/{rules → rule}/rule_applicator.py +0 -0
  773. /endoreg_db/{case_generator → utils/case_generator}/case_generator.py +0 -0
  774. /endoreg_db/{case_generator → utils/case_generator}/lab_sample_factory.py +0 -0
  775. /endoreg_db/{case_generator → utils/case_generator}/utils.py +0 -0
  776. /endoreg_db/views/{csrf.py → misc/csrf.py} +0 -0
  777. {endoreg_db-0.6.4.dist-info → endoreg_db-0.8.1.dist-info}/WHEEL +0 -0
  778. {endoreg_db-0.6.4.dist-info → endoreg_db-0.8.1.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,772 @@
1
+ import subprocess
2
+ import json
3
+ import logging
4
+ from pathlib import Path
5
+ from typing import List, Dict, Optional, Tuple
6
+ import cv2
7
+ from tqdm import tqdm
8
+ import shutil
9
+
10
+ logger = logging.getLogger("ffmpeg_wrapper")
11
+
12
+ # Global hardware acceleration cache
13
+ _nvenc_available = None
14
+ _preferred_encoder = None
15
+
16
+ def _detect_nvenc_support() -> bool:
17
+ """
18
+ Detect if NVIDIA NVENC hardware acceleration is available.
19
+
20
+ Returns:
21
+ True if NVENC is available, False otherwise
22
+ """
23
+ try:
24
+ # Test NVENC availability with a minimal command (minimum size for NVENC)
25
+ cmd = [
26
+ 'ffmpeg', '-f', 'lavfi', '-i', 'testsrc=duration=1:size=256x256:rate=1',
27
+ '-c:v', 'h264_nvenc', '-preset', 'p1', '-f', 'null', '-'
28
+ ]
29
+
30
+ result = subprocess.run(
31
+ cmd,
32
+ capture_output=True,
33
+ text=True,
34
+ timeout=15,
35
+ check=False
36
+ )
37
+
38
+ if result.returncode == 0:
39
+ logger.debug("NVENC h264 encoding test successful")
40
+ return True
41
+ else:
42
+ logger.debug(f"NVENC test failed: {result.stderr}")
43
+ return False
44
+
45
+ except (subprocess.TimeoutExpired, FileNotFoundError) as e:
46
+ logger.debug(f"NVENC detection failed: {e}")
47
+ return False
48
+ except Exception as e:
49
+ logger.warning(f"Unexpected error during NVENC detection: {e}")
50
+ return False
51
+
52
+ def _get_preferred_encoder() -> Dict[str, str]:
53
+ """
54
+ Get the preferred video encoder configuration based on available hardware.
55
+
56
+ Returns:
57
+ Dictionary with encoder configuration
58
+ """
59
+ global _nvenc_available, _preferred_encoder
60
+
61
+ if _nvenc_available is None:
62
+ _nvenc_available = _detect_nvenc_support()
63
+
64
+ if _preferred_encoder is None:
65
+ if _nvenc_available:
66
+ _preferred_encoder = {
67
+ 'name': 'h264_nvenc',
68
+ 'preset_param': '-preset',
69
+ 'preset_value': 'p4', # Medium quality/speed for NVENC
70
+ 'quality_param': '-cq',
71
+ 'quality_value': '20', # NVENC CQ mode
72
+ 'type': 'nvenc',
73
+ 'fallback_preset': 'p1' # Fastest NVENC preset for fallback
74
+ }
75
+ logger.info("Hardware acceleration: NVENC available")
76
+ else:
77
+ _preferred_encoder = {
78
+ 'name': 'libx264',
79
+ 'preset_param': '-preset',
80
+ 'preset_value': 'medium', # CPU preset
81
+ 'quality_param': '-crf',
82
+ 'quality_value': '23', # CPU CRF mode
83
+ 'type': 'cpu',
84
+ 'fallback_preset': 'ultrafast' # Fastest CPU preset for fallback
85
+ }
86
+ logger.info("Hardware acceleration: NVENC not available, using CPU")
87
+
88
+ return _preferred_encoder
89
+
90
+ def _build_encoder_args(quality_mode: str = 'balanced',
91
+ fallback: bool = False,
92
+ custom_crf: Optional[int] = None) -> Tuple[List[str], str]:
93
+ """
94
+ Build encoder command arguments based on available hardware and quality requirements.
95
+
96
+ Args:
97
+ quality_mode: 'fast', 'balanced', or 'quality'
98
+ fallback: Whether to use fallback settings for compatibility
99
+ custom_crf: Override quality setting (for backward compatibility)
100
+
101
+ Returns:
102
+ Tuple of (encoder_args, encoder_type)
103
+ """
104
+ encoder = _get_preferred_encoder()
105
+
106
+ if encoder['type'] == 'nvenc':
107
+ # NVIDIA NVENC configuration
108
+ if fallback:
109
+ preset = encoder['fallback_preset'] # p1 - fastest
110
+ quality = '28' # Lower quality for speed
111
+ elif quality_mode == 'fast':
112
+ preset = 'p2' # Faster preset
113
+ quality = '25'
114
+ elif quality_mode == 'quality':
115
+ preset = 'p6' # Higher quality preset
116
+ quality = '18'
117
+ else: # balanced
118
+ preset = encoder['preset_value'] # p4
119
+ quality = encoder['quality_value'] # 20
120
+
121
+ # Override with custom CRF if provided (for backward compatibility)
122
+ if custom_crf is not None:
123
+ quality = str(custom_crf)
124
+
125
+ return [
126
+ '-c:v', encoder['name'],
127
+ encoder['preset_param'], preset,
128
+ encoder['quality_param'], quality,
129
+ '-gpu', '0', # Use first GPU
130
+ '-rc', 'vbr', # Variable bitrate
131
+ '-profile:v', 'high'
132
+ ], encoder['type']
133
+ else:
134
+ # CPU libx264 configuration
135
+ if fallback:
136
+ preset = encoder['fallback_preset'] # ultrafast
137
+ quality = '28' # Lower quality for speed
138
+ elif quality_mode == 'fast':
139
+ preset = 'faster'
140
+ quality = '20'
141
+ elif quality_mode == 'quality':
142
+ preset = 'slow'
143
+ quality = '18'
144
+ else: # balanced
145
+ preset = encoder['preset_value'] # medium
146
+ quality = encoder['quality_value'] # 23
147
+
148
+ # Override with custom CRF if provided (for backward compatibility)
149
+ if custom_crf is not None:
150
+ quality = str(custom_crf)
151
+
152
+ return [
153
+ '-c:v', encoder['name'],
154
+ encoder['preset_param'], preset,
155
+ encoder['quality_param'], quality,
156
+ '-profile:v', 'high'
157
+ ], encoder['type']
158
+
159
+ def is_ffmpeg_available() -> bool:
160
+ """
161
+ Checks whether the FFmpeg executable is available in the system's PATH.
162
+
163
+ Returns:
164
+ True if FFmpeg is found in the PATH; otherwise, False.
165
+ """
166
+ return shutil.which("ffmpeg") is not None
167
+
168
+ def check_ffmpeg_availability():
169
+ """
170
+ Verifies that FFmpeg is installed and available in the system's PATH.
171
+
172
+ Raises:
173
+ FileNotFoundError: If FFmpeg is not found.
174
+
175
+ Returns:
176
+ True if FFmpeg is available.
177
+ """
178
+ if not is_ffmpeg_available():
179
+ error_msg = "FFmpeg is not available. Please install it and ensure it's in your PATH."
180
+ logger.error(error_msg)
181
+ raise FileNotFoundError(error_msg)
182
+ # logger.info("FFmpeg is available.") # Caller can log if needed
183
+ return True
184
+
185
+ def get_stream_info(file_path: Path) -> Optional[Dict]:
186
+ """
187
+ Retrieves video stream information from a file using ffprobe.
188
+
189
+ Runs ffprobe to extract stream metadata in JSON format from the specified video file. Returns a dictionary with stream information, or None if the file does not exist or if an error occurs during execution or parsing.
190
+ """
191
+ if not file_path.exists():
192
+ logger.error("File not found for ffprobe: %s", file_path)
193
+ return None
194
+
195
+ command = [
196
+ "ffprobe",
197
+ "-v", "quiet",
198
+ "-print_format", "json",
199
+ "-show_streams",
200
+ str(file_path),
201
+ ]
202
+ try:
203
+ result = subprocess.run(command, capture_output=True, text=True, check=True)
204
+ return json.loads(result.stdout)
205
+ except subprocess.CalledProcessError as e:
206
+ logger.error("ffprobe command failed for %s: %s\n%s", file_path, e, e.stderr)
207
+ return None
208
+ except json.JSONDecodeError as e:
209
+ logger.error("Failed to parse ffprobe JSON output for %s: %s", file_path, e)
210
+ return None
211
+ except Exception as e:
212
+ logger.error("Error running ffprobe for %s: %s", file_path, e, exc_info=True)
213
+ return None
214
+
215
+
216
+ def assemble_video_from_frames( # Renamed from assemble_video
217
+ frame_paths: List[Path],
218
+ output_path: Path,
219
+ fps: float,
220
+ width: Optional[int] = None,
221
+ height: Optional[int] = None,
222
+ ) -> Optional[Path]:
223
+ """
224
+ Assembles a video from a list of frame image paths using cv2.VideoWriter.
225
+ Determines dimensions from the first frame if not provided.
226
+ """
227
+ if not frame_paths:
228
+ logger.error("No frame paths provided for video assembly.")
229
+ return None
230
+
231
+ if width is None or height is None:
232
+ try:
233
+ first_frame = cv2.imread(str(frame_paths[0]))
234
+ if first_frame is None:
235
+ raise IOError(f"Could not read first frame: {frame_paths[0]}")
236
+ height, width, _ = first_frame.shape
237
+ logger.info("Determined video dimensions from first frame: %dx%d", width, height)
238
+ except Exception as e:
239
+ logger.error("Error reading first frame to determine dimensions: %s", e, exc_info=True)
240
+ return None
241
+
242
+ fourcc = cv2.VideoWriter_fourcc(*"mp4v") # type: ignore
243
+ output_path.parent.mkdir(parents=True, exist_ok=True)
244
+ video_writer = cv2.VideoWriter(str(output_path), fourcc, fps, (width, height))
245
+
246
+ if not video_writer.isOpened():
247
+ logger.error("Could not open video writer for path: %s", output_path)
248
+ return None
249
+
250
+ logger.info("Assembling video %s from %d frames...", output_path.name, len(frame_paths))
251
+ try:
252
+ for frame_path in tqdm(frame_paths, desc=f"Assembling {output_path.name}"):
253
+ frame = cv2.imread(str(frame_path))
254
+ if frame is None:
255
+ logger.warning("Could not read frame %s, skipping.", frame_path)
256
+ continue
257
+ # Ensure frame dimensions match - resize if necessary (or log error)
258
+ if frame.shape[1] != width or frame.shape[0] != height:
259
+ logger.warning(f"Frame {frame_path} has dimensions {frame.shape[1]}x{frame.shape[0]}, expected {width}x{height}. Resizing.")
260
+ frame = cv2.resize(frame, (width, height))
261
+ video_writer.write(frame)
262
+ finally:
263
+ video_writer.release()
264
+ logger.info("Finished assembling video: %s", output_path)
265
+
266
+ return output_path
267
+
268
+
269
+ def transcode_video(
270
+ input_path: Path,
271
+ output_path: Path,
272
+ codec: str = "auto", # Changed default to "auto" for automatic selection
273
+ crf: Optional[int] = None, # Will be determined automatically if None
274
+ preset: str = "auto", # Changed default to "auto" for automatic selection
275
+ audio_codec: str = "aac",
276
+ audio_bitrate: str = "128k",
277
+ extra_args: Optional[List[str]] = None,
278
+ quality_mode: str = "balanced", # New parameter: 'fast', 'balanced', 'quality'
279
+ force_cpu: bool = False, # New parameter to force CPU encoding
280
+ ) -> Optional[Path]:
281
+ """
282
+ Transcodes a video file using FFmpeg with automatic hardware acceleration.
283
+
284
+ Args:
285
+ input_path: Source video file path
286
+ output_path: Output video file path
287
+ codec: Video codec ('auto' for automatic selection, 'libx264', 'h264_nvenc')
288
+ crf: Constant Rate Factor (None for automatic selection)
289
+ preset: Encoder preset ('auto' for automatic selection)
290
+ audio_codec: Audio codec
291
+ audio_bitrate: Audio bitrate
292
+ extra_args: Additional FFmpeg arguments
293
+ quality_mode: Quality mode ('fast', 'balanced', 'quality')
294
+ force_cpu: Force CPU encoding even if NVENC is available
295
+
296
+ Returns:
297
+ Path to transcoded video or None if failed
298
+ """
299
+ if not input_path.exists():
300
+ logger.error("Input file not found for transcoding: %s", input_path)
301
+ return None
302
+
303
+ output_path.parent.mkdir(parents=True, exist_ok=True)
304
+
305
+ # Determine encoder configuration
306
+ if codec == "auto" or preset == "auto":
307
+ if force_cpu:
308
+ # Force CPU encoding
309
+ encoder_args, encoder_type = _build_encoder_args(quality_mode, fallback=False, custom_crf=crf)
310
+ # Override to use CPU encoder
311
+ encoder_args[1] = 'libx264' # Replace encoder name
312
+ encoder_args[3] = 'medium' if preset == "auto" else preset # Replace preset
313
+ if crf is not None:
314
+ encoder_args[5] = str(crf) # Replace quality value
315
+ else:
316
+ # Use automatic hardware detection
317
+ encoder_args, encoder_type = _build_encoder_args(quality_mode, fallback=False, custom_crf=crf)
318
+ else:
319
+ # Manual codec/preset specification (backward compatibility)
320
+ encoder_args = [
321
+ '-c:v', codec,
322
+ '-preset', preset,
323
+ '-crf' if codec == 'libx264' else '-cq', str(crf if crf is not None else 23),
324
+ ]
325
+ encoder_type = 'nvenc' if 'nvenc' in codec else 'cpu'
326
+
327
+ # Build complete command
328
+ command = [
329
+ "ffmpeg",
330
+ "-i", str(input_path),
331
+ *encoder_args,
332
+ "-c:a", audio_codec,
333
+ "-b:a", audio_bitrate,
334
+ "-y", # Overwrite output file if it exists
335
+ ]
336
+
337
+ if extra_args:
338
+ command.extend(extra_args)
339
+ command.append(str(output_path))
340
+
341
+ logger.info("Starting transcoding: %s -> %s (using %s)",
342
+ input_path.name, output_path.name, encoder_type)
343
+ logger.debug("FFmpeg command: %s", " ".join(command))
344
+
345
+ try:
346
+ process = subprocess.Popen(command, stderr=subprocess.PIPE, text=True, universal_newlines=True)
347
+
348
+ # Progress reporting and error handling
349
+ stderr_output = ""
350
+ if process.stderr:
351
+ for line in process.stderr:
352
+ stderr_output += line
353
+
354
+ process.wait()
355
+
356
+ if process.returncode == 0:
357
+ logger.info("Transcoding finished successfully: %s", output_path)
358
+ return output_path
359
+ else:
360
+ logger.error("FFmpeg transcoding failed for %s with return code %d.",
361
+ input_path.name, process.returncode)
362
+ logger.error("FFmpeg stderr:\n%s", stderr_output)
363
+
364
+ # Try fallback to CPU if NVENC failed
365
+ if encoder_type == 'nvenc' and not force_cpu:
366
+ logger.warning("NVENC transcoding failed, trying CPU fallback...")
367
+ return _transcode_video_fallback(
368
+ input_path, output_path, audio_codec, audio_bitrate,
369
+ extra_args, quality_mode, crf
370
+ )
371
+
372
+ # Clean up potentially corrupted output file
373
+ if output_path.exists():
374
+ try:
375
+ output_path.unlink()
376
+ except OSError as e:
377
+ logger.error("Failed to delete incomplete output file %s: %s", output_path, e)
378
+ return None
379
+
380
+ except FileNotFoundError:
381
+ logger.error("ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH.")
382
+ return None
383
+ except Exception as e:
384
+ logger.error("Error during transcoding of %s: %s", input_path.name, e, exc_info=True)
385
+ return None
386
+
387
+ def _transcode_video_fallback(
388
+ input_path: Path,
389
+ output_path: Path,
390
+ audio_codec: str,
391
+ audio_bitrate: str,
392
+ extra_args: Optional[List[str]],
393
+ quality_mode: str,
394
+ custom_crf: Optional[int]
395
+ ) -> Optional[Path]:
396
+ """
397
+ Fallback transcoding using CPU encoding.
398
+
399
+ Args:
400
+ input_path: Source video file path
401
+ output_path: Output video file path
402
+ audio_codec: Audio codec
403
+ audio_bitrate: Audio bitrate
404
+ extra_args: Additional FFmpeg arguments
405
+ quality_mode: Quality mode
406
+ custom_crf: Custom CRF value
407
+
408
+ Returns:
409
+ Path to transcoded video or None if failed
410
+ """
411
+ try:
412
+ # Build CPU encoder arguments
413
+ encoder_args, _ = _build_encoder_args(quality_mode, fallback=True, custom_crf=custom_crf)
414
+ # Force CPU encoder
415
+ encoder_args[1] = 'libx264'
416
+
417
+ command = [
418
+ "ffmpeg",
419
+ "-i", str(input_path),
420
+ *encoder_args,
421
+ "-c:a", audio_codec,
422
+ "-b:a", audio_bitrate,
423
+ "-y",
424
+ ]
425
+
426
+ if extra_args:
427
+ command.extend(extra_args)
428
+ command.append(str(output_path))
429
+
430
+ logger.info("CPU fallback transcoding: %s -> %s", input_path.name, output_path.name)
431
+ logger.debug("Fallback FFmpeg command: %s", " ".join(command))
432
+
433
+ process = subprocess.Popen(command, stderr=subprocess.PIPE, text=True, universal_newlines=True)
434
+ stderr_output = ""
435
+ if process.stderr:
436
+ for line in process.stderr:
437
+ stderr_output += line
438
+
439
+ process.wait()
440
+
441
+ if process.returncode == 0:
442
+ logger.info("CPU fallback transcoding successful: %s", output_path)
443
+ return output_path
444
+ else:
445
+ logger.error("CPU fallback transcoding also failed for %s", input_path.name)
446
+ logger.error("Fallback stderr:\n%s", stderr_output)
447
+ return None
448
+
449
+ except Exception as e:
450
+ logger.error("Error during CPU fallback transcoding: %s", e, exc_info=True)
451
+ return None
452
+
453
+ logger.info("Starting transcoding: %s -> %s", input_path.name, output_path.name)
454
+ logger.debug("FFmpeg command: %s", " ".join(command))
455
+
456
+ try:
457
+ process = subprocess.Popen(command, stderr=subprocess.PIPE, text=True, universal_newlines=True)
458
+
459
+ # Optional: Progress reporting (can be complex to parse ffmpeg output reliably)
460
+ # For simplicity, just wait and check the return code
461
+ stderr_output = ""
462
+ if process.stderr:
463
+ for line in process.stderr:
464
+ stderr_output += line
465
+ # Simple progress indication or detailed logging
466
+ # logger.debug(f"ffmpeg: {line.strip()}")
467
+
468
+ process.wait()
469
+
470
+ if process.returncode == 0:
471
+ logger.info("Transcoding finished successfully: %s", output_path)
472
+ return output_path
473
+ else:
474
+ logger.error("FFmpeg transcoding failed for %s with return code %d.", input_path.name, process.returncode)
475
+ logger.error("FFmpeg stderr:\n%s", stderr_output)
476
+ # Clean up potentially corrupted output file
477
+ if output_path.exists():
478
+ try:
479
+ output_path.unlink()
480
+ except OSError as e:
481
+ logger.error("Failed to delete incomplete output file %s: %s", output_path, e)
482
+ return None
483
+
484
+ except FileNotFoundError:
485
+ logger.error("ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH.")
486
+ return None
487
+ except Exception as e:
488
+ logger.error("Error during transcoding of %s: %s", input_path.name, e, exc_info=True)
489
+ return None
490
+
491
+ def transcode_videofile_if_required(
492
+ input_path: Path,
493
+ output_path: Path,
494
+ required_codec: str = "h264",
495
+ required_pixel_format: str = "yuv420p", # Changed default from yuvj420p
496
+ **transcode_options # Pass other options to transcode_video
497
+ ) -> Optional[Path]:
498
+ """
499
+ Checks if a video needs transcoding based on codec and pixel format,
500
+ and transcodes it using transcode_video if necessary.
501
+ Uses yuv420p with full color range (pc/jpeg) as the target format.
502
+ Returns the path to the compliant video (original or transcoded).
503
+ """
504
+ stream_info = get_stream_info(input_path)
505
+ if not stream_info or "streams" not in stream_info:
506
+ logger.error("Could not get stream info for %s to check if transcoding is required.", input_path)
507
+ return None
508
+
509
+ video_stream = next((s for s in stream_info["streams"] if s.get("codec_type") == "video"), None)
510
+
511
+ if not video_stream:
512
+ logger.error("No video stream found in %s.", input_path)
513
+ return None
514
+
515
+ codec_name = video_stream.get("codec_name")
516
+ pixel_format = video_stream.get("pix_fmt")
517
+ # Check color range as well, default is usually 'tv' (limited)
518
+ color_range = video_stream.get("color_range", "tv") # Default to tv if not specified
519
+
520
+ needs_transcoding = False
521
+ transcode_reason = []
522
+ if codec_name != required_codec:
523
+ reason = f"Codec mismatch ({codec_name} != {required_codec})"
524
+ logger.info("%s for %s. Transcoding required.", reason, input_path.name)
525
+ transcode_reason.append(reason)
526
+ needs_transcoding = True
527
+ # Check both pixel format and color range for yuv420p
528
+ if pixel_format != required_pixel_format or (pixel_format == "yuv420p" and color_range != "pc"):
529
+ reason = f"Pixel format/color range mismatch (pix_fmt: {pixel_format}, color_range: {color_range} != {required_pixel_format} with color_range=pc)"
530
+ logger.info("%s for %s. Transcoding required.", reason, input_path.name)
531
+ transcode_reason.append(reason)
532
+ needs_transcoding = True
533
+
534
+ if needs_transcoding:
535
+ logger.info("Transcoding %s to %s due to: %s", input_path.name, output_path.name, "; ".join(transcode_reason))
536
+ # Ensure codec and pixel format are set in options if not already present
537
+ transcode_options.setdefault('codec', 'libx264' if required_codec == 'h264' else required_codec)
538
+ transcode_options.setdefault('extra_args', [])
539
+
540
+ # Ensure pixel format and color range are correctly set in extra_args
541
+ extra_args = transcode_options['extra_args']
542
+ if '-pix_fmt' not in extra_args:
543
+ extra_args.extend(['-pix_fmt', required_pixel_format])
544
+ else:
545
+ # If pix_fmt is already set, ensure it's the required one
546
+ try:
547
+ pix_fmt_index = extra_args.index('-pix_fmt')
548
+ if extra_args[pix_fmt_index + 1] != required_pixel_format:
549
+ logger.warning("Overriding existing -pix_fmt '%s' with '%s'", extra_args[pix_fmt_index + 1], required_pixel_format)
550
+ extra_args[pix_fmt_index + 1] = required_pixel_format
551
+ except (ValueError, IndexError):
552
+ # Should not happen if '-pix_fmt' is in extra_args, but handle defensively
553
+ logger.error("Error processing existing -pix_fmt argument. Appending required format.")
554
+ extra_args.extend(['-pix_fmt', required_pixel_format])
555
+
556
+
557
+ if '-color_range' not in extra_args:
558
+ # Add color range 'pc' (which corresponds to 2 or 'jpeg') for yuv420p
559
+ extra_args.extend(['-color_range', 'pc'])
560
+ else:
561
+ # If color_range is already set, ensure it's 'pc'
562
+ try:
563
+ color_range_index = extra_args.index('-color_range')
564
+ if extra_args[color_range_index + 1] != 'pc':
565
+ logger.warning("Overriding existing -color_range '%s' with 'pc'", extra_args[color_range_index + 1])
566
+ extra_args[color_range_index + 1] = 'pc'
567
+ except (ValueError, IndexError):
568
+ logger.error("Error processing existing -color_range argument. Appending 'pc'.")
569
+ extra_args.extend(['-color_range', 'pc'])
570
+
571
+
572
+ return transcode_video(input_path, output_path, **transcode_options)
573
+ else:
574
+ logger.info("Video %s already meets requirements (%s, %s, color_range=pc). No transcoding needed.", input_path.name, required_codec, required_pixel_format)
575
+ # If no transcoding is needed, should we copy/link or just return the original path?
576
+ # For simplicity, let's assume the caller handles the file location.
577
+ # If the output_path is different, we might need to copy.
578
+ if input_path != output_path:
579
+ # Example: copy file if output path is different
580
+ try:
581
+ output_path.parent.mkdir(parents=True, exist_ok=True)
582
+ shutil.copy2(input_path, output_path)
583
+ logger.info("Copied %s to %s as it met requirements.", input_path.name, output_path.name)
584
+ return output_path
585
+ except Exception as e:
586
+ logger.error("Failed to copy %s to %s: %s", input_path.name, output_path.name, e)
587
+ return None
588
+ return input_path # Return original path if no copy needed
589
+
590
+ def extract_frames(
591
+ video_path: Path,
592
+ output_dir: Path,
593
+ quality: int,
594
+ ext: str = "jpg",
595
+ fps: Optional[float] = None
596
+ ) -> List[Path]:
597
+ """
598
+ Extracts frames from a video file using FFmpeg.
599
+
600
+ Args:
601
+ video_path: Path to the input video file.
602
+ output_dir: Directory to save the extracted frames.
603
+ quality: Quality factor for JPEG extraction (1-31, lower is better).
604
+ ext: Output frame image extension (e.g., 'jpg', 'png').
605
+ fps: Optional frames per second to extract. If None, extracts all frames.
606
+
607
+ Returns:
608
+ A list of Path objects for the extracted frames.
609
+ """
610
+ # Check if ffmpeg command exists
611
+ ffmpeg_executable = shutil.which("ffmpeg")
612
+ if not ffmpeg_executable:
613
+ error_msg = "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
614
+ logger.error(error_msg)
615
+ raise FileNotFoundError(error_msg)
616
+
617
+ output_dir.mkdir(parents=True, exist_ok=True)
618
+ output_pattern = output_dir / f"frame_%07d.{ext}"
619
+
620
+ cmd = [
621
+ ffmpeg_executable, # Use the found executable path
622
+ "-i", str(video_path),
623
+ "-qscale:v", str(quality), # Video quality scale
624
+ ]
625
+
626
+ if fps is not None:
627
+ cmd.extend(["-vf", f"fps={fps}"])
628
+
629
+ cmd.append(str(output_pattern))
630
+
631
+ logger.info("Running FFmpeg command: %s", " ".join(cmd))
632
+ try:
633
+ # Use subprocess.run for better error handling
634
+ result = subprocess.run(cmd, check=True, capture_output=True, text=True)
635
+ logger.debug("FFmpeg stdout:\n%s", result.stdout)
636
+ logger.debug("FFmpeg stderr:\n%s", result.stderr)
637
+ logger.info("FFmpeg frame extraction completed successfully.")
638
+ except FileNotFoundError as exc:
639
+ # This might be redundant now but kept for safety
640
+ error_msg = f"ffmpeg command not found at '{ffmpeg_executable}'. Ensure FFmpeg is installed and in the system's PATH."
641
+ logger.error(error_msg)
642
+ raise FileNotFoundError(error_msg) from exc
643
+ except subprocess.CalledProcessError as e:
644
+ logger.error("FFmpeg command failed with exit code %d.", e.returncode)
645
+ logger.error("FFmpeg stderr:\n%s", e.stderr)
646
+ logger.error("FFmpeg stdout:\n%s", e.stdout)
647
+ # Return empty list on error as frames were likely not created correctly
648
+ return []
649
+ except Exception as e:
650
+ logger.error("An unexpected error occurred during FFmpeg execution: %s", e, exc_info=True)
651
+ return []
652
+
653
+
654
+ # Collect paths of extracted frames
655
+ extracted_files = sorted(output_dir.glob(f"frame_*.{ext}"))
656
+ return extracted_files
657
+
658
+ def extract_frame_range(
659
+ video_path: Path,
660
+ output_dir: Path,
661
+ start_frame: int,
662
+ end_frame: int, # Exclusive end frame number
663
+ quality: int,
664
+ ext: str = "jpg",
665
+ ) -> List[Path]:
666
+ """
667
+ Extracts a specific range of frames from a video using FFmpeg.
668
+
669
+ Frames from start_frame (inclusive) to end_frame (exclusive) are saved as images
670
+ in the output directory, following the naming pattern 'frame_%07d.ext'. The
671
+ function ensures only the requested frames are returned, and cleans up partial
672
+ results on failure.
673
+
674
+ Args:
675
+ video_path: Path to the input video file.
676
+ output_dir: Directory where extracted frames will be saved.
677
+ start_frame: Index of the first frame to extract (inclusive, 0-based).
678
+ end_frame: Index at which to stop extraction (exclusive, 0-based).
679
+ quality: JPEG quality factor (1-31, lower is better).
680
+ ext: File extension for output images (e.g., 'jpg', 'png').
681
+
682
+ Returns:
683
+ List of Paths to the extracted frame image files within the specified range.
684
+
685
+ Raises:
686
+ FileNotFoundError: If the FFmpeg executable is not found.
687
+ ValueError: If start_frame is greater than or equal to end_frame.
688
+ RuntimeError: If FFmpeg fails to extract the requested frames.
689
+ """
690
+ if start_frame >= end_frame:
691
+ logger.warning("extract_frame_range called with start_frame (%d) >= end_frame (%d). No frames to extract.", start_frame, end_frame)
692
+ return []
693
+
694
+ ffmpeg_executable = shutil.which("ffmpeg")
695
+ if not ffmpeg_executable:
696
+ error_msg = "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
697
+ logger.error(error_msg)
698
+ raise FileNotFoundError(error_msg)
699
+
700
+ output_dir.mkdir(parents=True, exist_ok=True)
701
+ # Use a consistent naming convention, matching extract_frames
702
+ output_pattern = output_dir / f"frame_%07d.{ext}"
703
+
704
+ # Use select filter for precise frame range extraction
705
+ # 'select' uses 0-based indexing 'n'
706
+ # We want frames where start_frame <= n < end_frame
707
+ select_filter = f"select='between(n,{start_frame},{end_frame-1})'"
708
+
709
+ cmd = [
710
+ ffmpeg_executable,
711
+ "-i", str(video_path),
712
+ "-vf", select_filter,
713
+ "-vsync", "vfr", # Variable frame rate sync to handle selected frames
714
+ "-qscale:v", str(quality),
715
+ "-copyts", # Attempt to copy timestamps if needed, might not be accurate with select
716
+ str(output_pattern),
717
+ ]
718
+
719
+ logger.info("Running FFmpeg command for frame range extraction: %s", " ".join(cmd))
720
+ try:
721
+ result = subprocess.run(cmd, check=True, capture_output=True, text=True)
722
+ logger.debug("FFmpeg stdout:\n%s", result.stdout)
723
+ logger.debug("FFmpeg stderr:\n%s", result.stderr)
724
+ logger.info("FFmpeg frame range extraction completed successfully.")
725
+ except FileNotFoundError as exc:
726
+ error_msg = f"ffmpeg command not found at '{ffmpeg_executable}'. Ensure FFmpeg is installed and in the system's PATH."
727
+ logger.error(error_msg)
728
+ raise FileNotFoundError(error_msg) from exc
729
+ except subprocess.CalledProcessError as e:
730
+ logger.error("FFmpeg command failed with exit code %d.", e.returncode)
731
+ logger.error("FFmpeg stderr:\n%s", e.stderr)
732
+ logger.error("FFmpeg stdout:\n%s", e.stdout)
733
+ # Clean up potentially partially created files in the target directory within the expected range
734
+ logger.warning("Attempting cleanup of potentially incomplete frames in %s", output_dir)
735
+ for i in range(start_frame, end_frame):
736
+ potential_file = output_dir / f"frame_{i:07d}.{ext}"
737
+ if potential_file.exists():
738
+ try:
739
+ potential_file.unlink()
740
+ except OSError as unlink_err:
741
+ logger.error("Failed to delete potential frame %s during cleanup: %s", potential_file, unlink_err)
742
+ raise RuntimeError(f"FFmpeg frame range extraction failed for {video_path}") from e
743
+ except Exception as e:
744
+ logger.error("An unexpected error occurred during FFmpeg execution: %s", e, exc_info=True)
745
+ raise RuntimeError(f"Unexpected error during FFmpeg frame range extraction for {video_path}") from e
746
+
747
+ # Collect paths of extracted frames matching the pattern and expected range
748
+ # FFmpeg might create files outside the exact range depending on version/flags,
749
+ # so filter explicitly.
750
+ extracted_files = []
751
+ for i in range(start_frame, end_frame):
752
+ frame_file = output_dir / f"frame_{i:07d}.{ext}"
753
+ if frame_file.exists():
754
+ extracted_files.append(frame_file)
755
+ else:
756
+ # This might happen if ffmpeg fails silently for some frames or if the video ends early.
757
+ logger.warning("Expected frame file %s not found after extraction.", frame_file)
758
+
759
+
760
+ logger.info("Found %d extracted frame files in range [%d, %d) for video %s.", len(extracted_files), start_frame, end_frame, video_path.name)
761
+ return extracted_files
762
+
763
+ __all__ = [
764
+ "is_ffmpeg_available", # ADDED
765
+ "check_ffmpeg_availability", # ADDED
766
+ "get_stream_info",
767
+ "assemble_video_from_frames", # Updated name
768
+ "transcode_video",
769
+ "transcode_videofile_if_required",
770
+ "extract_frames",
771
+ "extract_frame_range", # Add new function to __all__
772
+ ]