endoreg-db 0.8.9.32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (787) hide show
  1. endoreg_db/__init__.py +0 -0
  2. endoreg_db/_version.py +34 -0
  3. endoreg_db/admin.py +97 -0
  4. endoreg_db/api/serializers/finding_descriptions.py +0 -0
  5. endoreg_db/api/views/finding_descriptions.py +0 -0
  6. endoreg_db/api_urls.py +4 -0
  7. endoreg_db/apps.py +17 -0
  8. endoreg_db/assets/dummy_model.ckpt +1 -0
  9. endoreg_db/authz/auth.py +78 -0
  10. endoreg_db/authz/backends.py +168 -0
  11. endoreg_db/authz/management/commands/list_routes.py +20 -0
  12. endoreg_db/authz/middleware.py +84 -0
  13. endoreg_db/authz/permissions.py +138 -0
  14. endoreg_db/authz/policy.py +224 -0
  15. endoreg_db/authz/settings.py +64 -0
  16. endoreg_db/authz/views_auth.py +70 -0
  17. endoreg_db/codemods/readme.md +88 -0
  18. endoreg_db/codemods/rename_datetime_fields.py +99 -0
  19. endoreg_db/config/__init__.py +0 -0
  20. endoreg_db/config/env.py +106 -0
  21. endoreg_db/config/settings/__init__.py +6 -0
  22. endoreg_db/config/settings/base.py +148 -0
  23. endoreg_db/config/settings/case_gen.py +32 -0
  24. endoreg_db/config/settings/dev.py +108 -0
  25. endoreg_db/config/settings/keycloak.py +177 -0
  26. endoreg_db/config/settings/prod.py +66 -0
  27. endoreg_db/config/settings/test.py +72 -0
  28. endoreg_db/data/__init__.py +135 -0
  29. endoreg_db/data/ai_model/data.yaml +7 -0
  30. endoreg_db/data/ai_model_label/label/data.yaml +88 -0
  31. endoreg_db/data/ai_model_label/label/polyp_classification.yaml +52 -0
  32. endoreg_db/data/ai_model_label/label-set/data.yaml +40 -0
  33. endoreg_db/data/ai_model_label/label-set/polyp_classifications.yaml +25 -0
  34. endoreg_db/data/ai_model_label/label-type/data.yaml +7 -0
  35. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +27 -0
  36. endoreg_db/data/ai_model_type/data.yaml +7 -0
  37. endoreg_db/data/ai_model_video_segmentation_label/base_segmentation.yaml +176 -0
  38. endoreg_db/data/ai_model_video_segmentation_labelset/data.yaml +20 -0
  39. endoreg_db/data/case_template/rule/00_patient_lab_sample_add_default_value.yaml +167 -0
  40. endoreg_db/data/case_template/rule/01_patient-set-age.yaml +8 -0
  41. endoreg_db/data/case_template/rule/01_patient-set-gender.yaml +9 -0
  42. endoreg_db/data/case_template/rule/11_create_patient_lab_sample.yaml +23 -0
  43. endoreg_db/data/case_template/rule/12_create-patient_medication-anticoagulation.yaml +19 -0
  44. endoreg_db/data/case_template/rule/13_create-patient_medication_schedule-anticoagulation.yaml +19 -0
  45. endoreg_db/data/case_template/rule/19_create_patient.yaml +17 -0
  46. endoreg_db/data/case_template/rule_type/base_types.yaml +35 -0
  47. endoreg_db/data/case_template/rule_value/.init +0 -0
  48. endoreg_db/data/case_template/rule_value_type/base_types.yaml +59 -0
  49. endoreg_db/data/case_template/template/base.yaml +8 -0
  50. endoreg_db/data/case_template/template_type/pre_endoscopy.yaml +3 -0
  51. endoreg_db/data/case_template/tmp/_rule_value +13 -0
  52. endoreg_db/data/case_template/tmp/rule/01_atrial_fibrillation.yaml +21 -0
  53. endoreg_db/data/case_template/tmp/rule/02_create_object.yaml +10 -0
  54. endoreg_db/data/case_template/tmp/template/atrial_fibrillation_low_risk.yaml +7 -0
  55. endoreg_db/data/center/data.yaml +99 -0
  56. endoreg_db/data/center_resource/green_endoscopy_dashboard_CenterResource.yaml +144 -0
  57. endoreg_db/data/center_shift/ukw.yaml +9 -0
  58. endoreg_db/data/center_waste/green_endoscopy_dashboard_CenterWaste.yaml +48 -0
  59. endoreg_db/data/contraindication/bleeding.yaml +11 -0
  60. endoreg_db/data/db_summary.csv +58 -0
  61. endoreg_db/data/db_summary.xlsx +0 -0
  62. endoreg_db/data/disease/cardiovascular.yaml +37 -0
  63. endoreg_db/data/disease/hepatology.yaml +5 -0
  64. endoreg_db/data/disease/misc.yaml +5 -0
  65. endoreg_db/data/disease/renal.yaml +5 -0
  66. endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +6 -0
  67. endoreg_db/data/disease_classification/coronary_vessel_disease.yaml +6 -0
  68. endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +41 -0
  69. endoreg_db/data/disease_classification_choice/coronary_vessel_disease.yaml +20 -0
  70. endoreg_db/data/distribution/date/patient.yaml +7 -0
  71. endoreg_db/data/distribution/multiple_categorical/.init +0 -0
  72. endoreg_db/data/distribution/numeric/data.yaml +14 -0
  73. endoreg_db/data/distribution/single_categorical/patient.yaml +7 -0
  74. endoreg_db/data/emission_factor/green_endoscopy_dashboard_EmissionFactor.yaml +132 -0
  75. endoreg_db/data/endoscope/data.yaml +93 -0
  76. endoreg_db/data/endoscope_type/data.yaml +11 -0
  77. endoreg_db/data/endoscopy_processor/data.yaml +50 -0
  78. endoreg_db/data/event/cardiology.yaml +15 -0
  79. endoreg_db/data/event/neurology.yaml +14 -0
  80. endoreg_db/data/event/surgery.yaml +13 -0
  81. endoreg_db/data/event/thrombembolism.yaml +20 -0
  82. endoreg_db/data/event_classification/data.yaml +4 -0
  83. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  84. endoreg_db/data/examination/examinations/data.yaml +172 -0
  85. endoreg_db/data/examination/time/data.yaml +48 -0
  86. endoreg_db/data/examination/time-type/data.yaml +5 -0
  87. endoreg_db/data/examination/type/data.yaml +17 -0
  88. endoreg_db/data/examination_indication/endoscopy.yaml +359 -0
  89. endoreg_db/data/examination_indication_classification/endoscopy.yaml +90 -0
  90. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +97 -0
  91. endoreg_db/data/examination_requirement_set/colonoscopy.yaml +15 -0
  92. endoreg_db/data/finding/00_generic.yaml +35 -0
  93. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  94. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  95. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  96. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  97. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  98. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  99. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  100. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  101. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  102. endoreg_db/data/finding_classification/02_colonoscopy_baseline.yaml +83 -0
  103. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  104. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  105. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  106. endoreg_db/data/finding_classification_choice/00_generic.yaml +15 -0
  107. endoreg_db/data/finding_classification_choice/00_generic_baseline.yaml +23 -0
  108. endoreg_db/data/finding_classification_choice/00_generic_complication.yaml +15 -0
  109. endoreg_db/data/finding_classification_choice/00_generic_histology.yaml +21 -0
  110. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  111. endoreg_db/data/finding_classification_choice/02_colonoscopy_bowel_preparation.yaml +49 -0
  112. endoreg_db/data/finding_classification_choice/02_colonoscopy_generic.yaml +19 -0
  113. endoreg_db/data/finding_classification_choice/02_colonoscopy_histology.yaml +20 -0
  114. endoreg_db/data/finding_classification_choice/02_colonoscopy_location.yaml +248 -0
  115. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  116. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  117. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_morphology.yaml +75 -0
  118. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  119. endoreg_db/data/finding_classification_type/00_generic.yaml +53 -0
  120. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  121. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  122. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  123. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  124. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  125. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  126. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  127. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  128. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  129. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  130. endoreg_db/data/finding_intervention_type/endoscopy.yaml +15 -0
  131. endoreg_db/data/finding_type/data.yaml +39 -0
  132. endoreg_db/data/gender/data.yaml +42 -0
  133. endoreg_db/data/information_source/annotation.yaml +6 -0
  134. endoreg_db/data/information_source/data.yaml +30 -0
  135. endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
  136. endoreg_db/data/information_source/medication.yaml +6 -0
  137. endoreg_db/data/information_source/prediction.yaml +7 -0
  138. endoreg_db/data/information_source_type/data.yaml +8 -0
  139. endoreg_db/data/lab_value/cardiac_enzymes.yaml +37 -0
  140. endoreg_db/data/lab_value/coagulation.yaml +54 -0
  141. endoreg_db/data/lab_value/electrolytes.yaml +228 -0
  142. endoreg_db/data/lab_value/gastrointestinal_function.yaml +133 -0
  143. endoreg_db/data/lab_value/hematology.yaml +184 -0
  144. endoreg_db/data/lab_value/hormones.yaml +59 -0
  145. endoreg_db/data/lab_value/lipids.yaml +53 -0
  146. endoreg_db/data/lab_value/misc.yaml +76 -0
  147. endoreg_db/data/lab_value/renal_function.yaml +12 -0
  148. endoreg_db/data/log_type/data.yaml +57 -0
  149. endoreg_db/data/lx_client_tag/base.yaml +54 -0
  150. endoreg_db/data/lx_client_type/base.yaml +30 -0
  151. endoreg_db/data/lx_permission/base.yaml +24 -0
  152. endoreg_db/data/lx_permission/endoreg.yaml +52 -0
  153. endoreg_db/data/material/material.yaml +91 -0
  154. endoreg_db/data/medication/anticoagulation.yaml +65 -0
  155. endoreg_db/data/medication/tah.yaml +70 -0
  156. endoreg_db/data/medication_indication/anticoagulation.yaml +115 -0
  157. endoreg_db/data/medication_indication_type/data.yaml +11 -0
  158. endoreg_db/data/medication_indication_type/thrombembolism.yaml +41 -0
  159. endoreg_db/data/medication_intake_time/base.yaml +31 -0
  160. endoreg_db/data/medication_schedule/apixaban.yaml +95 -0
  161. endoreg_db/data/medication_schedule/ass.yaml +12 -0
  162. endoreg_db/data/medication_schedule/enoxaparin.yaml +26 -0
  163. endoreg_db/data/names_first/first_names.yaml +54 -0
  164. endoreg_db/data/names_last/last_names.yaml +51 -0
  165. endoreg_db/data/network_device/data.yaml +59 -0
  166. endoreg_db/data/network_device_type/data.yaml +12 -0
  167. endoreg_db/data/organ/data.yaml +29 -0
  168. endoreg_db/data/patient_lab_sample_type/generic.yaml +6 -0
  169. endoreg_db/data/pdf_type/data.yaml +46 -0
  170. endoreg_db/data/product/green_endoscopy_dashboard_Product.yaml +66 -0
  171. endoreg_db/data/product_group/green_endoscopy_dashboard_ProductGroup.yaml +33 -0
  172. endoreg_db/data/product_material/green_endoscopy_dashboard_ProductMaterial.yaml +308 -0
  173. endoreg_db/data/product_weight/green_endoscopy_dashboard_ProductWeight.yaml +88 -0
  174. endoreg_db/data/profession/data.yaml +70 -0
  175. endoreg_db/data/qualification/endoscopy.yaml +36 -0
  176. endoreg_db/data/qualification/m2.yaml +39 -0
  177. endoreg_db/data/qualification/outpatient_clinic.yaml +35 -0
  178. endoreg_db/data/qualification/sonography.yaml +36 -0
  179. endoreg_db/data/qualification_type/base.yaml +29 -0
  180. endoreg_db/data/reference_product/green_endoscopy_dashboard_ReferenceProduct.yaml +55 -0
  181. endoreg_db/data/report_reader_flag/rkh-histology-generic.yaml +10 -0
  182. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +30 -0
  183. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +24 -0
  184. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  185. endoreg_db/data/requirement/old/colon_polyp_intervention.yaml +49 -0
  186. endoreg_db/data/requirement/old/colonoscopy_baseline_austria.yaml +45 -0
  187. endoreg_db/data/requirement/old/coloreg_colon_polyp.yaml +49 -0
  188. endoreg_db/data/requirement/old/disease_cardiovascular.yaml +79 -0
  189. endoreg_db/data/requirement/old/disease_classification_choice_cardiovascular.yaml +41 -0
  190. endoreg_db/data/requirement/old/disease_hepatology.yaml +12 -0
  191. endoreg_db/data/requirement/old/disease_misc.yaml +12 -0
  192. endoreg_db/data/requirement/old/disease_renal.yaml +96 -0
  193. endoreg_db/data/requirement/old/endoscopy_bleeding_risk.yaml +59 -0
  194. endoreg_db/data/requirement/old/event_cardiology.yaml +251 -0
  195. endoreg_db/data/requirement/old/event_requirements.yaml +145 -0
  196. endoreg_db/data/requirement/old/finding_colon_polyp.yaml +50 -0
  197. endoreg_db/data/requirement/old/gender.yaml +0 -0
  198. endoreg_db/data/requirement/old/lab_value.yaml +441 -0
  199. endoreg_db/data/requirement/old/medication.yaml +93 -0
  200. endoreg_db/data/requirement_operator/_old/age.yaml +13 -0
  201. endoreg_db/data/requirement_operator/_old/lab_operators.yaml +129 -0
  202. endoreg_db/data/requirement_operator/_old/model_operators.yaml +96 -0
  203. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  204. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +65 -0
  205. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  206. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  207. endoreg_db/data/requirement_set/90_coloreg.yaml +190 -0
  208. endoreg_db/data/requirement_set/_old_ +109 -0
  209. endoreg_db/data/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  210. endoreg_db/data/requirement_set_type/data.yaml +41 -0
  211. endoreg_db/data/requirement_type/requirement_types.yaml +165 -0
  212. endoreg_db/data/resource/green_endoscopy_dashboard_Resource.yaml +15 -0
  213. endoreg_db/data/risk/bleeding.yaml +26 -0
  214. endoreg_db/data/risk/thrombosis.yaml +37 -0
  215. endoreg_db/data/risk_type/data.yaml +27 -0
  216. endoreg_db/data/setup_config.yaml +38 -0
  217. endoreg_db/data/shift/endoscopy.yaml +21 -0
  218. endoreg_db/data/shift/m2.yaml +0 -0
  219. endoreg_db/data/shift_type/base.yaml +35 -0
  220. endoreg_db/data/tag/requirement_set_tags.yaml +32 -0
  221. endoreg_db/data/tmp/chronic_kidney_disease.yaml +0 -0
  222. endoreg_db/data/tmp/congestive_heart_failure.yaml +0 -0
  223. endoreg_db/data/transport_route/green_endoscopy_dashboard_TransportRoute.yaml +12 -0
  224. endoreg_db/data/unit/concentration.yaml +115 -0
  225. endoreg_db/data/unit/data.yaml +17 -0
  226. endoreg_db/data/unit/length.yaml +31 -0
  227. endoreg_db/data/unit/misc.yaml +20 -0
  228. endoreg_db/data/unit/rate.yaml +6 -0
  229. endoreg_db/data/unit/time.yaml +48 -0
  230. endoreg_db/data/unit/volume.yaml +35 -0
  231. endoreg_db/data/unit/weight.yaml +38 -0
  232. endoreg_db/data/waste/data.yaml +12 -0
  233. endoreg_db/exceptions.py +24 -0
  234. endoreg_db/export/frames/export.py +6 -0
  235. endoreg_db/export/frames/export_frames_with_labels.py +616 -0
  236. endoreg_db/factories/__init__.py +0 -0
  237. endoreg_db/forms/__init__.py +4 -0
  238. endoreg_db/forms/examination_form.py +12 -0
  239. endoreg_db/forms/patient_finding_intervention_form.py +40 -0
  240. endoreg_db/forms/patient_form.py +23 -0
  241. endoreg_db/forms/questionnaires/__init__.py +1 -0
  242. endoreg_db/forms/questionnaires/tto_questionnaire.py +23 -0
  243. endoreg_db/forms/settings/__init__.py +11 -0
  244. endoreg_db/forms/unit.py +7 -0
  245. endoreg_db/helpers/__init__.py +0 -0
  246. endoreg_db/helpers/count_db.py +48 -0
  247. endoreg_db/helpers/data_loader.py +280 -0
  248. endoreg_db/helpers/default_objects.py +414 -0
  249. endoreg_db/helpers/download_segmentation_model.py +32 -0
  250. endoreg_db/helpers/interact.py +1 -0
  251. endoreg_db/helpers/test_video_helper.py +127 -0
  252. endoreg_db/import_files/__init__.py +27 -0
  253. endoreg_db/import_files/context/__init__.py +7 -0
  254. endoreg_db/import_files/context/default_sensitive_meta.py +83 -0
  255. endoreg_db/import_files/context/ensure_center.py +17 -0
  256. endoreg_db/import_files/context/file_lock.py +66 -0
  257. endoreg_db/import_files/context/import_context.py +42 -0
  258. endoreg_db/import_files/context/validate_directories.py +57 -0
  259. endoreg_db/import_files/file_storage/__init__.py +15 -0
  260. endoreg_db/import_files/file_storage/create_report_file.py +99 -0
  261. endoreg_db/import_files/file_storage/create_video_file.py +104 -0
  262. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +42 -0
  263. endoreg_db/import_files/file_storage/state_management.py +463 -0
  264. endoreg_db/import_files/file_storage/storage.py +42 -0
  265. endoreg_db/import_files/import_service.md +26 -0
  266. endoreg_db/import_files/processing/__init__.py +11 -0
  267. endoreg_db/import_files/processing/report_processing/report_anonymization.py +99 -0
  268. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  269. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  270. endoreg_db/import_files/pseudonymization/__init__.py +0 -0
  271. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  272. endoreg_db/import_files/pseudonymization/k_anonymity.py +181 -0
  273. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +139 -0
  274. endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
  275. endoreg_db/import_files/report_import_service.py +147 -0
  276. endoreg_db/import_files/video_import_service.py +154 -0
  277. endoreg_db/logger_conf.py +156 -0
  278. endoreg_db/management/__init__.py +1 -0
  279. endoreg_db/management/commands/__init__.py +1 -0
  280. endoreg_db/management/commands/anonymize_video.py +0 -0
  281. endoreg_db/management/commands/check_auth.py +132 -0
  282. endoreg_db/management/commands/create_model_meta_from_huggingface.py +177 -0
  283. endoreg_db/management/commands/create_multilabel_model_meta.py +419 -0
  284. endoreg_db/management/commands/export_frame_annot.py +196 -0
  285. endoreg_db/management/commands/fix_missing_patient_data.py +206 -0
  286. endoreg_db/management/commands/fix_video_paths.py +186 -0
  287. endoreg_db/management/commands/import_report.py +361 -0
  288. endoreg_db/management/commands/list_routes.py +20 -0
  289. endoreg_db/management/commands/load_ai_model_data.py +83 -0
  290. endoreg_db/management/commands/load_ai_model_label_data.py +60 -0
  291. endoreg_db/management/commands/load_base_db_data.py +63 -0
  292. endoreg_db/management/commands/load_center_data.py +68 -0
  293. endoreg_db/management/commands/load_contraindication_data.py +39 -0
  294. endoreg_db/management/commands/load_disease_classification_choices_data.py +38 -0
  295. endoreg_db/management/commands/load_disease_classification_data.py +38 -0
  296. endoreg_db/management/commands/load_disease_data.py +59 -0
  297. endoreg_db/management/commands/load_distribution_data.py +63 -0
  298. endoreg_db/management/commands/load_endoscope_data.py +58 -0
  299. endoreg_db/management/commands/load_event_data.py +39 -0
  300. endoreg_db/management/commands/load_examination_data.py +78 -0
  301. endoreg_db/management/commands/load_examination_indication_data.py +85 -0
  302. endoreg_db/management/commands/load_finding_data.py +115 -0
  303. endoreg_db/management/commands/load_gender_data.py +37 -0
  304. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +142 -0
  305. endoreg_db/management/commands/load_information_source.py +46 -0
  306. endoreg_db/management/commands/load_lab_value_data.py +52 -0
  307. endoreg_db/management/commands/load_legacy_data.py +303 -0
  308. endoreg_db/management/commands/load_medication_data.py +104 -0
  309. endoreg_db/management/commands/load_name_data.py +36 -0
  310. endoreg_db/management/commands/load_organ_data.py +39 -0
  311. endoreg_db/management/commands/load_pdf_type_data.py +58 -0
  312. endoreg_db/management/commands/load_profession_data.py +40 -0
  313. endoreg_db/management/commands/load_qualification_data.py +56 -0
  314. endoreg_db/management/commands/load_report_reader_flag_data.py +40 -0
  315. endoreg_db/management/commands/load_requirement_data.py +207 -0
  316. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  317. endoreg_db/management/commands/load_risk_data.py +57 -0
  318. endoreg_db/management/commands/load_shift_data.py +57 -0
  319. endoreg_db/management/commands/load_tag_data.py +54 -0
  320. endoreg_db/management/commands/load_unit_data.py +40 -0
  321. endoreg_db/management/commands/load_user_groups.py +26 -0
  322. endoreg_db/management/commands/model_input.py +169 -0
  323. endoreg_db/management/commands/register_ai_model.py +70 -0
  324. endoreg_db/management/commands/setup_endoreg_db.py +459 -0
  325. endoreg_db/management/commands/start_filewatcher.py +115 -0
  326. endoreg_db/management/commands/storage_management.py +622 -0
  327. endoreg_db/management/commands/summarize_db_content.py +280 -0
  328. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  329. endoreg_db/management/commands/validate_video_files.py +189 -0
  330. endoreg_db/management/commands/video_validation.py +20 -0
  331. endoreg_db/mermaid/Overall_flow_patient_finding_intervention.md +10 -0
  332. endoreg_db/mermaid/anonymized_image_annotation.md +20 -0
  333. endoreg_db/mermaid/binary_classification_annotation.md +50 -0
  334. endoreg_db/mermaid/classification.md +8 -0
  335. endoreg_db/mermaid/examination.md +8 -0
  336. endoreg_db/mermaid/findings.md +7 -0
  337. endoreg_db/mermaid/image_classification.md +28 -0
  338. endoreg_db/mermaid/interventions.md +8 -0
  339. endoreg_db/mermaid/morphology.md +8 -0
  340. endoreg_db/mermaid/patient_creation.md +14 -0
  341. endoreg_db/mermaid/video_segmentation_annotation.md +17 -0
  342. endoreg_db/migrations/0001_initial.py +1953 -0
  343. endoreg_db/migrations/__init__.py +0 -0
  344. endoreg_db/models/__init__.py +322 -0
  345. endoreg_db/models/administration/__init__.py +95 -0
  346. endoreg_db/models/administration/ai/__init__.py +9 -0
  347. endoreg_db/models/administration/ai/active_model.py +35 -0
  348. endoreg_db/models/administration/ai/ai_model.py +180 -0
  349. endoreg_db/models/administration/ai/model_type.py +42 -0
  350. endoreg_db/models/administration/case/__init__.py +5 -0
  351. endoreg_db/models/administration/case/case.py +114 -0
  352. endoreg_db/models/administration/case/case_template/__init__.py +3 -0
  353. endoreg_db/models/administration/case/case_template/case_template.py +3 -0
  354. endoreg_db/models/administration/case/case_template/case_template_rule.py +3 -0
  355. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +3 -0
  356. endoreg_db/models/administration/case/case_template/case_template_type.py +3 -0
  357. endoreg_db/models/administration/center/__init__.py +13 -0
  358. endoreg_db/models/administration/center/center.py +85 -0
  359. endoreg_db/models/administration/center/center_product.py +67 -0
  360. endoreg_db/models/administration/center/center_resource.py +69 -0
  361. endoreg_db/models/administration/center/center_shift.py +94 -0
  362. endoreg_db/models/administration/center/center_waste.py +42 -0
  363. endoreg_db/models/administration/person/__init__.py +26 -0
  364. endoreg_db/models/administration/person/employee/__init__.py +3 -0
  365. endoreg_db/models/administration/person/employee/employee.py +40 -0
  366. endoreg_db/models/administration/person/employee/employee_qualification.py +44 -0
  367. endoreg_db/models/administration/person/employee/employee_type.py +50 -0
  368. endoreg_db/models/administration/person/examiner/__init__.py +4 -0
  369. endoreg_db/models/administration/person/examiner/examiner.py +64 -0
  370. endoreg_db/models/administration/person/names/__init__.py +0 -0
  371. endoreg_db/models/administration/person/names/first_name.py +20 -0
  372. endoreg_db/models/administration/person/names/last_name.py +20 -0
  373. endoreg_db/models/administration/person/patient/__init__.py +7 -0
  374. endoreg_db/models/administration/person/patient/patient.py +488 -0
  375. endoreg_db/models/administration/person/patient/patient_external_id.py +36 -0
  376. endoreg_db/models/administration/person/person.py +35 -0
  377. endoreg_db/models/administration/person/profession/__init__.py +28 -0
  378. endoreg_db/models/administration/person/user/__init__.py +5 -0
  379. endoreg_db/models/administration/person/user/portal_user_information.py +41 -0
  380. endoreg_db/models/administration/product/__init__.py +15 -0
  381. endoreg_db/models/administration/product/product.py +106 -0
  382. endoreg_db/models/administration/product/product_group.py +41 -0
  383. endoreg_db/models/administration/product/product_material.py +60 -0
  384. endoreg_db/models/administration/product/product_weight.py +51 -0
  385. endoreg_db/models/administration/product/reference_product.py +147 -0
  386. endoreg_db/models/administration/qualification/__init__.py +7 -0
  387. endoreg_db/models/administration/qualification/qualification.py +43 -0
  388. endoreg_db/models/administration/qualification/qualification_type.py +39 -0
  389. endoreg_db/models/administration/shift/__init__.py +9 -0
  390. endoreg_db/models/administration/shift/scheduled_days.py +72 -0
  391. endoreg_db/models/administration/shift/shift.py +57 -0
  392. endoreg_db/models/administration/shift/shift_type.py +108 -0
  393. endoreg_db/models/aidataset/__init__.py +5 -0
  394. endoreg_db/models/aidataset/aidataset.py +193 -0
  395. endoreg_db/models/label/__init__.py +23 -0
  396. endoreg_db/models/label/annotation/__init__.py +12 -0
  397. endoreg_db/models/label/annotation/image_classification.py +85 -0
  398. endoreg_db/models/label/annotation/video_segmentation_annotation.py +61 -0
  399. endoreg_db/models/label/label.py +91 -0
  400. endoreg_db/models/label/label_set.py +68 -0
  401. endoreg_db/models/label/label_type.py +29 -0
  402. endoreg_db/models/label/label_video_segment/__init__.py +3 -0
  403. endoreg_db/models/label/label_video_segment/_create_from_video.py +42 -0
  404. endoreg_db/models/label/label_video_segment/label_video_segment.py +611 -0
  405. endoreg_db/models/label/video_segmentation_label.py +35 -0
  406. endoreg_db/models/label/video_segmentation_labelset.py +28 -0
  407. endoreg_db/models/media/__init__.py +23 -0
  408. endoreg_db/models/media/frame/__init__.py +3 -0
  409. endoreg_db/models/media/frame/frame.py +137 -0
  410. endoreg_db/models/media/pdf/__init__.py +12 -0
  411. endoreg_db/models/media/pdf/raw_pdf.py +764 -0
  412. endoreg_db/models/media/pdf/report_file.py +162 -0
  413. endoreg_db/models/media/pdf/report_reader/__init__.py +7 -0
  414. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +85 -0
  415. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +46 -0
  416. endoreg_db/models/media/video/__init__.py +9 -0
  417. endoreg_db/models/media/video/create_from_file.py +402 -0
  418. endoreg_db/models/media/video/pipe_1.py +258 -0
  419. endoreg_db/models/media/video/pipe_2.py +129 -0
  420. endoreg_db/models/media/video/video_file.py +907 -0
  421. endoreg_db/models/media/video/video_file_ai.py +828 -0
  422. endoreg_db/models/media/video/video_file_anonymize.py +524 -0
  423. endoreg_db/models/media/video/video_file_frames/__init__.py +49 -0
  424. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +25 -0
  425. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +23 -0
  426. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +126 -0
  427. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +233 -0
  428. endoreg_db/models/media/video/video_file_frames/_get_frame.py +36 -0
  429. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +13 -0
  430. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +24 -0
  431. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +40 -0
  432. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +44 -0
  433. endoreg_db/models/media/video/video_file_frames/_get_frames.py +30 -0
  434. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +205 -0
  435. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +228 -0
  436. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +107 -0
  437. endoreg_db/models/media/video/video_file_io.py +272 -0
  438. endoreg_db/models/media/video/video_file_meta/__init__.py +22 -0
  439. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +58 -0
  440. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +62 -0
  441. endoreg_db/models/media/video/video_file_meta/get_fps.py +183 -0
  442. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +198 -0
  443. endoreg_db/models/media/video/video_file_meta/text_meta.py +178 -0
  444. endoreg_db/models/media/video/video_file_meta/video_meta.py +105 -0
  445. endoreg_db/models/media/video/video_file_segments.py +317 -0
  446. endoreg_db/models/media/video/video_metadata.py +67 -0
  447. endoreg_db/models/media/video/video_processing.py +192 -0
  448. endoreg_db/models/medical/__init__.py +136 -0
  449. endoreg_db/models/medical/contraindication/README.md +1 -0
  450. endoreg_db/models/medical/contraindication/__init__.py +29 -0
  451. endoreg_db/models/medical/disease.py +174 -0
  452. endoreg_db/models/medical/event.py +154 -0
  453. endoreg_db/models/medical/examination/__init__.py +20 -0
  454. endoreg_db/models/medical/examination/examination.py +183 -0
  455. endoreg_db/models/medical/examination/examination_indication.py +229 -0
  456. endoreg_db/models/medical/examination/examination_time.py +68 -0
  457. endoreg_db/models/medical/examination/examination_time_type.py +44 -0
  458. endoreg_db/models/medical/examination/examination_type.py +47 -0
  459. endoreg_db/models/medical/finding/__init__.py +20 -0
  460. endoreg_db/models/medical/finding/finding.py +113 -0
  461. endoreg_db/models/medical/finding/finding_classification.py +131 -0
  462. endoreg_db/models/medical/finding/finding_intervention.py +68 -0
  463. endoreg_db/models/medical/finding/finding_type.py +38 -0
  464. endoreg_db/models/medical/hardware/__init__.py +8 -0
  465. endoreg_db/models/medical/hardware/endoscope.py +77 -0
  466. endoreg_db/models/medical/hardware/endoscopy_processor.py +182 -0
  467. endoreg_db/models/medical/laboratory/__init__.py +5 -0
  468. endoreg_db/models/medical/laboratory/lab_value.py +490 -0
  469. endoreg_db/models/medical/medication/__init__.py +23 -0
  470. endoreg_db/models/medical/medication/medication.py +45 -0
  471. endoreg_db/models/medical/medication/medication_indication.py +78 -0
  472. endoreg_db/models/medical/medication/medication_indication_type.py +58 -0
  473. endoreg_db/models/medical/medication/medication_intake_time.py +58 -0
  474. endoreg_db/models/medical/medication/medication_schedule.py +58 -0
  475. endoreg_db/models/medical/organ/__init__.py +38 -0
  476. endoreg_db/models/medical/patient/__init__.py +48 -0
  477. endoreg_db/models/medical/patient/medication_examples.py +56 -0
  478. endoreg_db/models/medical/patient/patient_disease.py +72 -0
  479. endoreg_db/models/medical/patient/patient_event.py +80 -0
  480. endoreg_db/models/medical/patient/patient_examination.py +280 -0
  481. endoreg_db/models/medical/patient/patient_examination_indication.py +57 -0
  482. endoreg_db/models/medical/patient/patient_finding.py +416 -0
  483. endoreg_db/models/medical/patient/patient_finding_classification.py +231 -0
  484. endoreg_db/models/medical/patient/patient_finding_intervention.py +37 -0
  485. endoreg_db/models/medical/patient/patient_lab_sample.py +157 -0
  486. endoreg_db/models/medical/patient/patient_lab_value.py +247 -0
  487. endoreg_db/models/medical/patient/patient_medication.py +111 -0
  488. endoreg_db/models/medical/patient/patient_medication_schedule.py +152 -0
  489. endoreg_db/models/medical/risk/__init__.py +7 -0
  490. endoreg_db/models/medical/risk/risk.py +73 -0
  491. endoreg_db/models/medical/risk/risk_type.py +54 -0
  492. endoreg_db/models/metadata/__init__.py +19 -0
  493. endoreg_db/models/metadata/model_meta.py +266 -0
  494. endoreg_db/models/metadata/model_meta_logic.py +485 -0
  495. endoreg_db/models/metadata/pdf_meta.py +96 -0
  496. endoreg_db/models/metadata/sensitive_meta.py +345 -0
  497. endoreg_db/models/metadata/sensitive_meta_logic.py +1161 -0
  498. endoreg_db/models/metadata/video_meta.py +459 -0
  499. endoreg_db/models/metadata/video_prediction_logic.py +232 -0
  500. endoreg_db/models/metadata/video_prediction_meta.py +319 -0
  501. endoreg_db/models/operation_log.py +63 -0
  502. endoreg_db/models/other/__init__.py +40 -0
  503. endoreg_db/models/other/distribution/__init__.py +46 -0
  504. endoreg_db/models/other/distribution/base_value_distribution.py +22 -0
  505. endoreg_db/models/other/distribution/date_value_distribution.py +163 -0
  506. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +50 -0
  507. endoreg_db/models/other/distribution/numeric_value_distribution.py +211 -0
  508. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +23 -0
  509. endoreg_db/models/other/emission/__init__.py +5 -0
  510. endoreg_db/models/other/emission/emission_factor.py +110 -0
  511. endoreg_db/models/other/gender.py +32 -0
  512. endoreg_db/models/other/information_source.py +190 -0
  513. endoreg_db/models/other/material.py +34 -0
  514. endoreg_db/models/other/resource.py +24 -0
  515. endoreg_db/models/other/tag.py +32 -0
  516. endoreg_db/models/other/transport_route.py +40 -0
  517. endoreg_db/models/other/unit.py +40 -0
  518. endoreg_db/models/other/waste.py +28 -0
  519. endoreg_db/models/report/__init__.py +0 -0
  520. endoreg_db/models/report/images.py +0 -0
  521. endoreg_db/models/report/report.py +5 -0
  522. endoreg_db/models/requirement/__init__.py +11 -0
  523. endoreg_db/models/requirement/requirement.py +792 -0
  524. endoreg_db/models/requirement/requirement_error.py +84 -0
  525. endoreg_db/models/requirement/requirement_evaluation/__init__.py +6 -0
  526. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  527. endoreg_db/models/requirement/requirement_evaluation/get_values.py +40 -0
  528. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +6 -0
  529. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +137 -0
  530. endoreg_db/models/requirement/requirement_operator.py +187 -0
  531. endoreg_db/models/requirement/requirement_set.py +327 -0
  532. endoreg_db/models/state/__init__.py +13 -0
  533. endoreg_db/models/state/abstract.py +11 -0
  534. endoreg_db/models/state/anonymization.py +30 -0
  535. endoreg_db/models/state/audit_ledger.py +155 -0
  536. endoreg_db/models/state/label_video_segment.py +31 -0
  537. endoreg_db/models/state/processing_history/__init__.py +3 -0
  538. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  539. endoreg_db/models/state/raw_pdf.py +219 -0
  540. endoreg_db/models/state/sensitive_meta.py +50 -0
  541. endoreg_db/models/state/video.py +251 -0
  542. endoreg_db/models/upload_job.py +100 -0
  543. endoreg_db/models/utils.py +138 -0
  544. endoreg_db/queries/__init__.py +3 -0
  545. endoreg_db/queries/annotations/__init__.py +1 -0
  546. endoreg_db/queries/annotations/legacy.py +169 -0
  547. endoreg_db/queries/sanity/__init_.py +0 -0
  548. endoreg_db/root_urls.py +27 -0
  549. endoreg_db/schemas/__init__.py +0 -0
  550. endoreg_db/schemas/examination_evaluation.py +30 -0
  551. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +861 -0
  552. endoreg_db/serializers/__init__.py +104 -0
  553. endoreg_db/serializers/administration/__init__.py +13 -0
  554. endoreg_db/serializers/administration/ai/__init__.py +9 -0
  555. endoreg_db/serializers/administration/ai/active_model.py +12 -0
  556. endoreg_db/serializers/administration/ai/ai_model.py +20 -0
  557. endoreg_db/serializers/administration/ai/model_type.py +12 -0
  558. endoreg_db/serializers/administration/center.py +14 -0
  559. endoreg_db/serializers/administration/gender.py +11 -0
  560. endoreg_db/serializers/anonymization.py +77 -0
  561. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -0
  562. endoreg_db/serializers/examination/__init__.py +10 -0
  563. endoreg_db/serializers/examination/base.py +45 -0
  564. endoreg_db/serializers/examination/dropdown.py +20 -0
  565. endoreg_db/serializers/examination_serializer.py +9 -0
  566. endoreg_db/serializers/finding/__init__.py +5 -0
  567. endoreg_db/serializers/finding/finding.py +61 -0
  568. endoreg_db/serializers/finding_classification/__init__.py +7 -0
  569. endoreg_db/serializers/finding_classification/choice.py +19 -0
  570. endoreg_db/serializers/finding_classification/classification.py +11 -0
  571. endoreg_db/serializers/label_video_segment/__init__.py +9 -0
  572. endoreg_db/serializers/label_video_segment/image_classification_annotation.py +62 -0
  573. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  574. endoreg_db/serializers/label_video_segment/label/label.py +15 -0
  575. endoreg_db/serializers/label_video_segment/label_video_segment.py +427 -0
  576. endoreg_db/serializers/meta/__init__.py +13 -0
  577. endoreg_db/serializers/meta/sensitive_meta_detail.py +122 -0
  578. endoreg_db/serializers/meta/sensitive_meta_update.py +153 -0
  579. endoreg_db/serializers/meta/sensitive_meta_verification.py +62 -0
  580. endoreg_db/serializers/meta/video_meta.py +39 -0
  581. endoreg_db/serializers/misc/__init__.py +14 -0
  582. endoreg_db/serializers/misc/file_overview.py +72 -0
  583. endoreg_db/serializers/misc/sensitive_patient_data.py +144 -0
  584. endoreg_db/serializers/misc/stats.py +35 -0
  585. endoreg_db/serializers/misc/translatable_field_mix_in.py +44 -0
  586. endoreg_db/serializers/misc/upload_job.py +74 -0
  587. endoreg_db/serializers/patient/__init__.py +12 -0
  588. endoreg_db/serializers/patient/patient.py +103 -0
  589. endoreg_db/serializers/patient/patient_dropdown.py +35 -0
  590. endoreg_db/serializers/patient_examination/__init__.py +7 -0
  591. endoreg_db/serializers/patient_examination/patient_examination.py +168 -0
  592. endoreg_db/serializers/patient_finding/__init__.py +15 -0
  593. endoreg_db/serializers/patient_finding/patient_finding.py +32 -0
  594. endoreg_db/serializers/patient_finding/patient_finding_classification.py +47 -0
  595. endoreg_db/serializers/patient_finding/patient_finding_detail.py +62 -0
  596. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +28 -0
  597. endoreg_db/serializers/patient_finding/patient_finding_list.py +40 -0
  598. endoreg_db/serializers/patient_finding/patient_finding_write.py +135 -0
  599. endoreg_db/serializers/pdf/__init__.py +3 -0
  600. endoreg_db/serializers/pdf/anony_text_validation.py +101 -0
  601. endoreg_db/serializers/requirements/requirement_schema.py +20 -0
  602. endoreg_db/serializers/requirements/requirement_sets.py +99 -0
  603. endoreg_db/serializers/sensitive_meta_serializer.py +301 -0
  604. endoreg_db/serializers/video/__init__.py +7 -0
  605. endoreg_db/serializers/video/video_file.py +283 -0
  606. endoreg_db/serializers/video/video_file_brief.py +14 -0
  607. endoreg_db/serializers/video/video_file_detail.py +96 -0
  608. endoreg_db/serializers/video/video_file_list.py +100 -0
  609. endoreg_db/serializers/video/video_processing_history.py +172 -0
  610. endoreg_db/serializers/video_examination.py +198 -0
  611. endoreg_db/services/__init__.py +5 -0
  612. endoreg_db/services/anonymization.py +274 -0
  613. endoreg_db/services/examination_evaluation.py +172 -0
  614. endoreg_db/services/finding_description_service.py +0 -0
  615. endoreg_db/services/lookup_service.py +424 -0
  616. endoreg_db/services/lookup_store.py +266 -0
  617. endoreg_db/services/model_meta_from_hf.py +76 -0
  618. endoreg_db/services/pdf_import.py +0 -0
  619. endoreg_db/services/polling_coordinator.py +319 -0
  620. endoreg_db/services/pseudonym_service.py +94 -0
  621. endoreg_db/services/report_import.py +13 -0
  622. endoreg_db/services/segment_sync.py +171 -0
  623. endoreg_db/services/video_import.py +9 -0
  624. endoreg_db/templates/admin/patient_finding_intervention.html +253 -0
  625. endoreg_db/templates/admin/start_examination.html +12 -0
  626. endoreg_db/templates/timeline.html +176 -0
  627. endoreg_db/urls/__init__.py +56 -0
  628. endoreg_db/urls/ai.py +14 -0
  629. endoreg_db/urls/anonymization.py +78 -0
  630. endoreg_db/urls/auth.py +16 -0
  631. endoreg_db/urls/classification.py +34 -0
  632. endoreg_db/urls/examination.py +63 -0
  633. endoreg_db/urls/media.py +251 -0
  634. endoreg_db/urls/patient.py +23 -0
  635. endoreg_db/urls/requirements.py +15 -0
  636. endoreg_db/urls/root_urls.py +28 -0
  637. endoreg_db/urls/stats.py +54 -0
  638. endoreg_db/urls/upload.py +12 -0
  639. endoreg_db/urls.py +9 -0
  640. endoreg_db/utils/__init__.py +97 -0
  641. endoreg_db/utils/ai/__init__.py +9 -0
  642. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  643. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  644. endoreg_db/utils/ai/get.py +6 -0
  645. endoreg_db/utils/ai/inference_dataset.py +51 -0
  646. endoreg_db/utils/ai/model_training/config.py +117 -0
  647. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  648. endoreg_db/utils/ai/model_training/losses.py +68 -0
  649. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  650. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  651. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  652. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  653. endoreg_db/utils/ai/multilabel_classification_net.py +270 -0
  654. endoreg_db/utils/ai/postprocess.py +63 -0
  655. endoreg_db/utils/ai/predict.py +293 -0
  656. endoreg_db/utils/ai/preprocess.py +76 -0
  657. endoreg_db/utils/calc_duration_seconds.py +24 -0
  658. endoreg_db/utils/case_generator/__init__.py +3 -0
  659. endoreg_db/utils/case_generator/lab_sample_factory.py +32 -0
  660. endoreg_db/utils/check_video_files.py +175 -0
  661. endoreg_db/utils/cropping.py +30 -0
  662. endoreg_db/utils/dataloader.py +285 -0
  663. endoreg_db/utils/dates.py +59 -0
  664. endoreg_db/utils/defaults/set_default_center.py +33 -0
  665. endoreg_db/utils/env.py +37 -0
  666. endoreg_db/utils/extract_specific_frames.py +87 -0
  667. endoreg_db/utils/file_operations.py +70 -0
  668. endoreg_db/utils/fix_video_path_direct.py +157 -0
  669. endoreg_db/utils/frame_anonymization_utils.py +463 -0
  670. endoreg_db/utils/hashs.py +138 -0
  671. endoreg_db/utils/links/__init__.py +0 -0
  672. endoreg_db/utils/links/requirement_link.py +237 -0
  673. endoreg_db/utils/mime_types.py +0 -0
  674. endoreg_db/utils/names.py +82 -0
  675. endoreg_db/utils/ocr.py +195 -0
  676. endoreg_db/utils/operation_log.py +87 -0
  677. endoreg_db/utils/parse_and_generate_yaml.py +45 -0
  678. endoreg_db/utils/paths.py +159 -0
  679. endoreg_db/utils/permissions.py +160 -0
  680. endoreg_db/utils/pipelines/Readme.md +235 -0
  681. endoreg_db/utils/pipelines/__init__.py +0 -0
  682. endoreg_db/utils/pipelines/process_video_dir.py +144 -0
  683. endoreg_db/utils/product/__init__.py +0 -0
  684. endoreg_db/utils/product/sum_emissions.py +22 -0
  685. endoreg_db/utils/product/sum_weights.py +20 -0
  686. endoreg_db/utils/pydantic_models/__init__.py +5 -0
  687. endoreg_db/utils/pydantic_models/db_config.py +57 -0
  688. endoreg_db/utils/requirement_helpers.py +0 -0
  689. endoreg_db/utils/requirement_operator_logic/__init__.py +0 -0
  690. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +678 -0
  691. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +842 -0
  692. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +114 -0
  693. endoreg_db/utils/setup_config.py +196 -0
  694. endoreg_db/utils/storage.py +117 -0
  695. endoreg_db/utils/translation.py +31 -0
  696. endoreg_db/utils/uuid.py +5 -0
  697. endoreg_db/utils/validate_endo_roi.py +33 -0
  698. endoreg_db/utils/validate_subcategory_dict.py +93 -0
  699. endoreg_db/utils/validate_video_detailed.py +415 -0
  700. endoreg_db/utils/video/__init__.py +30 -0
  701. endoreg_db/utils/video/extract_frames.py +100 -0
  702. endoreg_db/utils/video/ffmpeg_wrapper.py +996 -0
  703. endoreg_db/utils/video/names.py +47 -0
  704. endoreg_db/utils/video/streaming_processor.py +386 -0
  705. endoreg_db/utils/video/video_splitter.py +105 -0
  706. endoreg_db/versioning.md +79 -0
  707. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +247 -0
  708. endoreg_db/views/__init__.py +157 -0
  709. endoreg_db/views/anonymization/__init__.py +31 -0
  710. endoreg_db/views/anonymization/media_management.py +486 -0
  711. endoreg_db/views/anonymization/overview.py +307 -0
  712. endoreg_db/views/anonymization/validate.py +310 -0
  713. endoreg_db/views/auth/__init__.py +13 -0
  714. endoreg_db/views/auth/keycloak.py +146 -0
  715. endoreg_db/views/examination/__init__.py +30 -0
  716. endoreg_db/views/examination/examination.py +37 -0
  717. endoreg_db/views/examination/examination_manifest_cache.py +26 -0
  718. endoreg_db/views/examination/get_finding_classification_choices.py +62 -0
  719. endoreg_db/views/examination/get_finding_classifications.py +38 -0
  720. endoreg_db/views/examination/get_findings.py +39 -0
  721. endoreg_db/views/examination/get_instruments.py +19 -0
  722. endoreg_db/views/examination/get_interventions.py +14 -0
  723. endoreg_db/views/finding/__init__.py +9 -0
  724. endoreg_db/views/finding/finding.py +116 -0
  725. endoreg_db/views/finding/get_classifications.py +14 -0
  726. endoreg_db/views/finding/get_interventions.py +17 -0
  727. endoreg_db/views/finding_classification/__init__.py +13 -0
  728. endoreg_db/views/finding_classification/base.py +0 -0
  729. endoreg_db/views/finding_classification/finding_classification.py +41 -0
  730. endoreg_db/views/finding_classification/get_classification_choices.py +54 -0
  731. endoreg_db/views/media/__init__.py +32 -0
  732. endoreg_db/views/media/pdf_media.py +411 -0
  733. endoreg_db/views/media/sensitive_metadata.py +372 -0
  734. endoreg_db/views/media/video_media.py +275 -0
  735. endoreg_db/views/meta/__init__.py +7 -0
  736. endoreg_db/views/meta/sensitive_meta_list.py +102 -0
  737. endoreg_db/views/meta/sensitive_meta_verification.py +74 -0
  738. endoreg_db/views/misc/__init__.py +29 -0
  739. endoreg_db/views/misc/center.py +14 -0
  740. endoreg_db/views/misc/csrf.py +8 -0
  741. endoreg_db/views/misc/gender.py +15 -0
  742. endoreg_db/views/misc/stats.py +255 -0
  743. endoreg_db/views/misc/upload_views.py +241 -0
  744. endoreg_db/views/patient/__init__.py +3 -0
  745. endoreg_db/views/patient/patient.py +253 -0
  746. endoreg_db/views/patient_examination/__init__.py +11 -0
  747. endoreg_db/views/patient_examination/patient_examination.py +141 -0
  748. endoreg_db/views/patient_examination/patient_examination_create.py +58 -0
  749. endoreg_db/views/patient_examination/patient_examination_detail.py +63 -0
  750. endoreg_db/views/patient_examination/patient_examination_list.py +72 -0
  751. endoreg_db/views/patient_examination/video.py +228 -0
  752. endoreg_db/views/patient_finding/__init__.py +7 -0
  753. endoreg_db/views/patient_finding/base.py +0 -0
  754. endoreg_db/views/patient_finding/patient_finding.py +71 -0
  755. endoreg_db/views/patient_finding/patient_finding_optimized.py +291 -0
  756. endoreg_db/views/patient_finding_classification/__init__.py +5 -0
  757. endoreg_db/views/patient_finding_classification/pfc_create.py +75 -0
  758. endoreg_db/views/report/__init__.py +7 -0
  759. endoreg_db/views/report/reimport.py +177 -0
  760. endoreg_db/views/report/report_stream.py +191 -0
  761. endoreg_db/views/requirement/__init__.py +11 -0
  762. endoreg_db/views/requirement/evaluate.py +278 -0
  763. endoreg_db/views/requirement/lookup.py +380 -0
  764. endoreg_db/views/requirement/lookup_store.py +183 -0
  765. endoreg_db/views/requirement/requirement_utils.py +87 -0
  766. endoreg_db/views/requirement_lookup/lookup.py +0 -0
  767. endoreg_db/views/requirement_lookup/lookup_store.py +0 -0
  768. endoreg_db/views/stats/__init__.py +13 -0
  769. endoreg_db/views/stats/stats_views.py +266 -0
  770. endoreg_db/views/video/__init__.py +49 -0
  771. endoreg_db/views/video/ai/__init__.py +8 -0
  772. endoreg_db/views/video/ai/label.py +159 -0
  773. endoreg_db/views/video/correction.py +529 -0
  774. endoreg_db/views/video/reimport.py +230 -0
  775. endoreg_db/views/video/segments_crud.py +709 -0
  776. endoreg_db/views/video/video_apply_mask.py +49 -0
  777. endoreg_db/views/video/video_correction.py +22 -0
  778. endoreg_db/views/video/video_download_processed.py +58 -0
  779. endoreg_db/views/video/video_examination_viewset.py +242 -0
  780. endoreg_db/views/video/video_metadata.py +101 -0
  781. endoreg_db/views/video/video_processing_history.py +25 -0
  782. endoreg_db/views/video/video_remove_frames.py +49 -0
  783. endoreg_db/views/video/video_stream.py +334 -0
  784. endoreg_db-0.8.9.32.dist-info/METADATA +404 -0
  785. endoreg_db-0.8.9.32.dist-info/RECORD +787 -0
  786. endoreg_db-0.8.9.32.dist-info/WHEEL +4 -0
  787. endoreg_db-0.8.9.32.dist-info/licenses/LICENSE +674 -0
@@ -0,0 +1,996 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ import shutil
5
+ import subprocess
6
+ from functools import lru_cache
7
+ from pathlib import Path
8
+ from typing import Dict, List, Optional, Tuple
9
+
10
+ import cv2
11
+ from tqdm import tqdm
12
+
13
+ logger = logging.getLogger("ffmpeg_wrapper")
14
+
15
+ # Global hardware acceleration cache
16
+ _nvenc_available = None
17
+ _preferred_encoder = None
18
+
19
+
20
+ @lru_cache(maxsize=1)
21
+ def _resolve_ffmpeg_executable() -> Optional[str]:
22
+ """Locate the ffmpeg executable using multiple discovery strategies."""
23
+ # 1) Explicit overrides via env vars
24
+ env_candidates = [
25
+ os.environ.get("FFMPEG_EXECUTABLE"),
26
+ os.environ.get("FFMPEG_BINARY"),
27
+ os.environ.get("FFMPEG_PATH"),
28
+ ]
29
+
30
+ # 2) Django settings overrides (if Django is configured)
31
+ try:
32
+ from django.conf import settings
33
+
34
+ env_candidates.extend(
35
+ getattr(settings, attr)
36
+ for attr in ("FFMPEG_EXECUTABLE", "FFMPEG_BINARY", "FFMPEG_PATH")
37
+ if hasattr(settings, attr)
38
+ )
39
+ except Exception:
40
+ # Django might not be configured for every consumer
41
+ pass
42
+
43
+ # Normalize and verify explicit candidates
44
+ for candidate in env_candidates:
45
+ if not candidate:
46
+ continue
47
+ candidate_path = Path(candidate)
48
+ if candidate_path.is_dir():
49
+ candidate_path = candidate_path / "ffmpeg"
50
+ if candidate_path.exists() and os.access(candidate_path, os.X_OK):
51
+ logger.debug("Using ffmpeg executable override at %s", candidate_path)
52
+ return str(candidate_path)
53
+
54
+ # 3) PATH lookup (shutil.which)
55
+ via_path = shutil.which("ffmpeg")
56
+ if via_path:
57
+ return via_path
58
+
59
+ # 4) Common fallback locations (useful for Nix-based environments)
60
+ nix_store = Path("/nix/store")
61
+ if nix_store.exists():
62
+ patterns = (
63
+ "*-ffmpeg-*/bin/ffmpeg",
64
+ "*-ffmpeg-headless-*/bin/ffmpeg",
65
+ "*-ffmpeg-headless*/bin/ffmpeg",
66
+ )
67
+ for pattern in patterns:
68
+ matches = sorted(nix_store.glob(pattern))
69
+ if matches:
70
+ logger.debug("Discovered ffmpeg in nix store at %s", matches[-1])
71
+ return str(matches[-1])
72
+
73
+ # 5) Final fallback to standard Unix locations
74
+ for fallback in (Path("/usr/bin/ffmpeg"), Path("/usr/local/bin/ffmpeg")):
75
+ if fallback.exists() and os.access(fallback, os.X_OK):
76
+ return str(fallback)
77
+
78
+ return None
79
+
80
+
81
+ def _detect_nvenc_support() -> bool:
82
+ """
83
+ Detect if NVIDIA NVENC hardware acceleration is available.
84
+
85
+ Returns:
86
+ True if NVENC is available, False otherwise
87
+ """
88
+ try:
89
+ # Test NVENC availability with a minimal command (minimum size for NVENC)
90
+ cmd = [
91
+ "ffmpeg",
92
+ "-f",
93
+ "lavfi",
94
+ "-i",
95
+ "testsrc=duration=1:size=256x256:rate=1",
96
+ "-c:v",
97
+ "h264_nvenc",
98
+ "-preset",
99
+ "p1",
100
+ "-f",
101
+ "null",
102
+ "-",
103
+ ]
104
+
105
+ result = subprocess.run(
106
+ cmd, capture_output=True, text=True, timeout=15, check=False
107
+ )
108
+
109
+ if result.returncode == 0:
110
+ logger.debug("NVENC h264 encoding test successful")
111
+ return True
112
+ else:
113
+ logger.debug(f"NVENC test failed: {result.stderr}")
114
+ return False
115
+
116
+ except (subprocess.TimeoutExpired, FileNotFoundError) as e:
117
+ logger.debug(f"NVENC detection failed: {e}")
118
+ return False
119
+ except Exception as e:
120
+ logger.warning(f"Unexpected error during NVENC detection: {e}")
121
+ return False
122
+
123
+
124
+ def _get_preferred_encoder() -> Dict[str, str]:
125
+ """
126
+ Get the preferred video encoder configuration based on available hardware.
127
+
128
+ Returns:
129
+ Dictionary with encoder configuration
130
+ """
131
+ global _nvenc_available, _preferred_encoder
132
+
133
+ if _nvenc_available is None:
134
+ _nvenc_available = _detect_nvenc_support()
135
+
136
+ if _preferred_encoder is None:
137
+ if _nvenc_available:
138
+ _preferred_encoder = {
139
+ "name": "h264_nvenc",
140
+ "preset_param": "-preset",
141
+ "preset_value": "p4", # Medium quality/speed for NVENC
142
+ "quality_param": "-cq",
143
+ "quality_value": "20", # NVENC CQ mode
144
+ "type": "nvenc",
145
+ "fallback_preset": "p1", # Fastest NVENC preset for fallback
146
+ }
147
+ logger.info("Hardware acceleration: NVENC available")
148
+ else:
149
+ _preferred_encoder = {
150
+ "name": "libx264",
151
+ "preset_param": "-preset",
152
+ "preset_value": "medium", # CPU preset
153
+ "quality_param": "-crf",
154
+ "quality_value": "23", # CPU CRF mode
155
+ "type": "cpu",
156
+ "fallback_preset": "ultrafast", # Fastest CPU preset for fallback
157
+ }
158
+ logger.info("Hardware acceleration: NVENC not available, using CPU")
159
+
160
+ return _preferred_encoder
161
+
162
+
163
+ def _build_encoder_args(
164
+ quality_mode: str = "balanced",
165
+ fallback: bool = False,
166
+ custom_crf: Optional[int] = None,
167
+ ) -> Tuple[List[str], str]:
168
+ """
169
+ Build encoder command arguments based on available hardware and quality requirements.
170
+
171
+ Args:
172
+ quality_mode: 'fast', 'balanced', or 'quality'
173
+ fallback: Whether to use fallback settings for compatibility
174
+ custom_crf: Override quality setting (for backward compatibility)
175
+
176
+ Returns:
177
+ Tuple of (encoder_args, encoder_type)
178
+ """
179
+ encoder = _get_preferred_encoder()
180
+
181
+ if encoder["type"] == "nvenc":
182
+ # NVIDIA NVENC configuration
183
+ if fallback:
184
+ preset = encoder["fallback_preset"] # p1 - fastest
185
+ quality = "28" # Lower quality for speed
186
+ elif quality_mode == "fast":
187
+ preset = "p2" # Faster preset
188
+ quality = "25"
189
+ elif quality_mode == "quality":
190
+ preset = "p6" # Higher quality preset
191
+ quality = "18"
192
+ else: # balanced
193
+ preset = encoder["preset_value"] # p4
194
+ quality = encoder["quality_value"] # 20
195
+
196
+ # Override with custom CRF if provided (for backward compatibility)
197
+ if custom_crf is not None:
198
+ quality = str(custom_crf)
199
+
200
+ return [
201
+ "-c:v",
202
+ encoder["name"],
203
+ encoder["preset_param"],
204
+ preset,
205
+ encoder["quality_param"],
206
+ quality,
207
+ "-gpu",
208
+ "0", # Use first GPU
209
+ "-rc",
210
+ "vbr", # Variable bitrate
211
+ "-profile:v",
212
+ "high",
213
+ ], encoder["type"]
214
+ else:
215
+ # CPU libx264 configuration
216
+ if fallback:
217
+ preset = encoder["fallback_preset"] # ultrafast
218
+ quality = "28" # Lower quality for speed
219
+ elif quality_mode == "fast":
220
+ preset = "faster"
221
+ quality = "20"
222
+ elif quality_mode == "quality":
223
+ preset = "slow"
224
+ quality = "18"
225
+ else: # balanced
226
+ preset = encoder["preset_value"] # medium
227
+ quality = encoder["quality_value"] # 23
228
+
229
+ # Override with custom CRF if provided (for backward compatibility)
230
+ if custom_crf is not None:
231
+ quality = str(custom_crf)
232
+
233
+ return [
234
+ "-c:v",
235
+ encoder["name"],
236
+ encoder["preset_param"],
237
+ preset,
238
+ encoder["quality_param"],
239
+ quality,
240
+ "-profile:v",
241
+ "high",
242
+ ], encoder["type"]
243
+
244
+
245
+ def is_ffmpeg_available() -> bool:
246
+ """
247
+ Checks whether the FFmpeg executable is available in the system's PATH.
248
+
249
+ Returns:
250
+ True if FFmpeg is found in the PATH; otherwise, False.
251
+ """
252
+ return _resolve_ffmpeg_executable() is not None
253
+
254
+
255
+ def check_ffmpeg_availability():
256
+ """
257
+ Verifies that FFmpeg is installed and available in the system's PATH.
258
+
259
+ Raises:
260
+ FileNotFoundError: If FFmpeg is not found.
261
+
262
+ Returns:
263
+ True if FFmpeg is available.
264
+ """
265
+ if not is_ffmpeg_available():
266
+ error_msg = (
267
+ "FFmpeg is not available. Please install it and ensure it's in your PATH."
268
+ )
269
+ logger.error(error_msg)
270
+ raise FileNotFoundError(error_msg)
271
+ # logger.info("FFmpeg is available.") # Caller can log if needed
272
+ return True
273
+
274
+
275
+ def get_stream_info(file_path: Path) -> Optional[Dict]:
276
+ """
277
+ Retrieves video stream information from a file using ffprobe.
278
+
279
+ Runs ffprobe to extract stream metadata in JSON format from the specified video file. Returns a dictionary with stream information, or None if the file does not exist or if an error occurs during execution or parsing.
280
+ """
281
+ if not file_path.exists():
282
+ logger.error("File not found for ffprobe: %s", file_path)
283
+ return None
284
+
285
+ command = [
286
+ "ffprobe",
287
+ "-v",
288
+ "quiet",
289
+ "-print_format",
290
+ "json",
291
+ "-show_streams",
292
+ str(file_path),
293
+ ]
294
+ try:
295
+ result = subprocess.run(command, capture_output=True, text=True, check=True)
296
+ return json.loads(result.stdout)
297
+ except subprocess.CalledProcessError as e:
298
+ logger.error("ffprobe command failed for %s: %s\n%s", file_path, e, e.stderr)
299
+ return None
300
+ except json.JSONDecodeError as e:
301
+ logger.error("Failed to parse ffprobe JSON output for %s: %s", file_path, e)
302
+ return None
303
+ except Exception as e:
304
+ logger.error("Error running ffprobe for %s: %s", file_path, e, exc_info=True)
305
+ return None
306
+
307
+
308
+ def assemble_video_from_frames( # Renamed from assemble_video
309
+ frame_paths: List[Path],
310
+ output_path: Path,
311
+ fps: float,
312
+ width: Optional[int] = None,
313
+ height: Optional[int] = None,
314
+ ) -> Optional[Path]:
315
+ """
316
+ Assembles a video from a list of frame image paths using cv2.VideoWriter.
317
+ Determines dimensions from the first frame if not provided.
318
+ """
319
+ if not frame_paths:
320
+ logger.error("No frame paths provided for video assembly.")
321
+ return None
322
+
323
+ if width is None or height is None:
324
+ try:
325
+ first_frame = cv2.imread(str(frame_paths[0]))
326
+ if first_frame is None:
327
+ raise IOError(f"Could not read first frame: {frame_paths[0]}")
328
+ height, width, _ = first_frame.shape
329
+ logger.info(
330
+ "Determined video dimensions from first frame: %dx%d", width, height
331
+ )
332
+ except Exception as e:
333
+ logger.error(
334
+ "Error reading first frame to determine dimensions: %s",
335
+ e,
336
+ exc_info=True,
337
+ )
338
+ return None
339
+
340
+ fourcc = cv2.VideoWriter_fourcc(*"mp4v")
341
+ output_path.parent.mkdir(parents=True, exist_ok=True)
342
+ video_writer = cv2.VideoWriter(str(output_path), fourcc, fps, (width, height))
343
+
344
+ if not video_writer.isOpened():
345
+ logger.error("Could not open video writer for path: %s", output_path)
346
+ return None
347
+
348
+ logger.info(
349
+ "Assembling video %s from %d frames...", output_path.name, len(frame_paths)
350
+ )
351
+ try:
352
+ for frame_path in tqdm(frame_paths, desc=f"Assembling {output_path.name}"):
353
+ frame = cv2.imread(str(frame_path))
354
+ if frame is None:
355
+ logger.warning("Could not read frame %s, skipping.", frame_path)
356
+ continue
357
+ # Ensure frame dimensions match - resize if necessary (or log error)
358
+ if frame.shape[1] != width or frame.shape[0] != height:
359
+ logger.warning(
360
+ f"Frame {frame_path} has dimensions {frame.shape[1]}x{frame.shape[0]}, expected {width}x{height}. Resizing."
361
+ )
362
+ frame = cv2.resize(frame, (width, height))
363
+ video_writer.write(frame)
364
+ finally:
365
+ video_writer.release()
366
+ logger.info("Finished assembling video: %s", output_path)
367
+
368
+ return output_path
369
+
370
+
371
+ def transcode_video(
372
+ input_path: Path,
373
+ output_path: Path,
374
+ codec: str = "auto", # Changed default to "auto" for automatic selection
375
+ crf: Optional[int] = None, # Will be determined automatically if None
376
+ preset: str = "auto", # Changed default to "auto" for automatic selection
377
+ audio_codec: str = "aac",
378
+ audio_bitrate: str = "128k",
379
+ extra_args: Optional[List[str]] = None,
380
+ quality_mode: str = "balanced", # New parameter: 'fast', 'balanced', 'quality'
381
+ force_cpu: bool = False, # New parameter to force CPU encoding
382
+ ) -> Optional[Path]:
383
+ """
384
+ Transcodes a video file using FFmpeg with automatic hardware acceleration.
385
+
386
+ Args:
387
+ input_path: Source video file path
388
+ output_path: Output video file path
389
+ codec: Video codec ('auto' for automatic selection, 'libx264', 'h264_nvenc')
390
+ crf: Constant Rate Factor (None for automatic selection)
391
+ preset: Encoder preset ('auto' for automatic selection)
392
+ audio_codec: Audio codec
393
+ audio_bitrate: Audio bitrate
394
+ extra_args: Additional FFmpeg arguments
395
+ quality_mode: Quality mode ('fast', 'balanced', 'quality')
396
+ force_cpu: Force CPU encoding even if NVENC is available
397
+
398
+ Returns:
399
+ Path to transcoded video or None if failed
400
+ """
401
+ if not input_path.exists():
402
+ logger.error("Input file not found for transcoding: %s", input_path)
403
+ return None
404
+
405
+ output_path.parent.mkdir(parents=True, exist_ok=True)
406
+
407
+ # Determine encoder configuration
408
+ if codec == "auto" or preset == "auto":
409
+ if force_cpu:
410
+ # Force CPU encoding
411
+ encoder_args, encoder_type = _build_encoder_args(
412
+ quality_mode, fallback=False, custom_crf=crf
413
+ )
414
+ # Override to use CPU encoder
415
+ encoder_args[1] = "libx264" # Replace encoder name
416
+ encoder_args[3] = "medium" if preset == "auto" else preset # Replace preset
417
+ if crf is not None:
418
+ encoder_args[5] = str(crf) # Replace quality value
419
+ else:
420
+ # Use automatic hardware detection
421
+ encoder_args, encoder_type = _build_encoder_args(
422
+ quality_mode, fallback=False, custom_crf=crf
423
+ )
424
+ else:
425
+ # Manual codec/preset specification (backward compatibility)
426
+ encoder_args = [
427
+ "-c:v",
428
+ codec,
429
+ "-preset",
430
+ preset,
431
+ "-crf" if codec == "libx264" else "-cq",
432
+ str(crf if crf is not None else 23),
433
+ ]
434
+ encoder_type = "nvenc" if "nvenc" in codec else "cpu"
435
+
436
+ # Build complete command
437
+ command = [
438
+ "ffmpeg",
439
+ "-i",
440
+ str(input_path),
441
+ *encoder_args,
442
+ "-c:a",
443
+ audio_codec,
444
+ "-b:a",
445
+ audio_bitrate,
446
+ "-y", # Overwrite output file if it exists
447
+ ]
448
+
449
+ if extra_args:
450
+ command.extend(extra_args)
451
+ command.append(str(output_path))
452
+
453
+ logger.info(
454
+ "Starting transcoding: %s -> %s (using %s)",
455
+ input_path.name,
456
+ output_path.name,
457
+ encoder_type,
458
+ )
459
+ logger.debug("FFmpeg command: %s", " ".join(command))
460
+
461
+ try:
462
+ process = subprocess.Popen(
463
+ command, stderr=subprocess.PIPE, text=True, universal_newlines=True
464
+ )
465
+
466
+ # Progress reporting and error handling
467
+ stderr_output = ""
468
+ if process.stderr:
469
+ for line in process.stderr:
470
+ stderr_output += line
471
+
472
+ process.wait()
473
+
474
+ if process.returncode == 0:
475
+ logger.info("Transcoding finished successfully: %s", output_path)
476
+ return output_path
477
+ else:
478
+ logger.error(
479
+ "FFmpeg transcoding failed for %s with return code %d.",
480
+ input_path.name,
481
+ process.returncode,
482
+ )
483
+ logger.error("FFmpeg stderr:\n%s", stderr_output)
484
+
485
+ # Try fallback to CPU if NVENC failed
486
+ if encoder_type == "nvenc" and not force_cpu:
487
+ logger.warning("NVENC transcoding failed, trying CPU fallback...")
488
+ return _transcode_video_fallback(
489
+ input_path,
490
+ output_path,
491
+ audio_codec,
492
+ audio_bitrate,
493
+ extra_args,
494
+ quality_mode,
495
+ crf,
496
+ )
497
+
498
+ # Clean up potentially corrupted output file
499
+ if output_path.exists():
500
+ try:
501
+ output_path.unlink()
502
+ except OSError as e:
503
+ logger.error(
504
+ "Failed to delete incomplete output file %s: %s", output_path, e
505
+ )
506
+ return None
507
+
508
+ except FileNotFoundError:
509
+ logger.error(
510
+ "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
511
+ )
512
+ return None
513
+ except Exception as e:
514
+ logger.error(
515
+ "Error during transcoding of %s: %s", input_path.name, e, exc_info=True
516
+ )
517
+ return None
518
+
519
+
520
+ def _transcode_video_fallback(
521
+ input_path: Path,
522
+ output_path: Path,
523
+ audio_codec: str,
524
+ audio_bitrate: str,
525
+ extra_args: Optional[List[str]],
526
+ quality_mode: str,
527
+ custom_crf: Optional[int],
528
+ ) -> Optional[Path]:
529
+ """
530
+ Fallback transcoding using CPU encoding.
531
+
532
+ Args:
533
+ input_path: Source video file path
534
+ output_path: Output video file path
535
+ audio_codec: Audio codec
536
+ audio_bitrate: Audio bitrate
537
+ extra_args: Additional FFmpeg arguments
538
+ quality_mode: Quality mode
539
+ custom_crf: Custom CRF value
540
+
541
+ Returns:
542
+ Path to transcoded video or None if failed
543
+ """
544
+ try:
545
+ # Build CPU encoder arguments
546
+ encoder_args, _ = _build_encoder_args(
547
+ quality_mode, fallback=True, custom_crf=custom_crf
548
+ )
549
+ # Force CPU encoder
550
+ encoder_args[1] = "libx264"
551
+
552
+ command = [
553
+ "ffmpeg",
554
+ "-i",
555
+ str(input_path),
556
+ *encoder_args,
557
+ "-c:a",
558
+ audio_codec,
559
+ "-b:a",
560
+ audio_bitrate,
561
+ "-y",
562
+ ]
563
+
564
+ if extra_args:
565
+ command.extend(extra_args)
566
+ command.append(str(output_path))
567
+
568
+ logger.info(
569
+ "CPU fallback transcoding: %s -> %s", input_path.name, output_path.name
570
+ )
571
+ logger.debug("Fallback FFmpeg command: %s", " ".join(command))
572
+
573
+ process = subprocess.Popen(
574
+ command, stderr=subprocess.PIPE, text=True, universal_newlines=True
575
+ )
576
+ stderr_output = ""
577
+ if process.stderr:
578
+ for line in process.stderr:
579
+ stderr_output += line
580
+
581
+ process.wait()
582
+
583
+ if process.returncode == 0:
584
+ logger.info("CPU fallback transcoding successful: %s", output_path)
585
+ return output_path
586
+ else:
587
+ logger.error("CPU fallback transcoding also failed for %s", input_path.name)
588
+ logger.error("Fallback stderr:\n%s", stderr_output)
589
+ return None
590
+
591
+ except Exception as e:
592
+ logger.error("Error during CPU fallback transcoding: %s", e, exc_info=True)
593
+ return None
594
+
595
+ logger.info("Starting transcoding: %s -> %s", input_path.name, output_path.name)
596
+ logger.debug("FFmpeg command: %s", " ".join(command))
597
+
598
+ try:
599
+ process = subprocess.Popen(
600
+ command, stderr=subprocess.PIPE, text=True, universal_newlines=True
601
+ )
602
+
603
+ # Optional: Progress reporting (can be complex to parse ffmpeg output reliably)
604
+ # For simplicity, just wait and check the return code
605
+ stderr_output = ""
606
+ if process.stderr:
607
+ for line in process.stderr:
608
+ stderr_output += line
609
+ # Simple progress indication or detailed logging
610
+ # logger.debug(f"ffmpeg: {line.strip()}")
611
+
612
+ process.wait()
613
+
614
+ if process.returncode == 0:
615
+ logger.info("Transcoding finished successfully: %s", output_path)
616
+ return output_path
617
+ else:
618
+ logger.error(
619
+ "FFmpeg transcoding failed for %s with return code %d.",
620
+ input_path.name,
621
+ process.returncode,
622
+ )
623
+ logger.error("FFmpeg stderr:\n%s", stderr_output)
624
+ # Clean up potentially corrupted output file
625
+ if output_path.exists():
626
+ try:
627
+ output_path.unlink()
628
+ except OSError as e:
629
+ logger.error(
630
+ "Failed to delete incomplete output file %s: %s", output_path, e
631
+ )
632
+ return None
633
+
634
+ except FileNotFoundError:
635
+ logger.error(
636
+ "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
637
+ )
638
+ return None
639
+ except Exception as e:
640
+ logger.error(
641
+ "Error during transcoding of %s: %s", input_path.name, e, exc_info=True
642
+ )
643
+ return None
644
+
645
+
646
+ def transcode_videofile_if_required(
647
+ input_path: Path,
648
+ output_path: Path,
649
+ required_codec: str = "h264",
650
+ required_pixel_format: str = "yuv420p", # Changed default from yuvj420p
651
+ **transcode_options, # Pass other options to transcode_video
652
+ ) -> Optional[Path]:
653
+ """
654
+ Checks if a video needs transcoding based on codec and pixel format,
655
+ and transcodes it using transcode_video if necessary.
656
+ Uses yuv420p with full color range (pc/jpeg) as the target format.
657
+ Returns the path to the compliant video (original or transcoded).
658
+ """
659
+ stream_info = get_stream_info(input_path)
660
+ if not stream_info or "streams" not in stream_info:
661
+ logger.error(
662
+ "Could not get stream info for %s to check if transcoding is required.",
663
+ input_path,
664
+ )
665
+ return None
666
+
667
+ video_stream = next(
668
+ (s for s in stream_info["streams"] if s.get("codec_type") == "video"), None
669
+ )
670
+
671
+ if not video_stream:
672
+ logger.error("No video stream found in %s.", input_path)
673
+ return None
674
+
675
+ codec_name = video_stream.get("codec_name")
676
+ pixel_format = video_stream.get("pix_fmt")
677
+ # Check color range as well, default is usually 'tv' (limited)
678
+ color_range = video_stream.get(
679
+ "color_range", "tv"
680
+ ) # Default to tv if not specified
681
+
682
+ needs_transcoding = False
683
+ transcode_reason = []
684
+ if codec_name != required_codec:
685
+ reason = f"Codec mismatch ({codec_name} != {required_codec})"
686
+ logger.info("%s for %s. Transcoding required.", reason, input_path.name)
687
+ transcode_reason.append(reason)
688
+ needs_transcoding = True
689
+ # Check both pixel format and color range for yuv420p
690
+ if pixel_format != required_pixel_format or (
691
+ pixel_format == "yuv420p" and color_range != "pc"
692
+ ):
693
+ reason = f"Pixel format/color range mismatch (pix_fmt: {pixel_format}, color_range: {color_range} != {required_pixel_format} with color_range=pc)"
694
+ logger.info("%s for %s. Transcoding required.", reason, input_path.name)
695
+ transcode_reason.append(reason)
696
+ needs_transcoding = True
697
+
698
+ if needs_transcoding:
699
+ logger.info(
700
+ "Transcoding %s to %s due to: %s",
701
+ input_path.name,
702
+ output_path.name,
703
+ "; ".join(transcode_reason),
704
+ )
705
+ # Ensure codec and pixel format are set in options if not already present
706
+ transcode_options.setdefault(
707
+ "codec", "libx264" if required_codec == "h264" else required_codec
708
+ )
709
+ transcode_options.setdefault("extra_args", [])
710
+
711
+ # Ensure pixel format and color range are correctly set in extra_args
712
+ extra_args = transcode_options["extra_args"]
713
+ if "-pix_fmt" not in extra_args:
714
+ extra_args.extend(["-pix_fmt", required_pixel_format])
715
+ else:
716
+ # If pix_fmt is already set, ensure it's the required one
717
+ try:
718
+ pix_fmt_index = extra_args.index("-pix_fmt")
719
+ if extra_args[pix_fmt_index + 1] != required_pixel_format:
720
+ logger.warning(
721
+ "Overriding existing -pix_fmt '%s' with '%s'",
722
+ extra_args[pix_fmt_index + 1],
723
+ required_pixel_format,
724
+ )
725
+ extra_args[pix_fmt_index + 1] = required_pixel_format
726
+ except (ValueError, IndexError):
727
+ # Should not happen if '-pix_fmt' is in extra_args, but handle defensively
728
+ logger.error(
729
+ "Error processing existing -pix_fmt argument. Appending required format."
730
+ )
731
+ extra_args.extend(["-pix_fmt", required_pixel_format])
732
+
733
+ if "-color_range" not in extra_args:
734
+ # Add color range 'pc' (which corresponds to 2 or 'jpeg') for yuv420p
735
+ extra_args.extend(["-color_range", "pc"])
736
+ else:
737
+ # If color_range is already set, ensure it's 'pc'
738
+ try:
739
+ color_range_index = extra_args.index("-color_range")
740
+ if extra_args[color_range_index + 1] != "pc":
741
+ logger.warning(
742
+ "Overriding existing -color_range '%s' with 'pc'",
743
+ extra_args[color_range_index + 1],
744
+ )
745
+ extra_args[color_range_index + 1] = "pc"
746
+ except (ValueError, IndexError):
747
+ logger.error(
748
+ "Error processing existing -color_range argument. Appending 'pc'."
749
+ )
750
+ extra_args.extend(["-color_range", "pc"])
751
+
752
+ return transcode_video(input_path, output_path, **transcode_options)
753
+ else:
754
+ logger.info(
755
+ "Video %s already meets requirements (%s, %s, color_range=pc). No transcoding needed.",
756
+ input_path.name,
757
+ required_codec,
758
+ required_pixel_format,
759
+ )
760
+ # If no transcoding is needed, should we copy/link or just return the original path?
761
+ # For simplicity, let's assume the caller handles the file location.
762
+ # If the output_path is different, we might need to copy.
763
+ if input_path != output_path:
764
+ # Example: copy file if output path is different
765
+ try:
766
+ output_path.parent.mkdir(parents=True, exist_ok=True)
767
+ shutil.copy2(input_path, output_path)
768
+ logger.info(
769
+ "Copied %s to %s as it met requirements.",
770
+ input_path.name,
771
+ output_path.name,
772
+ )
773
+ return output_path
774
+ except Exception as e:
775
+ logger.error(
776
+ "Failed to copy %s to %s: %s", input_path.name, output_path.name, e
777
+ )
778
+ return None
779
+ return input_path # Return original path if no copy needed
780
+
781
+
782
+ def extract_frames(
783
+ video_path: Path,
784
+ output_dir: Path,
785
+ quality: int,
786
+ ext: str = "jpg",
787
+ fps: Optional[float] = None,
788
+ ) -> List[Path]:
789
+ """
790
+ Extracts frames from a video file using FFmpeg.
791
+
792
+ Args:
793
+ video_path: Path to the input video file.
794
+ output_dir: Directory to save the extracted frames.
795
+ quality: Quality factor for JPEG extraction (1-31, lower is better).
796
+ ext: Output frame image extension (e.g., 'jpg', 'png').
797
+ fps: Optional frames per second to extract. If None, extracts all frames.
798
+
799
+ Returns:
800
+ A list of Path objects for the extracted frames.
801
+ """
802
+ # Resolve ffmpeg executable with multiple fallbacks
803
+ ffmpeg_executable = _resolve_ffmpeg_executable()
804
+ if not ffmpeg_executable:
805
+ error_msg = "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
806
+ logger.error(error_msg)
807
+ raise FileNotFoundError(error_msg)
808
+
809
+ output_dir.mkdir(parents=True, exist_ok=True)
810
+ output_pattern = output_dir / f"frame_%07d.{ext}"
811
+
812
+ cmd = [
813
+ ffmpeg_executable, # Use the found executable path
814
+ "-i",
815
+ str(video_path),
816
+ "-qscale:v",
817
+ str(quality), # Video quality scale
818
+ ]
819
+
820
+ if fps is not None:
821
+ cmd.extend(["-vf", f"fps={fps}"])
822
+
823
+ cmd.append(str(output_pattern))
824
+
825
+ logger.info("Running FFmpeg command: %s", " ".join(cmd))
826
+ try:
827
+ # Use subprocess.run for better error handling
828
+ result = subprocess.run(cmd, check=True, capture_output=True, text=True)
829
+ logger.debug("FFmpeg stdout:\n%s", result.stdout)
830
+ logger.debug("FFmpeg stderr:\n%s", result.stderr)
831
+ logger.info("FFmpeg frame extraction completed successfully.")
832
+ except FileNotFoundError as exc:
833
+ # This might be redundant now but kept for safety
834
+ error_msg = f"ffmpeg command not found at '{ffmpeg_executable}'. Ensure FFmpeg is installed and in the system's PATH."
835
+ logger.error(error_msg)
836
+ raise FileNotFoundError(error_msg) from exc
837
+ except subprocess.CalledProcessError as e:
838
+ logger.error("FFmpeg command failed with exit code %d.", e.returncode)
839
+ logger.error("FFmpeg stderr:\n%s", e.stderr)
840
+ logger.error("FFmpeg stdout:\n%s", e.stdout)
841
+ # Return empty list on error as frames were likely not created correctly
842
+ return []
843
+ except Exception as e:
844
+ logger.error(
845
+ "An unexpected error occurred during FFmpeg execution: %s", e, exc_info=True
846
+ )
847
+ return []
848
+
849
+ # Collect paths of extracted frames
850
+ extracted_files = sorted(output_dir.glob(f"frame_*.{ext}"))
851
+ return extracted_files
852
+
853
+
854
+ def extract_frame_range(
855
+ video_path: Path,
856
+ output_dir: Path,
857
+ start_frame: int,
858
+ end_frame: int, # Exclusive end frame number
859
+ quality: int,
860
+ ext: str = "jpg",
861
+ ) -> List[Path]:
862
+ """
863
+ Extracts a specific range of frames from a video using FFmpeg.
864
+
865
+ Frames from start_frame (inclusive) to end_frame (exclusive) are saved as images
866
+ in the output directory, following the naming pattern 'frame_%07d.ext'. The
867
+ function ensures only the requested frames are returned, and cleans up partial
868
+ results on failure.
869
+
870
+ Args:
871
+ video_path: Path to the input video file.
872
+ output_dir: Directory where extracted frames will be saved.
873
+ start_frame: Index of the first frame to extract (inclusive, 0-based).
874
+ end_frame: Index at which to stop extraction (exclusive, 0-based).
875
+ quality: JPEG quality factor (1-31, lower is better).
876
+ ext: File extension for output images (e.g., 'jpg', 'png').
877
+
878
+ Returns:
879
+ List of Paths to the extracted frame image files within the specified range.
880
+
881
+ Raises:
882
+ FileNotFoundError: If the FFmpeg executable is not found.
883
+ ValueError: If start_frame is greater than or equal to end_frame.
884
+ RuntimeError: If FFmpeg fails to extract the requested frames.
885
+ """
886
+ if start_frame >= end_frame:
887
+ logger.warning(
888
+ "extract_frame_range called with start_frame (%d) >= end_frame (%d). No frames to extract.",
889
+ start_frame,
890
+ end_frame,
891
+ )
892
+ return []
893
+
894
+ ffmpeg_executable = _resolve_ffmpeg_executable()
895
+ if not ffmpeg_executable:
896
+ error_msg = "ffmpeg command not found. Ensure FFmpeg is installed and in the system's PATH."
897
+ logger.error(error_msg)
898
+ raise FileNotFoundError(error_msg)
899
+
900
+ output_dir.mkdir(parents=True, exist_ok=True)
901
+ # Use a consistent naming convention, matching extract_frames
902
+ output_pattern = output_dir / f"frame_%07d.{ext}"
903
+
904
+ # Use select filter for precise frame range extraction
905
+ # 'select' uses 0-based indexing 'n'
906
+ # We want frames where start_frame <= n < end_frame
907
+ select_filter = f"select='between(n,{start_frame},{end_frame - 1})'"
908
+
909
+ cmd = [
910
+ ffmpeg_executable,
911
+ "-i",
912
+ str(video_path),
913
+ "-vf",
914
+ select_filter,
915
+ "-vsync",
916
+ "vfr", # Variable frame rate sync to handle selected frames
917
+ "-qscale:v",
918
+ str(quality),
919
+ "-copyts", # Attempt to copy timestamps if needed, might not be accurate with select
920
+ str(output_pattern),
921
+ ]
922
+
923
+ logger.info("Running FFmpeg command for frame range extraction: %s", " ".join(cmd))
924
+ try:
925
+ result = subprocess.run(cmd, check=True, capture_output=True, text=True)
926
+ logger.debug("FFmpeg stdout:\n%s", result.stdout)
927
+ logger.debug("FFmpeg stderr:\n%s", result.stderr)
928
+ logger.info("FFmpeg frame range extraction completed successfully.")
929
+ except FileNotFoundError as exc:
930
+ error_msg = f"ffmpeg command not found at '{ffmpeg_executable}'. Ensure FFmpeg is installed and in the system's PATH."
931
+ logger.error(error_msg)
932
+ raise FileNotFoundError(error_msg) from exc
933
+ except subprocess.CalledProcessError as e:
934
+ logger.error("FFmpeg command failed with exit code %d.", e.returncode)
935
+ logger.error("FFmpeg stderr:\n%s", e.stderr)
936
+ logger.error("FFmpeg stdout:\n%s", e.stdout)
937
+ # Clean up potentially partially created files in the target directory within the expected range
938
+ logger.warning(
939
+ "Attempting cleanup of potentially incomplete frames in %s", output_dir
940
+ )
941
+ for i in range(start_frame, end_frame):
942
+ potential_file = output_dir / f"frame_{i:07d}.{ext}"
943
+ if potential_file.exists():
944
+ try:
945
+ potential_file.unlink()
946
+ except OSError as unlink_err:
947
+ logger.error(
948
+ "Failed to delete potential frame %s during cleanup: %s",
949
+ potential_file,
950
+ unlink_err,
951
+ )
952
+ raise RuntimeError(
953
+ f"FFmpeg frame range extraction failed for {video_path}"
954
+ ) from e
955
+ except Exception as e:
956
+ logger.error(
957
+ "An unexpected error occurred during FFmpeg execution: %s", e, exc_info=True
958
+ )
959
+ raise RuntimeError(
960
+ f"Unexpected error during FFmpeg frame range extraction for {video_path}"
961
+ ) from e
962
+
963
+ # Collect paths of extracted frames matching the pattern and expected range
964
+ # FFmpeg might create files outside the exact range depending on version/flags,
965
+ # so filter explicitly.
966
+ extracted_files = []
967
+ for i in range(start_frame, end_frame):
968
+ frame_file = output_dir / f"frame_{i:07d}.{ext}"
969
+ if frame_file.exists():
970
+ extracted_files.append(frame_file)
971
+ else:
972
+ # This might happen if ffmpeg fails silently for some frames or if the video ends early.
973
+ logger.warning(
974
+ "Expected frame file %s not found after extraction.", frame_file
975
+ )
976
+
977
+ logger.info(
978
+ "Found %d extracted frame files in range [%d, %d) for video %s.",
979
+ len(extracted_files),
980
+ start_frame,
981
+ end_frame,
982
+ video_path.name,
983
+ )
984
+ return extracted_files
985
+
986
+
987
+ __all__ = [
988
+ "is_ffmpeg_available", # ADDED
989
+ "check_ffmpeg_availability", # ADDED
990
+ "get_stream_info",
991
+ "assemble_video_from_frames", # Updated name
992
+ "transcode_video",
993
+ "transcode_videofile_if_required",
994
+ "extract_frames",
995
+ "extract_frame_range", # Add new function to __all__
996
+ ]