endoreg-db 0.8.8.9__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (453) hide show
  1. endoreg_db/admin.py +10 -5
  2. endoreg_db/apps.py +4 -7
  3. endoreg_db/authz/auth.py +1 -0
  4. endoreg_db/authz/backends.py +1 -1
  5. endoreg_db/authz/management/commands/list_routes.py +2 -0
  6. endoreg_db/authz/middleware.py +8 -7
  7. endoreg_db/authz/permissions.py +21 -10
  8. endoreg_db/authz/policy.py +14 -19
  9. endoreg_db/authz/views_auth.py +14 -10
  10. endoreg_db/codemods/rename_datetime_fields.py +8 -1
  11. endoreg_db/exceptions.py +5 -2
  12. endoreg_db/forms/__init__.py +0 -1
  13. endoreg_db/forms/examination_form.py +4 -3
  14. endoreg_db/forms/patient_finding_intervention_form.py +30 -8
  15. endoreg_db/forms/patient_form.py +9 -13
  16. endoreg_db/forms/questionnaires/__init__.py +1 -1
  17. endoreg_db/forms/settings/__init__.py +4 -1
  18. endoreg_db/forms/unit.py +2 -1
  19. endoreg_db/helpers/count_db.py +17 -14
  20. endoreg_db/helpers/default_objects.py +2 -1
  21. endoreg_db/helpers/download_segmentation_model.py +4 -3
  22. endoreg_db/helpers/interact.py +0 -5
  23. endoreg_db/helpers/test_video_helper.py +33 -25
  24. endoreg_db/import_files/__init__.py +1 -1
  25. endoreg_db/import_files/context/__init__.py +1 -1
  26. endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
  27. endoreg_db/import_files/context/ensure_center.py +4 -4
  28. endoreg_db/import_files/context/file_lock.py +3 -3
  29. endoreg_db/import_files/context/import_context.py +11 -12
  30. endoreg_db/import_files/context/validate_directories.py +1 -0
  31. endoreg_db/import_files/file_storage/create_report_file.py +57 -34
  32. endoreg_db/import_files/file_storage/create_video_file.py +64 -35
  33. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
  34. endoreg_db/import_files/file_storage/state_management.py +146 -83
  35. endoreg_db/import_files/file_storage/storage.py +5 -1
  36. endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
  37. endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
  38. endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
  39. endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
  40. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
  41. endoreg_db/import_files/report_import_service.py +36 -30
  42. endoreg_db/import_files/video_import_service.py +27 -23
  43. endoreg_db/logger_conf.py +56 -40
  44. endoreg_db/management/__init__.py +1 -1
  45. endoreg_db/management/commands/__init__.py +1 -1
  46. endoreg_db/management/commands/check_auth.py +45 -38
  47. endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
  48. endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
  49. endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
  50. endoreg_db/management/commands/fix_video_paths.py +75 -54
  51. endoreg_db/management/commands/import_report.py +1 -3
  52. endoreg_db/management/commands/list_routes.py +2 -0
  53. endoreg_db/management/commands/load_ai_model_data.py +8 -2
  54. endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
  55. endoreg_db/management/commands/load_center_data.py +3 -3
  56. endoreg_db/management/commands/load_distribution_data.py +35 -38
  57. endoreg_db/management/commands/load_endoscope_data.py +0 -3
  58. endoreg_db/management/commands/load_examination_data.py +20 -4
  59. endoreg_db/management/commands/load_finding_data.py +18 -3
  60. endoreg_db/management/commands/load_gender_data.py +17 -24
  61. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
  62. endoreg_db/management/commands/load_information_source.py +0 -3
  63. endoreg_db/management/commands/load_lab_value_data.py +14 -3
  64. endoreg_db/management/commands/load_legacy_data.py +303 -0
  65. endoreg_db/management/commands/load_name_data.py +1 -2
  66. endoreg_db/management/commands/load_pdf_type_data.py +4 -8
  67. endoreg_db/management/commands/load_profession_data.py +0 -1
  68. endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
  69. endoreg_db/management/commands/load_requirement_data.py +6 -2
  70. endoreg_db/management/commands/load_unit_data.py +0 -4
  71. endoreg_db/management/commands/load_user_groups.py +5 -7
  72. endoreg_db/management/commands/model_input.py +169 -0
  73. endoreg_db/management/commands/register_ai_model.py +22 -16
  74. endoreg_db/management/commands/setup_endoreg_db.py +110 -32
  75. endoreg_db/management/commands/storage_management.py +14 -8
  76. endoreg_db/management/commands/summarize_db_content.py +154 -63
  77. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  78. endoreg_db/management/commands/validate_video_files.py +82 -50
  79. endoreg_db/management/commands/video_validation.py +4 -6
  80. endoreg_db/migrations/0001_initial.py +112 -63
  81. endoreg_db/migrations/__init__.py +0 -0
  82. endoreg_db/models/__init__.py +8 -0
  83. endoreg_db/models/administration/ai/active_model.py +5 -5
  84. endoreg_db/models/administration/ai/ai_model.py +41 -18
  85. endoreg_db/models/administration/ai/model_type.py +1 -0
  86. endoreg_db/models/administration/case/case.py +22 -22
  87. endoreg_db/models/administration/center/__init__.py +5 -5
  88. endoreg_db/models/administration/center/center.py +6 -2
  89. endoreg_db/models/administration/center/center_resource.py +18 -4
  90. endoreg_db/models/administration/center/center_shift.py +3 -1
  91. endoreg_db/models/administration/center/center_waste.py +6 -2
  92. endoreg_db/models/administration/person/__init__.py +1 -1
  93. endoreg_db/models/administration/person/employee/__init__.py +1 -1
  94. endoreg_db/models/administration/person/employee/employee_type.py +3 -1
  95. endoreg_db/models/administration/person/examiner/__init__.py +1 -1
  96. endoreg_db/models/administration/person/examiner/examiner.py +10 -2
  97. endoreg_db/models/administration/person/names/first_name.py +6 -4
  98. endoreg_db/models/administration/person/names/last_name.py +4 -3
  99. endoreg_db/models/administration/person/patient/__init__.py +1 -1
  100. endoreg_db/models/administration/person/patient/patient.py +0 -1
  101. endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
  102. endoreg_db/models/administration/person/person.py +1 -1
  103. endoreg_db/models/administration/product/__init__.py +7 -6
  104. endoreg_db/models/administration/product/product.py +6 -2
  105. endoreg_db/models/administration/product/product_group.py +9 -7
  106. endoreg_db/models/administration/product/product_material.py +9 -2
  107. endoreg_db/models/administration/product/reference_product.py +64 -15
  108. endoreg_db/models/administration/qualification/qualification.py +3 -1
  109. endoreg_db/models/administration/shift/shift.py +3 -1
  110. endoreg_db/models/administration/shift/shift_type.py +12 -4
  111. endoreg_db/models/aidataset/__init__.py +5 -0
  112. endoreg_db/models/aidataset/aidataset.py +193 -0
  113. endoreg_db/models/label/__init__.py +1 -1
  114. endoreg_db/models/label/label.py +10 -2
  115. endoreg_db/models/label/label_set.py +3 -1
  116. endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
  117. endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
  118. endoreg_db/models/media/__init__.py +12 -5
  119. endoreg_db/models/media/frame/__init__.py +1 -1
  120. endoreg_db/models/media/frame/frame.py +34 -8
  121. endoreg_db/models/media/pdf/__init__.py +2 -1
  122. endoreg_db/models/media/pdf/raw_pdf.py +11 -4
  123. endoreg_db/models/media/pdf/report_file.py +6 -2
  124. endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
  125. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
  126. endoreg_db/models/media/video/create_from_file.py +20 -41
  127. endoreg_db/models/media/video/pipe_1.py +75 -30
  128. endoreg_db/models/media/video/pipe_2.py +37 -12
  129. endoreg_db/models/media/video/video_file.py +36 -24
  130. endoreg_db/models/media/video/video_file_ai.py +235 -70
  131. endoreg_db/models/media/video/video_file_anonymize.py +240 -65
  132. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
  133. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
  134. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
  135. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
  136. endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
  137. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
  138. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
  139. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
  140. endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
  141. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
  142. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
  143. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
  144. endoreg_db/models/media/video/video_file_io.py +85 -33
  145. endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
  146. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
  147. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
  148. endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
  149. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
  150. endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
  151. endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
  152. endoreg_db/models/media/video/video_file_segments.py +118 -27
  153. endoreg_db/models/media/video/video_metadata.py +25 -6
  154. endoreg_db/models/media/video/video_processing.py +54 -15
  155. endoreg_db/models/medical/__init__.py +3 -13
  156. endoreg_db/models/medical/contraindication/__init__.py +3 -1
  157. endoreg_db/models/medical/disease.py +18 -6
  158. endoreg_db/models/medical/event.py +6 -2
  159. endoreg_db/models/medical/examination/__init__.py +5 -1
  160. endoreg_db/models/medical/examination/examination.py +22 -6
  161. endoreg_db/models/medical/examination/examination_indication.py +23 -7
  162. endoreg_db/models/medical/examination/examination_time.py +6 -2
  163. endoreg_db/models/medical/finding/__init__.py +3 -1
  164. endoreg_db/models/medical/finding/finding.py +37 -12
  165. endoreg_db/models/medical/finding/finding_classification.py +27 -8
  166. endoreg_db/models/medical/finding/finding_intervention.py +19 -6
  167. endoreg_db/models/medical/finding/finding_type.py +3 -1
  168. endoreg_db/models/medical/hardware/__init__.py +1 -1
  169. endoreg_db/models/medical/hardware/endoscope.py +14 -2
  170. endoreg_db/models/medical/laboratory/__init__.py +1 -1
  171. endoreg_db/models/medical/laboratory/lab_value.py +139 -39
  172. endoreg_db/models/medical/medication/__init__.py +7 -3
  173. endoreg_db/models/medical/medication/medication.py +3 -1
  174. endoreg_db/models/medical/medication/medication_indication.py +3 -1
  175. endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
  176. endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
  177. endoreg_db/models/medical/medication/medication_schedule.py +3 -1
  178. endoreg_db/models/medical/patient/__init__.py +2 -10
  179. endoreg_db/models/medical/patient/medication_examples.py +3 -14
  180. endoreg_db/models/medical/patient/patient_disease.py +17 -5
  181. endoreg_db/models/medical/patient/patient_event.py +12 -4
  182. endoreg_db/models/medical/patient/patient_examination.py +52 -15
  183. endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
  184. endoreg_db/models/medical/patient/patient_finding.py +105 -29
  185. endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
  186. endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
  187. endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
  188. endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
  189. endoreg_db/models/medical/patient/patient_medication.py +25 -7
  190. endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
  191. endoreg_db/models/metadata/model_meta.py +40 -12
  192. endoreg_db/models/metadata/model_meta_logic.py +51 -16
  193. endoreg_db/models/metadata/sensitive_meta.py +65 -28
  194. endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
  195. endoreg_db/models/metadata/video_meta.py +146 -39
  196. endoreg_db/models/metadata/video_prediction_logic.py +70 -21
  197. endoreg_db/models/metadata/video_prediction_meta.py +80 -27
  198. endoreg_db/models/operation_log.py +63 -0
  199. endoreg_db/models/other/__init__.py +10 -10
  200. endoreg_db/models/other/distribution/__init__.py +9 -7
  201. endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
  202. endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
  203. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
  204. endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
  205. endoreg_db/models/other/emission/__init__.py +1 -1
  206. endoreg_db/models/other/emission/emission_factor.py +9 -3
  207. endoreg_db/models/other/information_source.py +15 -5
  208. endoreg_db/models/other/material.py +3 -1
  209. endoreg_db/models/other/transport_route.py +3 -1
  210. endoreg_db/models/other/unit.py +6 -2
  211. endoreg_db/models/report/report.py +0 -1
  212. endoreg_db/models/requirement/requirement.py +84 -27
  213. endoreg_db/models/requirement/requirement_error.py +5 -6
  214. endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
  215. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
  216. endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
  217. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
  218. endoreg_db/models/requirement/requirement_operator.py +28 -8
  219. endoreg_db/models/requirement/requirement_set.py +34 -11
  220. endoreg_db/models/state/__init__.py +1 -0
  221. endoreg_db/models/state/audit_ledger.py +9 -2
  222. endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
  223. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  224. endoreg_db/models/state/raw_pdf.py +0 -1
  225. endoreg_db/models/state/video.py +2 -3
  226. endoreg_db/models/utils.py +4 -2
  227. endoreg_db/queries/__init__.py +2 -6
  228. endoreg_db/queries/annotations/__init__.py +1 -3
  229. endoreg_db/queries/annotations/legacy.py +37 -26
  230. endoreg_db/root_urls.py +3 -4
  231. endoreg_db/schemas/examination_evaluation.py +3 -0
  232. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
  233. endoreg_db/serializers/__init__.py +2 -8
  234. endoreg_db/serializers/administration/__init__.py +1 -2
  235. endoreg_db/serializers/administration/ai/__init__.py +0 -1
  236. endoreg_db/serializers/administration/ai/active_model.py +3 -1
  237. endoreg_db/serializers/administration/ai/ai_model.py +5 -3
  238. endoreg_db/serializers/administration/ai/model_type.py +3 -1
  239. endoreg_db/serializers/administration/center.py +7 -2
  240. endoreg_db/serializers/administration/gender.py +4 -2
  241. endoreg_db/serializers/anonymization.py +13 -13
  242. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
  243. endoreg_db/serializers/examination/__init__.py +1 -1
  244. endoreg_db/serializers/examination/base.py +12 -13
  245. endoreg_db/serializers/examination/dropdown.py +6 -7
  246. endoreg_db/serializers/examination_serializer.py +3 -6
  247. endoreg_db/serializers/finding/__init__.py +1 -1
  248. endoreg_db/serializers/finding/finding.py +14 -7
  249. endoreg_db/serializers/finding_classification/__init__.py +3 -3
  250. endoreg_db/serializers/finding_classification/choice.py +3 -3
  251. endoreg_db/serializers/finding_classification/classification.py +2 -4
  252. endoreg_db/serializers/label_video_segment/__init__.py +5 -3
  253. endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
  254. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  255. endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
  256. endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
  257. endoreg_db/serializers/meta/__init__.py +1 -2
  258. endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
  259. endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
  260. endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
  261. endoreg_db/serializers/misc/__init__.py +2 -2
  262. endoreg_db/serializers/misc/file_overview.py +11 -7
  263. endoreg_db/serializers/misc/stats.py +10 -8
  264. endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
  265. endoreg_db/serializers/misc/upload_job.py +32 -29
  266. endoreg_db/serializers/patient/__init__.py +2 -1
  267. endoreg_db/serializers/patient/patient.py +32 -15
  268. endoreg_db/serializers/patient/patient_dropdown.py +11 -3
  269. endoreg_db/serializers/patient_examination/__init__.py +1 -1
  270. endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
  271. endoreg_db/serializers/patient_finding/__init__.py +1 -1
  272. endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
  273. endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
  274. endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
  275. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
  276. endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
  277. endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
  278. endoreg_db/serializers/pdf/__init__.py +1 -3
  279. endoreg_db/serializers/requirements/requirement_schema.py +1 -6
  280. endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
  281. endoreg_db/serializers/video/__init__.py +2 -2
  282. endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
  283. endoreg_db/serializers/video/video_file_brief.py +6 -2
  284. endoreg_db/serializers/video/video_file_detail.py +36 -23
  285. endoreg_db/serializers/video/video_file_list.py +4 -2
  286. endoreg_db/serializers/video/video_processing_history.py +54 -50
  287. endoreg_db/services/__init__.py +1 -1
  288. endoreg_db/services/anonymization.py +2 -2
  289. endoreg_db/services/examination_evaluation.py +40 -17
  290. endoreg_db/services/model_meta_from_hf.py +76 -0
  291. endoreg_db/services/polling_coordinator.py +101 -70
  292. endoreg_db/services/pseudonym_service.py +27 -22
  293. endoreg_db/services/report_import.py +6 -3
  294. endoreg_db/services/segment_sync.py +75 -59
  295. endoreg_db/services/video_import.py +6 -7
  296. endoreg_db/urls/__init__.py +2 -2
  297. endoreg_db/urls/ai.py +7 -25
  298. endoreg_db/urls/anonymization.py +61 -15
  299. endoreg_db/urls/auth.py +4 -4
  300. endoreg_db/urls/classification.py +4 -9
  301. endoreg_db/urls/examination.py +27 -18
  302. endoreg_db/urls/media.py +27 -34
  303. endoreg_db/urls/patient.py +11 -7
  304. endoreg_db/urls/requirements.py +3 -1
  305. endoreg_db/urls/root_urls.py +2 -3
  306. endoreg_db/urls/stats.py +24 -16
  307. endoreg_db/urls/upload.py +3 -11
  308. endoreg_db/utils/__init__.py +14 -15
  309. endoreg_db/utils/ai/__init__.py +1 -1
  310. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  311. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  312. endoreg_db/utils/ai/get.py +2 -1
  313. endoreg_db/utils/ai/inference_dataset.py +14 -15
  314. endoreg_db/utils/ai/model_training/config.py +117 -0
  315. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  316. endoreg_db/utils/ai/model_training/losses.py +68 -0
  317. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  318. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  319. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  320. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  321. endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
  322. endoreg_db/utils/ai/predict.py +4 -4
  323. endoreg_db/utils/ai/preprocess.py +19 -11
  324. endoreg_db/utils/calc_duration_seconds.py +4 -4
  325. endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
  326. endoreg_db/utils/check_video_files.py +74 -47
  327. endoreg_db/utils/cropping.py +10 -9
  328. endoreg_db/utils/dataloader.py +11 -3
  329. endoreg_db/utils/dates.py +3 -4
  330. endoreg_db/utils/defaults/set_default_center.py +7 -6
  331. endoreg_db/utils/env.py +6 -2
  332. endoreg_db/utils/extract_specific_frames.py +24 -9
  333. endoreg_db/utils/file_operations.py +30 -18
  334. endoreg_db/utils/fix_video_path_direct.py +57 -41
  335. endoreg_db/utils/frame_anonymization_utils.py +157 -157
  336. endoreg_db/utils/hashs.py +3 -18
  337. endoreg_db/utils/links/requirement_link.py +96 -52
  338. endoreg_db/utils/ocr.py +30 -25
  339. endoreg_db/utils/operation_log.py +61 -0
  340. endoreg_db/utils/parse_and_generate_yaml.py +12 -13
  341. endoreg_db/utils/paths.py +6 -6
  342. endoreg_db/utils/permissions.py +40 -24
  343. endoreg_db/utils/pipelines/process_video_dir.py +50 -26
  344. endoreg_db/utils/product/sum_emissions.py +5 -3
  345. endoreg_db/utils/product/sum_weights.py +4 -2
  346. endoreg_db/utils/pydantic_models/__init__.py +3 -4
  347. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
  348. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
  349. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
  350. endoreg_db/utils/setup_config.py +21 -5
  351. endoreg_db/utils/storage.py +3 -1
  352. endoreg_db/utils/translation.py +19 -15
  353. endoreg_db/utils/uuid.py +1 -0
  354. endoreg_db/utils/validate_endo_roi.py +12 -4
  355. endoreg_db/utils/validate_subcategory_dict.py +26 -24
  356. endoreg_db/utils/validate_video_detailed.py +207 -149
  357. endoreg_db/utils/video/__init__.py +7 -3
  358. endoreg_db/utils/video/extract_frames.py +30 -18
  359. endoreg_db/utils/video/ffmpeg_wrapper.py +217 -52
  360. endoreg_db/utils/video/names.py +11 -6
  361. endoreg_db/utils/video/streaming_processor.py +175 -101
  362. endoreg_db/utils/video/video_splitter.py +30 -19
  363. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
  364. endoreg_db/views/__init__.py +0 -20
  365. endoreg_db/views/anonymization/__init__.py +6 -2
  366. endoreg_db/views/anonymization/media_management.py +2 -6
  367. endoreg_db/views/anonymization/overview.py +34 -1
  368. endoreg_db/views/anonymization/validate.py +79 -18
  369. endoreg_db/views/auth/__init__.py +1 -1
  370. endoreg_db/views/auth/keycloak.py +16 -14
  371. endoreg_db/views/examination/__init__.py +12 -15
  372. endoreg_db/views/examination/examination.py +5 -5
  373. endoreg_db/views/examination/examination_manifest_cache.py +5 -5
  374. endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
  375. endoreg_db/views/examination/get_finding_classifications.py +9 -7
  376. endoreg_db/views/examination/get_findings.py +8 -10
  377. endoreg_db/views/examination/get_instruments.py +3 -2
  378. endoreg_db/views/examination/get_interventions.py +1 -1
  379. endoreg_db/views/finding/__init__.py +2 -2
  380. endoreg_db/views/finding/finding.py +58 -54
  381. endoreg_db/views/finding/get_classifications.py +1 -1
  382. endoreg_db/views/finding/get_interventions.py +1 -1
  383. endoreg_db/views/finding_classification/__init__.py +5 -5
  384. endoreg_db/views/finding_classification/finding_classification.py +5 -6
  385. endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
  386. endoreg_db/views/media/__init__.py +13 -13
  387. endoreg_db/views/media/pdf_media.py +9 -9
  388. endoreg_db/views/media/sensitive_metadata.py +10 -7
  389. endoreg_db/views/media/video_media.py +4 -4
  390. endoreg_db/views/meta/__init__.py +1 -1
  391. endoreg_db/views/meta/sensitive_meta_list.py +20 -22
  392. endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
  393. endoreg_db/views/misc/__init__.py +6 -34
  394. endoreg_db/views/misc/center.py +2 -1
  395. endoreg_db/views/misc/csrf.py +2 -1
  396. endoreg_db/views/misc/gender.py +2 -1
  397. endoreg_db/views/misc/stats.py +141 -106
  398. endoreg_db/views/patient/__init__.py +1 -3
  399. endoreg_db/views/patient/patient.py +141 -99
  400. endoreg_db/views/patient_examination/__init__.py +5 -5
  401. endoreg_db/views/patient_examination/patient_examination.py +43 -42
  402. endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
  403. endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
  404. endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
  405. endoreg_db/views/patient_examination/video.py +114 -80
  406. endoreg_db/views/patient_finding/__init__.py +1 -1
  407. endoreg_db/views/patient_finding/patient_finding.py +17 -10
  408. endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
  409. endoreg_db/views/patient_finding_classification/__init__.py +1 -1
  410. endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
  411. endoreg_db/views/report/reimport.py +1 -1
  412. endoreg_db/views/report/report_stream.py +5 -8
  413. endoreg_db/views/requirement/__init__.py +2 -1
  414. endoreg_db/views/requirement/evaluate.py +7 -9
  415. endoreg_db/views/requirement/lookup.py +2 -3
  416. endoreg_db/views/requirement/lookup_store.py +0 -1
  417. endoreg_db/views/requirement/requirement_utils.py +2 -4
  418. endoreg_db/views/stats/__init__.py +4 -4
  419. endoreg_db/views/stats/stats_views.py +152 -115
  420. endoreg_db/views/video/__init__.py +18 -27
  421. endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
  422. endoreg_db/views/{ai → video/ai}/label.py +20 -16
  423. endoreg_db/views/video/correction.py +5 -6
  424. endoreg_db/views/video/reimport.py +134 -99
  425. endoreg_db/views/video/segments_crud.py +134 -44
  426. endoreg_db/views/video/video_apply_mask.py +13 -12
  427. endoreg_db/views/video/video_correction.py +2 -1
  428. endoreg_db/views/video/video_download_processed.py +15 -15
  429. endoreg_db/views/video/video_meta_stats.py +7 -6
  430. endoreg_db/views/video/video_processing_history.py +3 -2
  431. endoreg_db/views/video/video_remove_frames.py +13 -12
  432. endoreg_db/views/video/video_stream.py +110 -82
  433. {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
  434. {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +436 -433
  435. endoreg_db/import_files/processing/video_processing/video_cleanup_on_error.py +0 -119
  436. endoreg_db/management/commands/import_fallback_video.py +0 -203
  437. endoreg_db/management/commands/import_video.py +0 -422
  438. endoreg_db/management/commands/import_video_with_classification.py +0 -367
  439. endoreg_db/models/media/processing_history/processing_history.py +0 -96
  440. endoreg_db/serializers/label/__init__.py +0 -7
  441. endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
  442. endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
  443. endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
  444. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
  445. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
  446. endoreg_db/services/__old/pdf_import.py +0 -1487
  447. endoreg_db/services/__old/video_import.py +0 -1306
  448. endoreg_db/tasks/upload_tasks.py +0 -216
  449. endoreg_db/tasks/video_ingest.py +0 -161
  450. endoreg_db/tasks/video_processing_tasks.py +0 -327
  451. endoreg_db/views/misc/translation.py +0 -182
  452. {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
  453. {endoreg_db-0.8.8.9.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
@@ -4,12 +4,13 @@ from endoreg_db.models import Label, LabelVideoSegment
4
4
  from itertools import combinations
5
5
  from pathlib import Path
6
6
  from django.conf import settings
7
+
7
8
  # === CONFIGURABLE PARAMETERS - ForNiceClassificationSerializer ===
8
9
  POLYP_LABEL_NAME = "polyp"
9
10
  CHROMO_LABEL_NAMES = ["digital_chromo_endoscopy", "nbi"]
10
11
  FPS = 50 # Frames per second- should fetch dynamically from videofile table
11
12
  # Sequence-level filtering
12
- MIN_SEQUENCE_GAP_SECONDS = 0.5 # Enforce diversity between sequences
13
+ MIN_SEQUENCE_GAP_SECONDS = 0.5 # Enforce diversity between sequences
13
14
  MIN_SEQUENCE_GAP_FRAMES = FPS * MIN_SEQUENCE_GAP_SECONDS # Convert to frame count
14
15
  # Minimum length of a segment in seconds
15
16
  MIN_SEGMENT_LENGTH_SECONDS = 0.5
@@ -27,13 +28,20 @@ FRAME_SELECTION_RULES = [
27
28
  lambda pred: pred.get("low_quality", 1.0) < 0.1,
28
29
  lambda pred: pred.get("outside", 1.0) < 0.1,
29
30
  lambda pred: pred.get("snare", 1.0) < 0.1,
30
-
31
31
  # Add more rules easily here
32
32
  ]
33
33
 
34
- REQUIRED_FRAME_KEYS = ["low_quality", "outside", "snare"] # need tp update when adding rules
35
-
36
- POLYP_CONFIDENCE_THRESHOLDS = [0.9, 0.8, 0.7] # basically this is the prediction value/score, we are using for frame selection
34
+ REQUIRED_FRAME_KEYS = [
35
+ "low_quality",
36
+ "outside",
37
+ "snare",
38
+ ] # need tp update when adding rules
39
+
40
+ POLYP_CONFIDENCE_THRESHOLDS = [
41
+ 0.9,
42
+ 0.8,
43
+ 0.7,
44
+ ] # basically this is the prediction value/score, we are using for frame selection
37
45
  INSTRUMENT_LABEL_NAMES = ["instrument", "snare", "needle"] # Add more as needed later
38
46
 
39
47
 
@@ -42,11 +50,11 @@ class BaseClassificationSerializer(serializers.Serializer):
42
50
  Base class for NICE and PARIS serializers.
43
51
  Handles label lookup, chromo/instrument segment filtering, and shared utilities.
44
52
  """
45
- #TODO add create method
46
- #TODO add update method
53
+
54
+ # TODO add create method
55
+ # TODO add update method
47
56
  LABEL_NAME = "polyp" # default (can be overridden)
48
- INSTRUMENT_LABEL_NAME = "instrument" #TODO @Hamzaukw we should define frequently used labels in a utils file
49
-
57
+ INSTRUMENT_LABEL_NAME = "instrument" # TODO @Hamzaukw we should define frequently used labels in a utils file
50
58
 
51
59
  def get_label_id_by_name(self, label_name):
52
60
  """
@@ -57,10 +65,10 @@ class BaseClassificationSerializer(serializers.Serializer):
57
65
  label = Label.objects.get(name=label_name)
58
66
  return label.id
59
67
  except Label.DoesNotExist:
60
- raise serializers.ValidationError({
61
- "error": f"Label with name '{label_name}' does not exist."
62
- })
63
-
68
+ raise serializers.ValidationError(
69
+ {"error": f"Label with name '{label_name}' does not exist."}
70
+ )
71
+
64
72
  def get_label_ids_by_names(self, label_names):
65
73
  """
66
74
  Get a list of label IDs for a given list of label names.
@@ -71,12 +79,12 @@ class BaseClassificationSerializer(serializers.Serializer):
71
79
 
72
80
  missing = set(label_names) - set(label_map.keys())
73
81
  if missing:
74
- raise serializers.ValidationError({
75
- "error": f"Labels not found: {', '.join(missing)}"
76
- })
82
+ raise serializers.ValidationError(
83
+ {"error": f"Labels not found: {', '.join(missing)}"}
84
+ )
77
85
 
78
86
  return list(label_map.values())
79
-
87
+
80
88
  def get_chromo_segments(self, video_id):
81
89
  """
82
90
  Fetch all segments that match chromo-like labels (e.g., chromo or NBI).
@@ -85,10 +93,10 @@ class BaseClassificationSerializer(serializers.Serializer):
85
93
  - PARIS: chromo overlap is disqualifying
86
94
  """
87
95
  chromo_label_ids = self.get_label_ids_by_names(CHROMO_LABEL_NAMES)
88
- return LabelVideoSegment.objects.filter(label_id__in=chromo_label_ids, video_file_id=video_id)
89
-
96
+ return LabelVideoSegment.objects.filter(
97
+ label_id__in=chromo_label_ids, video_file_id=video_id
98
+ )
90
99
 
91
-
92
100
  def get_filtered_polyp_segments(self, video_id):
93
101
  """
94
102
  Return polyp segments that do NOT overlap with 'instrument' segments.
@@ -96,7 +104,12 @@ class BaseClassificationSerializer(serializers.Serializer):
96
104
  polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
97
105
  instrument_label_id = self.get_label_id_by_name(self.INSTRUMENT_LABEL_NAME)
98
106
 
99
- print("polyp label id is :-", polyp_label_id,"and - instrument_label_id is ,", instrument_label_id)
107
+ print(
108
+ "polyp label id is :-",
109
+ polyp_label_id,
110
+ "and - instrument_label_id is ,",
111
+ instrument_label_id,
112
+ )
100
113
 
101
114
  polyp_segments = LabelVideoSegment.objects.filter(
102
115
  label_id=polyp_label_id, video_file_id=video_id
@@ -105,39 +118,61 @@ class BaseClassificationSerializer(serializers.Serializer):
105
118
  instrument_segments = LabelVideoSegment.objects.filter(
106
119
  label_id=instrument_label_id, video_file_id=video_id
107
120
  )
108
- print("------------------------ --------------------------------- -------------------------------")
109
- print("polyp label id is :-", polyp_label_id,"and - instrument_label_id is ,", instrument_label_id)
110
- print("polyp segments are",polyp_segments , "instrument_segments are", instrument_segments)
121
+ print(
122
+ "------------------------ --------------------------------- -------------------------------"
123
+ )
124
+ print(
125
+ "polyp label id is :-",
126
+ polyp_label_id,
127
+ "and - instrument_label_id is ,",
128
+ instrument_label_id,
129
+ )
130
+ print(
131
+ "polyp segments are",
132
+ polyp_segments,
133
+ "instrument_segments are",
134
+ instrument_segments,
135
+ )
111
136
 
112
137
  def overlaps(seg1, seg2):
113
- return seg1.start_frame_number <= seg2.end_frame_number and seg1.end_frame_number >= seg2.start_frame_number
138
+ return (
139
+ seg1.start_frame_number <= seg2.end_frame_number
140
+ and seg1.end_frame_number >= seg2.start_frame_number
141
+ )
114
142
 
115
143
  filtered_polyp_segments = []
116
144
  for polyp_seg in polyp_segments:
117
- if not any(overlaps(polyp_seg, instr_seg) for instr_seg in instrument_segments):
145
+ if not any(
146
+ overlaps(polyp_seg, instr_seg) for instr_seg in instrument_segments
147
+ ):
118
148
  filtered_polyp_segments.append(polyp_seg)
119
149
 
120
150
  return filtered_polyp_segments
121
151
 
122
152
  def get_polyp_segments(self, video_id):
123
153
  polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
124
- return LabelVideoSegment.objects.filter(label_id=polyp_label_id, video_file_id=video_id)
154
+ return LabelVideoSegment.objects.filter(
155
+ label_id=polyp_label_id, video_file_id=video_id
156
+ )
125
157
 
126
158
  def apply_sequence_diversity(self, matching_segments):
127
159
  if not matching_segments:
128
160
  return []
129
161
 
130
- matching_segments.sort(key=lambda seg: seg['polyp'].start_frame_number)
162
+ matching_segments.sort(key=lambda seg: seg["polyp"].start_frame_number)
131
163
 
132
164
  valid_segments = [
133
- seg for seg in matching_segments
134
- if (seg['polyp'].end_frame_number - seg['polyp'].start_frame_number) >= MIN_SEGMENT_LENGTH_FRAMES
165
+ seg
166
+ for seg in matching_segments
167
+ if (seg["polyp"].end_frame_number - seg["polyp"].start_frame_number)
168
+ >= MIN_SEGMENT_LENGTH_FRAMES
135
169
  ]
136
170
 
137
171
  if len(valid_segments) < 3:
138
172
  valid_segments.sort(
139
- key=lambda seg: seg['polyp'].end_frame_number - seg['polyp'].start_frame_number,
140
- reverse=True
173
+ key=lambda seg: seg["polyp"].end_frame_number
174
+ - seg["polyp"].start_frame_number,
175
+ reverse=True,
141
176
  )
142
177
  return valid_segments[:3]
143
178
 
@@ -145,12 +180,14 @@ class BaseClassificationSerializer(serializers.Serializer):
145
180
  best_gap_sum = -1
146
181
 
147
182
  for combo in combinations(valid_segments, 3):
148
- sorted_combo = sorted(combo, key=lambda seg: seg['polyp'].start_frame_number)
183
+ sorted_combo = sorted(
184
+ combo, key=lambda seg: seg["polyp"].start_frame_number
185
+ )
149
186
  s1, s2, s3 = sorted_combo
150
- s1_end = s1['polyp'].end_frame_number
151
- s2_start = s2['polyp'].start_frame_number
152
- s2_end = s2['polyp'].end_frame_number
153
- s3_start = s3['polyp'].start_frame_number
187
+ s1_end = s1["polyp"].end_frame_number
188
+ s2_start = s2["polyp"].start_frame_number
189
+ s2_end = s2["polyp"].end_frame_number
190
+ s3_start = s3["polyp"].start_frame_number
154
191
  gap1 = s2_start - s1_end
155
192
  gap2 = s3_start - s2_end
156
193
 
@@ -164,18 +201,19 @@ class BaseClassificationSerializer(serializers.Serializer):
164
201
  return list(best_combo)
165
202
 
166
203
  valid_segments.sort(
167
- key=lambda seg: seg['polyp'].end_frame_number - seg['polyp'].start_frame_number,
168
- reverse=True
204
+ key=lambda seg: seg["polyp"].end_frame_number
205
+ - seg["polyp"].start_frame_number,
206
+ reverse=True,
169
207
  )
170
208
  return valid_segments[:3]
171
-
209
+
172
210
  def select_frames_for_sequence(self, sequence):
173
211
  """
174
212
  Selects evenly spaced representative frames from a polyp segment.
175
-
213
+
176
214
  Frames are chosen from the segment's frame range, ensuring a minimum gap between selected frames. Returns a list of dictionaries containing frame numbers and their corresponding file paths. If the video's frame directory is unavailable, returns an empty list.
177
215
  """
178
- polyp_sequence = sequence['polyp']
216
+ polyp_sequence = sequence["polyp"]
179
217
  video = polyp_sequence.video_file
180
218
 
181
219
  start_frame = polyp_sequence.start_frame_number
@@ -190,28 +228,32 @@ class BaseClassificationSerializer(serializers.Serializer):
190
228
  segment_frames = [
191
229
  {
192
230
  "frame_number": idx,
193
- "frame_path": f"{frame_dir}/frame_{str(idx).zfill(7)}.png"
231
+ "frame_path": f"{frame_dir}/frame_{str(idx).zfill(7)}.png",
194
232
  }
195
233
  for idx in range(start_frame, end_frame + 1)
196
234
  ]
197
235
 
198
236
  selected_frames = []
199
- last_selected_frame = -float('inf')
237
+ last_selected_frame = -float("inf")
200
238
 
201
239
  for frame in segment_frames:
202
240
  if len(selected_frames) >= FRAMES_PER_SEQUENCE:
203
241
  break
204
242
 
205
243
  if frame["frame_number"] - last_selected_frame >= MIN_FRAME_GAP_IN_SEQUENCE:
206
- selected_frames.append({
207
- "frame_number": frame["frame_number"],
208
- "frame_path": frame["frame_path"]
209
- })
244
+ selected_frames.append(
245
+ {
246
+ "frame_number": frame["frame_number"],
247
+ "frame_path": frame["frame_path"],
248
+ }
249
+ )
210
250
  last_selected_frame = frame["frame_number"]
211
251
 
212
252
  return selected_frames
213
-
214
- def extract_and_save_selected_frames(self, video, frame_numbers, classification_type: str):
253
+
254
+ def extract_and_save_selected_frames(
255
+ self, video, frame_numbers, classification_type: str
256
+ ):
215
257
  """
216
258
  Extract specific frames from the original video file and save them
217
259
  into a structured folder path based on the classification type.
@@ -224,7 +266,13 @@ class BaseClassificationSerializer(serializers.Serializer):
224
266
  # Resolve the path to the original video
225
267
  original_path = Path(video.original_file_name)
226
268
  if not original_path.is_absolute():
227
- base_video_dir = settings.BASE_DIR.parent.parent / "production_test" / "endoreg-db" / "data" / "coloreg_first_test_batch"
269
+ base_video_dir = (
270
+ settings.BASE_DIR.parent.parent
271
+ / "production_test"
272
+ / "endoreg-db"
273
+ / "data"
274
+ / "coloreg_first_test_batch"
275
+ )
228
276
  original_path = base_video_dir / original_path
229
277
 
230
278
  # Define output directory based on classification and video ID
@@ -235,11 +283,10 @@ class BaseClassificationSerializer(serializers.Serializer):
235
283
  video_path=original_path,
236
284
  frame_numbers=frame_numbers,
237
285
  output_dir=output_path,
238
- fps=FPS
286
+ fps=FPS,
239
287
  )
240
288
 
241
-
242
- ''' # # If any error occurs in this function, use the commented one below(fallback:select_frames_for_sequence).
289
+ """ # # If any error occurs in this function, use the commented one below(fallback:select_frames_for_sequence).
243
290
  def select_frames_for_sequence(self, sequence):
244
291
  # Extract the polyp segment from the current sequence
245
292
  polyp_sequence = sequence['polyp']
@@ -318,11 +365,10 @@ class BaseClassificationSerializer(serializers.Serializer):
318
365
  return selected_frames
319
366
 
320
367
  # If no threshold yielded enough frames, return what was found in the last attempt (could be empty)
321
- return selected_frames if selected_frames else []'''
322
-
368
+ return selected_frames if selected_frames else []"""
323
369
 
324
370
  # fallback:select_frames_for_sequence
325
- '''
371
+ """
326
372
  def select_frames_for_sequence(self, sequence):
327
373
  print("----------------------in selected_frames fro sequnces funtion ----------------------------------------")
328
374
  polyp_sequence = sequence['polyp']
@@ -365,7 +411,9 @@ class BaseClassificationSerializer(serializers.Serializer):
365
411
  })
366
412
  last_selected_frame = frame["frame_number"]
367
413
 
368
- return selected_frames'''
414
+ return selected_frames"""
415
+
416
+
369
417
  class ForNiceClassificationSerializer(BaseClassificationSerializer):
370
418
  """
371
419
  NICE classification logic:
@@ -374,6 +422,7 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
374
422
  - Requires overlap with chromo-like labels (chromo/NBI)
375
423
  - Selects diverse sequences and representative frames from each
376
424
  """
425
+
377
426
  def get_matching_sequences(self, video_id):
378
427
  """
379
428
  1. Fetch all polyp segments in a video
@@ -383,20 +432,28 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
383
432
  """
384
433
  polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
385
434
  instrument_label_ids = self.get_label_ids_by_names(INSTRUMENT_LABEL_NAMES)
386
- #chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
435
+ # chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
387
436
  chromo_segments = self.get_chromo_segments(video_id)
388
437
 
438
+ polyp_segments = LabelVideoSegment.objects.filter(
439
+ label_id=polyp_label_id, video_file_id=video_id
440
+ )
441
+ instrument_segments = LabelVideoSegment.objects.filter(
442
+ label_id__in=instrument_label_ids, video_file_id=video_id
443
+ )
389
444
 
390
- polyp_segments = LabelVideoSegment.objects.filter(label_id=polyp_label_id, video_file_id=video_id)
391
- instrument_segments = LabelVideoSegment.objects.filter(label_id__in=instrument_label_ids, video_file_id=video_id)
392
-
393
- #chromo_segments = LabelVideoSegment.objects.filter(label_id=chromo_label_id, video_file_id=video_id)
394
- print("polyp_label_id is", polyp_label_id, "and ployp_segment is ", polyp_segments)
445
+ # chromo_segments = LabelVideoSegment.objects.filter(label_id=chromo_label_id, video_file_id=video_id)
446
+ print(
447
+ "polyp_label_id is", polyp_label_id, "and ployp_segment is ", polyp_segments
448
+ )
395
449
  print("instrument_segments is", instrument_segments)
396
- #print("chromo_segments is ",chromo_segments)
450
+ # print("chromo_segments is ",chromo_segments)
397
451
 
398
452
  def overlaps(seg1, seg2):
399
- return seg1.start_frame_number <= seg2.end_frame_number and seg1.end_frame_number >= seg2.start_frame_number
453
+ return (
454
+ seg1.start_frame_number <= seg2.end_frame_number
455
+ and seg1.end_frame_number >= seg2.start_frame_number
456
+ )
400
457
 
401
458
  def subtract_overlap(seg, overlaps):
402
459
  start = seg.start_frame_number
@@ -424,7 +481,9 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
424
481
  matching_segments = []
425
482
 
426
483
  for polyp in polyp_segments:
427
- overlapping_instr = [seg for seg in instrument_segments if overlaps(polyp, seg)]
484
+ overlapping_instr = [
485
+ seg for seg in instrument_segments if overlaps(polyp, seg)
486
+ ]
428
487
  safe_ranges = subtract_overlap(polyp, overlapping_instr)
429
488
 
430
489
  for start, end in safe_ranges:
@@ -432,7 +491,10 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
432
491
  continue
433
492
 
434
493
  for chromo in chromo_segments:
435
- if chromo.start_frame_number <= end and chromo.end_frame_number >= start:
494
+ if (
495
+ chromo.start_frame_number <= end
496
+ and chromo.end_frame_number >= start
497
+ ):
436
498
  overlap_start = max(start, chromo.start_frame_number)
437
499
  overlap_end = min(end, chromo.end_frame_number)
438
500
 
@@ -440,9 +502,9 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
440
502
  overlapping_segment = LabelVideoSegment(
441
503
  video_file=polyp.video_file,
442
504
  start_frame_number=overlap_start,
443
- end_frame_number=overlap_end
505
+ end_frame_number=overlap_end,
444
506
  )
445
- matching_segments.append({'polyp': overlapping_segment})
507
+ matching_segments.append({"polyp": overlapping_segment})
446
508
 
447
509
  return matching_segments
448
510
 
@@ -465,18 +527,22 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
465
527
  matching_segments = self.get_matching_sequences(video_id)
466
528
  print("matching_segments are", matching_segments)
467
529
  if not matching_segments:
468
- results.append({
469
- "video_id": video_id,
470
- "message": "No valid polyp segments overlapping with chromo and not overlapping with instrument."
471
- })
530
+ results.append(
531
+ {
532
+ "video_id": video_id,
533
+ "message": "No valid polyp segments overlapping with chromo and not overlapping with instrument.",
534
+ }
535
+ )
472
536
  continue
473
537
 
474
538
  diverse_segments = self.apply_sequence_diversity(matching_segments)
475
539
  if not diverse_segments:
476
- results.append({
477
- "video_id": video_id,
478
- "message": "No diverse NICE sequences found — either too short or too close together."
479
- })
540
+ results.append(
541
+ {
542
+ "video_id": video_id,
543
+ "message": "No diverse NICE sequences found — either too short or too close together.",
544
+ }
545
+ )
480
546
  continue
481
547
 
482
548
  # Collect all frame numbers to extract only once per video
@@ -487,49 +553,54 @@ class ForNiceClassificationSerializer(BaseClassificationSerializer):
487
553
  frames = self.select_frames_for_sequence(segment)
488
554
 
489
555
  if not frames:
490
- segment_results.append({
491
- "video_id": video_id,
492
- "segment_start": segment['polyp'].start_frame_number,
493
- "segment_end": segment['polyp'].end_frame_number,
494
- "message": "No valid frames passed filtering rules (low_quality, outside, snare)."
495
- })
556
+ segment_results.append(
557
+ {
558
+ "video_id": video_id,
559
+ "segment_start": segment["polyp"].start_frame_number,
560
+ "segment_end": segment["polyp"].end_frame_number,
561
+ "message": "No valid frames passed filtering rules (low_quality, outside, snare).",
562
+ }
563
+ )
496
564
  else:
497
- segment_results.append({
498
- "video_id": video_id,
499
- "segment_start": segment['polyp'].start_frame_number,
500
- "segment_end": segment['polyp'].end_frame_number,
501
- "frames": frames
502
- })
565
+ segment_results.append(
566
+ {
567
+ "video_id": video_id,
568
+ "segment_start": segment["polyp"].start_frame_number,
569
+ "segment_end": segment["polyp"].end_frame_number,
570
+ "frames": frames,
571
+ }
572
+ )
503
573
  all_frames.extend(frames)
504
574
 
505
575
  # Extract once per video
506
576
  if all_frames:
507
- unique_frame_numbers = sorted({f["frame_number"] for f in all_frames})
508
- classification_type = "nice" if isinstance(self, ForNiceClassificationSerializer) else "paris"
509
- self.extract_and_save_selected_frames(video, unique_frame_numbers, classification_type)
577
+ unique_frame_numbers = sorted(
578
+ {f["frame_number"] for f in all_frames}
579
+ )
580
+ classification_type = (
581
+ "nice"
582
+ if isinstance(self, ForNiceClassificationSerializer)
583
+ else "paris"
584
+ )
585
+ self.extract_and_save_selected_frames(
586
+ video, unique_frame_numbers, classification_type
587
+ )
510
588
 
511
589
  results.extend(segment_results)
512
590
 
513
591
  except Exception as e:
514
- results.append({
515
- "video_id": video.id,
516
- "error": str(e)
517
- })
592
+ results.append({"video_id": video.id, "error": str(e)})
518
593
 
519
594
  if not results:
520
- return [{
521
- "message": "No classification data generated for any videos."
522
- }]
595
+ return [{"message": "No classification data generated for any videos."}]
523
596
 
524
597
  return {
525
- "message": "NICE classification data generated." if isinstance(self, ForNiceClassificationSerializer)
598
+ "message": "NICE classification data generated."
599
+ if isinstance(self, ForNiceClassificationSerializer)
526
600
  else "PARIS classification data generated.",
527
- "data": results
601
+ "data": results,
528
602
  }
529
603
 
530
-
531
-
532
-
533
604
 
534
605
  class ForParisClassificationSerializer(BaseClassificationSerializer):
535
606
  """
@@ -540,26 +611,32 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
540
611
  - Enforces minimum length and sequence diversity
541
612
  - Selects frames from each valid segment
542
613
  """
543
-
614
+
544
615
  def get_filtered_polyp_segments(self, video_id):
545
616
  """
546
617
  1. Fetch all polyp segments
547
618
  2. Subtract any overlap with chromo or instrument segments
548
619
  3. Keep only trimmed segments longer than the minimum threshold
549
620
  """
550
-
621
+
551
622
  polyp_label_id = self.get_label_id_by_name(self.LABEL_NAME)
552
- instrument_label_ids = self.get_label_ids_by_names(INSTRUMENT_LABEL_NAMES)
553
- #chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
623
+ instrument_label_ids = self.get_label_ids_by_names(INSTRUMENT_LABEL_NAMES)
624
+ # chromo_label_id = self.get_label_id_by_name(CHROMO_LABEL_NAME)
554
625
 
555
- polyp_segments = LabelVideoSegment.objects.filter(label_id=polyp_label_id, video_file_id=video_id)
556
- instrument_segments = LabelVideoSegment.objects.filter(label_id__in=instrument_label_ids, video_file_id=video_id)
557
- #chromo_segments = LabelVideoSegment.objects.filter(label_id=chromo_label_id, video_file_id=video_id)
626
+ polyp_segments = LabelVideoSegment.objects.filter(
627
+ label_id=polyp_label_id, video_file_id=video_id
628
+ )
629
+ instrument_segments = LabelVideoSegment.objects.filter(
630
+ label_id__in=instrument_label_ids, video_file_id=video_id
631
+ )
632
+ # chromo_segments = LabelVideoSegment.objects.filter(label_id=chromo_label_id, video_file_id=video_id)
558
633
  chromo_segments = self.get_chromo_segments(video_id)
559
634
 
560
-
561
635
  def overlaps(seg1, seg2):
562
- return seg1.start_frame_number <= seg2.end_frame_number and seg1.end_frame_number >= seg2.start_frame_number
636
+ return (
637
+ seg1.start_frame_number <= seg2.end_frame_number
638
+ and seg1.end_frame_number >= seg2.start_frame_number
639
+ )
563
640
 
564
641
  def subtract_overlap(seg, overlaps):
565
642
  """
@@ -594,7 +671,8 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
594
671
  for polyp_seg in polyp_segments:
595
672
  # Collect all overlapping regions with instrument or chromo
596
673
  overlapping = [
597
- seg for seg in (list(instrument_segments) + list(chromo_segments))
674
+ seg
675
+ for seg in (list(instrument_segments) + list(chromo_segments))
598
676
  if overlaps(polyp_seg, seg)
599
677
  ]
600
678
 
@@ -602,18 +680,23 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
602
680
 
603
681
  for start, end in safe_ranges:
604
682
  if end - start + 1 >= MIN_SEGMENT_LENGTH_FRAMES:
605
- trimmed_segments.append(LabelVideoSegment(
606
- video_file=polyp_seg.video_file,
607
- start_frame_number=start,
608
- end_frame_number=end
609
- ))
683
+ trimmed_segments.append(
684
+ LabelVideoSegment(
685
+ video_file=polyp_seg.video_file,
686
+ start_frame_number=start,
687
+ end_frame_number=end,
688
+ )
689
+ )
610
690
 
611
691
  return trimmed_segments
612
692
 
613
693
  filtered_segments = []
614
694
  for polyp_seg in polyp_segments:
615
- if not any(overlaps(polyp_seg, instr_seg) for instr_seg in instrument_segments) and \
616
- not any(overlaps(polyp_seg, chromo_seg) for chromo_seg in chromo_segments):
695
+ if not any(
696
+ overlaps(polyp_seg, instr_seg) for instr_seg in instrument_segments
697
+ ) and not any(
698
+ overlaps(polyp_seg, chromo_seg) for chromo_seg in chromo_segments
699
+ ):
617
700
  filtered_segments.append(polyp_seg)
618
701
 
619
702
  return filtered_segments
@@ -624,9 +707,8 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
624
707
  This is needed for consistent downstream processing.
625
708
  """
626
709
  segments = self.get_filtered_polyp_segments(video_id)
627
- return [{'polyp': seg} for seg in segments]
710
+ return [{"polyp": seg} for seg in segments]
628
711
 
629
-
630
712
  def to_representation(self, videos):
631
713
  """
632
714
  Processes videos for PARIS classification.
@@ -642,18 +724,22 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
642
724
  matching_segments = self.get_matching_sequences(video_id)
643
725
 
644
726
  if not matching_segments:
645
- results.append({
646
- "video_id": video_id,
647
- "message": "No matching polyp segments found — possibly filtered due to overlap with 'instrument' label."
648
- })
727
+ results.append(
728
+ {
729
+ "video_id": video_id,
730
+ "message": "No matching polyp segments found — possibly filtered due to overlap with 'instrument' label.",
731
+ }
732
+ )
649
733
  continue
650
734
 
651
735
  diverse_segments = self.apply_sequence_diversity(matching_segments)
652
736
  if not diverse_segments:
653
- results.append({
654
- "video_id": video_id,
655
- "message": "No diverse sequences found — either too short or too close to each other."
656
- })
737
+ results.append(
738
+ {
739
+ "video_id": video_id,
740
+ "message": "No diverse sequences found — either too short or too close to each other.",
741
+ }
742
+ )
657
743
  continue
658
744
 
659
745
  all_frames = []
@@ -663,43 +749,46 @@ class ForParisClassificationSerializer(BaseClassificationSerializer):
663
749
  frames = self.select_frames_for_sequence(segment)
664
750
 
665
751
  if not frames:
666
- segment_results.append({
667
- "video_id": video_id,
668
- "segment_start": segment['polyp'].start_frame_number,
669
- "segment_end": segment['polyp'].end_frame_number,
670
- "message": "No valid frames selected due to prediction rules (e.g., quality, outside, snare)."
671
- })
752
+ segment_results.append(
753
+ {
754
+ "video_id": video_id,
755
+ "segment_start": segment["polyp"].start_frame_number,
756
+ "segment_end": segment["polyp"].end_frame_number,
757
+ "message": "No valid frames selected due to prediction rules (e.g., quality, outside, snare).",
758
+ }
759
+ )
672
760
  else:
673
- segment_results.append({
674
- "video_id": video_id,
675
- "segment_start": segment['polyp'].start_frame_number,
676
- "segment_end": segment['polyp'].end_frame_number,
677
- "frames": frames
678
- })
761
+ segment_results.append(
762
+ {
763
+ "video_id": video_id,
764
+ "segment_start": segment["polyp"].start_frame_number,
765
+ "segment_end": segment["polyp"].end_frame_number,
766
+ "frames": frames,
767
+ }
768
+ )
679
769
  all_frames.extend(frames)
680
770
 
681
771
  if all_frames:
682
- unique_frame_numbers = sorted({f["frame_number"] for f in all_frames})
683
- self.extract_and_save_selected_frames(video, unique_frame_numbers, classification_type="paris")
772
+ unique_frame_numbers = sorted(
773
+ {f["frame_number"] for f in all_frames}
774
+ )
775
+ self.extract_and_save_selected_frames(
776
+ video, unique_frame_numbers, classification_type="paris"
777
+ )
684
778
 
685
779
  results.extend(segment_results)
686
780
 
687
781
  except Exception as e:
688
- results.append({
689
- "video_id": video.id,
690
- "error": str(e)
691
- })
782
+ results.append({"video_id": video.id, "error": str(e)})
692
783
 
693
784
  if not results:
694
- return [{
695
- "message": "No valid classification results could be generated for any video."
696
- }]
697
-
698
- return {
699
- "message": "PARIS classification data generated.",
700
- "data": results
701
- }
785
+ return [
786
+ {
787
+ "message": "No valid classification results could be generated for any video."
788
+ }
789
+ ]
702
790
 
791
+ return {"message": "PARIS classification data generated.", "data": results}
703
792
 
704
793
 
705
794
  """
@@ -750,10 +839,7 @@ Returns:
750
839
  """
751
840
 
752
841
 
753
-
754
-
755
-
756
- '''
842
+ """
757
843
  await import('https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js');
758
844
  const fetchNiceClassification = async () => {
759
845
  try {
@@ -772,4 +858,4 @@ const fetchNiceClassification = async () => {
772
858
  };
773
859
 
774
860
  fetchNiceClassification().then(data => console.log("Final Output:", data));
775
- '''
861
+ """