endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (372) hide show
  1. endoreg_db/authz/auth.py +74 -0
  2. endoreg_db/authz/backends.py +168 -0
  3. endoreg_db/authz/management/commands/list_routes.py +18 -0
  4. endoreg_db/authz/middleware.py +83 -0
  5. endoreg_db/authz/permissions.py +127 -0
  6. endoreg_db/authz/policy.py +218 -0
  7. endoreg_db/authz/views_auth.py +66 -0
  8. endoreg_db/config/env.py +13 -8
  9. endoreg_db/data/__init__.py +8 -31
  10. endoreg_db/data/_examples/disease.yaml +55 -0
  11. endoreg_db/data/_examples/disease_classification.yaml +13 -0
  12. endoreg_db/data/_examples/disease_classification_choice.yaml +62 -0
  13. endoreg_db/data/_examples/event.yaml +64 -0
  14. endoreg_db/data/_examples/examination.yaml +72 -0
  15. endoreg_db/data/_examples/finding/anatomy_colon.yaml +128 -0
  16. endoreg_db/data/_examples/finding/colonoscopy.yaml +40 -0
  17. endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +56 -0
  18. endoreg_db/data/_examples/finding/complication.yaml +16 -0
  19. endoreg_db/data/_examples/finding/data.yaml +105 -0
  20. endoreg_db/data/_examples/finding/examination_setting.yaml +16 -0
  21. endoreg_db/data/_examples/finding/medication_related.yaml +18 -0
  22. endoreg_db/data/_examples/finding/outcome.yaml +12 -0
  23. endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +68 -0
  24. endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +22 -0
  25. endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +25 -0
  26. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +20 -0
  27. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +24 -0
  28. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +68 -0
  29. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +20 -0
  30. endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +80 -0
  31. endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +21 -0
  32. endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +20 -0
  33. endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +26 -0
  34. endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +22 -0
  35. endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +53 -0
  36. endoreg_db/data/_examples/finding_classification/complication_generic.yaml +25 -0
  37. endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +40 -0
  38. endoreg_db/data/_examples/finding_classification/histology_colo.yaml +51 -0
  39. endoreg_db/data/_examples/finding_classification/intervention_required.yaml +26 -0
  40. endoreg_db/data/_examples/finding_classification/medication_related.yaml +23 -0
  41. endoreg_db/data/_examples/finding_classification/visualized.yaml +33 -0
  42. endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +78 -0
  43. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +32 -0
  44. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +15 -0
  45. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +23 -0
  46. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +15 -0
  47. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +17 -0
  48. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_paris.yaml +57 -0
  49. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +49 -0
  50. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +14 -0
  51. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +36 -0
  52. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_location.yaml +229 -0
  53. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml +19 -0
  54. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +82 -0
  55. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +15 -0
  56. endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +15 -0
  57. endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +15 -0
  58. endoreg_db/data/_examples/finding_classification_choice/histology.yaml +24 -0
  59. endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +20 -0
  60. endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +19 -0
  61. endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +11 -0
  62. endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +48 -0
  63. endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +43 -0
  64. endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +168 -0
  65. endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +128 -0
  66. endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +32 -0
  67. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +9 -0
  68. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +36 -0
  69. endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +15 -0
  70. endoreg_db/data/_examples/finding_type/data.yaml +43 -0
  71. endoreg_db/data/_examples/requirement/age.yaml +26 -0
  72. endoreg_db/data/_examples/requirement/colonoscopy_baseline_austria.yaml +45 -0
  73. endoreg_db/data/_examples/requirement/disease_cardiovascular.yaml +79 -0
  74. endoreg_db/data/_examples/requirement/disease_classification_choice_cardiovascular.yaml +41 -0
  75. endoreg_db/data/_examples/requirement/disease_hepatology.yaml +12 -0
  76. endoreg_db/data/_examples/requirement/disease_misc.yaml +12 -0
  77. endoreg_db/data/_examples/requirement/disease_renal.yaml +96 -0
  78. endoreg_db/data/_examples/requirement/endoscopy_bleeding_risk.yaml +59 -0
  79. endoreg_db/data/_examples/requirement/event_cardiology.yaml +251 -0
  80. endoreg_db/data/_examples/requirement/event_requirements.yaml +145 -0
  81. endoreg_db/data/_examples/requirement/finding_colon_polyp.yaml +50 -0
  82. endoreg_db/data/_examples/requirement/gender.yaml +25 -0
  83. endoreg_db/data/_examples/requirement/lab_value.yaml +441 -0
  84. endoreg_db/data/_examples/requirement/medication.yaml +93 -0
  85. endoreg_db/data/_examples/requirement_operator/age.yaml +13 -0
  86. endoreg_db/data/_examples/requirement_operator/lab_operators.yaml +129 -0
  87. endoreg_db/data/_examples/requirement_operator/model_operators.yaml +96 -0
  88. endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +48 -0
  89. endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +57 -0
  90. endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
  91. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +4 -3
  92. endoreg_db/data/event_classification/data.yaml +4 -0
  93. endoreg_db/data/event_classification_choice/data.yaml +9 -0
  94. endoreg_db/data/finding_classification/colonoscopy_bowel_preparation.yaml +43 -70
  95. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +22 -52
  96. endoreg_db/data/finding_classification/colonoscopy_location.yaml +31 -62
  97. endoreg_db/data/finding_classification/histology_colo.yaml +28 -36
  98. endoreg_db/data/requirement/colon_polyp_intervention.yaml +49 -0
  99. endoreg_db/data/requirement/coloreg_colon_polyp.yaml +49 -0
  100. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +31 -12
  101. endoreg_db/data/requirement_set/01_laboratory.yaml +13 -0
  102. endoreg_db/data/requirement_set/02_endoscopy_bleeding_risk.yaml +46 -0
  103. endoreg_db/data/requirement_set/90_coloreg.yaml +178 -0
  104. endoreg_db/data/requirement_set/_old_ +109 -0
  105. endoreg_db/data/requirement_set_type/data.yaml +21 -0
  106. endoreg_db/data/setup_config.yaml +4 -4
  107. endoreg_db/data/tag/requirement_set_tags.yaml +21 -0
  108. endoreg_db/exceptions.py +5 -2
  109. endoreg_db/helpers/data_loader.py +1 -1
  110. endoreg_db/management/commands/create_model_meta_from_huggingface.py +21 -10
  111. endoreg_db/management/commands/create_multilabel_model_meta.py +299 -129
  112. endoreg_db/management/commands/import_video.py +9 -10
  113. endoreg_db/management/commands/import_video_with_classification.py +1 -1
  114. endoreg_db/management/commands/init_default_ai_model.py +1 -1
  115. endoreg_db/management/commands/list_routes.py +18 -0
  116. endoreg_db/management/commands/load_ai_model_data.py +2 -1
  117. endoreg_db/management/commands/load_center_data.py +12 -12
  118. endoreg_db/management/commands/load_requirement_data.py +60 -31
  119. endoreg_db/management/commands/load_requirement_set_tags.py +95 -0
  120. endoreg_db/management/commands/setup_endoreg_db.py +14 -10
  121. endoreg_db/management/commands/storage_management.py +271 -203
  122. endoreg_db/migrations/0001_initial.py +1799 -1300
  123. endoreg_db/migrations/0002_requirementset_depends_on.py +18 -0
  124. endoreg_db/migrations/_old/0001_initial.py +1857 -0
  125. endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +68 -0
  126. endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +77 -0
  127. endoreg_db/migrations/_old/0005_merge_20251111_1003.py +14 -0
  128. endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +68 -0
  129. endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +89 -0
  130. endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +27 -0
  131. endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +21 -0
  132. endoreg_db/models/__init__.py +78 -123
  133. endoreg_db/models/administration/__init__.py +21 -42
  134. endoreg_db/models/administration/ai/active_model.py +2 -2
  135. endoreg_db/models/administration/ai/ai_model.py +7 -6
  136. endoreg_db/models/administration/case/__init__.py +1 -15
  137. endoreg_db/models/administration/case/case.py +3 -3
  138. endoreg_db/models/administration/case/case_template/__init__.py +2 -14
  139. endoreg_db/models/administration/case/case_template/case_template.py +2 -124
  140. endoreg_db/models/administration/case/case_template/case_template_rule.py +2 -268
  141. endoreg_db/models/administration/case/case_template/case_template_rule_value.py +2 -85
  142. endoreg_db/models/administration/case/case_template/case_template_type.py +2 -25
  143. endoreg_db/models/administration/center/center.py +33 -19
  144. endoreg_db/models/administration/center/center_product.py +12 -9
  145. endoreg_db/models/administration/center/center_resource.py +25 -19
  146. endoreg_db/models/administration/center/center_shift.py +21 -17
  147. endoreg_db/models/administration/center/center_waste.py +16 -8
  148. endoreg_db/models/administration/person/__init__.py +2 -0
  149. endoreg_db/models/administration/person/employee/employee.py +10 -5
  150. endoreg_db/models/administration/person/employee/employee_qualification.py +9 -4
  151. endoreg_db/models/administration/person/employee/employee_type.py +12 -6
  152. endoreg_db/models/administration/person/examiner/examiner.py +13 -11
  153. endoreg_db/models/administration/person/patient/__init__.py +2 -0
  154. endoreg_db/models/administration/person/patient/patient.py +103 -100
  155. endoreg_db/models/administration/person/patient/patient_external_id.py +37 -0
  156. endoreg_db/models/administration/person/person.py +4 -0
  157. endoreg_db/models/administration/person/profession/__init__.py +8 -4
  158. endoreg_db/models/administration/person/user/portal_user_information.py +11 -7
  159. endoreg_db/models/administration/product/product.py +20 -15
  160. endoreg_db/models/administration/product/product_material.py +17 -18
  161. endoreg_db/models/administration/product/product_weight.py +12 -8
  162. endoreg_db/models/administration/product/reference_product.py +23 -55
  163. endoreg_db/models/administration/qualification/qualification.py +7 -3
  164. endoreg_db/models/administration/qualification/qualification_type.py +7 -3
  165. endoreg_db/models/administration/shift/scheduled_days.py +8 -5
  166. endoreg_db/models/administration/shift/shift.py +16 -12
  167. endoreg_db/models/administration/shift/shift_type.py +23 -31
  168. endoreg_db/models/label/__init__.py +7 -8
  169. endoreg_db/models/label/annotation/image_classification.py +10 -9
  170. endoreg_db/models/label/annotation/video_segmentation_annotation.py +8 -5
  171. endoreg_db/models/label/label.py +15 -15
  172. endoreg_db/models/label/label_set.py +19 -6
  173. endoreg_db/models/label/label_type.py +1 -1
  174. endoreg_db/models/label/label_video_segment/_create_from_video.py +5 -8
  175. endoreg_db/models/label/label_video_segment/label_video_segment.py +76 -102
  176. endoreg_db/models/label/video_segmentation_label.py +4 -0
  177. endoreg_db/models/label/video_segmentation_labelset.py +4 -3
  178. endoreg_db/models/media/frame/frame.py +22 -22
  179. endoreg_db/models/media/pdf/raw_pdf.py +249 -177
  180. endoreg_db/models/media/pdf/report_file.py +25 -29
  181. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +30 -46
  182. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +23 -7
  183. endoreg_db/models/media/video/__init__.py +1 -0
  184. endoreg_db/models/media/video/create_from_file.py +48 -56
  185. endoreg_db/models/media/video/pipe_1.py +30 -33
  186. endoreg_db/models/media/video/pipe_2.py +8 -9
  187. endoreg_db/models/media/video/video_file.py +359 -204
  188. endoreg_db/models/media/video/video_file_ai.py +288 -74
  189. endoreg_db/models/media/video/video_file_anonymize.py +38 -38
  190. endoreg_db/models/media/video/video_file_frames/__init__.py +3 -1
  191. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -8
  192. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +7 -9
  193. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +9 -8
  194. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +38 -45
  195. endoreg_db/models/media/video/video_file_frames/_get_frame.py +6 -8
  196. endoreg_db/models/media/video/video_file_frames/_get_frame_number.py +4 -18
  197. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -3
  198. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +7 -6
  199. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +6 -8
  200. endoreg_db/models/media/video/video_file_frames/_get_frames.py +6 -8
  201. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +15 -25
  202. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +26 -23
  203. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +23 -14
  204. endoreg_db/models/media/video/video_file_io.py +109 -62
  205. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +3 -3
  206. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +5 -3
  207. endoreg_db/models/media/video/video_file_meta/get_fps.py +37 -34
  208. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +19 -25
  209. endoreg_db/models/media/video/video_file_meta/text_meta.py +41 -38
  210. endoreg_db/models/media/video/video_file_meta/video_meta.py +14 -7
  211. endoreg_db/models/media/video/video_file_segments.py +24 -17
  212. endoreg_db/models/media/video/video_metadata.py +19 -35
  213. endoreg_db/models/media/video/video_processing.py +96 -95
  214. endoreg_db/models/medical/contraindication/__init__.py +13 -3
  215. endoreg_db/models/medical/disease.py +22 -16
  216. endoreg_db/models/medical/event.py +31 -18
  217. endoreg_db/models/medical/examination/__init__.py +13 -6
  218. endoreg_db/models/medical/examination/examination.py +17 -18
  219. endoreg_db/models/medical/examination/examination_indication.py +26 -25
  220. endoreg_db/models/medical/examination/examination_time.py +16 -6
  221. endoreg_db/models/medical/examination/examination_time_type.py +9 -6
  222. endoreg_db/models/medical/examination/examination_type.py +3 -4
  223. endoreg_db/models/medical/finding/finding.py +38 -39
  224. endoreg_db/models/medical/finding/finding_classification.py +37 -48
  225. endoreg_db/models/medical/finding/finding_intervention.py +27 -22
  226. endoreg_db/models/medical/finding/finding_type.py +13 -12
  227. endoreg_db/models/medical/hardware/endoscope.py +20 -26
  228. endoreg_db/models/medical/hardware/endoscopy_processor.py +2 -2
  229. endoreg_db/models/medical/laboratory/lab_value.py +62 -91
  230. endoreg_db/models/medical/medication/medication.py +22 -10
  231. endoreg_db/models/medical/medication/medication_indication.py +29 -3
  232. endoreg_db/models/medical/medication/medication_indication_type.py +25 -14
  233. endoreg_db/models/medical/medication/medication_intake_time.py +31 -19
  234. endoreg_db/models/medical/medication/medication_schedule.py +27 -16
  235. endoreg_db/models/medical/organ/__init__.py +15 -12
  236. endoreg_db/models/medical/patient/medication_examples.py +1 -5
  237. endoreg_db/models/medical/patient/patient_disease.py +20 -23
  238. endoreg_db/models/medical/patient/patient_event.py +19 -22
  239. endoreg_db/models/medical/patient/patient_examination.py +48 -54
  240. endoreg_db/models/medical/patient/patient_examination_indication.py +16 -14
  241. endoreg_db/models/medical/patient/patient_finding.py +122 -139
  242. endoreg_db/models/medical/patient/patient_finding_classification.py +44 -49
  243. endoreg_db/models/medical/patient/patient_finding_intervention.py +8 -19
  244. endoreg_db/models/medical/patient/patient_lab_sample.py +28 -23
  245. endoreg_db/models/medical/patient/patient_lab_value.py +82 -89
  246. endoreg_db/models/medical/patient/patient_medication.py +27 -38
  247. endoreg_db/models/medical/patient/patient_medication_schedule.py +28 -36
  248. endoreg_db/models/medical/risk/risk.py +7 -6
  249. endoreg_db/models/medical/risk/risk_type.py +8 -5
  250. endoreg_db/models/metadata/model_meta.py +60 -29
  251. endoreg_db/models/metadata/model_meta_logic.py +139 -18
  252. endoreg_db/models/metadata/pdf_meta.py +19 -24
  253. endoreg_db/models/metadata/sensitive_meta.py +102 -85
  254. endoreg_db/models/metadata/sensitive_meta_logic.py +383 -43
  255. endoreg_db/models/metadata/video_meta.py +51 -31
  256. endoreg_db/models/metadata/video_prediction_logic.py +16 -23
  257. endoreg_db/models/metadata/video_prediction_meta.py +29 -33
  258. endoreg_db/models/other/distribution/date_value_distribution.py +89 -29
  259. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +21 -5
  260. endoreg_db/models/other/distribution/numeric_value_distribution.py +114 -53
  261. endoreg_db/models/other/distribution/single_categorical_value_distribution.py +4 -3
  262. endoreg_db/models/other/emission/emission_factor.py +18 -8
  263. endoreg_db/models/other/gender.py +10 -5
  264. endoreg_db/models/other/information_source.py +25 -25
  265. endoreg_db/models/other/material.py +9 -5
  266. endoreg_db/models/other/resource.py +6 -4
  267. endoreg_db/models/other/tag.py +10 -5
  268. endoreg_db/models/other/transport_route.py +13 -8
  269. endoreg_db/models/other/unit.py +10 -6
  270. endoreg_db/models/other/waste.py +6 -5
  271. endoreg_db/models/requirement/requirement.py +580 -272
  272. endoreg_db/models/requirement/requirement_error.py +85 -0
  273. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +268 -0
  274. endoreg_db/models/requirement/requirement_evaluation/operator_evaluation_models.py +3 -6
  275. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +90 -64
  276. endoreg_db/models/requirement/requirement_operator.py +36 -33
  277. endoreg_db/models/requirement/requirement_set.py +74 -57
  278. endoreg_db/models/state/__init__.py +4 -4
  279. endoreg_db/models/state/abstract.py +2 -2
  280. endoreg_db/models/state/anonymization.py +12 -0
  281. endoreg_db/models/state/audit_ledger.py +46 -47
  282. endoreg_db/models/state/label_video_segment.py +9 -0
  283. endoreg_db/models/state/raw_pdf.py +40 -46
  284. endoreg_db/models/state/sensitive_meta.py +6 -2
  285. endoreg_db/models/state/video.py +58 -53
  286. endoreg_db/models/upload_job.py +32 -55
  287. endoreg_db/models/utils.py +1 -2
  288. endoreg_db/root_urls.py +21 -2
  289. endoreg_db/serializers/__init__.py +26 -57
  290. endoreg_db/serializers/anonymization.py +18 -10
  291. endoreg_db/serializers/meta/report_meta.py +1 -1
  292. endoreg_db/serializers/meta/sensitive_meta_detail.py +63 -118
  293. endoreg_db/serializers/misc/__init__.py +1 -1
  294. endoreg_db/serializers/misc/file_overview.py +33 -91
  295. endoreg_db/serializers/misc/{vop_patient_data.py → sensitive_patient_data.py} +1 -1
  296. endoreg_db/serializers/requirements/requirement_sets.py +92 -22
  297. endoreg_db/serializers/video/segmentation.py +2 -1
  298. endoreg_db/serializers/video/video_processing_history.py +20 -5
  299. endoreg_db/serializers/video_examination.py +198 -0
  300. endoreg_db/services/anonymization.py +75 -73
  301. endoreg_db/services/lookup_service.py +256 -73
  302. endoreg_db/services/lookup_store.py +174 -30
  303. endoreg_db/services/pdf_import.py +711 -310
  304. endoreg_db/services/storage_aware_video_processor.py +140 -114
  305. endoreg_db/services/video_import.py +266 -117
  306. endoreg_db/urls/__init__.py +27 -27
  307. endoreg_db/urls/label_video_segments.py +2 -0
  308. endoreg_db/urls/media.py +108 -66
  309. endoreg_db/urls/root_urls.py +29 -0
  310. endoreg_db/utils/__init__.py +15 -5
  311. endoreg_db/utils/ai/multilabel_classification_net.py +116 -20
  312. endoreg_db/utils/case_generator/__init__.py +3 -0
  313. endoreg_db/utils/dataloader.py +88 -16
  314. endoreg_db/utils/defaults/set_default_center.py +32 -0
  315. endoreg_db/utils/names.py +22 -16
  316. endoreg_db/utils/permissions.py +2 -1
  317. endoreg_db/utils/pipelines/process_video_dir.py +1 -1
  318. endoreg_db/utils/requirement_operator_logic/model_evaluators.py +414 -127
  319. endoreg_db/utils/setup_config.py +8 -5
  320. endoreg_db/utils/storage.py +115 -0
  321. endoreg_db/utils/validate_endo_roi.py +8 -2
  322. endoreg_db/utils/video/ffmpeg_wrapper.py +184 -188
  323. endoreg_db/views/__init__.py +5 -12
  324. endoreg_db/views/anonymization/media_management.py +198 -163
  325. endoreg_db/views/anonymization/overview.py +4 -1
  326. endoreg_db/views/anonymization/validate.py +174 -40
  327. endoreg_db/views/media/__init__.py +2 -0
  328. endoreg_db/views/media/pdf_media.py +131 -150
  329. endoreg_db/views/media/sensitive_metadata.py +46 -6
  330. endoreg_db/views/media/video_media.py +89 -82
  331. endoreg_db/views/media/video_segments.py +187 -260
  332. endoreg_db/views/meta/sensitive_meta_detail.py +0 -63
  333. endoreg_db/views/patient/patient.py +5 -4
  334. endoreg_db/views/pdf/__init__.py +5 -8
  335. endoreg_db/views/pdf/pdf_stream.py +186 -0
  336. endoreg_db/views/pdf/pdf_stream_views.py +0 -127
  337. endoreg_db/views/pdf/reimport.py +86 -91
  338. endoreg_db/views/requirement/evaluate.py +188 -187
  339. endoreg_db/views/requirement/lookup.py +186 -288
  340. endoreg_db/views/requirement/requirement_utils.py +89 -0
  341. endoreg_db/views/video/__init__.py +0 -4
  342. endoreg_db/views/video/correction.py +2 -2
  343. endoreg_db/views/video/video_examination_viewset.py +202 -289
  344. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/METADATA +7 -3
  345. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/RECORD +350 -255
  346. endoreg_db/models/administration/permissions/__init__.py +0 -44
  347. endoreg_db/models/media/video/refactor_plan.md +0 -0
  348. endoreg_db/models/media/video/video_file_frames.py +0 -0
  349. endoreg_db/models/metadata/frame_ocr_result.py +0 -0
  350. endoreg_db/models/rule/__init__.py +0 -13
  351. endoreg_db/models/rule/rule.py +0 -27
  352. endoreg_db/models/rule/rule_applicator.py +0 -224
  353. endoreg_db/models/rule/rule_attribute_dtype.py +0 -17
  354. endoreg_db/models/rule/rule_type.py +0 -20
  355. endoreg_db/models/rule/ruleset.py +0 -17
  356. endoreg_db/serializers/video/video_metadata.py +0 -105
  357. endoreg_db/urls/report.py +0 -48
  358. endoreg_db/urls/video.py +0 -61
  359. endoreg_db/utils/case_generator/case_generator.py +0 -159
  360. endoreg_db/utils/case_generator/utils.py +0 -30
  361. endoreg_db/views/pdf/pdf_media.py +0 -239
  362. endoreg_db/views/report/__init__.py +0 -9
  363. endoreg_db/views/report/report_list.py +0 -112
  364. endoreg_db/views/report/report_with_secure_url.py +0 -28
  365. endoreg_db/views/report/start_examination.py +0 -7
  366. endoreg_db/views/video/video_media.py +0 -158
  367. endoreg_db/views.py +0 -0
  368. /endoreg_db/data/{requirement_set → _examples/requirement_set}/endoscopy_bleeding_risk.yaml +0 -0
  369. /endoreg_db/migrations/{0002_add_video_correction_models.py → _old/0002_add_video_correction_models.py} +0 -0
  370. /endoreg_db/migrations/{0003_add_center_display_name.py → _old/0003_add_center_display_name.py} +0 -0
  371. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/WHEEL +0 -0
  372. {endoreg_db-0.8.4.4.dist-info → endoreg_db-0.8.8.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,10 +1,10 @@
1
- from django.db import models
2
1
  import logging
3
2
  from pathlib import Path
4
- from typing import Optional, TYPE_CHECKING
3
+ from typing import TYPE_CHECKING, Optional
5
4
 
6
5
  # import endoreg_center_id from django settings
7
6
  from django.conf import settings
7
+ from django.db import models
8
8
 
9
9
  # check if endoreg_center_id is set
10
10
  if not hasattr(settings, "ENDOREG_CENTER_ID"):
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
19
19
 
20
20
  if TYPE_CHECKING:
21
21
  from ..administration import Center
22
- from ..medical.hardware import EndoscopyProcessor, Endoscope
22
+ from ..medical.hardware import Endoscope, EndoscopyProcessor
23
23
 
24
24
 
25
25
  # VideoMeta
@@ -29,19 +29,40 @@ class VideoMeta(models.Model):
29
29
 
30
30
  Links to hardware (processor, endoscope), center, import details, and FFmpeg technical specs.
31
31
  """
32
- processor = models.ForeignKey(
33
- "EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
34
- )
35
- endoscope = models.ForeignKey(
36
- "Endoscope", on_delete=models.CASCADE, blank=True, null=True
37
- )
32
+
33
+ processor = models.ForeignKey("EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True)
34
+ endoscope = models.ForeignKey("Endoscope", on_delete=models.CASCADE, blank=True, null=True)
38
35
  center = models.ForeignKey("Center", on_delete=models.CASCADE)
39
- import_meta = models.OneToOneField(
40
- "VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True
41
- )
42
- ffmpeg_meta = models.OneToOneField(
43
- "FFMpegMeta", on_delete=models.CASCADE, blank=True, null=True
44
- )
36
+ import_meta = models.OneToOneField("VideoImportMeta", on_delete=models.CASCADE, blank=True, null=True)
37
+ ffmpeg_meta = models.OneToOneField("FFMpegMeta", on_delete=models.CASCADE, blank=True, null=True)
38
+
39
+ if TYPE_CHECKING:
40
+ processor: models.ForeignKey["EndoscopyProcessor|None"]
41
+ endoscope: models.ForeignKey["Endoscope|None"]
42
+ center: models.ForeignKey["Center|None"]
43
+ import_meta: models.OneToOneField["VideoImportMeta|None"]
44
+ ffmpeg_meta: models.OneToOneField["FFMpegMeta|None"]
45
+
46
+ @property
47
+ def center_safe(self) -> "Center":
48
+ center = self.center
49
+ if not center:
50
+ raise Center.DoesNotExist("Center does not exist for this VideoMeta instance.")
51
+ return center
52
+
53
+ @property
54
+ def processor_safe(self) -> "EndoscopyProcessor":
55
+ processor = self.processor
56
+ if not processor:
57
+ raise EndoscopyProcessor.DoesNotExist("EndoscopyProcessor does not exist for this VideoMeta instance.")
58
+ return processor
59
+
60
+ @property
61
+ def ffmpeg_meta_safe(self) -> "FFMpegMeta":
62
+ ffmpeg_meta = self.ffmpeg_meta
63
+ if not ffmpeg_meta:
64
+ raise FFMpegMeta.DoesNotExist("FFMpegMeta does not exist for this VideoMeta instance.")
65
+ return ffmpeg_meta
45
66
 
46
67
  @classmethod
47
68
  def create_from_file(
@@ -71,7 +92,7 @@ class VideoMeta(models.Model):
71
92
  raise RuntimeError(f"Failed to initialize FFMpeg metadata for {video_path.name}") from e
72
93
 
73
94
  if save_instance:
74
- meta.save() # This ensures VideoImportMeta is created too
95
+ meta.save() # This ensures VideoImportMeta is created too
75
96
  logger.info("Created and saved VideoMeta instance PK %s from %s", meta.pk, video_path.name)
76
97
  else:
77
98
  logger.info("Instantiated VideoMeta from %s (not saved yet)", video_path.name)
@@ -120,8 +141,8 @@ class VideoMeta(models.Model):
120
141
  # If the VideoMeta instance is already saved, save the link immediately.
121
142
  # Otherwise, the link will be saved when VideoMeta itself is saved.
122
143
  if self.pk:
123
- self.save(update_fields=['ffmpeg_meta'])
124
- logger.info("Successfully created and linked FFMpegMeta PK %s", self.ffmpeg_meta.pk)
144
+ self.save(update_fields=["ffmpeg_meta"])
145
+ logger.info("Successfully created and linked FFMpegMeta PK %s", self.ffmpeg_meta_safe.pk)
125
146
 
126
147
  except Exception as e:
127
148
  # Log the error and re-raise it
@@ -140,8 +161,8 @@ class VideoMeta(models.Model):
140
161
  logger.debug("Deleting existing FFMpegMeta PK %s before update.", existing_ffmpeg_pk)
141
162
  # Nullify the relation first before deleting the related object
142
163
  self.ffmpeg_meta = None
143
- self.save(update_fields=['ffmpeg_meta']) # Save the null relation
144
- FFMpegMeta.objects.filter(pk=existing_ffmpeg_pk).delete() # Delete the old object
164
+ self.save(update_fields=["ffmpeg_meta"]) # Save the null relation
165
+ FFMpegMeta.objects.filter(pk=existing_ffmpeg_pk).delete() # Delete the old object
145
166
 
146
167
  # initialize_ffmpeg_meta handles creation, linking, saving the link, and raises exceptions
147
168
  self.initialize_ffmpeg_meta(video_path)
@@ -150,7 +171,7 @@ class VideoMeta(models.Model):
150
171
  """Retrieves the endoscope region of interest (ROI) from the associated processor."""
151
172
  from ..medical.hardware import EndoscopyProcessor
152
173
 
153
- processor: EndoscopyProcessor = self.processor
174
+ processor: EndoscopyProcessor = self.processor_safe
154
175
  endo_roi = processor.get_roi_endoscope_image()
155
176
  return endo_roi
156
177
 
@@ -189,6 +210,7 @@ class FFMpegMeta(models.Model):
189
210
  """
190
211
  Stores technical video stream information extracted using FFmpeg (ffprobe).
191
212
  """
213
+
192
214
  width = models.IntegerField(null=True, blank=True)
193
215
  height = models.IntegerField(null=True, blank=True)
194
216
  duration = models.FloatField(null=True, blank=True) # Duration in seconds
@@ -219,7 +241,6 @@ class FFMpegMeta(models.Model):
219
241
  logger.error("ffprobe execution failed for %s: %s", file_path, probe_err, exc_info=True)
220
242
  raise RuntimeError(f"ffprobe execution failed for {file_path}") from probe_err
221
243
 
222
-
223
244
  if not probe_data or "streams" not in probe_data:
224
245
  logger.error("Failed to get valid stream info from ffprobe for %s", file_path)
225
246
  # Raise exception instead of returning None
@@ -237,8 +258,8 @@ class FFMpegMeta(models.Model):
237
258
  height = video_stream.get("height")
238
259
  duration_str = video_stream.get("duration")
239
260
  # --- FIX: Handle potential format key ---
240
- if duration_str is None and 'format' in probe_data and 'duration' in probe_data['format']:
241
- duration_str = probe_data['format']['duration']
261
+ if duration_str is None and "format" in probe_data and "duration" in probe_data["format"]:
262
+ duration_str = probe_data["format"]["duration"]
242
263
  logger.debug("Using duration from format block: %s", duration_str)
243
264
  # --- End Fix ---
244
265
  duration = float(duration_str) if duration_str else None
@@ -253,10 +274,10 @@ class FFMpegMeta(models.Model):
253
274
  frame_rate_num, frame_rate_den = None, None
254
275
  if frame_rate_str and "/" in frame_rate_str:
255
276
  try:
256
- num_str, den_str = frame_rate_str.split('/')
277
+ num_str, den_str = frame_rate_str.split("/")
257
278
  frame_rate_num = int(num_str)
258
279
  frame_rate_den = int(den_str)
259
- if frame_rate_den == 0: # Avoid division by zero
280
+ if frame_rate_den == 0: # Avoid division by zero
260
281
  logger.warning("Invalid frame rate denominator (0) for %s", file_path)
261
282
  frame_rate_num, frame_rate_den = None, None
262
283
  except ValueError:
@@ -267,8 +288,8 @@ class FFMpegMeta(models.Model):
267
288
  pixel_format = video_stream.get("pix_fmt")
268
289
  bit_rate_str = video_stream.get("bit_rate")
269
290
  # --- FIX: Handle potential format key for bit_rate ---
270
- if bit_rate_str is None and 'format' in probe_data and 'bit_rate' in probe_data['format']:
271
- bit_rate_str = probe_data['format']['bit_rate']
291
+ if bit_rate_str is None and "format" in probe_data and "bit_rate" in probe_data["format"]:
292
+ bit_rate_str = probe_data["format"]["bit_rate"]
272
293
  logger.debug("Using bit_rate from format block: %s", bit_rate_str)
273
294
  # --- End Fix ---
274
295
  bit_rate = int(bit_rate_str) if bit_rate_str else None
@@ -311,6 +332,7 @@ class VideoImportMeta(models.Model):
311
332
  """
312
333
  Stores metadata related to the import and processing status of a video.
313
334
  """
335
+
314
336
  file_name = models.CharField(max_length=255, blank=True, null=True)
315
337
  video_anonymized = models.BooleanField(default=False)
316
338
  video_patient_data_detected = models.BooleanField(default=False)
@@ -323,9 +345,7 @@ class VideoImportMeta(models.Model):
323
345
  result_html = ""
324
346
 
325
347
  result_html += f"Video anonymized: {self.video_anonymized}\n"
326
- result_html += (
327
- f"Video patient data detected: {self.video_patient_data_detected}\n"
328
- )
348
+ result_html += f"Video patient data detected: {self.video_patient_data_detected}\n"
329
349
  result_html += f"Outside detected: {self.outside_detected}\n"
330
350
  result_html += f"Patient data removed: {self.patient_data_removed}\n"
331
351
  result_html += f"Outside removed: {self.outside_removed}\n"
@@ -1,20 +1,22 @@
1
1
  import logging
2
- from typing import TYPE_CHECKING, List, Tuple, Optional
2
+ from typing import TYPE_CHECKING, List, Optional, Tuple
3
+
3
4
  import numpy as np
4
5
 
5
- # Import necessary models and utils used by the logic
6
- from ..utils import find_segments_in_prediction_array
7
6
  from ..label.label_video_segment import LabelVideoSegment
8
7
 
8
+ # Import necessary models and utils used by the logic
9
+ from ..utils import find_segments_in_prediction_array
9
10
 
10
11
  logger = logging.getLogger(__name__)
11
12
 
13
+ # TODO configure via settings
12
14
  DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
13
15
  DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
14
16
 
15
17
  if TYPE_CHECKING:
16
- from .video_prediction_meta import VideoPredictionMeta
17
18
  from ..label import Label
19
+ from .video_prediction_meta import VideoPredictionMeta
18
20
 
19
21
 
20
22
  def apply_running_mean_logic(instance: "VideoPredictionMeta", confidence_array: np.ndarray, window_size_in_seconds: Optional[float] = None) -> np.ndarray:
@@ -63,6 +65,7 @@ def calculate_prediction_array_logic(instance: "VideoPredictionMeta", window_siz
63
65
  Does not save the array itself.
64
66
  """
65
67
  from ..label import ImageClassificationAnnotation
68
+
66
69
  video_obj = instance.get_video()
67
70
  model_meta = instance.model_meta
68
71
  label_list = instance.get_label_list()
@@ -78,15 +81,10 @@ def calculate_prediction_array_logic(instance: "VideoPredictionMeta", window_siz
78
81
 
79
82
  prediction_array = np.zeros((num_frames, len(label_list)))
80
83
 
81
- base_pred_qs = ImageClassificationAnnotation.objects.filter(
82
- model_meta=model_meta,
83
- frame__video_file=video_obj
84
- )
84
+ base_pred_qs = ImageClassificationAnnotation.objects.filter(model_meta=model_meta, frame__video_file=video_obj)
85
85
 
86
86
  for i, label in enumerate(label_list):
87
- predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list(
88
- "frame__frame_number", "confidence"
89
- )
87
+ predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list("frame__frame_number", "confidence")
90
88
 
91
89
  # Initialize with 0.5 (neutral confidence)
92
90
  confidences = np.full(num_frames, 0.5)
@@ -96,14 +94,12 @@ def calculate_prediction_array_logic(instance: "VideoPredictionMeta", window_siz
96
94
  confidences[frame_num] = confidence
97
95
  found_predictions = True
98
96
  else:
99
- logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames-1}). Skipping.")
97
+ logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames - 1}). Skipping.")
100
98
 
101
99
  if not found_predictions:
102
100
  logger.warning(f"No predictions found for label '{label.name}' in {video_obj}. Using default confidence.")
103
101
 
104
- smooth_confidences = apply_running_mean_logic(
105
- instance, confidences, window_size_in_seconds
106
- )
102
+ smooth_confidences = apply_running_mean_logic(instance, confidences, window_size_in_seconds)
107
103
  # Threshold smoothed confidences
108
104
  binary_predictions = smooth_confidences > 0.5
109
105
  prediction_array[:, i] = binary_predictions
@@ -116,6 +112,7 @@ def create_video_segments_for_label_logic(instance: "VideoPredictionMeta", segme
116
112
  Creates LabelVideoSegment instances for the given label and segments.
117
113
  """
118
114
  from ..other import InformationSource
115
+
119
116
  video_obj = instance.get_video()
120
117
  information_source, _ = InformationSource.objects.get_or_create(name="prediction")
121
118
 
@@ -131,11 +128,7 @@ def create_video_segments_for_label_logic(instance: "VideoPredictionMeta", segme
131
128
  }
132
129
  # Check for existence before creating the object instance
133
130
  if not LabelVideoSegment.objects.filter(
134
- video_file=video_obj,
135
- prediction_meta=instance,
136
- label=label,
137
- start_frame_number=start_frame,
138
- end_frame_number=end_frame
131
+ video_file=video_obj, prediction_meta=instance, label=label, start_frame_number=start_frame, end_frame_number=end_frame
139
132
  ).exists():
140
133
  segments_to_create.append(LabelVideoSegment(**segment_data))
141
134
 
@@ -161,7 +154,7 @@ def create_video_segments_logic(instance: "VideoPredictionMeta", segment_length_
161
154
  return
162
155
 
163
156
  min_frame_length = int(segment_length_threshold_in_s * fps)
164
- min_frame_length = max(min_frame_length, 1) # Ensure minimum length is at least 1
157
+ min_frame_length = max(min_frame_length, 1) # Ensure minimum length is at least 1
165
158
 
166
159
  label_list = instance.get_label_list()
167
160
  if not label_list:
@@ -171,8 +164,8 @@ def create_video_segments_logic(instance: "VideoPredictionMeta", segment_length_
171
164
  prediction_array = instance.get_prediction_array()
172
165
  if prediction_array is None:
173
166
  logger.info(f"Prediction array not found for {instance}. Calculating...")
174
- instance.calculate_prediction_array() # This will save the array internally
175
- prediction_array = instance.get_prediction_array() # Fetch again
167
+ instance.calculate_prediction_array() # This will save the array internally
168
+ prediction_array = instance.get_prediction_array() # Fetch again
176
169
  if prediction_array is None:
177
170
  logger.error(f"Failed to get or calculate prediction array for {instance}. Cannot create segments.")
178
171
  return
@@ -1,25 +1,25 @@
1
- from typing import TYPE_CHECKING, Optional, List, Tuple
2
- from django.db import models
3
1
  import logging
2
+ import pickle
3
+ from typing import TYPE_CHECKING, List, Optional, Tuple
4
+
5
+ import numpy as np
6
+ from django.db import models
4
7
 
5
8
  from endoreg_db.models.label import LabelSet
6
9
 
7
10
  from ..label.label_video_segment import (
8
11
  LabelVideoSegment,
9
-
10
12
  )
11
13
  from ..utils import find_segments_in_prediction_array
12
14
 
13
- import numpy as np
14
- import pickle
15
-
16
15
  logger = logging.getLogger(__name__)
17
16
 
18
17
  DEFAULT_WINDOW_SIZE_IN_SECONDS_FOR_RUNNING_MEAN = 1.5
19
18
  DEFAULT_VIDEO_SEGMENT_LENGTH_THRESHOLD_IN_S = 1.0
20
19
 
21
20
  if TYPE_CHECKING:
22
- from endoreg_db.models import ModelMeta, Label
21
+ from endoreg_db.models import Label, ModelMeta
22
+
23
23
  from ..media.video.video_file import VideoFile
24
24
 
25
25
 
@@ -29,6 +29,7 @@ class VideoPredictionMeta(models.Model):
29
29
 
30
30
  Must be associated with exactly one `VideoFile`.
31
31
  """
32
+
32
33
  model_meta = models.ForeignKey("ModelMeta", on_delete=models.CASCADE)
33
34
  date_created = models.DateTimeField(auto_now_add=True)
34
35
  date_modified = models.DateTimeField(auto_now=True)
@@ -43,18 +44,24 @@ class VideoPredictionMeta(models.Model):
43
44
  )
44
45
 
45
46
  if TYPE_CHECKING:
46
- model_meta: "ModelMeta"
47
- video_file: "VideoFile"
47
+ model_meta: models.ForeignKey["ModelMeta"]
48
+ video_file: models.ForeignKey["VideoFile|None"]
48
49
  label_video_segments: "models.Manager[LabelVideoSegment]"
49
50
 
50
51
  class Meta:
51
- constraints = [
52
- models.UniqueConstraint(fields=['model_meta', 'video_file'], name='unique_prediction_per_video_model')
53
- ]
52
+ constraints = [models.UniqueConstraint(fields=["model_meta", "video_file"], name="unique_prediction_per_video_model")]
54
53
  indexes = [
55
54
  models.Index(fields=["model_meta", "video_file"]),
56
55
  ]
57
56
 
57
+ @property
58
+ def video_file_safe(self) -> "VideoFile":
59
+ """Returns the associated VideoFile instance."""
60
+ video_file = self.video_file
61
+ if not video_file:
62
+ raise ValueError("VideoPredictionMeta is not associated with a VideoFile.")
63
+ return video_file
64
+
58
65
  def get_video(self) -> "VideoFile":
59
66
  """Returns the associated VideoFile instance."""
60
67
  if self.video_file:
@@ -83,12 +90,12 @@ class VideoPredictionMeta(models.Model):
83
90
  return labelset.get_labels_in_order()
84
91
  return []
85
92
 
86
- def save_prediction_array(self, prediction_array: np.array):
93
+ def save_prediction_array(self, prediction_array: np.typing.NDArray):
87
94
  """
88
95
  Save the prediction array to the database.
89
96
  """
90
97
  self.prediction_array = pickle.dumps(prediction_array)
91
- self.save(update_fields=['prediction_array', 'date_modified'])
98
+ self.save(update_fields=["prediction_array", "date_modified"])
92
99
 
93
100
  def get_prediction_array(self):
94
101
  """
@@ -103,7 +110,7 @@ class VideoPredictionMeta(models.Model):
103
110
  logger.error(f"Error unpickling prediction array for {self}: {e}")
104
111
  return None
105
112
 
106
- def calculate_prediction_array(self, window_size_in_seconds: int = None):
113
+ def calculate_prediction_array(self, window_size_in_seconds: Optional[int] = None):
107
114
  """
108
115
  Fetches all predictions for the associated video, labelset, and model meta,
109
116
  applies smoothing, and saves the resulting binary prediction array.
@@ -125,15 +132,10 @@ class VideoPredictionMeta(models.Model):
125
132
 
126
133
  prediction_array = np.zeros((num_frames, len(label_list)))
127
134
 
128
- base_pred_qs = ImageClassificationAnnotation.objects.filter(
129
- model_meta=model_meta,
130
- frame__video_file=video_obj
131
- )
135
+ base_pred_qs = ImageClassificationAnnotation.objects.filter(model_meta=model_meta, frame__video_file=video_obj)
132
136
 
133
137
  for i, label in enumerate(label_list):
134
- predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list(
135
- "frame__frame_number", "float_value"
136
- )
138
+ predictions = base_pred_qs.filter(label=label).order_by("frame__frame_number").values_list("frame__frame_number", "float_value")
137
139
 
138
140
  confidences = np.full(num_frames, 0.5)
139
141
  found_predictions = False
@@ -142,21 +144,19 @@ class VideoPredictionMeta(models.Model):
142
144
  confidences[frame_num] = confidence
143
145
  found_predictions = True
144
146
  else:
145
- logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames-1}). Skipping.")
147
+ logger.warning(f"Prediction found for out-of-bounds frame number {frame_num} (max: {num_frames - 1}). Skipping.")
146
148
 
147
149
  if not found_predictions:
148
150
  logger.warning(f"No predictions found for label '{label.name}' in {video_obj}. Using default confidence.")
149
151
 
150
- smooth_confidences = self.apply_running_mean(
151
- confidences, window_size_in_seconds
152
- )
152
+ smooth_confidences = self.apply_running_mean(confidences, window_size_in_seconds)
153
153
  binary_predictions = smooth_confidences > 0.5
154
154
  prediction_array[:, i] = binary_predictions
155
155
 
156
156
  self.save_prediction_array(prediction_array)
157
157
  logger.info(f"Calculated and saved prediction array for {self}")
158
158
 
159
- def apply_running_mean(self, confidence_array, window_size_in_seconds: int = None):
159
+ def apply_running_mean(self, confidence_array, window_size_in_seconds: Optional[float] = None):
160
160
  """
161
161
  Apply a running mean filter to the confidence array for smoothing, assuming a padding
162
162
  of 0.5 for the edges.
@@ -215,11 +215,7 @@ class VideoPredictionMeta(models.Model):
215
215
  "video_file": video_obj,
216
216
  }
217
217
  if not LabelVideoSegment.objects.filter(
218
- video_file=video_obj,
219
- prediction_meta=self,
220
- label=label,
221
- start_frame_number=start_frame,
222
- end_frame_number=end_frame
218
+ video_file=video_obj, prediction_meta=self, label=label, start_frame_number=start_frame, end_frame_number=end_frame
223
219
  ).exists():
224
220
  segments_to_create.append(LabelVideoSegment(**segment_data))
225
221
 
@@ -229,7 +225,7 @@ class VideoPredictionMeta(models.Model):
229
225
  else:
230
226
  logger.info(f"No new video segments needed for label '{label.name}' in {video_obj}.")
231
227
 
232
- def create_video_segments(self, segment_length_threshold_in_s: float = None):
228
+ def create_video_segments(self, segment_length_threshold_in_s: Optional[float] = None):
233
229
  """
234
230
  Generates LabelVideoSegments based on the stored prediction array.
235
231
  """
@@ -1,15 +1,18 @@
1
- '''Model for date value distribution'''
1
+ """Model for date value distribution"""
2
2
 
3
3
  from datetime import date, timedelta
4
- from django.db import models
4
+
5
5
  import numpy as np
6
+ from django.db import models
6
7
 
7
8
  from .base_value_distribution import BaseValueDistribution
8
9
 
10
+
9
11
  class DateValueDistributionManager(models.Manager):
10
- '''Object manager for DateValueDistribution'''
12
+ """Object manager for DateValueDistribution"""
13
+
11
14
  def get_by_natural_key(self, name):
12
- '''Retrieve a DateValueDistribution by its natural key.'''
15
+ """Retrieve a DateValueDistribution by its natural key."""
13
16
  return self.get(name=name)
14
17
 
15
18
 
@@ -20,16 +23,17 @@ class DateValueDistribution(BaseValueDistribution):
20
23
  date_min, date_max, date_mean, date_std_dev or
21
24
  timedelta_days_min, timedelta_days_max, timedelta_days_mean, timedelta_days_std_dev
22
25
  """
26
+
23
27
  objects = DateValueDistributionManager()
24
28
  name = models.CharField(max_length=100)
25
29
  description = models.TextField(blank=True, null=True)
26
30
  DISTRIBUTION_CHOICES = [
27
- ('uniform', 'Uniform'),
28
- ('normal', 'Normal'),
31
+ ("uniform", "Uniform"),
32
+ ("normal", "Normal"),
29
33
  ]
30
34
  MODE_CHOICES = [
31
- ('date', 'Date'),
32
- ('timedelta', 'Timedelta'),
35
+ ("date", "Date"),
36
+ ("timedelta", "Timedelta"),
33
37
  ]
34
38
 
35
39
  distribution_type = models.CharField(max_length=20, choices=DISTRIBUTION_CHOICES)
@@ -47,43 +51,99 @@ class DateValueDistribution(BaseValueDistribution):
47
51
  timedelta_days_mean = models.IntegerField(blank=True, null=True)
48
52
  timedelta_days_std_dev = models.IntegerField(blank=True, null=True)
49
53
 
54
+ # create *_safe properties for dates and timedeltas
55
+ @property
56
+ def date_min_safe(self):
57
+ _date = self.date_min
58
+ if _date is None:
59
+ raise ValueError("date_min is not set")
60
+ return _date
61
+
62
+ @property
63
+ def date_max_safe(self):
64
+ _date = self.date_max
65
+ if _date is None:
66
+ raise ValueError("date_max is not set")
67
+ return _date
68
+
69
+ @property
70
+ def date_mean_safe(self):
71
+ _date = self.date_mean
72
+ if _date is None:
73
+ raise ValueError("date_mean is not set")
74
+ return _date
75
+
76
+ @property
77
+ def date_std_dev_safe(self):
78
+ _std_dev = self.date_std_dev
79
+ if _std_dev is None:
80
+ raise ValueError("date_std_dev is not set")
81
+ return _std_dev
82
+
83
+ @property
84
+ def timedelta_days_min_safe(self):
85
+ _min = self.timedelta_days_min
86
+ if _min is None:
87
+ raise ValueError("timedelta_days_min is not set")
88
+ return _min
89
+
90
+ @property
91
+ def timedelta_days_max_safe(self):
92
+ _max = self.timedelta_days_max
93
+ if _max is None:
94
+ raise ValueError("timedelta_days_max is not set")
95
+ return _max
96
+
97
+ @property
98
+ def timedelta_days_mean_safe(self):
99
+ _mean = self.timedelta_days_mean
100
+ if _mean is None:
101
+ raise ValueError("timedelta_days_mean is not set")
102
+ return _mean
103
+
104
+ @property
105
+ def timedelta_days_std_dev_safe(self):
106
+ _std_dev = self.timedelta_days_std_dev
107
+ if _std_dev is None:
108
+ raise ValueError("timedelta_days_std_dev is not set")
109
+ return _std_dev
110
+
50
111
  def generate_value(self):
51
- if self.mode == 'date':
112
+ if self.mode == "date":
52
113
  return self._generate_date_value()
53
- elif self.mode == 'timedelta':
114
+ elif self.mode == "timedelta":
54
115
  return self._generate_timedelta_value()
55
116
  else:
56
117
  raise ValueError("Unsupported mode")
57
118
 
58
119
  def _generate_date_value(self):
59
- #UNTESTED
60
- if self.distribution_type == 'uniform':
61
- start_date = self.date_min.toordinal()
62
- end_date = self.date_max.toordinal()
120
+ # UNTESTED
121
+ if self.distribution_type == "uniform":
122
+ start_date = self.date_min_safe.toordinal()
123
+ end_date = self.date_max_safe.toordinal()
63
124
  random_ordinal = np.random.randint(start_date, end_date)
64
125
  return date.fromordinal(random_ordinal)
65
- elif self.distribution_type == 'normal':
66
- mean_ordinal = self.date_mean.toordinal()
67
- std_dev_days = self.date_std_dev
126
+ elif self.distribution_type == "normal":
127
+ mean_ordinal = self.date_mean_safe.toordinal()
128
+ std_dev_days = self.date_std_dev_safe
68
129
  random_ordinal = int(np.random.normal(mean_ordinal, std_dev_days))
69
- random_ordinal = np.clip(random_ordinal, self.date_min.toordinal(), self.date_max.toordinal())
130
+ random_ordinal = np.clip(random_ordinal, self.date_min_safe.toordinal(), self.date_max_safe.toordinal())
70
131
  return date.fromordinal(random_ordinal)
71
132
  else:
72
133
  raise ValueError("Unsupported distribution type")
73
134
 
74
135
  def _generate_timedelta_value(self):
75
- if self.distribution_type == 'uniform':
76
- random_days = np.random.randint(self.timedelta_days_min, self.timedelta_days_max + 1)
77
-
78
-
79
- elif self.distribution_type == 'normal':
80
- random_days = int(np.random.normal(self.timedelta_days_mean, self.timedelta_days_std_dev))
81
- random_days = np.clip(random_days, self.timedelta_days_min, self.timedelta_days_max)
82
-
136
+ if self.distribution_type == "uniform":
137
+ random_days = np.random.randint(self.timedelta_days_min_safe, self.timedelta_days_max_safe + 1)
138
+
139
+ elif self.distribution_type == "normal":
140
+ random_days = int(np.random.normal(self.timedelta_days_mean_safe, self.timedelta_days_std_dev_safe))
141
+ random_days = np.clip(random_days, self.timedelta_days_min_safe, self.timedelta_days_max_safe)
142
+
83
143
  else:
84
144
  raise ValueError("Unsupported distribution type")
85
-
145
+
86
146
  current_date = date.today()
87
147
  generated_date = current_date - timedelta(days=random_days)
88
- print(generated_date)
89
- return(generated_date)
148
+
149
+ return generated_date
@@ -1,29 +1,45 @@
1
- from django.db import models
2
1
  import numpy as np
2
+ from django.db import models
3
+
3
4
  from .base_value_distribution import BaseValueDistribution
4
5
 
6
+
5
7
  class MultipleCategoricalValueDistributionManager(models.Manager):
6
8
  def get_by_natural_key(self, name):
7
9
  return self.get(name=name)
8
10
 
11
+
9
12
  class MultipleCategoricalValueDistribution(BaseValueDistribution):
10
13
  """
11
14
  Multiple categorical value distribution model.
12
15
  Assigns a specific number or varying number of values based on probabilities.
13
16
  """
17
+
14
18
  objects = MultipleCategoricalValueDistributionManager()
15
19
  categories = models.JSONField() # { "category": "probability", ... }
16
20
  min_count = models.IntegerField()
17
21
  max_count = models.IntegerField()
18
- count_distribution_type = models.CharField(max_length=20, choices=[('uniform', 'Uniform'), ('normal', 'Normal')])
22
+ count_distribution_type = models.CharField(max_length=20, choices=[("uniform", "Uniform"), ("normal", "Normal")])
19
23
  count_mean = models.FloatField(null=True, blank=True)
20
24
  count_std_dev = models.FloatField(null=True, blank=True)
21
25
 
26
+ @property
27
+ def count_mean_safe(self):
28
+ if self.count_mean is None:
29
+ raise ValueError("count_mean is not set")
30
+ return self.count_mean
31
+
32
+ @property
33
+ def count_std_dev_safe(self):
34
+ if self.count_std_dev is None:
35
+ raise ValueError("count_std_dev is not set")
36
+ return self.count_std_dev
37
+
22
38
  def generate_value(self):
23
- if self.count_distribution_type == 'uniform':
39
+ if self.count_distribution_type == "uniform":
24
40
  count = np.random.randint(self.min_count, self.max_count + 1)
25
- elif self.count_distribution_type == 'normal':
26
- count = int(np.random.normal(self.count_mean, self.count_std_dev))
41
+ elif self.count_distribution_type == "normal":
42
+ count = int(np.random.normal(self.count_mean_safe, self.count_std_dev_safe))
27
43
  count = np.clip(count, self.min_count, self.max_count)
28
44
  else:
29
45
  raise ValueError("Unsupported count distribution type")