endoreg-db 0.8.9.2__py3-none-any.whl → 0.8.9.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (450) hide show
  1. endoreg_db/admin.py +10 -5
  2. endoreg_db/apps.py +4 -7
  3. endoreg_db/authz/auth.py +1 -0
  4. endoreg_db/authz/backends.py +1 -1
  5. endoreg_db/authz/management/commands/list_routes.py +2 -0
  6. endoreg_db/authz/middleware.py +8 -7
  7. endoreg_db/authz/permissions.py +21 -10
  8. endoreg_db/authz/policy.py +14 -19
  9. endoreg_db/authz/views_auth.py +14 -10
  10. endoreg_db/codemods/rename_datetime_fields.py +8 -1
  11. endoreg_db/exceptions.py +5 -2
  12. endoreg_db/forms/__init__.py +0 -1
  13. endoreg_db/forms/examination_form.py +4 -3
  14. endoreg_db/forms/patient_finding_intervention_form.py +30 -8
  15. endoreg_db/forms/patient_form.py +9 -13
  16. endoreg_db/forms/questionnaires/__init__.py +1 -1
  17. endoreg_db/forms/settings/__init__.py +4 -1
  18. endoreg_db/forms/unit.py +2 -1
  19. endoreg_db/helpers/count_db.py +17 -14
  20. endoreg_db/helpers/default_objects.py +2 -1
  21. endoreg_db/helpers/download_segmentation_model.py +4 -3
  22. endoreg_db/helpers/interact.py +0 -5
  23. endoreg_db/helpers/test_video_helper.py +33 -25
  24. endoreg_db/import_files/__init__.py +1 -1
  25. endoreg_db/import_files/context/__init__.py +1 -1
  26. endoreg_db/import_files/context/default_sensitive_meta.py +11 -9
  27. endoreg_db/import_files/context/ensure_center.py +4 -4
  28. endoreg_db/import_files/context/file_lock.py +3 -3
  29. endoreg_db/import_files/context/import_context.py +11 -12
  30. endoreg_db/import_files/context/validate_directories.py +1 -0
  31. endoreg_db/import_files/file_storage/create_report_file.py +57 -34
  32. endoreg_db/import_files/file_storage/create_video_file.py +64 -35
  33. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +5 -2
  34. endoreg_db/import_files/file_storage/state_management.py +89 -122
  35. endoreg_db/import_files/file_storage/storage.py +5 -1
  36. endoreg_db/import_files/processing/report_processing/report_anonymization.py +24 -19
  37. endoreg_db/import_files/processing/sensitive_meta_adapter.py +3 -3
  38. endoreg_db/import_files/processing/video_processing/video_anonymization.py +18 -18
  39. endoreg_db/import_files/pseudonymization/k_anonymity.py +8 -9
  40. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +16 -5
  41. endoreg_db/import_files/report_import_service.py +36 -30
  42. endoreg_db/import_files/video_import_service.py +27 -23
  43. endoreg_db/logger_conf.py +56 -40
  44. endoreg_db/management/__init__.py +1 -1
  45. endoreg_db/management/commands/__init__.py +1 -1
  46. endoreg_db/management/commands/check_auth.py +45 -38
  47. endoreg_db/management/commands/create_model_meta_from_huggingface.py +53 -2
  48. endoreg_db/management/commands/create_multilabel_model_meta.py +54 -19
  49. endoreg_db/management/commands/fix_missing_patient_data.py +105 -71
  50. endoreg_db/management/commands/fix_video_paths.py +75 -54
  51. endoreg_db/management/commands/import_report.py +1 -3
  52. endoreg_db/management/commands/list_routes.py +2 -0
  53. endoreg_db/management/commands/load_ai_model_data.py +8 -2
  54. endoreg_db/management/commands/load_ai_model_label_data.py +0 -1
  55. endoreg_db/management/commands/load_center_data.py +3 -3
  56. endoreg_db/management/commands/load_distribution_data.py +35 -38
  57. endoreg_db/management/commands/load_endoscope_data.py +0 -3
  58. endoreg_db/management/commands/load_examination_data.py +20 -4
  59. endoreg_db/management/commands/load_finding_data.py +18 -3
  60. endoreg_db/management/commands/load_gender_data.py +17 -24
  61. endoreg_db/management/commands/load_green_endoscopy_wuerzburg_data.py +95 -85
  62. endoreg_db/management/commands/load_information_source.py +0 -3
  63. endoreg_db/management/commands/load_lab_value_data.py +14 -3
  64. endoreg_db/management/commands/load_legacy_data.py +303 -0
  65. endoreg_db/management/commands/load_name_data.py +1 -2
  66. endoreg_db/management/commands/load_pdf_type_data.py +4 -8
  67. endoreg_db/management/commands/load_profession_data.py +0 -1
  68. endoreg_db/management/commands/load_report_reader_flag_data.py +0 -4
  69. endoreg_db/management/commands/load_requirement_data.py +6 -2
  70. endoreg_db/management/commands/load_unit_data.py +0 -4
  71. endoreg_db/management/commands/load_user_groups.py +5 -7
  72. endoreg_db/management/commands/model_input.py +169 -0
  73. endoreg_db/management/commands/register_ai_model.py +22 -16
  74. endoreg_db/management/commands/setup_endoreg_db.py +110 -32
  75. endoreg_db/management/commands/storage_management.py +14 -8
  76. endoreg_db/management/commands/summarize_db_content.py +154 -63
  77. endoreg_db/management/commands/train_image_multilabel_model.py +144 -0
  78. endoreg_db/management/commands/validate_video_files.py +82 -50
  79. endoreg_db/management/commands/video_validation.py +4 -6
  80. endoreg_db/migrations/0001_initial.py +112 -63
  81. endoreg_db/models/__init__.py +8 -0
  82. endoreg_db/models/administration/ai/active_model.py +5 -5
  83. endoreg_db/models/administration/ai/ai_model.py +41 -18
  84. endoreg_db/models/administration/ai/model_type.py +1 -0
  85. endoreg_db/models/administration/case/case.py +22 -22
  86. endoreg_db/models/administration/center/__init__.py +5 -5
  87. endoreg_db/models/administration/center/center.py +6 -2
  88. endoreg_db/models/administration/center/center_resource.py +18 -4
  89. endoreg_db/models/administration/center/center_shift.py +3 -1
  90. endoreg_db/models/administration/center/center_waste.py +6 -2
  91. endoreg_db/models/administration/person/__init__.py +1 -1
  92. endoreg_db/models/administration/person/employee/__init__.py +1 -1
  93. endoreg_db/models/administration/person/employee/employee_type.py +3 -1
  94. endoreg_db/models/administration/person/examiner/__init__.py +1 -1
  95. endoreg_db/models/administration/person/examiner/examiner.py +10 -2
  96. endoreg_db/models/administration/person/names/first_name.py +6 -4
  97. endoreg_db/models/administration/person/names/last_name.py +4 -3
  98. endoreg_db/models/administration/person/patient/__init__.py +1 -1
  99. endoreg_db/models/administration/person/patient/patient.py +0 -1
  100. endoreg_db/models/administration/person/patient/patient_external_id.py +0 -1
  101. endoreg_db/models/administration/person/person.py +1 -1
  102. endoreg_db/models/administration/product/__init__.py +7 -6
  103. endoreg_db/models/administration/product/product.py +6 -2
  104. endoreg_db/models/administration/product/product_group.py +9 -7
  105. endoreg_db/models/administration/product/product_material.py +9 -2
  106. endoreg_db/models/administration/product/reference_product.py +64 -15
  107. endoreg_db/models/administration/qualification/qualification.py +3 -1
  108. endoreg_db/models/administration/shift/shift.py +3 -1
  109. endoreg_db/models/administration/shift/shift_type.py +12 -4
  110. endoreg_db/models/aidataset/__init__.py +5 -0
  111. endoreg_db/models/aidataset/aidataset.py +193 -0
  112. endoreg_db/models/label/__init__.py +1 -1
  113. endoreg_db/models/label/label.py +10 -2
  114. endoreg_db/models/label/label_set.py +3 -1
  115. endoreg_db/models/label/label_video_segment/_create_from_video.py +6 -2
  116. endoreg_db/models/label/label_video_segment/label_video_segment.py +148 -44
  117. endoreg_db/models/media/__init__.py +12 -5
  118. endoreg_db/models/media/frame/__init__.py +1 -1
  119. endoreg_db/models/media/frame/frame.py +34 -8
  120. endoreg_db/models/media/pdf/__init__.py +2 -1
  121. endoreg_db/models/media/pdf/raw_pdf.py +11 -4
  122. endoreg_db/models/media/pdf/report_file.py +6 -2
  123. endoreg_db/models/media/pdf/report_reader/__init__.py +3 -3
  124. endoreg_db/models/media/pdf/report_reader/report_reader_flag.py +15 -5
  125. endoreg_db/models/media/video/create_from_file.py +20 -41
  126. endoreg_db/models/media/video/pipe_1.py +75 -30
  127. endoreg_db/models/media/video/pipe_2.py +37 -12
  128. endoreg_db/models/media/video/video_file.py +36 -24
  129. endoreg_db/models/media/video/video_file_ai.py +235 -70
  130. endoreg_db/models/media/video/video_file_anonymize.py +240 -65
  131. endoreg_db/models/media/video/video_file_frames/_bulk_create_frames.py +6 -1
  132. endoreg_db/models/media/video/video_file_frames/_create_frame_object.py +3 -1
  133. endoreg_db/models/media/video/video_file_frames/_delete_frames.py +30 -9
  134. endoreg_db/models/media/video/video_file_frames/_extract_frames.py +95 -29
  135. endoreg_db/models/media/video/video_file_frames/_get_frame.py +13 -3
  136. endoreg_db/models/media/video/video_file_frames/_get_frame_path.py +4 -1
  137. endoreg_db/models/media/video/video_file_frames/_get_frame_paths.py +15 -3
  138. endoreg_db/models/media/video/video_file_frames/_get_frame_range.py +15 -3
  139. endoreg_db/models/media/video/video_file_frames/_get_frames.py +7 -2
  140. endoreg_db/models/media/video/video_file_frames/_initialize_frames.py +109 -23
  141. endoreg_db/models/media/video/video_file_frames/_manage_frame_range.py +111 -27
  142. endoreg_db/models/media/video/video_file_frames/_mark_frames_extracted_status.py +46 -13
  143. endoreg_db/models/media/video/video_file_io.py +85 -33
  144. endoreg_db/models/media/video/video_file_meta/__init__.py +6 -6
  145. endoreg_db/models/media/video/video_file_meta/get_crop_template.py +17 -4
  146. endoreg_db/models/media/video/video_file_meta/get_endo_roi.py +28 -7
  147. endoreg_db/models/media/video/video_file_meta/get_fps.py +46 -13
  148. endoreg_db/models/media/video/video_file_meta/initialize_video_specs.py +81 -20
  149. endoreg_db/models/media/video/video_file_meta/text_meta.py +61 -20
  150. endoreg_db/models/media/video/video_file_meta/video_meta.py +40 -12
  151. endoreg_db/models/media/video/video_file_segments.py +118 -27
  152. endoreg_db/models/media/video/video_metadata.py +25 -6
  153. endoreg_db/models/media/video/video_processing.py +54 -15
  154. endoreg_db/models/medical/__init__.py +3 -13
  155. endoreg_db/models/medical/contraindication/__init__.py +3 -1
  156. endoreg_db/models/medical/disease.py +18 -6
  157. endoreg_db/models/medical/event.py +6 -2
  158. endoreg_db/models/medical/examination/__init__.py +5 -1
  159. endoreg_db/models/medical/examination/examination.py +22 -6
  160. endoreg_db/models/medical/examination/examination_indication.py +23 -7
  161. endoreg_db/models/medical/examination/examination_time.py +6 -2
  162. endoreg_db/models/medical/finding/__init__.py +3 -1
  163. endoreg_db/models/medical/finding/finding.py +37 -12
  164. endoreg_db/models/medical/finding/finding_classification.py +27 -8
  165. endoreg_db/models/medical/finding/finding_intervention.py +19 -6
  166. endoreg_db/models/medical/finding/finding_type.py +3 -1
  167. endoreg_db/models/medical/hardware/__init__.py +1 -1
  168. endoreg_db/models/medical/hardware/endoscope.py +14 -2
  169. endoreg_db/models/medical/laboratory/__init__.py +1 -1
  170. endoreg_db/models/medical/laboratory/lab_value.py +139 -39
  171. endoreg_db/models/medical/medication/__init__.py +7 -3
  172. endoreg_db/models/medical/medication/medication.py +3 -1
  173. endoreg_db/models/medical/medication/medication_indication.py +3 -1
  174. endoreg_db/models/medical/medication/medication_indication_type.py +11 -3
  175. endoreg_db/models/medical/medication/medication_intake_time.py +3 -1
  176. endoreg_db/models/medical/medication/medication_schedule.py +3 -1
  177. endoreg_db/models/medical/patient/__init__.py +2 -10
  178. endoreg_db/models/medical/patient/medication_examples.py +3 -14
  179. endoreg_db/models/medical/patient/patient_disease.py +17 -5
  180. endoreg_db/models/medical/patient/patient_event.py +12 -4
  181. endoreg_db/models/medical/patient/patient_examination.py +52 -15
  182. endoreg_db/models/medical/patient/patient_examination_indication.py +15 -4
  183. endoreg_db/models/medical/patient/patient_finding.py +105 -29
  184. endoreg_db/models/medical/patient/patient_finding_classification.py +41 -12
  185. endoreg_db/models/medical/patient/patient_finding_intervention.py +11 -3
  186. endoreg_db/models/medical/patient/patient_lab_sample.py +6 -2
  187. endoreg_db/models/medical/patient/patient_lab_value.py +42 -10
  188. endoreg_db/models/medical/patient/patient_medication.py +25 -7
  189. endoreg_db/models/medical/patient/patient_medication_schedule.py +34 -10
  190. endoreg_db/models/metadata/model_meta.py +40 -12
  191. endoreg_db/models/metadata/model_meta_logic.py +51 -16
  192. endoreg_db/models/metadata/sensitive_meta.py +65 -28
  193. endoreg_db/models/metadata/sensitive_meta_logic.py +28 -26
  194. endoreg_db/models/metadata/video_meta.py +146 -39
  195. endoreg_db/models/metadata/video_prediction_logic.py +70 -21
  196. endoreg_db/models/metadata/video_prediction_meta.py +80 -27
  197. endoreg_db/models/operation_log.py +63 -0
  198. endoreg_db/models/other/__init__.py +10 -10
  199. endoreg_db/models/other/distribution/__init__.py +9 -7
  200. endoreg_db/models/other/distribution/base_value_distribution.py +3 -1
  201. endoreg_db/models/other/distribution/date_value_distribution.py +19 -5
  202. endoreg_db/models/other/distribution/multiple_categorical_value_distribution.py +3 -1
  203. endoreg_db/models/other/distribution/numeric_value_distribution.py +34 -9
  204. endoreg_db/models/other/emission/__init__.py +1 -1
  205. endoreg_db/models/other/emission/emission_factor.py +9 -3
  206. endoreg_db/models/other/information_source.py +15 -5
  207. endoreg_db/models/other/material.py +3 -1
  208. endoreg_db/models/other/transport_route.py +3 -1
  209. endoreg_db/models/other/unit.py +6 -2
  210. endoreg_db/models/report/report.py +0 -1
  211. endoreg_db/models/requirement/requirement.py +84 -27
  212. endoreg_db/models/requirement/requirement_error.py +5 -6
  213. endoreg_db/models/requirement/requirement_evaluation/__init__.py +1 -1
  214. endoreg_db/models/requirement/requirement_evaluation/evaluate_with_dependencies.py +8 -8
  215. endoreg_db/models/requirement/requirement_evaluation/get_values.py +3 -3
  216. endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +24 -8
  217. endoreg_db/models/requirement/requirement_operator.py +28 -8
  218. endoreg_db/models/requirement/requirement_set.py +34 -11
  219. endoreg_db/models/state/__init__.py +1 -0
  220. endoreg_db/models/state/audit_ledger.py +9 -2
  221. endoreg_db/models/{media → state}/processing_history/__init__.py +1 -3
  222. endoreg_db/models/state/processing_history/processing_history.py +136 -0
  223. endoreg_db/models/state/raw_pdf.py +0 -1
  224. endoreg_db/models/state/video.py +2 -4
  225. endoreg_db/models/utils.py +4 -2
  226. endoreg_db/queries/__init__.py +2 -6
  227. endoreg_db/queries/annotations/__init__.py +1 -3
  228. endoreg_db/queries/annotations/legacy.py +37 -26
  229. endoreg_db/root_urls.py +3 -4
  230. endoreg_db/schemas/examination_evaluation.py +3 -0
  231. endoreg_db/serializers/Frames_NICE_and_PARIS_classifications.py +249 -163
  232. endoreg_db/serializers/__init__.py +2 -8
  233. endoreg_db/serializers/administration/__init__.py +1 -2
  234. endoreg_db/serializers/administration/ai/__init__.py +0 -1
  235. endoreg_db/serializers/administration/ai/active_model.py +3 -1
  236. endoreg_db/serializers/administration/ai/ai_model.py +5 -3
  237. endoreg_db/serializers/administration/ai/model_type.py +3 -1
  238. endoreg_db/serializers/administration/center.py +7 -2
  239. endoreg_db/serializers/administration/gender.py +4 -2
  240. endoreg_db/serializers/anonymization.py +13 -13
  241. endoreg_db/serializers/evaluation/examination_evaluation.py +0 -1
  242. endoreg_db/serializers/examination/__init__.py +1 -1
  243. endoreg_db/serializers/examination/base.py +12 -13
  244. endoreg_db/serializers/examination/dropdown.py +6 -7
  245. endoreg_db/serializers/examination_serializer.py +3 -6
  246. endoreg_db/serializers/finding/__init__.py +1 -1
  247. endoreg_db/serializers/finding/finding.py +14 -7
  248. endoreg_db/serializers/finding_classification/__init__.py +3 -3
  249. endoreg_db/serializers/finding_classification/choice.py +3 -3
  250. endoreg_db/serializers/finding_classification/classification.py +2 -4
  251. endoreg_db/serializers/label_video_segment/__init__.py +5 -3
  252. endoreg_db/serializers/{label → label_video_segment}/image_classification_annotation.py +5 -5
  253. endoreg_db/serializers/label_video_segment/label/__init__.py +6 -0
  254. endoreg_db/serializers/{label → label_video_segment/label}/label.py +1 -1
  255. endoreg_db/serializers/label_video_segment/label_video_segment.py +338 -228
  256. endoreg_db/serializers/meta/__init__.py +1 -2
  257. endoreg_db/serializers/meta/sensitive_meta_detail.py +28 -13
  258. endoreg_db/serializers/meta/sensitive_meta_update.py +51 -46
  259. endoreg_db/serializers/meta/sensitive_meta_verification.py +19 -16
  260. endoreg_db/serializers/misc/__init__.py +2 -2
  261. endoreg_db/serializers/misc/file_overview.py +11 -7
  262. endoreg_db/serializers/misc/stats.py +10 -8
  263. endoreg_db/serializers/misc/translatable_field_mix_in.py +6 -6
  264. endoreg_db/serializers/misc/upload_job.py +32 -29
  265. endoreg_db/serializers/patient/__init__.py +2 -1
  266. endoreg_db/serializers/patient/patient.py +32 -15
  267. endoreg_db/serializers/patient/patient_dropdown.py +11 -3
  268. endoreg_db/serializers/patient_examination/__init__.py +1 -1
  269. endoreg_db/serializers/patient_examination/patient_examination.py +67 -40
  270. endoreg_db/serializers/patient_finding/__init__.py +1 -1
  271. endoreg_db/serializers/patient_finding/patient_finding.py +2 -1
  272. endoreg_db/serializers/patient_finding/patient_finding_classification.py +17 -9
  273. endoreg_db/serializers/patient_finding/patient_finding_detail.py +26 -17
  274. endoreg_db/serializers/patient_finding/patient_finding_intervention.py +7 -5
  275. endoreg_db/serializers/patient_finding/patient_finding_list.py +10 -11
  276. endoreg_db/serializers/patient_finding/patient_finding_write.py +36 -27
  277. endoreg_db/serializers/pdf/__init__.py +1 -3
  278. endoreg_db/serializers/requirements/requirement_schema.py +1 -6
  279. endoreg_db/serializers/sensitive_meta_serializer.py +100 -81
  280. endoreg_db/serializers/video/__init__.py +2 -2
  281. endoreg_db/serializers/video/{segmentation.py → video_file.py} +66 -47
  282. endoreg_db/serializers/video/video_file_brief.py +6 -2
  283. endoreg_db/serializers/video/video_file_detail.py +36 -23
  284. endoreg_db/serializers/video/video_file_list.py +4 -2
  285. endoreg_db/serializers/video/video_processing_history.py +54 -50
  286. endoreg_db/services/__init__.py +1 -1
  287. endoreg_db/services/anonymization.py +2 -2
  288. endoreg_db/services/examination_evaluation.py +40 -17
  289. endoreg_db/services/model_meta_from_hf.py +76 -0
  290. endoreg_db/services/polling_coordinator.py +101 -70
  291. endoreg_db/services/pseudonym_service.py +27 -22
  292. endoreg_db/services/report_import.py +6 -3
  293. endoreg_db/services/segment_sync.py +75 -59
  294. endoreg_db/services/video_import.py +6 -7
  295. endoreg_db/urls/__init__.py +2 -2
  296. endoreg_db/urls/ai.py +7 -25
  297. endoreg_db/urls/anonymization.py +61 -15
  298. endoreg_db/urls/auth.py +4 -4
  299. endoreg_db/urls/classification.py +4 -9
  300. endoreg_db/urls/examination.py +27 -18
  301. endoreg_db/urls/media.py +27 -34
  302. endoreg_db/urls/patient.py +11 -7
  303. endoreg_db/urls/requirements.py +3 -1
  304. endoreg_db/urls/root_urls.py +2 -3
  305. endoreg_db/urls/stats.py +24 -16
  306. endoreg_db/urls/upload.py +3 -11
  307. endoreg_db/utils/__init__.py +14 -15
  308. endoreg_db/utils/ai/__init__.py +1 -1
  309. endoreg_db/utils/ai/data_loader_for_model_input.py +262 -0
  310. endoreg_db/utils/ai/data_loader_for_model_training.py +262 -0
  311. endoreg_db/utils/ai/get.py +2 -1
  312. endoreg_db/utils/ai/inference_dataset.py +14 -15
  313. endoreg_db/utils/ai/model_training/config.py +117 -0
  314. endoreg_db/utils/ai/model_training/dataset.py +74 -0
  315. endoreg_db/utils/ai/model_training/losses.py +68 -0
  316. endoreg_db/utils/ai/model_training/metrics.py +78 -0
  317. endoreg_db/utils/ai/model_training/model_backbones.py +155 -0
  318. endoreg_db/utils/ai/model_training/model_gastronet_resnet.py +118 -0
  319. endoreg_db/utils/ai/model_training/trainer_gastronet_multilabel.py +771 -0
  320. endoreg_db/utils/ai/multilabel_classification_net.py +21 -6
  321. endoreg_db/utils/ai/predict.py +4 -4
  322. endoreg_db/utils/ai/preprocess.py +19 -11
  323. endoreg_db/utils/calc_duration_seconds.py +4 -4
  324. endoreg_db/utils/case_generator/lab_sample_factory.py +3 -4
  325. endoreg_db/utils/check_video_files.py +74 -47
  326. endoreg_db/utils/cropping.py +10 -9
  327. endoreg_db/utils/dataloader.py +11 -3
  328. endoreg_db/utils/dates.py +3 -4
  329. endoreg_db/utils/defaults/set_default_center.py +7 -6
  330. endoreg_db/utils/env.py +6 -2
  331. endoreg_db/utils/extract_specific_frames.py +24 -9
  332. endoreg_db/utils/file_operations.py +30 -18
  333. endoreg_db/utils/fix_video_path_direct.py +57 -41
  334. endoreg_db/utils/frame_anonymization_utils.py +157 -157
  335. endoreg_db/utils/hashs.py +3 -18
  336. endoreg_db/utils/links/requirement_link.py +96 -52
  337. endoreg_db/utils/ocr.py +30 -25
  338. endoreg_db/utils/operation_log.py +61 -0
  339. endoreg_db/utils/parse_and_generate_yaml.py +12 -13
  340. endoreg_db/utils/paths.py +6 -6
  341. endoreg_db/utils/permissions.py +40 -24
  342. endoreg_db/utils/pipelines/process_video_dir.py +50 -26
  343. endoreg_db/utils/product/sum_emissions.py +5 -3
  344. endoreg_db/utils/product/sum_weights.py +4 -2
  345. endoreg_db/utils/pydantic_models/__init__.py +3 -4
  346. endoreg_db/utils/requirement_operator_logic/_old/lab_value_operators.py +207 -107
  347. endoreg_db/utils/requirement_operator_logic/_old/model_evaluators.py +252 -65
  348. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +27 -10
  349. endoreg_db/utils/setup_config.py +21 -5
  350. endoreg_db/utils/storage.py +3 -1
  351. endoreg_db/utils/translation.py +19 -15
  352. endoreg_db/utils/uuid.py +1 -0
  353. endoreg_db/utils/validate_endo_roi.py +12 -4
  354. endoreg_db/utils/validate_subcategory_dict.py +26 -24
  355. endoreg_db/utils/validate_video_detailed.py +207 -149
  356. endoreg_db/utils/video/__init__.py +7 -3
  357. endoreg_db/utils/video/extract_frames.py +30 -18
  358. endoreg_db/utils/video/names.py +11 -6
  359. endoreg_db/utils/video/streaming_processor.py +175 -101
  360. endoreg_db/utils/video/video_splitter.py +30 -19
  361. endoreg_db/views/Frames_NICE_and_PARIS_classifications_views.py +59 -50
  362. endoreg_db/views/__init__.py +0 -20
  363. endoreg_db/views/anonymization/__init__.py +6 -2
  364. endoreg_db/views/anonymization/media_management.py +2 -6
  365. endoreg_db/views/anonymization/overview.py +34 -1
  366. endoreg_db/views/anonymization/validate.py +79 -18
  367. endoreg_db/views/auth/__init__.py +1 -1
  368. endoreg_db/views/auth/keycloak.py +16 -14
  369. endoreg_db/views/examination/__init__.py +12 -15
  370. endoreg_db/views/examination/examination.py +5 -5
  371. endoreg_db/views/examination/examination_manifest_cache.py +5 -5
  372. endoreg_db/views/examination/get_finding_classification_choices.py +8 -5
  373. endoreg_db/views/examination/get_finding_classifications.py +9 -7
  374. endoreg_db/views/examination/get_findings.py +8 -10
  375. endoreg_db/views/examination/get_instruments.py +3 -2
  376. endoreg_db/views/examination/get_interventions.py +1 -1
  377. endoreg_db/views/finding/__init__.py +2 -2
  378. endoreg_db/views/finding/finding.py +58 -54
  379. endoreg_db/views/finding/get_classifications.py +1 -1
  380. endoreg_db/views/finding/get_interventions.py +1 -1
  381. endoreg_db/views/finding_classification/__init__.py +5 -5
  382. endoreg_db/views/finding_classification/finding_classification.py +5 -6
  383. endoreg_db/views/finding_classification/get_classification_choices.py +3 -4
  384. endoreg_db/views/media/__init__.py +13 -13
  385. endoreg_db/views/media/pdf_media.py +9 -9
  386. endoreg_db/views/media/sensitive_metadata.py +10 -7
  387. endoreg_db/views/media/video_media.py +4 -4
  388. endoreg_db/views/meta/__init__.py +1 -1
  389. endoreg_db/views/meta/sensitive_meta_list.py +20 -22
  390. endoreg_db/views/meta/sensitive_meta_verification.py +14 -11
  391. endoreg_db/views/misc/__init__.py +6 -34
  392. endoreg_db/views/misc/center.py +2 -1
  393. endoreg_db/views/misc/csrf.py +2 -1
  394. endoreg_db/views/misc/gender.py +2 -1
  395. endoreg_db/views/misc/stats.py +141 -106
  396. endoreg_db/views/patient/__init__.py +1 -3
  397. endoreg_db/views/patient/patient.py +141 -99
  398. endoreg_db/views/patient_examination/__init__.py +5 -5
  399. endoreg_db/views/patient_examination/patient_examination.py +43 -42
  400. endoreg_db/views/patient_examination/patient_examination_create.py +10 -15
  401. endoreg_db/views/patient_examination/patient_examination_detail.py +12 -15
  402. endoreg_db/views/patient_examination/patient_examination_list.py +21 -17
  403. endoreg_db/views/patient_examination/video.py +114 -80
  404. endoreg_db/views/patient_finding/__init__.py +1 -1
  405. endoreg_db/views/patient_finding/patient_finding.py +17 -10
  406. endoreg_db/views/patient_finding/patient_finding_optimized.py +127 -95
  407. endoreg_db/views/patient_finding_classification/__init__.py +1 -1
  408. endoreg_db/views/patient_finding_classification/pfc_create.py +35 -27
  409. endoreg_db/views/report/reimport.py +1 -1
  410. endoreg_db/views/report/report_stream.py +5 -8
  411. endoreg_db/views/requirement/__init__.py +2 -1
  412. endoreg_db/views/requirement/evaluate.py +7 -9
  413. endoreg_db/views/requirement/lookup.py +2 -3
  414. endoreg_db/views/requirement/lookup_store.py +0 -1
  415. endoreg_db/views/requirement/requirement_utils.py +2 -4
  416. endoreg_db/views/stats/__init__.py +4 -4
  417. endoreg_db/views/stats/stats_views.py +152 -115
  418. endoreg_db/views/video/__init__.py +18 -27
  419. endoreg_db/views/{ai → video/ai}/__init__.py +2 -2
  420. endoreg_db/views/{ai → video/ai}/label.py +20 -16
  421. endoreg_db/views/video/correction.py +5 -6
  422. endoreg_db/views/video/reimport.py +134 -99
  423. endoreg_db/views/video/segments_crud.py +134 -44
  424. endoreg_db/views/video/video_apply_mask.py +13 -12
  425. endoreg_db/views/video/video_correction.py +2 -1
  426. endoreg_db/views/video/video_download_processed.py +15 -15
  427. endoreg_db/views/video/video_meta_stats.py +7 -6
  428. endoreg_db/views/video/video_processing_history.py +3 -2
  429. endoreg_db/views/video/video_remove_frames.py +13 -12
  430. endoreg_db/views/video/video_stream.py +110 -82
  431. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/METADATA +9 -3
  432. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/RECORD +434 -431
  433. endoreg_db/management/commands/import_fallback_video.py +0 -203
  434. endoreg_db/management/commands/import_video.py +0 -422
  435. endoreg_db/management/commands/import_video_with_classification.py +0 -367
  436. endoreg_db/models/media/processing_history/processing_history.py +0 -96
  437. endoreg_db/serializers/label/__init__.py +0 -7
  438. endoreg_db/serializers/label_video_segment/_lvs_create.py +0 -149
  439. endoreg_db/serializers/label_video_segment/_lvs_update.py +0 -138
  440. endoreg_db/serializers/label_video_segment/_lvs_validate.py +0 -149
  441. endoreg_db/serializers/label_video_segment/label_video_segment_annotation.py +0 -99
  442. endoreg_db/serializers/label_video_segment/label_video_segment_update.py +0 -163
  443. endoreg_db/services/__old/pdf_import.py +0 -1487
  444. endoreg_db/services/__old/video_import.py +0 -1306
  445. endoreg_db/tasks/upload_tasks.py +0 -216
  446. endoreg_db/tasks/video_ingest.py +0 -161
  447. endoreg_db/tasks/video_processing_tasks.py +0 -327
  448. endoreg_db/views/misc/translation.py +0 -182
  449. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/WHEEL +0 -0
  450. {endoreg_db-0.8.9.2.dist-info → endoreg_db-0.8.9.10.dist-info}/licenses/LICENSE +0 -0
@@ -1,7 +1,8 @@
1
- from endoreg_db.models.media.processing_history.processing_history import ProcessingHistory
2
- from endoreg_db.utils.paths import IMPORT_REPORT_DIR, IMPORT_VIDEO_DIR, ANONYM_REPORT_DIR, ANONYM_VIDEO_DIR
1
+ from endoreg_db.models.state.processing_history.processing_history import (
2
+ ProcessingHistory,
3
+ )
4
+ from endoreg_db.utils.paths import ANONYM_REPORT_DIR, ANONYM_VIDEO_DIR
3
5
 
4
- import os
5
6
  import logging
6
7
  import shutil
7
8
  from pathlib import Path
@@ -13,11 +14,24 @@ from endoreg_db.import_files.context.import_context import ImportContext
13
14
  from endoreg_db.models.media import RawPdfFile, VideoFile
14
15
  from endoreg_db.models.state import RawPdfState, VideoState
15
16
  from endoreg_db.utils import paths as path_utils
17
+ from endoreg_db.utils.file_operations import sha256_file
16
18
 
17
19
  logger = logging.getLogger(__name__)
18
20
 
19
21
 
20
- def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[Union[RawPdfState, VideoState]]:
22
+ def _get_history_filename(ctx: ImportContext) -> str:
23
+ """
24
+ Prefer original_path.name if provided, otherwise fall back to file_path.name.
25
+ """
26
+ if ctx.original_path is not None:
27
+ return ctx.original_path.name
28
+ # ctx.file_path is always present and already a Path in your tests
29
+ return Path(ctx.file_path).name
30
+
31
+
32
+ def _ensure_instance_state(
33
+ instance: Union[VideoFile, RawPdfFile],
34
+ ) -> Optional[Union[RawPdfState, VideoState]]:
21
35
  """
22
36
  Helper: ensure instance.state exists and return it.
23
37
  Mirrors PdfImportService._ensure_state.
@@ -26,7 +40,7 @@ def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[U
26
40
  state = getattr(instance, "state", None)
27
41
  else:
28
42
  state = getattr(instance, "state", None)
29
-
43
+
30
44
  if state is not None:
31
45
  return state
32
46
 
@@ -37,14 +51,15 @@ def _ensure_instance_state(instance: Union[VideoFile, RawPdfFile]) -> Optional[U
37
51
 
38
52
  return None
39
53
 
40
- def mark_instance_processing_started(
54
+
55
+ def mark_instance_processing_started(
41
56
  instance: Union[RawPdfFile, VideoFile],
42
- ctx: ImportContext,):
57
+ ctx: ImportContext,
58
+ ):
43
59
  state = _ensure_instance_state(instance)
44
60
 
45
61
  with transaction.atomic():
46
62
  if state is not None:
47
-
48
63
  # In the old code, processing_started was set earlier; we guard here
49
64
  if not getattr(state, "processing_started", False) and hasattr(
50
65
  state, "mark_processing_started"
@@ -125,28 +140,12 @@ def finalize_report_success(
125
140
  if current_name != relative_name:
126
141
  instance.processed_file.name = relative_name
127
142
  logger.info("Updated processed_file to %s", relative_name)
128
- try:
129
- relative_name = str(ctx.anonymized_path)
130
- except ValueError:
131
- # Fallback: absolute path if outside STORAGE_DIR
132
- relative_name = str(final_path)
133
-
134
- current_name = getattr(instance.processed_file, "name", None)
135
- if current_name != relative_name:
136
- instance.processed_file.name = relative_name
137
- logger.info(
138
- "Updated processed_file reference to: %s",
139
- instance.processed_file.name,
140
- )
141
-
142
143
 
143
144
  # --- Update RawPdfState flags (mirrors _finalize_processing) ---
144
145
  state = _ensure_instance_state(instance)
145
146
 
146
147
  with transaction.atomic():
147
148
  if state is not None:
148
-
149
- # In the old code, processing_started was set earlier; we guard here
150
149
  if not getattr(state, "processing_started", False) and hasattr(
151
150
  state, "mark_processing_started"
152
151
  ):
@@ -165,17 +164,20 @@ def finalize_report_success(
165
164
  # --- ProcessingHistory entry ---
166
165
  try:
167
166
  with transaction.atomic():
168
- ProcessingHistory.get_or_create_for_object(
167
+ if not isinstance(ctx.file_hash, str):
168
+ ctx.file_hash = sha256_file(ctx.file_path)
169
+ ProcessingHistory.get_or_create_for_hash(
169
170
  obj=instance,
171
+ file_hash=ctx.file_hash,
170
172
  success=True,
171
173
  )
172
174
  except Exception as e:
173
175
  logger.debug(
174
- "Saving not possible; %s"
175
- f"skipping ProcessingHistory.{e}",
176
+ f"Saving not possible; %sskipping ProcessingHistory.{e}",
176
177
  instance.pk,
177
178
  )
178
179
 
180
+
179
181
  def finalize_video_success(
180
182
  ctx: ImportContext,
181
183
  ) -> None:
@@ -187,8 +189,7 @@ def finalize_video_success(
187
189
  - Mark VideoState as anonymized + sensitive_meta_processed
188
190
  - Mark ProcessingHistory.success = True
189
191
  """
190
- nuke = nuke_transcoding_dir()
191
- assert(nuke is True)
192
+
192
193
  instance = ctx.current_video
193
194
  if not isinstance(instance, VideoFile):
194
195
  logger.warning("finalize_video_success called with non-VideoFile instance")
@@ -265,10 +266,13 @@ def finalize_video_success(
265
266
  instance.processed_file.name = relative_name
266
267
  logger.info("Updated video processed_file to %s", relative_name)
267
268
 
269
+ if not nuke_transcoding_dir():
270
+ logger.warning(
271
+ "Transcoding directory cleanup returned False after finalize_video_success; there may be leftover files."
272
+ )
273
+
268
274
  # --- Update VideoState flags (mirrors report) ---
269
275
  state = _ensure_instance_state(instance)
270
-
271
-
272
276
 
273
277
  with transaction.atomic():
274
278
  if state is not None:
@@ -289,8 +293,10 @@ def finalize_video_success(
289
293
  # --- ProcessingHistory entry ---
290
294
  try:
291
295
  with transaction.atomic():
292
- ProcessingHistory.get_or_create_for_object(
293
- obj=instance,
296
+ if not isinstance(ctx.file_hash, str):
297
+ ctx.file_hash = sha256_file(ctx.file_path)
298
+ ProcessingHistory.get_or_create_for_hash(
299
+ file_hash=ctx.file_hash,
294
300
  success=True,
295
301
  )
296
302
  except Exception as e:
@@ -309,7 +315,9 @@ def finalize_failure(
309
315
 
310
316
  - Reset RawPdfState flags to "not processed"
311
317
  - Mark ProcessingHistory.success = False
318
+ - Delete all associated files
312
319
  """
320
+
313
321
  if ctx.instance is None:
314
322
  if isinstance(ctx.current_report, RawPdfFile):
315
323
  ctx.instance = ctx.current_report
@@ -317,6 +325,15 @@ def finalize_failure(
317
325
  ctx.instance = ctx.current_video
318
326
  else:
319
327
  raise Exception
328
+
329
+ # History entry with success=False
330
+ if not isinstance(ctx.file_hash, str):
331
+ ctx.file_hash = sha256_file(ctx.file_path)
332
+ ProcessingHistory.get_or_create_for_hash(
333
+ file_hash=ctx.file_hash,
334
+ success=False,
335
+ )
336
+
320
337
  # Reset state flags similar to _mark_processing_incomplete / _cleanup_on_error
321
338
  state = _ensure_instance_state(ctx.instance)
322
339
 
@@ -336,29 +353,17 @@ def finalize_failure(
336
353
  e,
337
354
  )
338
355
 
339
- try:
340
- delete_associated_files(ctx)
341
- except Exception as e:
342
- logger.warning(f"There might be files remaining. {e}")
343
-
344
- # History entry with success=False
345
- if ctx.file_hash:
346
- ProcessingHistory.get_or_create_for_object(
347
- obj=ctx.instance,
348
- success=False,
349
- )
350
- else:
351
- logger.debug(
352
- "No file_hash in context for instance %s when finalizing failure; "
353
- "skipping ProcessingHistory.",
354
- ctx.instance.pk,
355
- )
356
+ try:
357
+ delete_associated_files(ctx)
358
+ except Exception as e:
359
+ logger.warning(f"There might be files remaining. {e}")
356
360
 
357
361
  logger.error(
358
- "Report processing failed for %s",
362
+ "File processing failed for %s - state reset, ready for retry.",
359
363
  ctx.file_path,
360
364
  )
361
365
 
366
+
362
367
  def delete_associated_files(ctx: ImportContext) -> None:
363
368
  """
364
369
  Best-effort cleanup of anonymized, sensitive and transcoding artefacts.
@@ -373,92 +378,46 @@ def delete_associated_files(ctx: ImportContext) -> None:
373
378
  Only restoration of the original import file is treated as critical.
374
379
  """
375
380
 
376
- # --- 1. Restore original import file if needed (critical) ---
377
- original_missing = not isinstance(ctx.original_path, Path) or not ctx.original_path.exists()
378
- if original_missing:
379
- logger.warning(
380
- "Original file missing in ctx (file_type=%s); "
381
- "trying to restore from sensitive copy.",
382
- ctx.file_type,
383
- )
384
-
385
- if not isinstance(ctx.sensitive_path, Path) or not ctx.sensitive_path.exists():
386
- # This is serious: we lost both original and sensitive copy
387
- msg = (
388
- f"Cannot restore original file for {ctx.file_type}: "
389
- "sensitive copy missing as well."
390
- )
391
- logger.error(msg)
392
- raise RuntimeError(msg)
393
-
394
- try:
395
- if ctx.file_type == "video":
396
- target_dir = IMPORT_VIDEO_DIR
397
- elif ctx.file_type == "report":
398
- target_dir = IMPORT_REPORT_DIR
399
- else:
400
- raise ValueError(f"Unknown file_type in context: {ctx.file_type}")
401
-
402
- target_dir.mkdir(parents=True, exist_ok=True)
403
- restored_path = shutil.copy2(ctx.sensitive_path, target_dir)
404
- ctx.original_path = Path(restored_path)
405
- logger.info("Restored original file for %s to %s", ctx.file_type, ctx.original_path)
406
- except Exception as e:
407
- logger.error("Error during safety copy / restore of original file: %s", e, exc_info=True)
408
- raise
409
-
410
- # --- 2. Delete anonymized file (best-effort) ---
381
+ # --- Delete anonymized file (best-effort) ---
411
382
  if isinstance(ctx.anonymized_path, Path):
412
383
  try:
413
- if ctx.anonymized_path.exists() and isinstance(ctx.anonymized_path, Path):
384
+ if ctx.anonymized_path.exists():
414
385
  ctx.anonymized_path.unlink()
415
386
  logger.info("Deleted anonymized file %s", ctx.anonymized_path)
416
387
  except Exception as e:
417
- logger.error("Error when unlinking anonymized path %s: %s", ctx.anonymized_path, e, exc_info=True)
418
- if ctx.anonymized_path.exists() and isinstance( ctx.anonymized_path, str):
419
- if isinstance(ctx.current_video, VideoFile):
420
- p = Path(path_utils.data_paths["anonym_video" / ctx.anonymized_path])
421
- p.unlink()
422
- elif isinstance(ctx.current_report, RawPdfFile):
423
- p = Path(path_utils.data_paths["anonym_report" / ctx.anonymized_path])
424
- p.unlink()
425
- if ctx.anonymized_path.exists():
426
- ctx.anonymized_path.rmdir()
388
+ logger.error(
389
+ "Error when unlinking anonymized path %s: %s",
390
+ ctx.anonymized_path,
391
+ e,
392
+ exc_info=True,
393
+ )
427
394
  finally:
428
- if ctx.anonymized_path.exists():
429
- raise AssertionError("Anonym file remains after all deletion attempts.")
430
395
  ctx.anonymized_path = None
431
396
 
432
- # --- 3. Nuke transcoding directory (best-effort) ---
397
+ # --- Nuke transcoding directory (best-effort) ---
433
398
  if not nuke_transcoding_dir():
434
- logger.warning("Transcoding directory cleanup returned False; there may be leftover files.")
399
+ logger.warning(
400
+ "Transcoding directory cleanup returned False; there may be leftover files."
401
+ )
435
402
 
436
- # --- 4. Delete sensitive file (best-effort) ---
403
+ # --- Delete sensitive file (best-effort) ---
437
404
  if isinstance(ctx.sensitive_path, Path):
438
405
  try:
439
406
  if ctx.sensitive_path.exists():
440
407
  ctx.sensitive_path.unlink()
441
408
  logger.info("Deleted sensitive file %s", ctx.sensitive_path)
442
409
  except Exception as e:
443
- logger.error("Error when unlinking sensitive path %s: %s", ctx.sensitive_path, e, exc_info=True)
444
- if ctx.sensitive_path.exists() and isinstance( ctx.sensitive_path, str):
445
- if isinstance(ctx.current_video, VideoFile):
446
- p = Path(path_utils.data_paths["sensitive_video" / ctx.sensitive_path])
447
- p.unlink()
448
- elif isinstance(ctx.current_report, RawPdfFile):
449
- p = Path(path_utils.data_paths["sensitive_report" / ctx.sensitive_path])
450
- p.unlink()
451
- if ctx.sensitive_path.exists():
452
- ctx.sensitive_path.rmdir()
410
+ logger.error(
411
+ "Error when unlinking sensitive path %s: %s",
412
+ ctx.sensitive_path,
413
+ e,
414
+ exc_info=True,
415
+ )
453
416
  finally:
454
- if ctx.sensitive_path.exists():
455
- raise AssertionError("Sensitive file remains after all deletion attempts.")
456
417
  ctx.sensitive_path = None
457
418
 
458
-
459
- def nuke_transcoding_dir(
460
- transcoding_dir: Union[str, Path, None] = None
461
- ) -> bool:
419
+
420
+ def nuke_transcoding_dir(transcoding_dir: Union[str, Path, None] = None) -> bool:
462
421
  """
463
422
  Delete all files and subdirectories inside the transcoding directory.
464
423
 
@@ -473,11 +432,15 @@ def nuke_transcoding_dir(
473
432
  transcoding_dir = Path(transcoding_dir)
474
433
 
475
434
  if not transcoding_dir.exists():
476
- logger.info("Transcoding dir %s does not exist; nothing to clean.", transcoding_dir)
435
+ logger.info(
436
+ "Transcoding dir %s does not exist; nothing to clean.", transcoding_dir
437
+ )
477
438
  return True
478
439
 
479
440
  if not transcoding_dir.is_dir():
480
- logger.error("Configured transcoding path %s is not a directory.", transcoding_dir)
441
+ logger.error(
442
+ "Configured transcoding path %s is not a directory.", transcoding_dir
443
+ )
481
444
  return False
482
445
 
483
446
  for entry in transcoding_dir.iterdir():
@@ -487,10 +450,14 @@ def nuke_transcoding_dir(
487
450
  elif entry.is_dir():
488
451
  shutil.rmtree(entry)
489
452
  except Exception as e:
490
- logger.warning("Failed to remove entry %s in transcoding dir: %s", entry, e)
453
+ logger.warning(
454
+ "Failed to remove entry %s in transcoding dir: %s", entry, e
455
+ )
491
456
  # Continue trying to delete other entries
492
457
  return True
493
458
 
494
459
  except Exception as e:
495
- logger.error("Unexpected error while nuking transcoding dir: %s", e, exc_info=True)
460
+ logger.error(
461
+ "Unexpected error while nuking transcoding dir: %s", e, exc_info=True
462
+ )
496
463
  return False
@@ -1,6 +1,10 @@
1
1
  import logging
2
2
  from pathlib import Path
3
- from endoreg_db.models.media.video.create_from_file import atomic_copy_with_fallback, atomic_move_with_fallback
3
+ from endoreg_db.models.media.video.create_from_file import (
4
+ atomic_copy_with_fallback,
5
+ atomic_move_with_fallback,
6
+ )
7
+
4
8
  logger = logging.getLogger(__name__)
5
9
 
6
10
 
@@ -2,13 +2,14 @@ import logging
2
2
  import os
3
3
  import sys
4
4
  from pathlib import Path
5
- from typing import Any, Callable, Literal, NoReturn
6
5
 
7
6
  from lx_anonymizer import ReportReader
8
7
  from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
9
8
 
10
9
  from endoreg_db.import_files.context import ImportContext
11
- from endoreg_db.import_files.file_storage.sensitive_meta_storage import sensitive_meta_storage
10
+ from endoreg_db.import_files.file_storage.sensitive_meta_storage import (
11
+ sensitive_meta_storage,
12
+ )
12
13
  from endoreg_db.utils.paths import ANONYM_REPORT_DIR
13
14
 
14
15
 
@@ -20,10 +21,8 @@ class ReportAnonymizer:
20
21
  self._report_reader_class = None
21
22
  self._ensure_report_reading_available()
22
23
  self.storage = False
23
-
24
24
 
25
25
  def anonymize_report(self, ctx: ImportContext):
26
-
27
26
  # Setup anonymized directory
28
27
  anonymized_dir = ANONYM_REPORT_DIR
29
28
  anonymized_dir.mkdir(parents=True, exist_ok=True)
@@ -32,29 +31,35 @@ class ReportAnonymizer:
32
31
  pdf_hash = ctx.current_report.pdf_hash
33
32
  anonymized_output_path = anonymized_dir / f"{pdf_hash}.pdf"
34
33
  self._report_reader_class = ReportReader()
35
-
34
+
36
35
  assert isinstance(self._report_reader_class, ReportReader)
37
36
 
38
37
  # Process with enhanced process_report method (returns 4-tuple now)
39
- ctx.original_text, ctx.anonymized_text, extracted_metadata, ctx.anonymized_path = self._report_reader_class.process_report(
40
- pdf_path=ctx.file_path,
41
- create_anonymized_pdf=True,
42
- anonymized_pdf_output_path=str(anonymized_output_path),
43
- )
44
-
38
+ (
39
+ ctx.original_text,
40
+ ctx.anonymized_text,
41
+ extracted_metadata,
42
+ ctx.anonymized_path,
43
+ ) = self._report_reader_class.process_report(
44
+ pdf_path=ctx.file_path,
45
+ create_anonymized_pdf=True,
46
+ anonymized_pdf_output_path=str(anonymized_output_path),
47
+ )
48
+
45
49
  if ctx.anonymized_path:
46
- logger.info("DEBUG: after anonymizer, ctx.anonymized_path=%s (exists=%s)",
47
- ctx.anonymized_path, isinstance(ctx.anonymized_path, str))
50
+ logger.info(
51
+ "DEBUG: after anonymizer, ctx.anonymized_path=%s (exists=%s)",
52
+ ctx.anonymized_path,
53
+ isinstance(ctx.anonymized_path, str),
54
+ )
48
55
 
49
56
  sm = LxSM()
50
- sm.safe_update(extracted_metadata)
51
-
57
+ sm.safe_update(extracted_metadata)
58
+
52
59
  self.storage = sensitive_meta_storage(sm, ctx.current_report)
53
60
  return ctx
54
-
55
- def _ensure_report_reading_available(
56
- self
57
- ) -> None:
61
+
62
+ def _ensure_report_reading_available(self) -> None:
58
63
  """
59
64
  Ensure report reading modules are available by adding lx-anonymizer to path.
60
65
 
@@ -21,10 +21,10 @@ def normalize_lx_sensitive_meta(meta: LxSensitiveMeta) -> Dict[str, Any]:
21
21
  "file_path",
22
22
  "patient_first_name",
23
23
  "patient_last_name",
24
- "patient_dob", # string; logic has parsing
24
+ "patient_dob", # string; logic has parsing
25
25
  "casenumber",
26
- "examination_date", # string; logic has parsing
27
- "examination_time", # string "HH:MM" is fine
26
+ "examination_date", # string; logic has parsing
27
+ "examination_time", # string "HH:MM" is fine
28
28
  "examiner_first_name",
29
29
  "examiner_last_name",
30
30
  "text",
@@ -1,16 +1,16 @@
1
- from typing import List, Dict, Any, Tuple, Optional
2
-
3
1
  import logging
2
+
4
3
  logger = logging.getLogger(__name__)
5
4
 
6
5
  from lx_anonymizer import FrameCleaner
7
6
  from lx_anonymizer.sensitive_meta_interface import SensitiveMeta as LxSM
8
7
 
9
-
10
- from endoreg_db.import_files.file_storage.sensitive_meta_storage import sensitive_meta_storage
11
8
  from endoreg_db.import_files.context import ImportContext
12
- from endoreg_db.utils.paths import ANONYM_VIDEO_DIR
9
+ from endoreg_db.import_files.file_storage.sensitive_meta_storage import (
10
+ sensitive_meta_storage,
11
+ )
13
12
  from endoreg_db.models import EndoscopyProcessor, VideoFile
13
+ from endoreg_db.utils.paths import ANONYM_VIDEO_DIR
14
14
 
15
15
 
16
16
  class VideoAnonymizer:
@@ -20,35 +20,35 @@ class VideoAnonymizer:
20
20
  self._frame_cleaning_class = None
21
21
  self.storage = False
22
22
 
23
-
24
23
  def anonymize_video(self, ctx: ImportContext):
25
24
  # Setup anonymized directory
26
25
  anonymized_dir = ANONYM_VIDEO_DIR
27
26
  anonymized_dir.mkdir(parents=True, exist_ok=True)
28
27
  assert ctx.current_video is not None
29
28
  # Generate output path for anonymized report
30
-
29
+
31
30
  video_hash = ctx.current_video.video_hash
32
31
  anonymized_output_path = anonymized_dir / f"{video_hash}.mp4"
33
-
32
+
34
33
  self._frame_cleaning_class = FrameCleaner()
35
-
34
+
36
35
  assert isinstance(self._frame_cleaning_class, FrameCleaner)
37
36
  endoscope_roi, endoscope_roi_nested = self._get_processor_roi_info(ctx)
38
37
  # Process with enhanced process_report method (returns 4-tuple now)
39
- ctx.anonymized_path, extracted_metadata = self._frame_cleaning_class.clean_video(
38
+ ctx.anonymized_path, extracted_metadata = (
39
+ self._frame_cleaning_class.clean_video(
40
40
  video_path=ctx.file_path,
41
41
  endoscope_image_roi=endoscope_roi,
42
42
  endoscope_data_roi_nested=endoscope_roi_nested,
43
- output_path=anonymized_output_path
44
-
43
+ output_path=anonymized_output_path,
45
44
  )
45
+ )
46
46
  sm = LxSM()
47
47
  sm.safe_update(extracted_metadata)
48
-
48
+
49
49
  self.storage = sensitive_meta_storage(sm, ctx.current_video)
50
50
  return ctx
51
-
51
+
52
52
  def _ensure_frame_cleaning_available(self):
53
53
  """
54
54
  Ensure frame cleaning modules are available by adding lx-anonymizer to path.
@@ -68,12 +68,12 @@ class VideoAnonymizer:
68
68
  self._frame_cleaning_class = FrameCleaner()
69
69
  self._frame_cleaning_available = True
70
70
 
71
-
72
71
  def _get_processor_roi_info(
73
72
  self,
74
73
  ctx: ImportContext,
75
- ) -> tuple[dict[str, int | None] | None,
76
- dict[str, dict[str, int | None] | None] | None]:
74
+ ) -> tuple[
75
+ dict[str, int | None] | None, dict[str, dict[str, int | None] | None] | None
76
+ ]:
77
77
  """Get processor ROI information for masking and data extraction."""
78
78
  endoscope_data_roi_nested = None
79
79
  endoscope_image_roi = None
@@ -98,7 +98,7 @@ class VideoAnonymizer:
98
98
  else:
99
99
  logger.warning(
100
100
  "No processor found for video %s, proceeding without ROI masking",
101
- video.uuid,
101
+ video.video_hash,
102
102
  )
103
103
  except Exception as exc:
104
104
  logger.error("Failed to retrieve processor ROI information: %s", exc)
@@ -1,4 +1,4 @@
1
- from endoreg_db.models import Patient, SensitiveMeta, Center, Gender
1
+ from endoreg_db.models import SensitiveMeta
2
2
  import logging
3
3
  from datetime import timedelta
4
4
  from typing import Tuple
@@ -6,7 +6,7 @@ from typing import Tuple
6
6
  from django.db.models import QuerySet
7
7
 
8
8
  from itertools import combinations
9
- from typing import Dict, Tuple, List
9
+ from typing import Dict
10
10
 
11
11
  logger = logging.getLogger(__name__)
12
12
 
@@ -68,7 +68,7 @@ def get_k_anonymity(pk, k=3):
68
68
  pk (_type_): _description_
69
69
  k (int, optional): _description_. Defaults to 3.
70
70
  """
71
- return get_k_anonymity_for_sensitive_meta(pk=pk, k=k, dob_year_tolerance=1)
71
+ return get_k_anonymity_for_sensitive_meta(pk=pk, k=k, dob_year_tolerance=1)
72
72
 
73
73
 
74
74
  def _build_sensitive_meta_qi_queryset(
@@ -99,20 +99,19 @@ def _build_sensitive_meta_qi_queryset(
99
99
  A Django QuerySet for further aggregation.
100
100
  """
101
101
  qs = SensitiveMeta.objects.all()
102
-
102
+
103
103
  if use_first_name and instance.patient_first_name is not None:
104
104
  qs = qs.filter(patient_first_name=instance.patient_first_name)
105
-
106
- if use_last_name and instance.patient_first_name is not None:
107
- qs = qs.filter(patient_first_name=instance.patient_first_name)
108
-
105
+
106
+ if use_last_name and instance.patient_last_name is not None:
107
+ qs = qs.filter(patient_last_name=instance.patient_last_name)
109
108
  # --- Center ---
110
109
  if use_center and instance.center is not None:
111
110
  if instance.center.pk is not None:
112
111
  qs = qs.filter(center=instance.center.pk)
113
112
 
114
113
  # --- Gender ---
115
- if use_gender and instance.patient_gender is not None
114
+ if use_gender and instance.patient_gender is not None:
116
115
  if instance.patient_gender.pk is not None:
117
116
  qs = qs.filter(patient_gender_id=instance.patient_gender)
118
117
 
@@ -1,13 +1,16 @@
1
1
  from typing import Optional, Tuple
2
+ import logging
2
3
 
3
4
  from datetime import date as Date
4
- import datetime
5
5
 
6
6
  from .k_anonymity import _build_sensitive_meta_qi_queryset
7
7
  from .fake import fake_name_with_similar_dob_and_gender
8
8
 
9
9
  from endoreg_db.models import SensitiveMeta
10
10
 
11
+ logger = logging.getLogger(__name__)
12
+
13
+
11
14
  def k_pseudonymize(
12
15
  instance: SensitiveMeta,
13
16
  *,
@@ -44,10 +47,16 @@ def k_pseudonymize(
44
47
  Returns:
45
48
  (instance, k_value_after, is_k_anonymous_after)
46
49
  """
47
- if qi_subset is None:
48
- qi_subset = tuple(QI_FLAGS)
49
50
 
50
51
  # --- 1) Compute k for the requested subset BEFORE pseudonymization ---
52
+ if qi_subset is None:
53
+ qi_subset = ("first_name", "last_name", "center", "gender", "dob_band")
54
+ # --- 1) Compute k for the requested subset BEFORE pseudonymization ---
55
+ use_first_name = "first_name" in qi_subset
56
+ use_last_name = "last_name" in qi_subset
57
+ use_center = "center" in qi_subset
58
+ use_gender = "gender" in qi_subset
59
+ use_dob_band = "dob_band" in qi_subset
51
60
  use_first_name = "first_name" in qi_subset
52
61
  use_last_name = "last_name" in qi_subset
53
62
  use_center = "center" in qi_subset
@@ -95,12 +104,14 @@ def k_pseudonymize(
95
104
  # Assign to instance (SensitiveMeta.patient_dob is a DateTimeField)
96
105
  instance.patient_first_name = first_name
97
106
  instance.patient_last_name = last_name
98
- instance.patient_dob = datetime(
107
+ instance.patient_dob = Date(
99
108
  fake_dob.year, fake_dob.month, fake_dob.day
100
109
  ) # naive is usually fine for DOB
101
110
 
102
111
  if save:
103
- instance.save(update_fields=["patient_first_name", "patient_last_name", "patient_dob"])
112
+ instance.save(
113
+ update_fields=["patient_first_name", "patient_last_name", "patient_dob"]
114
+ )
104
115
 
105
116
  # --- 3) Recompute k AFTER pseudonymization ---
106
117
  qs_after = _build_sensitive_meta_qi_queryset(