endoreg-db 0.8.8.0__py3-none-any.whl → 0.8.9.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of endoreg-db might be problematic. Click here for more details.

Files changed (402) hide show
  1. endoreg_db/data/__init__.py +22 -8
  2. endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +0 -1
  3. endoreg_db/data/examination/examinations/data.yaml +114 -14
  4. endoreg_db/data/examination/time-type/data.yaml +0 -3
  5. endoreg_db/data/examination_indication/endoscopy.yaml +108 -173
  6. endoreg_db/data/examination_indication_classification/endoscopy.yaml +0 -70
  7. endoreg_db/data/examination_indication_classification_choice/endoscopy.yaml +33 -37
  8. endoreg_db/data/finding/00_generic.yaml +35 -0
  9. endoreg_db/data/finding/00_generic_complication.yaml +9 -0
  10. endoreg_db/data/finding/01_gastroscopy_baseline.yaml +88 -0
  11. endoreg_db/data/finding/01_gastroscopy_observation.yaml +113 -0
  12. endoreg_db/data/finding/02_colonoscopy_baseline.yaml +53 -0
  13. endoreg_db/data/finding/02_colonoscopy_hidden.yaml +119 -0
  14. endoreg_db/data/finding/02_colonoscopy_observation.yaml +152 -0
  15. endoreg_db/data/finding_classification/00_generic.yaml +44 -0
  16. endoreg_db/data/finding_classification/00_generic_histology.yaml +28 -0
  17. endoreg_db/data/finding_classification/00_generic_lesion.yaml +52 -0
  18. endoreg_db/data/finding_classification/{colonoscopy_bowel_preparation.yaml → 02_colonoscopy_baseline.yaml} +35 -20
  19. endoreg_db/data/finding_classification/02_colonoscopy_histology.yaml +13 -0
  20. endoreg_db/data/finding_classification/02_colonoscopy_other.yaml +12 -0
  21. endoreg_db/data/finding_classification/02_colonoscopy_polyp.yaml +101 -0
  22. endoreg_db/data/finding_classification_choice/{yes_no_na.yaml → 00_generic.yaml} +5 -1
  23. endoreg_db/data/finding_classification_choice/{examination_setting_generic_types.yaml → 00_generic_baseline.yaml} +10 -2
  24. endoreg_db/data/finding_classification_choice/{complication_generic_types.yaml → 00_generic_complication.yaml} +1 -1
  25. endoreg_db/data/finding_classification_choice/{histology.yaml → 00_generic_histology.yaml} +1 -4
  26. endoreg_db/data/finding_classification_choice/00_generic_lesion.yaml +158 -0
  27. endoreg_db/data/finding_classification_choice/{bowel_preparation.yaml → 02_colonoscopy_bowel_preparation.yaml} +1 -30
  28. endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_not_complete_reason.yaml → finding_classification_choice/02_colonoscopy_generic.yaml} +1 -1
  29. endoreg_db/data/finding_classification_choice/{histology_polyp.yaml → 02_colonoscopy_histology.yaml} +1 -1
  30. endoreg_db/data/{_examples/finding_classification_choice/colonoscopy_location.yaml → finding_classification_choice/02_colonoscopy_location.yaml} +23 -4
  31. endoreg_db/data/finding_classification_choice/02_colonoscopy_other.yaml +34 -0
  32. endoreg_db/data/finding_classification_choice/02_colonoscopy_polyp_advanced_imaging.yaml +76 -0
  33. endoreg_db/data/{_examples/finding_classification_choice/colon_lesion_paris.yaml → finding_classification_choice/02_colonoscopy_polyp_morphology.yaml} +26 -8
  34. endoreg_db/data/finding_classification_choice/02_colonoscopy_size.yaml +27 -0
  35. endoreg_db/data/finding_classification_type/{colonoscopy_basic.yaml → 00_generic.yaml} +18 -13
  36. endoreg_db/data/finding_classification_type/02_colonoscopy.yaml +9 -0
  37. endoreg_db/data/finding_intervention/00_generic_endoscopy.yaml +59 -0
  38. endoreg_db/data/finding_intervention/00_generic_endoscopy_ablation.yaml +44 -0
  39. endoreg_db/data/finding_intervention/00_generic_endoscopy_bleeding.yaml +55 -0
  40. endoreg_db/data/finding_intervention/00_generic_endoscopy_resection.yaml +85 -0
  41. endoreg_db/data/finding_intervention/00_generic_endoscopy_stenosis.yaml +17 -0
  42. endoreg_db/data/finding_intervention/00_generic_endoscopy_stent.yaml +9 -0
  43. endoreg_db/data/finding_intervention/01_gastroscopy.yaml +19 -0
  44. endoreg_db/data/finding_intervention/04_eus.yaml +39 -0
  45. endoreg_db/data/finding_intervention/05_ercp.yaml +3 -0
  46. endoreg_db/data/finding_type/data.yaml +8 -12
  47. endoreg_db/data/requirement/01_patient_data.yaml +93 -0
  48. endoreg_db/data/requirement_operator/new_operators.yaml +36 -0
  49. endoreg_db/data/requirement_set/01_endoscopy_generic.yaml +0 -2
  50. endoreg_db/data/requirement_set/90_coloreg.yaml +20 -8
  51. endoreg_db/exceptions.py +0 -1
  52. endoreg_db/forms/examination_form.py +1 -1
  53. endoreg_db/helpers/data_loader.py +124 -52
  54. endoreg_db/helpers/default_objects.py +116 -81
  55. endoreg_db/import_files/__init__.py +27 -0
  56. endoreg_db/import_files/context/__init__.py +7 -0
  57. endoreg_db/import_files/context/default_sensitive_meta.py +81 -0
  58. endoreg_db/import_files/context/ensure_center.py +17 -0
  59. endoreg_db/import_files/context/file_lock.py +66 -0
  60. endoreg_db/import_files/context/import_context.py +43 -0
  61. endoreg_db/import_files/context/validate_directories.py +56 -0
  62. endoreg_db/import_files/file_storage/__init__.py +15 -0
  63. endoreg_db/import_files/file_storage/create_report_file.py +76 -0
  64. endoreg_db/import_files/file_storage/create_video_file.py +75 -0
  65. endoreg_db/import_files/file_storage/sensitive_meta_storage.py +39 -0
  66. endoreg_db/import_files/file_storage/state_management.py +496 -0
  67. endoreg_db/import_files/file_storage/storage.py +36 -0
  68. endoreg_db/import_files/import_service.md +26 -0
  69. endoreg_db/import_files/processing/__init__.py +11 -0
  70. endoreg_db/import_files/processing/report_processing/report_anonymization.py +94 -0
  71. endoreg_db/import_files/processing/sensitive_meta_adapter.py +51 -0
  72. endoreg_db/import_files/processing/video_processing/video_anonymization.py +107 -0
  73. endoreg_db/import_files/pseudonymization/fake.py +52 -0
  74. endoreg_db/import_files/pseudonymization/k_anonymity.py +182 -0
  75. endoreg_db/import_files/pseudonymization/k_pseudonymity.py +128 -0
  76. endoreg_db/import_files/pseudonymization/pseudonymize.py +0 -0
  77. endoreg_db/import_files/report_import_service.py +141 -0
  78. endoreg_db/import_files/video_import_service.py +150 -0
  79. endoreg_db/management/commands/import_report.py +130 -65
  80. endoreg_db/management/commands/import_video_with_classification.py +1 -1
  81. endoreg_db/management/commands/load_ai_model_data.py +5 -5
  82. endoreg_db/management/commands/load_ai_model_label_data.py +9 -7
  83. endoreg_db/management/commands/load_base_db_data.py +5 -134
  84. endoreg_db/management/commands/load_contraindication_data.py +14 -16
  85. endoreg_db/management/commands/load_disease_classification_choices_data.py +15 -18
  86. endoreg_db/management/commands/load_disease_classification_data.py +15 -18
  87. endoreg_db/management/commands/load_disease_data.py +25 -28
  88. endoreg_db/management/commands/load_endoscope_data.py +20 -27
  89. endoreg_db/management/commands/load_event_data.py +14 -16
  90. endoreg_db/management/commands/load_examination_data.py +31 -44
  91. endoreg_db/management/commands/load_examination_indication_data.py +20 -21
  92. endoreg_db/management/commands/load_finding_data.py +52 -80
  93. endoreg_db/management/commands/load_information_source.py +21 -23
  94. endoreg_db/management/commands/load_lab_value_data.py +17 -26
  95. endoreg_db/management/commands/load_medication_data.py +13 -12
  96. endoreg_db/management/commands/load_organ_data.py +15 -19
  97. endoreg_db/management/commands/load_pdf_type_data.py +19 -18
  98. endoreg_db/management/commands/load_profession_data.py +14 -17
  99. endoreg_db/management/commands/load_qualification_data.py +20 -23
  100. endoreg_db/management/commands/load_report_reader_flag_data.py +17 -19
  101. endoreg_db/management/commands/load_requirement_data.py +14 -20
  102. endoreg_db/management/commands/load_risk_data.py +7 -6
  103. endoreg_db/management/commands/load_shift_data.py +20 -23
  104. endoreg_db/management/commands/load_tag_data.py +8 -11
  105. endoreg_db/management/commands/load_unit_data.py +17 -19
  106. endoreg_db/management/commands/start_filewatcher.py +46 -37
  107. endoreg_db/management/commands/validate_video_files.py +1 -5
  108. endoreg_db/migrations/0001_initial.py +1360 -1812
  109. endoreg_db/models/administration/person/patient/patient.py +72 -46
  110. endoreg_db/models/label/__init__.py +2 -2
  111. endoreg_db/models/label/annotation/video_segmentation_annotation.py +18 -26
  112. endoreg_db/models/label/label_video_segment/label_video_segment.py +23 -1
  113. endoreg_db/models/media/pdf/raw_pdf.py +136 -64
  114. endoreg_db/models/media/pdf/report_reader/report_reader_config.py +34 -10
  115. endoreg_db/models/media/processing_history/__init__.py +5 -0
  116. endoreg_db/models/media/processing_history/processing_history.py +96 -0
  117. endoreg_db/models/media/video/create_from_file.py +101 -31
  118. endoreg_db/models/media/video/video_file.py +125 -105
  119. endoreg_db/models/media/video/video_file_io.py +31 -26
  120. endoreg_db/models/medical/contraindication/README.md +1 -0
  121. endoreg_db/models/medical/examination/examination.py +28 -8
  122. endoreg_db/models/medical/examination/examination_indication.py +13 -79
  123. endoreg_db/models/medical/examination/examination_time.py +8 -3
  124. endoreg_db/models/medical/finding/finding.py +5 -12
  125. endoreg_db/models/medical/finding/finding_classification.py +18 -37
  126. endoreg_db/models/medical/finding/finding_intervention.py +7 -9
  127. endoreg_db/models/medical/hardware/endoscope.py +6 -0
  128. endoreg_db/models/medical/patient/medication_examples.py +5 -1
  129. endoreg_db/models/medical/patient/patient_finding.py +1 -1
  130. endoreg_db/models/metadata/pdf_meta.py +22 -10
  131. endoreg_db/models/metadata/sensitive_meta.py +3 -0
  132. endoreg_db/models/metadata/sensitive_meta_logic.py +200 -124
  133. endoreg_db/models/other/information_source.py +27 -6
  134. endoreg_db/models/report/__init__.py +0 -0
  135. endoreg_db/models/report/images.py +0 -0
  136. endoreg_db/models/report/report.py +6 -0
  137. endoreg_db/models/requirement/requirement.py +59 -399
  138. endoreg_db/models/requirement/requirement_operator.py +86 -98
  139. endoreg_db/models/state/audit_ledger.py +4 -5
  140. endoreg_db/models/state/raw_pdf.py +69 -30
  141. endoreg_db/models/state/video.py +65 -49
  142. endoreg_db/models/upload_job.py +33 -9
  143. endoreg_db/models/utils.py +27 -23
  144. endoreg_db/queries/__init__.py +3 -1
  145. endoreg_db/schemas/examination_evaluation.py +1 -1
  146. endoreg_db/serializers/__init__.py +2 -8
  147. endoreg_db/serializers/label_video_segment/label_video_segment.py +2 -29
  148. endoreg_db/serializers/meta/__init__.py +1 -6
  149. endoreg_db/serializers/misc/sensitive_patient_data.py +50 -26
  150. endoreg_db/serializers/patient_examination/patient_examination.py +3 -3
  151. endoreg_db/serializers/pdf/anony_text_validation.py +39 -23
  152. endoreg_db/serializers/video/video_file_list.py +65 -34
  153. endoreg_db/services/__old/pdf_import.py +1487 -0
  154. endoreg_db/services/__old/video_import.py +1306 -0
  155. endoreg_db/services/anonymization.py +63 -26
  156. endoreg_db/services/lookup_service.py +28 -28
  157. endoreg_db/services/lookup_store.py +2 -2
  158. endoreg_db/services/pdf_import.py +0 -1480
  159. endoreg_db/services/report_import.py +10 -0
  160. endoreg_db/services/video_import.py +6 -1165
  161. endoreg_db/tasks/upload_tasks.py +79 -70
  162. endoreg_db/tasks/video_ingest.py +8 -4
  163. endoreg_db/urls/__init__.py +0 -14
  164. endoreg_db/urls/ai.py +32 -0
  165. endoreg_db/urls/media.py +21 -24
  166. endoreg_db/utils/dataloader.py +87 -57
  167. endoreg_db/utils/paths.py +110 -46
  168. endoreg_db/utils/pipelines/Readme.md +1 -1
  169. endoreg_db/utils/requirement_operator_logic/new_operator_logic.py +97 -0
  170. endoreg_db/utils/video/ffmpeg_wrapper.py +217 -52
  171. endoreg_db/views/__init__.py +85 -173
  172. endoreg_db/views/ai/__init__.py +8 -0
  173. endoreg_db/views/ai/label.py +155 -0
  174. endoreg_db/views/anonymization/media_management.py +8 -7
  175. endoreg_db/views/anonymization/overview.py +97 -68
  176. endoreg_db/views/anonymization/validate.py +25 -21
  177. endoreg_db/views/media/__init__.py +5 -20
  178. endoreg_db/views/media/pdf_media.py +109 -65
  179. endoreg_db/views/media/sensitive_metadata.py +163 -148
  180. endoreg_db/views/meta/__init__.py +0 -8
  181. endoreg_db/views/misc/__init__.py +1 -7
  182. endoreg_db/views/misc/upload_views.py +94 -93
  183. endoreg_db/views/report/__init__.py +7 -0
  184. endoreg_db/views/{pdf → report}/reimport.py +45 -24
  185. endoreg_db/views/{pdf/pdf_stream.py → report/report_stream.py} +40 -32
  186. endoreg_db/views/requirement/lookup_store.py +22 -90
  187. endoreg_db/views/video/__init__.py +23 -22
  188. endoreg_db/views/video/correction.py +201 -172
  189. endoreg_db/views/video/reimport.py +1 -1
  190. endoreg_db/views/{media/video_segments.py → video/segments_crud.py} +75 -37
  191. endoreg_db/views/video/{video_meta.py → video_meta_stats.py} +2 -2
  192. endoreg_db/views/video/video_stream.py +7 -8
  193. {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/METADATA +2 -2
  194. {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/RECORD +217 -335
  195. {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/WHEEL +1 -1
  196. endoreg_db/data/_examples/disease.yaml +0 -55
  197. endoreg_db/data/_examples/disease_classification.yaml +0 -13
  198. endoreg_db/data/_examples/disease_classification_choice.yaml +0 -62
  199. endoreg_db/data/_examples/event.yaml +0 -64
  200. endoreg_db/data/_examples/examination.yaml +0 -72
  201. endoreg_db/data/_examples/finding/anatomy_colon.yaml +0 -128
  202. endoreg_db/data/_examples/finding/colonoscopy.yaml +0 -40
  203. endoreg_db/data/_examples/finding/colonoscopy_bowel_prep.yaml +0 -56
  204. endoreg_db/data/_examples/finding/complication.yaml +0 -16
  205. endoreg_db/data/_examples/finding/data.yaml +0 -105
  206. endoreg_db/data/_examples/finding/examination_setting.yaml +0 -16
  207. endoreg_db/data/_examples/finding/medication_related.yaml +0 -18
  208. endoreg_db/data/_examples/finding/outcome.yaml +0 -12
  209. endoreg_db/data/_examples/finding_classification/colonoscopy_bowel_preparation.yaml +0 -68
  210. endoreg_db/data/_examples/finding_classification/colonoscopy_jnet.yaml +0 -22
  211. endoreg_db/data/_examples/finding_classification/colonoscopy_kudo.yaml +0 -25
  212. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
  213. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
  214. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_size.yaml +0 -68
  215. endoreg_db/data/_examples/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
  216. endoreg_db/data/_examples/finding_classification/colonoscopy_location.yaml +0 -80
  217. endoreg_db/data/_examples/finding_classification/colonoscopy_lst.yaml +0 -21
  218. endoreg_db/data/_examples/finding_classification/colonoscopy_nice.yaml +0 -20
  219. endoreg_db/data/_examples/finding_classification/colonoscopy_paris.yaml +0 -26
  220. endoreg_db/data/_examples/finding_classification/colonoscopy_sano.yaml +0 -22
  221. endoreg_db/data/_examples/finding_classification/colonoscopy_summary.yaml +0 -53
  222. endoreg_db/data/_examples/finding_classification/complication_generic.yaml +0 -25
  223. endoreg_db/data/_examples/finding_classification/examination_setting_generic.yaml +0 -40
  224. endoreg_db/data/_examples/finding_classification/histology_colo.yaml +0 -51
  225. endoreg_db/data/_examples/finding_classification/intervention_required.yaml +0 -26
  226. endoreg_db/data/_examples/finding_classification/medication_related.yaml +0 -23
  227. endoreg_db/data/_examples/finding_classification/visualized.yaml +0 -33
  228. endoreg_db/data/_examples/finding_classification_choice/bowel_preparation.yaml +0 -78
  229. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
  230. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
  231. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
  232. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_lst.yaml +0 -15
  233. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_nice.yaml +0 -17
  234. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
  235. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_sano.yaml +0 -14
  236. endoreg_db/data/_examples/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
  237. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_size.yaml +0 -82
  238. endoreg_db/data/_examples/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
  239. endoreg_db/data/_examples/finding_classification_choice/complication_generic_types.yaml +0 -15
  240. endoreg_db/data/_examples/finding_classification_choice/examination_setting_generic_types.yaml +0 -15
  241. endoreg_db/data/_examples/finding_classification_choice/histology.yaml +0 -24
  242. endoreg_db/data/_examples/finding_classification_choice/histology_polyp.yaml +0 -20
  243. endoreg_db/data/_examples/finding_classification_choice/outcome.yaml +0 -19
  244. endoreg_db/data/_examples/finding_classification_choice/yes_no_na.yaml +0 -11
  245. endoreg_db/data/_examples/finding_classification_type/colonoscopy_basic.yaml +0 -48
  246. endoreg_db/data/_examples/finding_intervention/endoscopy.yaml +0 -43
  247. endoreg_db/data/_examples/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
  248. endoreg_db/data/_examples/finding_intervention/endoscopy_egd.yaml +0 -128
  249. endoreg_db/data/_examples/finding_intervention/endoscopy_ercp.yaml +0 -32
  250. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_lower.yaml +0 -9
  251. endoreg_db/data/_examples/finding_intervention/endoscopy_eus_upper.yaml +0 -36
  252. endoreg_db/data/_examples/finding_intervention_type/endoscopy.yaml +0 -15
  253. endoreg_db/data/_examples/finding_type/data.yaml +0 -43
  254. endoreg_db/data/_examples/requirement/age.yaml +0 -26
  255. endoreg_db/data/_examples/requirement/gender.yaml +0 -25
  256. endoreg_db/data/_examples/requirement_set/01_endoscopy_generic.yaml +0 -48
  257. endoreg_db/data/_examples/requirement_set/colonoscopy_austria_screening.yaml +0 -57
  258. endoreg_db/data/_examples/requirement_set/endoscopy_bleeding_risk.yaml +0 -52
  259. endoreg_db/data/_examples/yaml_examples.xlsx +0 -0
  260. endoreg_db/data/finding/anatomy_colon.yaml +0 -128
  261. endoreg_db/data/finding/colonoscopy.yaml +0 -40
  262. endoreg_db/data/finding/colonoscopy_bowel_prep.yaml +0 -56
  263. endoreg_db/data/finding/complication.yaml +0 -16
  264. endoreg_db/data/finding/data.yaml +0 -105
  265. endoreg_db/data/finding/examination_setting.yaml +0 -16
  266. endoreg_db/data/finding/medication_related.yaml +0 -18
  267. endoreg_db/data/finding/outcome.yaml +0 -12
  268. endoreg_db/data/finding_classification/colonoscopy_jnet.yaml +0 -22
  269. endoreg_db/data/finding_classification/colonoscopy_kudo.yaml +0 -25
  270. endoreg_db/data/finding_classification/colonoscopy_lesion_circularity.yaml +0 -20
  271. endoreg_db/data/finding_classification/colonoscopy_lesion_planarity.yaml +0 -24
  272. endoreg_db/data/finding_classification/colonoscopy_lesion_size.yaml +0 -38
  273. endoreg_db/data/finding_classification/colonoscopy_lesion_surface.yaml +0 -20
  274. endoreg_db/data/finding_classification/colonoscopy_location.yaml +0 -49
  275. endoreg_db/data/finding_classification/colonoscopy_lst.yaml +0 -21
  276. endoreg_db/data/finding_classification/colonoscopy_nice.yaml +0 -20
  277. endoreg_db/data/finding_classification/colonoscopy_paris.yaml +0 -26
  278. endoreg_db/data/finding_classification/colonoscopy_sano.yaml +0 -22
  279. endoreg_db/data/finding_classification/colonoscopy_summary.yaml +0 -53
  280. endoreg_db/data/finding_classification/complication_generic.yaml +0 -25
  281. endoreg_db/data/finding_classification/examination_setting_generic.yaml +0 -40
  282. endoreg_db/data/finding_classification/histology_colo.yaml +0 -43
  283. endoreg_db/data/finding_classification/intervention_required.yaml +0 -26
  284. endoreg_db/data/finding_classification/medication_related.yaml +0 -23
  285. endoreg_db/data/finding_classification/visualized.yaml +0 -33
  286. endoreg_db/data/finding_classification_choice/colon_lesion_circularity_default.yaml +0 -32
  287. endoreg_db/data/finding_classification_choice/colon_lesion_jnet.yaml +0 -15
  288. endoreg_db/data/finding_classification_choice/colon_lesion_kudo.yaml +0 -23
  289. endoreg_db/data/finding_classification_choice/colon_lesion_lst.yaml +0 -15
  290. endoreg_db/data/finding_classification_choice/colon_lesion_nice.yaml +0 -17
  291. endoreg_db/data/finding_classification_choice/colon_lesion_paris.yaml +0 -57
  292. endoreg_db/data/finding_classification_choice/colon_lesion_planarity_default.yaml +0 -49
  293. endoreg_db/data/finding_classification_choice/colon_lesion_sano.yaml +0 -14
  294. endoreg_db/data/finding_classification_choice/colon_lesion_surface_intact_default.yaml +0 -36
  295. endoreg_db/data/finding_classification_choice/colonoscopy_location.yaml +0 -229
  296. endoreg_db/data/finding_classification_choice/colonoscopy_not_complete_reason.yaml +0 -19
  297. endoreg_db/data/finding_classification_choice/colonoscopy_size.yaml +0 -82
  298. endoreg_db/data/finding_classification_choice/colonoscopy_summary_worst_finding.yaml +0 -15
  299. endoreg_db/data/finding_classification_choice/outcome.yaml +0 -19
  300. endoreg_db/data/finding_intervention/endoscopy.yaml +0 -43
  301. endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +0 -168
  302. endoreg_db/data/finding_intervention/endoscopy_egd.yaml +0 -128
  303. endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +0 -32
  304. endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +0 -9
  305. endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +0 -36
  306. endoreg_db/data/finding_morphology_classification_type/colonoscopy.yaml +0 -79
  307. endoreg_db/data/requirement/age.yaml +0 -26
  308. endoreg_db/data/requirement/colonoscopy_baseline_austria.yaml +0 -45
  309. endoreg_db/data/requirement/disease_cardiovascular.yaml +0 -79
  310. endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +0 -41
  311. endoreg_db/data/requirement/disease_hepatology.yaml +0 -12
  312. endoreg_db/data/requirement/disease_misc.yaml +0 -12
  313. endoreg_db/data/requirement/disease_renal.yaml +0 -96
  314. endoreg_db/data/requirement/endoscopy_bleeding_risk.yaml +0 -59
  315. endoreg_db/data/requirement/event_cardiology.yaml +0 -251
  316. endoreg_db/data/requirement/event_requirements.yaml +0 -145
  317. endoreg_db/data/requirement/finding_colon_polyp.yaml +0 -50
  318. endoreg_db/data/requirement/gender.yaml +0 -25
  319. endoreg_db/data/requirement/lab_value.yaml +0 -441
  320. endoreg_db/data/requirement/medication.yaml +0 -93
  321. endoreg_db/data/requirement_operator/age.yaml +0 -13
  322. endoreg_db/data/requirement_operator/lab_operators.yaml +0 -129
  323. endoreg_db/data/requirement_operator/model_operators.yaml +0 -96
  324. endoreg_db/management/commands/init_default_ai_model.py +0 -112
  325. endoreg_db/management/commands/reset_celery_schedule.py +0 -9
  326. endoreg_db/management/commands/validate_video.py +0 -204
  327. endoreg_db/migrations/0002_requirementset_depends_on.py +0 -18
  328. endoreg_db/migrations/_old/0001_initial.py +0 -1857
  329. endoreg_db/migrations/_old/0002_add_video_correction_models.py +0 -52
  330. endoreg_db/migrations/_old/0003_add_center_display_name.py +0 -30
  331. endoreg_db/migrations/_old/0004_employee_city_employee_post_code_employee_street_and_more.py +0 -68
  332. endoreg_db/migrations/_old/0004_remove_casetemplate_rules_and_more.py +0 -77
  333. endoreg_db/migrations/_old/0005_merge_20251111_1003.py +0 -14
  334. endoreg_db/migrations/_old/0006_sensitivemeta_anonymized_text_and_more.py +0 -68
  335. endoreg_db/migrations/_old/0007_remove_rule_attribute_dtype_remove_rule_rule_type_and_more.py +0 -89
  336. endoreg_db/migrations/_old/0008_remove_event_event_classification_and_more.py +0 -27
  337. endoreg_db/migrations/_old/0009_alter_modelmeta_options_and_more.py +0 -21
  338. endoreg_db/renames.yml +0 -8
  339. endoreg_db/serializers/_old/raw_pdf_meta_validation.py +0 -223
  340. endoreg_db/serializers/_old/raw_video_meta_validation.py +0 -179
  341. endoreg_db/serializers/_old/video.py +0 -71
  342. endoreg_db/serializers/meta/pdf_file_meta_extraction.py +0 -115
  343. endoreg_db/serializers/meta/report_meta.py +0 -53
  344. endoreg_db/serializers/report/__init__.py +0 -9
  345. endoreg_db/serializers/report/mixins.py +0 -45
  346. endoreg_db/serializers/report/report.py +0 -105
  347. endoreg_db/serializers/report/report_list.py +0 -22
  348. endoreg_db/serializers/report/secure_file_url.py +0 -26
  349. endoreg_db/services/requirements_object.py +0 -147
  350. endoreg_db/services/storage_aware_video_processor.py +0 -370
  351. endoreg_db/urls/files.py +0 -6
  352. endoreg_db/urls/label_video_segment_validate.py +0 -33
  353. endoreg_db/urls/label_video_segments.py +0 -46
  354. endoreg_db/views/label/__init__.py +0 -5
  355. endoreg_db/views/label/label.py +0 -15
  356. endoreg_db/views/label_video_segment/__init__.py +0 -16
  357. endoreg_db/views/label_video_segment/create_lvs_from_annotation.py +0 -44
  358. endoreg_db/views/label_video_segment/get_lvs_by_name_and_video.py +0 -50
  359. endoreg_db/views/label_video_segment/label_video_segment.py +0 -77
  360. endoreg_db/views/label_video_segment/label_video_segment_by_label.py +0 -174
  361. endoreg_db/views/label_video_segment/label_video_segment_detail.py +0 -73
  362. endoreg_db/views/label_video_segment/update_lvs_from_annotation.py +0 -46
  363. endoreg_db/views/label_video_segment/validate.py +0 -226
  364. endoreg_db/views/media/segments.py +0 -71
  365. endoreg_db/views/meta/available_files_list.py +0 -146
  366. endoreg_db/views/meta/report_meta.py +0 -53
  367. endoreg_db/views/meta/sensitive_meta_detail.py +0 -85
  368. endoreg_db/views/misc/secure_file_serving_view.py +0 -80
  369. endoreg_db/views/misc/secure_file_url_view.py +0 -84
  370. endoreg_db/views/misc/secure_url_validate.py +0 -79
  371. endoreg_db/views/patient_examination/DEPRECATED_video_backup.py +0 -164
  372. endoreg_db/views/patient_finding_location/__init__.py +0 -5
  373. endoreg_db/views/patient_finding_location/pfl_create.py +0 -70
  374. endoreg_db/views/patient_finding_morphology/__init__.py +0 -5
  375. endoreg_db/views/patient_finding_morphology/pfm_create.py +0 -70
  376. endoreg_db/views/pdf/__init__.py +0 -8
  377. endoreg_db/views/video/segmentation.py +0 -274
  378. endoreg_db/views/video/task_status.py +0 -49
  379. endoreg_db/views/video/timeline.py +0 -46
  380. endoreg_db/views/video/video_analyze.py +0 -52
  381. /endoreg_db/data/requirement/{colon_polyp_intervention.yaml → old/colon_polyp_intervention.yaml} +0 -0
  382. /endoreg_db/data/{_examples/requirement → requirement/old}/colonoscopy_baseline_austria.yaml +0 -0
  383. /endoreg_db/data/requirement/{coloreg_colon_polyp.yaml → old/coloreg_colon_polyp.yaml} +0 -0
  384. /endoreg_db/data/{_examples/requirement → requirement/old}/disease_cardiovascular.yaml +0 -0
  385. /endoreg_db/data/{_examples/requirement → requirement/old}/disease_classification_choice_cardiovascular.yaml +0 -0
  386. /endoreg_db/data/{_examples/requirement → requirement/old}/disease_hepatology.yaml +0 -0
  387. /endoreg_db/data/{_examples/requirement → requirement/old}/disease_misc.yaml +0 -0
  388. /endoreg_db/data/{_examples/requirement → requirement/old}/disease_renal.yaml +0 -0
  389. /endoreg_db/data/{_examples/requirement → requirement/old}/endoscopy_bleeding_risk.yaml +0 -0
  390. /endoreg_db/data/{_examples/requirement → requirement/old}/event_cardiology.yaml +0 -0
  391. /endoreg_db/data/{_examples/requirement → requirement/old}/event_requirements.yaml +0 -0
  392. /endoreg_db/data/{_examples/requirement → requirement/old}/finding_colon_polyp.yaml +0 -0
  393. /endoreg_db/{urls/sensitive_meta.py → data/requirement/old/gender.yaml} +0 -0
  394. /endoreg_db/data/{_examples/requirement → requirement/old}/lab_value.yaml +0 -0
  395. /endoreg_db/data/{_examples/requirement → requirement/old}/medication.yaml +0 -0
  396. /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/age.yaml +0 -0
  397. /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/lab_operators.yaml +0 -0
  398. /endoreg_db/data/{_examples/requirement_operator → requirement_operator/_old}/model_operators.yaml +0 -0
  399. /endoreg_db/{views/pdf/pdf_stream_views.py → import_files/pseudonymization/__init__.py} +0 -0
  400. /endoreg_db/utils/requirement_operator_logic/{lab_value_operators.py → _old/lab_value_operators.py} +0 -0
  401. /endoreg_db/utils/requirement_operator_logic/{model_evaluators.py → _old/model_evaluators.py} +0 -0
  402. {endoreg_db-0.8.8.0.dist-info → endoreg_db-0.8.9.2.dist-info}/licenses/LICENSE +0 -0
@@ -13,7 +13,11 @@ from django.db import models
13
13
 
14
14
  from endoreg_db.utils.file_operations import get_uuid_filename
15
15
  from endoreg_db.utils.hashs import get_pdf_hash
16
- from endoreg_db.utils.paths import PDF_DIR
16
+ from endoreg_db.utils.paths import (
17
+ ANONYM_REPORT_DIR,
18
+ IMPORT_REPORT_DIR,
19
+ SENSITIVE_REPORT_DIR,
20
+ )
17
21
  from endoreg_db.utils.storage import (
18
22
  delete_field_file,
19
23
  ensure_local_file,
@@ -26,7 +30,6 @@ if TYPE_CHECKING:
26
30
 
27
31
  from endoreg_db.models.state import RawPdfState
28
32
 
29
- # setup logging to pdf_import.log
30
33
  import logging
31
34
  from pathlib import Path
32
35
 
@@ -67,14 +70,14 @@ class RawPdfFile(models.Model):
67
70
  text = models.TextField(blank=True, null=True)
68
71
  date_created = models.DateTimeField(auto_now_add=True)
69
72
  date_modified = models.DateTimeField(auto_now=True)
70
- anonymized = models.BooleanField(default=False, help_text="True if the PDF has been anonymized.")
73
+
71
74
  file = models.FileField(
72
- # Use the relative path from the specific PDF_DIR
73
- upload_to=PDF_DIR.name,
75
+ # Use the relative path from the specific REPORT_DIR
76
+ upload_to=SENSITIVE_REPORT_DIR.name,
74
77
  validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
75
78
  )
76
- anonymized_file = models.FileField(
77
- upload_to=PDF_DIR.name,
79
+ processed_file = models.FileField(
80
+ upload_to=ANONYM_REPORT_DIR.name,
78
81
  validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
79
82
  null=True,
80
83
  blank=True,
@@ -130,9 +133,11 @@ class RawPdfFile(models.Model):
130
133
  state: models.ForeignKey["RawPdfState | None"]
131
134
  patient: models.ForeignKey["Patient | None"]
132
135
  sensitive_meta: models.ForeignKey["SensitiveMeta | None"]
133
- anonym_examination_report: models.OneToOneField["AnonymExaminationReport | None"]
136
+ anonym_examination_report: models.OneToOneField[
137
+ "AnonymExaminationReport | None"
138
+ ]
134
139
  file = cast(FieldFile, file)
135
- anonymized_file = cast(FieldFile, anonymized_file)
140
+ processed_file = cast(FieldFile, processed_file)
136
141
 
137
142
  @property
138
143
  def uuid(self):
@@ -147,7 +152,7 @@ class RawPdfFile(models.Model):
147
152
  @property
148
153
  def file_path(self) -> Path | None:
149
154
  """
150
- Returns the file path of the stored PDF file if available; otherwise, returns None.
155
+ Returns the file path of the stored report file if available; otherwise, returns None.
151
156
  """
152
157
  from django.db.models.fields.files import FieldFile
153
158
 
@@ -162,7 +167,7 @@ class RawPdfFile(models.Model):
162
167
 
163
168
  def set_file_path(self, file_path: Path):
164
169
  """
165
- Sets the file path of the stored PDF file.
170
+ Sets the file path of the stored report file.
166
171
  """
167
172
  if not file_path.exists():
168
173
  raise FileNotFoundError(f"File path does not exist: {file_path}")
@@ -173,30 +178,30 @@ class RawPdfFile(models.Model):
173
178
  @property
174
179
  def anonymized_file_path(self) -> Path | None:
175
180
  """
176
- Returns the file path of the anonymized PDF file if available; otherwise, returns None.
181
+ Returns the file path of the anonymized report file if available; otherwise, returns None.
177
182
  """
178
- if self.anonymized_file and self.anonymized_file.name:
183
+ if self.processed_file and self.processed_file.name:
179
184
  try:
180
- return Path(self.anonymized_file.path)
185
+ return Path(self.processed_file.path)
181
186
  except (ValueError, AttributeError, NotImplementedError):
182
187
  return None
183
188
  return None
184
189
 
185
190
  def set_anonymized_file_path(self, file_path: Path):
186
191
  """
187
- Sets the file path of the anonymized PDF file.
192
+ Sets the file path of the anonymized report file.
188
193
  """
189
194
  if not file_path.exists():
190
195
  raise FileNotFoundError(f"File path does not exist: {file_path}")
191
196
 
192
- save_local_file(self.anonymized_file, file_path, name=file_path.name, save=False)
193
- self.save(update_fields=["anonymized_file"])
197
+ save_local_file(self.processed_file, file_path, name=file_path.name, save=False)
198
+ self.save(update_fields=["processed_file"])
194
199
 
195
200
  def get_raw_file_path(self) -> Optional[Path]:
196
201
  """
197
- Get the path to the raw PDF file, searching common locations.
202
+ Get the path to the raw report file, searching common locations.
198
203
 
199
- This method attempts to find the original raw PDF file by checking:
204
+ This method attempts to find the original raw report file by checking:
200
205
  1. Direct hash-based path in raw_pdfs/
201
206
  2. Scanning raw_pdfs/ directory for files matching the hash
202
207
  3. Checking the file field if it exists
@@ -211,17 +216,17 @@ class RawPdfFile(models.Model):
211
216
  try:
212
217
  file_path = Path(self.file.path)
213
218
  if file_path.exists():
214
- logger.debug(f"Found raw PDF via file field: {file_path}")
219
+ logger.debug(f"Found raw report via file field: {file_path}")
215
220
  return file_path
216
221
  except (ValueError, AttributeError, NotImplementedError):
217
222
  pass
218
223
 
219
224
  # Define potential raw directories
220
225
  raw_dirs = [
221
- PDF_DIR / "sensitive", # Files might be in sensitive dir
222
- Path(settings.BASE_DIR) / "data" / "raw_pdfs",
226
+ SENSITIVE_REPORT_DIR, # Files might be in sensitive dir
227
+ Path(settings.BASE_DIR) / "data" / "temporary_reports",
223
228
  Path(settings.BASE_DIR) / "data" / "pdfs" / "raw",
224
- PDF_DIR, # General PDF directory
229
+ IMPORT_REPORT_DIR, # General report directory
225
230
  ]
226
231
 
227
232
  # Check direct hash-based name in each directory
@@ -231,7 +236,7 @@ class RawPdfFile(models.Model):
231
236
 
232
237
  hash_path = raw_dir / f"{self.pdf_hash}.pdf"
233
238
  if hash_path.exists():
234
- logger.debug(f"Found raw PDF at: {hash_path}")
239
+ logger.debug(f"Found raw report at: {hash_path}")
235
240
  return hash_path
236
241
 
237
242
  # Scan directories for matching hash
@@ -243,19 +248,19 @@ class RawPdfFile(models.Model):
243
248
  try:
244
249
  file_hash = get_pdf_hash(file_path)
245
250
  if file_hash == self.pdf_hash:
246
- logger.debug(f"Found matching PDF by hash: {file_path}")
251
+ logger.debug(f"Found matching report by hash: {file_path}")
247
252
  return file_path
248
253
  except Exception as e:
249
254
  logger.debug(f"Error checking {file_path}: {e}")
250
255
  continue
251
256
 
252
- logger.warning(f"No raw file found for PDF hash: {self.pdf_hash}")
257
+ logger.warning(f"No raw file found for report hash: {self.pdf_hash}")
253
258
  return None
254
259
 
255
260
  @property
256
261
  def file_url(self):
257
262
  """
258
- Returns the URL of the stored PDF file if available; otherwise, returns None.
263
+ Returns the URL of the stored report file if available; otherwise, returns None.
259
264
  """
260
265
  try:
261
266
  return self.file.url if self.file and self.file.name else None
@@ -265,16 +270,20 @@ class RawPdfFile(models.Model):
265
270
  @property
266
271
  def anonymized_file_url(self):
267
272
  """
268
- Returns the URL of the stored PDF file if available; otherwise, returns None.
273
+ Returns the URL of the stored report file if available; otherwise, returns None.
269
274
  """
270
275
  try:
271
- return self.anonymized_file.url if self.anonymized_file and self.anonymized_file.name else None
276
+ return (
277
+ self.processed_file.url
278
+ if self.processed_file and self.processed_file.name
279
+ else None
280
+ )
272
281
  except (ValueError, AttributeError):
273
282
  return None
274
283
 
275
284
  def __str__(self):
276
285
  """
277
- Return a string representation of the RawPdfFile, including its PDF hash, type, and center.
286
+ Return a string representation of the RawPdfFile, including its report hash, type, and center.
278
287
  """
279
288
  str_repr = f"{self.pdf_hash} ({self.pdf_type}, {self.center})"
280
289
  return str_repr
@@ -283,19 +292,57 @@ class RawPdfFile(models.Model):
283
292
  """
284
293
  Deletes the RawPdfFile instance from the database and removes the associated file from storage if it exists.
285
294
 
286
- This method ensures that the physical PDF file is deleted from the file system after the database record is removed. Logs warnings or errors if the file cannot be found or deleted.
295
+ This method ensures that the physical report file is deleted from the file system after the database record is removed. Logs warnings or errors if the file cannot be found or deleted.
287
296
  """
288
297
  primary_name = self.file.name if self.file and self.file.name else None
289
- anonymized_name = self.anonymized_file.name if self.anonymized_file and self.anonymized_file.name else None
298
+ anonymized_name = (
299
+ self.processed_file.name
300
+ if self.processed_file and self.processed_file.name
301
+ else None
302
+ )
290
303
 
291
304
  if delete_field_file(self.file, missing_ok=True, save=False):
292
305
  logger.info("Original file removed from storage: %s", primary_name)
293
- if delete_field_file(self.anonymized_file, missing_ok=True, save=False):
306
+ if delete_field_file(self.processed_file, missing_ok=True, save=False):
294
307
  logger.info("Anonymized file removed from storage: %s", anonymized_name)
295
308
 
296
309
  super().delete(*args, **kwargs)
297
310
 
298
- def validate_metadata_annotation(self, extracted_data_dict: Optional[dict] = None) -> bool:
311
+ # --- Convenience state/meta helpers used in tests and admin workflows ---
312
+
313
+ def mark_sensitive_meta_processed(self, *, save: bool = True) -> "RawPdfFile":
314
+ """
315
+ Mark this video's processing state as having its sensitive meta fully processed.
316
+ This proxies to the related VideoState and persists by default.
317
+ """
318
+ sm = self.sensitive_meta
319
+ from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
320
+
321
+ if not isinstance(sm, SensitiveMeta):
322
+ raise AttributeError()
323
+ state = self.get_or_create_state()
324
+ state.mark_sensitive_meta_processed(save=save)
325
+ return self
326
+
327
+ def mark_sensitive_meta_verified(self) -> "RawPdfFile":
328
+ """
329
+ Mark the associated SensitiveMeta as verified by setting both DOB and names as verified.
330
+ Ensures the SensitiveMeta and its state exist.
331
+ """
332
+ sm = self.sensitive_meta
333
+ # Use SensitiveMeta methods to update underlying SensitiveMetaState
334
+ from endoreg_db.models.metadata.sensitive_meta import SensitiveMeta
335
+
336
+ if not isinstance(sm, SensitiveMeta):
337
+ raise AttributeError()
338
+
339
+ sm.mark_dob_verified()
340
+ sm.mark_names_verified()
341
+ return self
342
+
343
+ def validate_metadata_annotation(
344
+ self, extracted_data_dict: Optional[dict] = None
345
+ ) -> bool:
299
346
  """
300
347
  Validate the metadata of the RawPdf instance.
301
348
 
@@ -303,9 +350,8 @@ class RawPdfFile(models.Model):
303
350
  It also ensures the video file is properly saved after the metadata update.
304
351
  """
305
352
 
306
- if not self.sensitive_meta:
307
- logger.error("No sensitive meta data associated with this PDF file.")
308
- return False
353
+ self.mark_sensitive_meta_processed()
354
+ self.mark_sensitive_meta_verified()
309
355
 
310
356
  if not extracted_data_dict:
311
357
  logger.error("No extracted data provided for validation.")
@@ -322,16 +368,22 @@ class RawPdfFile(models.Model):
322
368
  # Save the RawPdfFile instance to ensure all changes are saved
323
369
  self.save()
324
370
 
325
- logger.info(f"Metadata for PDF {self.pk} validated and updated successfully.")
371
+ logger.info(
372
+ f"Metadata for report {self.pk} validated and updated successfully."
373
+ )
326
374
 
327
375
  deleted_original = delete_field_file(self.file, missing_ok=True, save=False)
328
- deleted_anonymized = delete_field_file(self.anonymized_file, missing_ok=True, save=False)
376
+ deleted_anonymized = delete_field_file(
377
+ self.processed_file, missing_ok=True, save=False
378
+ )
329
379
  self.get_or_create_state().mark_anonymization_validated()
330
380
 
331
381
  if deleted_original or deleted_anonymized:
332
- self.save(update_fields=["file", "anonymized_file"]) # Persist cleared fields
382
+ self.save(
383
+ update_fields=["file", "processed_file"]
384
+ ) # Persist cleared fields
333
385
 
334
- logger.info(f"Files for PDF {self.pk} deleted successfully.")
386
+ logger.info(f"Files for report {self.pk} deleted successfully.")
335
387
  return True
336
388
 
337
389
  @classmethod
@@ -345,8 +397,8 @@ class RawPdfFile(models.Model):
345
397
  Creates a RawPdfFile instance from a file and center name, ensuring an associated RawPdfState exists.
346
398
 
347
399
  Parameters:
348
- file_path (Path): Path to the source PDF file.
349
- center_name (str): Name of the center to associate with the PDF.
400
+ file_path (Path): Path to the source report file.
401
+ center_name (str): Name of the center to associate with the report.
350
402
  delete_source (bool): Whether to delete the source file after processing. Defaults to True.
351
403
 
352
404
  Returns:
@@ -370,12 +422,12 @@ class RawPdfFile(models.Model):
370
422
  delete_source=True,
371
423
  ):
372
424
  """
373
- Creates or retrieves a RawPdfFile instance from a given PDF file path and center name.
425
+ Creates or retrieves a RawPdfFile instance from a given report file path and center name.
374
426
 
375
- If a RawPdfFile with the same PDF hash already exists, verifies the file exists in storage and restores it if missing. Otherwise, creates a new RawPdfFile, assigns the file, and saves it to storage. Optionally deletes the source file after processing.
427
+ If a RawPdfFile with the same report hash already exists, verifies the file exists in storage and restores it if missing. Otherwise, creates a new RawPdfFile, assigns the file, and saves it to storage. Optionally deletes the source file after processing.
376
428
 
377
429
  Parameters:
378
- file_path (Path): Path to the source PDF file.
430
+ file_path (Path): Path to the source report file.
379
431
  center_name (str): Name of the center to associate with the file.
380
432
  save (bool, optional): Deprecated; saving occurs internally.
381
433
  delete_source (bool, optional): Whether to delete the source file after processing (default True).
@@ -386,7 +438,7 @@ class RawPdfFile(models.Model):
386
438
  Raises:
387
439
  FileNotFoundError: If the source file does not exist.
388
440
  Center.DoesNotExist: If the specified center is not found.
389
- ValueError: If the PDF hash cannot be calculated.
441
+ ValueError: If the report hash cannot be calculated.
390
442
  IOError: If the file fails to save to storage.
391
443
  """
392
444
  from endoreg_db.models.administration import Center
@@ -427,9 +479,13 @@ class RawPdfFile(models.Model):
427
479
  )
428
480
  # Re-save the file from the source to potentially fix it
429
481
  with file_path.open("rb") as f:
430
- django_file = File(f, name=Path(_file.name).name) # Use existing name if possible
482
+ django_file = File(
483
+ f, name=Path(_file.name).name
484
+ ) # Use existing name if possible
431
485
  existing_pdf_file.file = django_file
432
- existing_pdf_file.save(update_fields=["file"]) # Only update file field
486
+ existing_pdf_file.save(
487
+ update_fields=["file"]
488
+ ) # Only update file field
433
489
  else:
434
490
  pass
435
491
  # logger.debug("File for existing RawPdfFile %s already exists in storage.", pdf_hash)
@@ -485,7 +541,9 @@ class RawPdfFile(models.Model):
485
541
  "File was not saved correctly to storage path %s after model save.",
486
542
  _file.name,
487
543
  )
488
- raise IOError(f"File not found at expected storage path after save: {_file.name}")
544
+ raise IOError(
545
+ f"File not found at expected storage path after save: {_file.name}"
546
+ )
489
547
 
490
548
  try:
491
549
  logger.info("File saved to absolute path: %s", _file.path)
@@ -496,14 +554,18 @@ class RawPdfFile(models.Model):
496
554
  )
497
555
 
498
556
  except Exception as e:
499
- logger.error("Error processing or saving file %s for new record: %s", file_path, e)
557
+ logger.error(
558
+ "Error processing or saving file %s for new record: %s", file_path, e
559
+ )
500
560
  raise
501
561
 
502
562
  # Delete source file *after* successful save and verification
503
563
  if delete_source:
504
564
  try:
505
565
  file_path.unlink()
506
- logger.info("Deleted source file %s after creating new record.", file_path)
566
+ logger.info(
567
+ "Deleted source file %s after creating new record.", file_path
568
+ )
507
569
  except OSError as e:
508
570
  logger.error("Error deleting source file %s: %s", file_path, e)
509
571
 
@@ -514,15 +576,17 @@ class RawPdfFile(models.Model):
514
576
  # Ensure hash is calculated before the first save if possible and not already set
515
577
  # This is primarily a fallback if instance created manually without using create_from_file
516
578
  """
517
- Saves the RawPdfFile instance, ensuring the PDF hash is set and related fields are derived from metadata.
579
+ Saves the RawPdfFile instance, ensuring the report hash is set and related fields are derived from metadata.
518
580
 
519
- If the PDF hash is missing, attempts to calculate it from the file before saving. Validates that the file has a `.pdf` extension. If related fields such as patient, examination, center, or examiner are unset but available in the associated sensitive metadata, they are populated accordingly before saving.
581
+ If the report hash is missing, attempts to calculate it from the file before saving. Validates that the file has a `.pdf` extension. If related fields such as patient, examination, center, or examiner are unset but available in the associated sensitive metadata, they are populated accordingly before saving.
520
582
  """
521
583
  if not self.pk and not self.pdf_hash and self.file:
522
584
  try:
523
585
  with ensure_local_file(self.file) as local_path:
524
586
  self.pdf_hash = get_pdf_hash(local_path)
525
- logger.info("Calculated hash during pre-save for %s", self.file.name)
587
+ logger.info(
588
+ "Calculated hash during pre-save for %s", self.file.name
589
+ )
526
590
  except Exception as exc:
527
591
  logger.warning(
528
592
  "Could not calculate hash before initial save for %s: %s",
@@ -531,7 +595,7 @@ class RawPdfFile(models.Model):
531
595
  )
532
596
 
533
597
  if self.file and not self.file.name.endswith(".pdf"):
534
- raise ValidationError("Only PDF files are allowed")
598
+ raise ValidationError("Only report files are allowed")
535
599
 
536
600
  # If hash is still missing after potential creation logic (e.g., direct instantiation)
537
601
  # and the file exists in storage, try calculating it from storage path.
@@ -539,7 +603,9 @@ class RawPdfFile(models.Model):
539
603
  if not self.pdf_hash and self.pk and self.file and file_exists(self.file):
540
604
  try:
541
605
  with ensure_local_file(self.file) as local_path:
542
- logger.warning("Hash missing for saved file %s. Recalculating.", self.file.name)
606
+ logger.warning(
607
+ "Hash missing for saved file %s. Recalculating.", self.file.name
608
+ )
543
609
  self.pdf_hash = get_pdf_hash(local_path)
544
610
  except Exception as exc:
545
611
  logger.error(
@@ -585,7 +651,7 @@ class RawPdfFile(models.Model):
585
651
  # This method might still be useful if called explicitly, but create_from_file now handles restoration
586
652
  # Ensure fallback_file is a Path object.
587
653
  """
588
- Checks if the stored PDF file exists in storage and attempts to restore it from a fallback file path if missing.
654
+ Checks if the stored report file exists in storage and attempts to restore it from a fallback file path if missing.
589
655
 
590
656
  Parameters:
591
657
  fallback_file: Path or string representing the fallback file location to restore from if the stored file is missing.
@@ -597,12 +663,18 @@ class RawPdfFile(models.Model):
597
663
  assert _file is not None
598
664
  try:
599
665
  if not _file.field.storage.exists(_file.name):
600
- logger.warning(f"File missing at storage path {_file.name}. Attempting copy from fallback {fallback_file}")
666
+ logger.warning(
667
+ f"File missing at storage path {_file.name}. Attempting copy from fallback {fallback_file}"
668
+ )
601
669
  if fallback_file.exists():
602
670
  with fallback_file.open("rb") as f:
603
671
  # Use save method which handles storage backend
604
- _file.save(Path(_file.name).name, File(f), save=True) # Re-save the file content
605
- logger.info(f"Successfully restored file from fallback {fallback_file} to {_file.name}")
672
+ _file.save(
673
+ Path(_file.name).name, File(f), save=True
674
+ ) # Re-save the file content
675
+ logger.info(
676
+ f"Successfully restored file from fallback {fallback_file} to {_file.name}"
677
+ )
606
678
  else:
607
679
  logger.error(f"Fallback file {fallback_file} does not exist.")
608
680
  except Exception as e:
@@ -678,8 +750,8 @@ class RawPdfFile(models.Model):
678
750
  return settings_dict
679
751
 
680
752
  @staticmethod
681
- def get_pdf_by_id(pdf_id: int) -> "RawPdfFile":
753
+ def get_pdf_by_pk(pk: int) -> "RawPdfFile":
682
754
  try:
683
- return RawPdfFile.objects.get(pk=pdf_id)
755
+ return RawPdfFile.objects.get(pk=pk)
684
756
  except RawPdfFile.DoesNotExist:
685
- raise ValueError(f"PDF with ID {pdf_id} does not exist.")
757
+ raise ValueError(f"report with ID {pdf_id} does not exist.")
@@ -12,21 +12,41 @@ if TYPE_CHECKING:
12
12
 
13
13
  class ReportReaderConfig(models.Model):
14
14
  """
15
- Configuration settings for parsing PDF reports (ReportReader).
15
+ Configuration settings for parsing report reports (ReportReader).
16
16
 
17
17
  Stores locale, name lists, date format, and flags used to identify key information lines
18
18
  and text sections to ignore.
19
19
  """
20
20
 
21
21
  locale = models.CharField(default="de_DE", max_length=10)
22
- first_names = models.ManyToManyField("FirstName", related_name="report_reader_configs")
23
- last_names = models.ManyToManyField("LastName", related_name="report_reader_configs")
22
+ first_names = models.ManyToManyField(
23
+ "FirstName", related_name="report_reader_configs"
24
+ )
25
+ last_names = models.ManyToManyField(
26
+ "LastName", related_name="report_reader_configs"
27
+ )
24
28
  text_date_format = models.CharField(default="%d.%m.%Y", max_length=10)
25
- patient_info_line_flag = models.ForeignKey("ReportReaderFlag", related_name="report_reader_configs_patient_info_line", on_delete=models.CASCADE)
26
- endoscope_info_line_flag = models.ForeignKey("ReportReaderFlag", related_name="report_reader_configs_endoscope_info_line", on_delete=models.CASCADE)
27
- examiner_info_line_flag = models.ForeignKey("ReportReaderFlag", related_name="report_reader_configs_examiner_info_line", on_delete=models.CASCADE)
28
- cut_off_below = models.ManyToManyField("ReportReaderFlag", related_name="report_reader_configs_cut_off_below")
29
- cut_off_above = models.ManyToManyField("ReportReaderFlag", related_name="report_reader_configs_cut_off_above")
29
+ patient_info_line_flag = models.ForeignKey(
30
+ "ReportReaderFlag",
31
+ related_name="report_reader_configs_patient_info_line",
32
+ on_delete=models.CASCADE,
33
+ )
34
+ endoscope_info_line_flag = models.ForeignKey(
35
+ "ReportReaderFlag",
36
+ related_name="report_reader_configs_endoscope_info_line",
37
+ on_delete=models.CASCADE,
38
+ )
39
+ examiner_info_line_flag = models.ForeignKey(
40
+ "ReportReaderFlag",
41
+ related_name="report_reader_configs_examiner_info_line",
42
+ on_delete=models.CASCADE,
43
+ )
44
+ cut_off_below = models.ManyToManyField(
45
+ "ReportReaderFlag", related_name="report_reader_configs_cut_off_below"
46
+ )
47
+ cut_off_above = models.ManyToManyField(
48
+ "ReportReaderFlag", related_name="report_reader_configs_cut_off_above"
49
+ )
30
50
 
31
51
  if TYPE_CHECKING:
32
52
  patient_info_line_flag = models.ForeignKey["ReportReaderFlag"]
@@ -35,8 +55,12 @@ class ReportReaderConfig(models.Model):
35
55
 
36
56
  first_names = cast(models.manager.RelatedManager["FirstName"], first_names)
37
57
  last_names = cast(models.manager.RelatedManager["LastName"], last_names)
38
- cut_off_below = cast(models.manager.RelatedManager["ReportReaderFlag"], cut_off_below)
39
- cut_off_above = cast(models.manager.RelatedManager["ReportReaderFlag"], cut_off_above)
58
+ cut_off_below = cast(
59
+ models.manager.RelatedManager["ReportReaderFlag"], cut_off_below
60
+ )
61
+ cut_off_above = cast(
62
+ models.manager.RelatedManager["ReportReaderFlag"], cut_off_above
63
+ )
40
64
 
41
65
  def __str__(self):
42
66
  """Returns a string representation including the locale and primary key."""
@@ -0,0 +1,5 @@
1
+ from .processing_history import ProcessingHistory
2
+
3
+ __all__ = [
4
+ "ProcessingHistory"
5
+ ]
@@ -0,0 +1,96 @@
1
+ from typing import Optional
2
+
3
+ from django.contrib.contenttypes.fields import GenericForeignKey
4
+ from django.contrib.contenttypes.models import ContentType
5
+ from django.db import models
6
+
7
+
8
+ class ProcessingHistory(models.Model):
9
+ """
10
+ Generic processing history for media files (video, pdf, ...).
11
+
12
+ Stores:
13
+ - which object (VideoFile/RawPdfFile/other) this entry belongs to
14
+ - whether processing/anonymization succeeded
15
+ - timestamps
16
+ """
17
+
18
+ created_at = models.DateTimeField(auto_now_add=True)
19
+ success = models.BooleanField(default=False, blank=True)
20
+
21
+ content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
22
+ object_id = models.PositiveBigIntegerField()
23
+ content_object = GenericForeignKey("content_type", "object_id")
24
+
25
+ class Meta:
26
+ ordering = ["-created_at"]
27
+
28
+ def __str__(self) -> str:
29
+ return f"{self.content_type} #{self.object_id}, Success: {self.success}"
30
+
31
+ # --- public helpers that work with *objects* ---
32
+
33
+ @classmethod
34
+ def get_or_create_for_object(
35
+ cls,
36
+ *,
37
+ obj: models.Model,
38
+ success: Optional[bool],
39
+ ) -> "ProcessingHistory":
40
+ """
41
+ Get or create a ProcessingHistory entry for a concrete model instance.
42
+
43
+ - Returns existing entry for (object_id, content_type) if present
44
+ - Otherwise creates one with the given success flag (default False)
45
+ - If an entry exists and `success` is provided, it updates `success`
46
+ """
47
+ ct = ContentType.objects.get_for_model(obj, for_concrete_model=False)
48
+
49
+ ph, created = cls.objects.get_or_create(
50
+ object_id=obj.pk,
51
+ content_type=ct,
52
+ defaults={"success": success},
53
+ )
54
+
55
+ if not created and success is not None and ph.success != success:
56
+ ph.success = success
57
+ ph.save(update_fields=["success"])
58
+
59
+ return ph
60
+
61
+ @classmethod
62
+ def has_history(
63
+ cls,
64
+ *,
65
+ object_id: int,
66
+ content_type: ContentType,
67
+ success: Optional[bool] = None,
68
+ ) -> bool:
69
+ """
70
+ Return True if there is at least one ProcessingHistory entry for the
71
+ given (object_id, content_type) combination.
72
+
73
+ If `success` is not None, only entries matching that success flag are
74
+ considered.
75
+ """
76
+ qs = cls.objects.filter(object_id=object_id, content_type=content_type)
77
+ if success is not None:
78
+ qs = qs.filter(success=success)
79
+ return qs.exists()
80
+
81
+ @classmethod
82
+ def has_history_for_object(
83
+ cls,
84
+ *,
85
+ obj: models.Model,
86
+ success: Optional[bool] = None,
87
+ ) -> bool:
88
+ """
89
+ Convenience wrapper for model instances.
90
+ """
91
+ ct = ContentType.objects.get_for_model(obj, for_concrete_model=False)
92
+ return cls.has_history(
93
+ object_id=obj.pk,
94
+ content_type=ct,
95
+ success=success,
96
+ )