endoreg-db 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. endoreg_db/data/__init__.py +3 -1
  2. endoreg_db/data/center/data.yaml +45 -0
  3. endoreg_db/data/pdf_type/data.yaml +28 -0
  4. endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +26 -0
  5. endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +19 -0
  6. endoreg_db/management/commands/load_base_db_data.py +12 -3
  7. endoreg_db/management/commands/load_center_data.py +3 -3
  8. endoreg_db/management/commands/load_pdf_type_data.py +61 -0
  9. endoreg_db/management/commands/load_report_reader_flag.py +46 -0
  10. endoreg_db/management/commands/reset_celery_schedule.py +9 -0
  11. endoreg_db/migrations/0013_rawpdffile.py +31 -0
  12. endoreg_db/migrations/0014_pdftype_alter_rawpdffile_file_pdfmeta.py +38 -0
  13. endoreg_db/migrations/0015_rename_report_processed_rawpdffile_state_report_processed_and_more.py +31 -0
  14. endoreg_db/migrations/0016_rawpdffile_state_report_processing_required.py +18 -0
  15. endoreg_db/migrations/0017_firstname_lastname_center_first_names_and_more.py +37 -0
  16. endoreg_db/migrations/0018_reportreaderflag_reportreaderconfig.py +37 -0
  17. endoreg_db/migrations/0019_pdftype_cut_off_above_lines_and_more.py +42 -0
  18. endoreg_db/migrations/0020_rename_endoscopy_info_line_pdftype_endoscope_info_line.py +18 -0
  19. endoreg_db/migrations/0021_alter_pdftype_endoscope_info_line.py +19 -0
  20. endoreg_db/migrations/0022_alter_pdftype_endoscope_info_line.py +19 -0
  21. endoreg_db/models/__init__.py +2 -0
  22. endoreg_db/models/center.py +6 -0
  23. endoreg_db/models/data_file/__init__.py +2 -3
  24. endoreg_db/models/data_file/import_classes/__init__.py +1 -0
  25. endoreg_db/models/data_file/import_classes/processing_functions/__init__.py +35 -0
  26. endoreg_db/models/data_file/import_classes/processing_functions/pdf.py +28 -0
  27. endoreg_db/models/data_file/import_classes/{processing_functions.py → processing_functions/video.py} +6 -7
  28. endoreg_db/models/data_file/import_classes/raw_pdf.py +185 -0
  29. endoreg_db/models/data_file/import_classes/raw_video.py +7 -5
  30. endoreg_db/models/data_file/metadata/__init__.py +2 -132
  31. endoreg_db/models/data_file/metadata/pdf_meta.py +70 -0
  32. endoreg_db/models/data_file/metadata/sensitive_meta.py +19 -1
  33. endoreg_db/models/data_file/metadata/video_meta.py +132 -0
  34. endoreg_db/models/data_file/report_file.py +1 -0
  35. endoreg_db/models/persons/__init__.py +3 -1
  36. endoreg_db/models/persons/first_name.py +18 -0
  37. endoreg_db/models/persons/last_name.py +20 -0
  38. endoreg_db/models/report_reader/__init__.py +2 -0
  39. endoreg_db/models/report_reader/report_reader_config.py +53 -0
  40. endoreg_db/models/report_reader/report_reader_flag.py +20 -0
  41. endoreg_db/utils/dataloader.py +172 -56
  42. endoreg_db/utils/hashs.py +18 -0
  43. {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/METADATA +2 -1
  44. {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/RECORD +46 -20
  45. {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/LICENSE +0 -0
  46. {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/WHEEL +0 -0
@@ -0,0 +1,35 @@
1
+ from .video import (
2
+ get_videos_scheduled_for_frame_extraction,
3
+ extract_frames_from_video,
4
+ extract_frames_from_videos,
5
+
6
+ get_videos_scheduled_for_ocr,
7
+ videos_scheduled_for_ocr_preflight,
8
+ perform_ocr_on_video,
9
+ perform_ocr_on_videos,
10
+
11
+ videos_scheduled_for_initial_prediction_preflight,
12
+ get_videos_scheduled_for_initial_prediction,
13
+ get_multilabel_model,
14
+ get_multilabel_classifier,
15
+ get_crops,
16
+ perform_initial_prediction_on_video,
17
+ perform_initial_prediction_on_videos,
18
+
19
+ videos_scheduled_for_prediction_import_preflight,
20
+ get_videos_scheduled_for_prediction_import,
21
+ import_predictions_for_video,
22
+ import_predictions_for_videos,
23
+
24
+
25
+ delete_frames_preflight,
26
+ get_videos_scheduled_for_frame_deletion,
27
+ delete_frames_for_video,
28
+ delete_frames,
29
+ )
30
+
31
+ from .pdf import (
32
+ get_pdf_files_scheduled_for_processing,
33
+ process_pdf_file,
34
+ process_pdf_files,
35
+ )
@@ -0,0 +1,28 @@
1
+ from ..raw_pdf import RawPdfFile
2
+ from logging import getLogger
3
+ #
4
+ # # setup logging to pdf_import.log
5
+ logger = getLogger('examination_pdf_import')
6
+
7
+ def get_pdf_files_scheduled_for_processing():
8
+ reports = RawPdfFile.objects.filter(
9
+ state_report_processing_required=True
10
+ )
11
+ return reports
12
+
13
+ def process_pdf_file(report:RawPdfFile):
14
+ if report.update(save=True, verbose=True):
15
+ logger.info(f"Report {report} processed successfully")
16
+ return True
17
+ else:
18
+ logger.error(f"Report {report} processing failed")
19
+ return False
20
+
21
+
22
+ def process_pdf_files():
23
+ reports = get_pdf_files_scheduled_for_processing()
24
+ for report in reports:
25
+ process_pdf_file(report)
26
+
27
+
28
+
@@ -1,4 +1,4 @@
1
- from .raw_video import RawVideoFile
1
+ from ..raw_video import RawVideoFile
2
2
  import json
3
3
 
4
4
  # # Starting point
@@ -201,8 +201,8 @@ def videos_scheduled_for_prediction_import_preflight():
201
201
 
202
202
  def get_videos_scheduled_for_prediction_import():
203
203
  return RawVideoFile.objects.filter(
204
- state_prediction_import_required=True,
205
- state_prediction_import_completed=False,
204
+ state_initial_prediction_import_required=True,
205
+ state_initial_prediction_import_completed=False,
206
206
  state_initial_prediction_completed=True
207
207
  )
208
208
 
@@ -211,9 +211,9 @@ def import_predictions_for_video(video:RawVideoFile):
211
211
  pass
212
212
 
213
213
  # update state_prediction_import_completed
214
- video.state_prediction_import_required = False
215
- video.state_prediction_import_completed = True
216
- video.save()
214
+ # video.state_initial_prediction_import_required = False
215
+ # video.state_initial_prediction_import_completed = True
216
+ # video.save()
217
217
 
218
218
  return video
219
219
 
@@ -222,7 +222,6 @@ def import_predictions_for_videos():
222
222
  for video in videos:
223
223
  import_predictions_for_video(video)
224
224
 
225
-
226
225
  # # Step 4 - Delete Frames if not needed anymore
227
226
  # function to query for videos scheduled for frame deletion,
228
227
  # first we need to set state_frames_required = False for videos with:
@@ -0,0 +1,185 @@
1
+ # models/data_file/import_classes/raw_pdf.py
2
+ # django db model "RawPdf"
3
+ # Class to store raw pdf file using django file field
4
+ # Class contains classmethod to create object from pdf file
5
+ # objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
6
+ # ------------------------------------------------------------------------------
7
+
8
+ from django.db import models
9
+ from django.core.files.storage import FileSystemStorage
10
+ from django.core.files import File
11
+ from django.conf import settings
12
+ from django.utils import timezone
13
+ from django.core.exceptions import ValidationError
14
+ from django.core.validators import FileExtensionValidator
15
+ from endoreg_db.utils.file_operations import get_uuid_filename
16
+
17
+ from agl_report_reader.report_reader import ReportReader
18
+
19
+ from endoreg_db.utils.hashs import get_pdf_hash
20
+ from ..metadata import SensitiveMeta
21
+
22
+ # setup logging to pdf_import.log
23
+ import logging
24
+ logger = logging.getLogger('pdf_import')
25
+
26
+ import shutil
27
+
28
+ class RawPdfFile(models.Model):
29
+ file = models.FileField(
30
+ upload_to='raw_pdf/',
31
+ validators=[FileExtensionValidator(allowed_extensions=['pdf'])],
32
+ storage=FileSystemStorage(location=settings.PSEUDO_DIR_RAW_PDF.resolve().as_posix()),
33
+ )
34
+
35
+ pdf_hash = models.CharField(max_length=255, unique=True)
36
+ pdf_type = models.ForeignKey('PdfType', on_delete=models.CASCADE)
37
+ center = models.ForeignKey('Center', on_delete=models.CASCADE)
38
+
39
+ state_report_processing_required = models.BooleanField(default = True)
40
+ state_report_processed = models.BooleanField(default=False)
41
+
42
+ # report_file = models.OneToOneField("ReportFile", on_delete=models.CASCADE, null=True, blank=True)
43
+ sensitive_meta = models.OneToOneField(
44
+ 'SensitiveMeta',
45
+ on_delete=models.CASCADE,
46
+ related_name='raw_pdf_file',
47
+ null=True,
48
+ blank=True,
49
+ )
50
+
51
+ text = models.TextField(blank=True, null=True)
52
+ anonymized_text = models.TextField(blank=True, null=True)
53
+
54
+ raw_meta = models.JSONField(blank=True, null=True)
55
+
56
+ created_at = models.DateTimeField(auto_now_add=True)
57
+
58
+ def __str__(self):
59
+ str_repr = f"RawPdfFile: {self.file.name}"
60
+ return str_repr
61
+
62
+ @classmethod
63
+ def create_from_file(
64
+ cls,
65
+ file_path,
66
+ center_name,
67
+ pdf_type_name,
68
+ destination_dir,
69
+ save=True,
70
+ ):
71
+ from endoreg_db.models import PdfType, Center
72
+ logger.info(f"Creating RawPdfFile object from file: {file_path}")
73
+ original_file_name = file_path.name
74
+
75
+ new_file_name, uuid = get_uuid_filename(file_path)
76
+
77
+ if not destination_dir.exists():
78
+ destination_dir.mkdir(parents=True)
79
+
80
+ pdf_hash = get_pdf_hash(file_path)
81
+
82
+ # check if pdf file already exists
83
+ if cls.objects.filter(pdf_hash=pdf_hash).exists():
84
+ logger.warning(f"RawPdfFile with hash {pdf_hash} already exists")
85
+ return None
86
+
87
+ assert pdf_type_name is not None, "pdf_type_name is required"
88
+ assert center_name is not None, "center_name is required"
89
+
90
+ pdf_type = PdfType.objects.get(name=pdf_type_name)
91
+ center = Center.objects.get(name=center_name)
92
+
93
+ new_file_path = destination_dir / new_file_name
94
+
95
+ logger.info(f"Copying file to {new_file_path}")
96
+ success = shutil.copy(file_path, new_file_path)
97
+
98
+ # validate copy operation by comparing hashs
99
+ assert get_pdf_hash(new_file_path) == pdf_hash, "Copy operation failed"
100
+
101
+ raw_pdf = cls(
102
+ file=new_file_path.resolve().as_posix(),
103
+ pdf_hash=pdf_hash,
104
+ pdf_type=pdf_type,
105
+ center=center,
106
+ )
107
+ logger.info(f"RawPdfFile object created: {raw_pdf}")
108
+
109
+ # remove source file
110
+ file_path.unlink()
111
+ logger.info(f"Source file removed: {file_path}")
112
+
113
+ if save:
114
+ raw_pdf.save()
115
+
116
+
117
+ return raw_pdf
118
+
119
+ def process_file(self, verbose = False):
120
+
121
+ pdf_path = self.file.path
122
+ rr_config = self.get_report_reader_config()
123
+
124
+ rr = ReportReader(**rr_config) #FIXME In future we need to pass a configuration file
125
+ # This configuration file should be associated with pdf type
126
+
127
+ text, anonymized_text, report_meta = rr.process_report(pdf_path, verbose=verbose)
128
+ if not self.sensitive_meta:
129
+ sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
130
+ sensitive_meta.save()
131
+ self.sensitive_meta = sensitive_meta
132
+
133
+ else:
134
+ # update existing sensitive meta
135
+ sensitive_meta = self.sensitive_meta
136
+ sensitive_meta.update_from_dict(report_meta)
137
+
138
+ return text, anonymized_text, report_meta
139
+
140
+ def update(self, save=True, verbose = True):
141
+ try:
142
+ self.text, self.anonymized_text, self.raw_meta = self.process_file(verbose = verbose)
143
+ self.state_report_processed = True
144
+ self.state_report_processing_required = False
145
+
146
+ if save:
147
+
148
+ self.save()
149
+
150
+ return True
151
+
152
+ except:
153
+ logger.error(f"Error processing file: {self.file.path}")
154
+ return False
155
+
156
+ def save(self, *args, **kwargs):
157
+ if not self.file.name.endswith('.pdf'):
158
+ raise ValidationError('Only PDF files are allowed')
159
+
160
+ if not self.pdf_hash:
161
+ self.pdf_hash = get_pdf_hash(self.file.path)
162
+
163
+ super().save(*args, **kwargs)
164
+
165
+
166
+ def get_report_reader_config(self):
167
+ if self.pdf_type.endoscope_info_line:
168
+ endoscope_info_line = self.pdf_type.endoscope_info_line.value
169
+ else:
170
+ endoscope_info_line = None
171
+ settings_dict = {
172
+ "locale": "de_DE",
173
+ "employee_first_names": [_.name for _ in self.center.first_names.all()],
174
+ "employee_last_names": [_.name for _ in self.center.last_names.all()],
175
+ "text_date_format":'%d.%m.%Y',
176
+ "flags": {
177
+ "patient_info_line": self.pdf_type.patient_info_line.value,
178
+ "endoscope_info_line": endoscope_info_line,
179
+ "examiner_info_line": self.pdf_type.examiner_info_line.value,
180
+ "cut_off_below": [_.value for _ in self.pdf_type.cut_off_below_lines.all()],
181
+ "cut_off_above": [_.value for _ in self.pdf_type.cut_off_above_lines.all()],
182
+ }
183
+ }
184
+
185
+ return settings_dict
@@ -15,9 +15,11 @@ from ..metadata import VideoMeta, SensitiveMeta
15
15
  class RawVideoFile(models.Model):
16
16
  uuid = models.UUIDField()
17
17
  file = models.FileField(upload_to="raw_data/")
18
+
18
19
  sensitive_meta = models.OneToOneField(
19
20
  "SensitiveMeta", on_delete=models.CASCADE, blank=True, null=True
20
- )
21
+ )
22
+
21
23
  center = models.ForeignKey("Center", on_delete=models.CASCADE)
22
24
  processor = models.ForeignKey(
23
25
  "EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
@@ -69,7 +71,7 @@ class RawVideoFile(models.Model):
69
71
  def create_from_file(
70
72
  cls,
71
73
  file_path: Path,
72
- video_dir_parent: Path,
74
+ video_dir: Path,
73
75
  center_name: str,
74
76
  processor_name: str,
75
77
  frame_dir_parent: Path,
@@ -87,8 +89,8 @@ class RawVideoFile(models.Model):
87
89
  if not framedir.exists():
88
90
  framedir.mkdir(parents=True, exist_ok=True)
89
91
 
90
- if not video_dir_parent.exists():
91
- video_dir_parent.mkdir(parents=True, exist_ok=True)
92
+ if not video_dir.exists():
93
+ video_dir.mkdir(parents=True, exist_ok=True)
92
94
 
93
95
  video_hash = get_video_hash(file_path)
94
96
 
@@ -98,7 +100,7 @@ class RawVideoFile(models.Model):
98
100
  processor = EndoscopyProcessor.objects.get(name=processor_name)
99
101
  assert processor is not None, "Processor must exist"
100
102
 
101
- new_filepath = video_dir_parent / new_file_name
103
+ new_filepath = video_dir / new_file_name
102
104
 
103
105
  print(f"Moving {file_path} to {new_filepath}")
104
106
  shutil.move(file_path.resolve().as_posix(), new_filepath.resolve().as_posix())
@@ -1,133 +1,3 @@
1
- from django.db import models
2
- import ffmpeg
3
- from pathlib import Path
4
1
  from .sensitive_meta import SensitiveMeta
5
-
6
- # import endoreg_center_id from django settings
7
- from django.conf import settings
8
-
9
- # check if endoreg_center_id is set
10
- if not hasattr(settings, 'ENDOREG_CENTER_ID'):
11
- ENDOREG_CENTER_ID = 9999
12
- else:
13
- ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
14
-
15
- # VideoMeta
16
- class VideoMeta(models.Model):
17
- processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
18
- endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
19
- center = models.ForeignKey('Center', on_delete=models.CASCADE)
20
- import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
21
- ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
22
-
23
- def __str__(self):
24
-
25
- processor_name = self.processor.name if self.processor is not None else "None"
26
- endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
27
- center_name = self.center.name if self.center is not None else "None"
28
-
29
- result_html = ""
30
-
31
- result_html += f"Processor: {processor_name}<br>"
32
- result_html += f"Endoscope: {endoscope_name}<br>"
33
- result_html += f"Center: {center_name}<br>"
34
-
35
- return result_html
36
-
37
- # import meta should be created when video meta is created
38
- def save(self, *args, **kwargs):
39
- if self.import_meta is None:
40
- self.import_meta = VideoImportMeta.objects.create()
41
- super(VideoMeta, self).save(*args, **kwargs)
42
-
43
- def initialize_ffmpeg_meta(self, file_path):
44
- """Initializes FFMpeg metadata for the video file if not already done."""
45
- self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
46
- self.save()
47
-
48
- def update_meta(self, file_path):
49
- """Updates the video metadata from the file."""
50
- self.initialize_ffmpeg_meta(file_path)
51
- self.save()
52
-
53
- def get_endo_roi(self):
54
- endo_roi = self.processor.get_roi_endoscope_image()
55
- return endo_roi
56
-
57
- def get_fps(self):
58
- if not self.ffmpeg_meta:
59
- return None
60
-
61
- return self.ffmpeg_meta.frame_rate
62
-
63
-
64
- class FFMpegMeta(models.Model):
65
- # Existing fields
66
- duration = models.FloatField(blank=True, null=True)
67
- width = models.IntegerField(blank=True, null=True)
68
- height = models.IntegerField(blank=True, null=True)
69
- frame_rate = models.FloatField(blank=True, null=True)
70
-
71
- # New fields for comprehensive information
72
- video_codec = models.CharField(max_length=50, blank=True, null=True)
73
- audio_codec = models.CharField(max_length=50, blank=True, null=True)
74
- audio_channels = models.IntegerField(blank=True, null=True)
75
- audio_sample_rate = models.IntegerField(blank=True, null=True)
76
-
77
- # Existing __str__ method can be updated to include new fields
78
-
79
- @classmethod
80
- def create_from_file(cls, file_path: Path):
81
- """Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
82
- try:
83
- probe = ffmpeg.probe(str(file_path))
84
- except ffmpeg.Error as e:
85
- print(e.stderr)
86
- return None
87
-
88
- video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
89
- audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
90
-
91
- # Check for the existence of a video stream
92
- if video_stream is None:
93
- print(f"No video stream found in {file_path}")
94
- return None
95
-
96
- # Extract and store video metadata
97
- metadata = {
98
- 'duration': float(video_stream.get('duration', 0)),
99
- 'width': int(video_stream.get('width', 0)),
100
- 'height': int(video_stream.get('height', 0)),
101
- 'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
102
- 'video_codec': video_stream.get('codec_name', ''),
103
- }
104
-
105
- # If there are audio streams, extract and store audio metadata from the first stream
106
- if audio_streams:
107
- first_audio_stream = audio_streams[0]
108
- metadata.update({
109
- 'audio_codec': first_audio_stream.get('codec_name', ''),
110
- 'audio_channels': int(first_audio_stream.get('channels', 0)),
111
- 'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
112
- })
113
-
114
- # Create and return the FFMpegMeta instance
115
- return cls.objects.create(**metadata)
116
-
117
- class VideoImportMeta(models.Model):
118
-
119
- video_anonymized = models.BooleanField(default=False)
120
- video_patient_data_detected = models.BooleanField(default=False)
121
- outside_detected = models.BooleanField(default=False)
122
- patient_data_removed = models.BooleanField(default=False)
123
- outside_removed = models.BooleanField(default=False)
124
-
125
- def __str__(self):
126
- result_html = ""
127
-
128
- result_html += f"Video anonymized: {self.video_anonymized}<br>"
129
- result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
130
- result_html += f"Outside detected: {self.outside_detected}<br>"
131
- result_html += f"Patient data removed: {self.patient_data_removed}<br>"
132
- result_html += f"Outside removed: {self.outside_removed}<br>"
133
- return result_html
2
+ from .pdf_meta import PdfMeta, PdfType
3
+ from .video_meta import VideoMeta, FFMpegMeta, VideoImportMeta
@@ -0,0 +1,70 @@
1
+ from django.db import models
2
+
3
+ # import endoreg_center_id from django settings
4
+ from django.conf import settings
5
+
6
+
7
+ # import File class
8
+ from django.core.files import File
9
+
10
+ # # check if endoreg_center_id is set
11
+ # if not hasattr(settings, 'ENDOREG_CENTER_ID'):
12
+ # ENDOREG_CENTER_ID = 9999
13
+ # else:
14
+ # ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
15
+
16
+ class PdfType(models.Model):
17
+ name = models.CharField(max_length=255)
18
+
19
+ patient_info_line = models.ForeignKey(
20
+ "ReportReaderFlag",
21
+ related_name="pdf_type_patient_info_line",
22
+ on_delete=models.CASCADE
23
+ )
24
+ endoscope_info_line = models.ForeignKey(
25
+ "ReportReaderFlag",
26
+ related_name="pdf_type_endoscopy_info_line",
27
+ on_delete=models.CASCADE,
28
+ )
29
+ examiner_info_line = models.ForeignKey(
30
+ "ReportReaderFlag",
31
+ related_name="pdf_type_examiner_info_line",
32
+ on_delete=models.CASCADE
33
+ )
34
+ cut_off_above_lines = models.ManyToManyField(
35
+ "ReportReaderFlag",
36
+ related_name="pdf_type_cut_off_above_lines",
37
+ )
38
+ cut_off_below_lines = models.ManyToManyField(
39
+ "ReportReaderFlag",
40
+ related_name="pdf_type_cut_off_below_lines",
41
+ )
42
+
43
+
44
+ def __str__(self):
45
+ summary = f"{self.name}"
46
+ # add lines to summary
47
+ summary += f"\nPatient Info Line: {self.patient_info_line.value}"
48
+ summary += f"\nEndoscope Info Line: {self.endoscope_info_line.value}"
49
+ summary += f"\nExaminer Info Line: {self.examiner_info_line.value}"
50
+ summary += f"\nCut Off Above Lines: {[_.value for _ in self.cut_off_above_lines.all()]}"
51
+ summary += f"\nCut Off Below Lines: {[_.value for _ in self.cut_off_below_lines.all()]}"
52
+
53
+ return summary
54
+
55
+ class PdfMeta(models.Model):
56
+ pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
57
+ date = models.DateField()
58
+ time = models.TimeField()
59
+ pdf_hash = models.CharField(max_length=255, unique=True)
60
+
61
+ def __str__(self):
62
+ return self.pdf_hash
63
+
64
+ @classmethod
65
+ def create_from_file(cls, pdf_file):
66
+ pdf_file = File(pdf_file)
67
+ pdf_meta = cls(file=pdf_file)
68
+ pdf_meta.save()
69
+ return pdf_meta
70
+
@@ -10,4 +10,22 @@ class SensitiveMeta(models.Model):
10
10
 
11
11
  @classmethod
12
12
  def create_from_dict(cls, data: dict):
13
- return cls.objects.create(**data)
13
+ # data can contain more fields than the model has
14
+ field_names = [_.name for _ in cls._meta.fields]
15
+ selected_data = {k: v for k, v in data.items() if k in field_names}
16
+
17
+ return cls.objects.create(**selected_data)
18
+
19
+ def update_from_dict(self, data: dict):
20
+ # data can contain more fields than the model has
21
+ field_names = [_.name for _ in self._meta.fields]
22
+ selected_data = {k: v for k, v in data.items() if k in field_names}
23
+
24
+ for k, v in selected_data.items():
25
+ setattr(self, k, v)
26
+
27
+ self.save()
28
+
29
+ def __str__(self):
30
+ return f"SensitiveMeta: {self.examination_date} {self.patient_first_name} {self.patient_last_name} (*{self.patient_dob})"
31
+
@@ -0,0 +1,132 @@
1
+ from django.db import models
2
+ import ffmpeg
3
+ from pathlib import Path
4
+
5
+ # import endoreg_center_id from django settings
6
+ from django.conf import settings
7
+
8
+ # check if endoreg_center_id is set
9
+ if not hasattr(settings, 'ENDOREG_CENTER_ID'):
10
+ ENDOREG_CENTER_ID = 9999
11
+ else:
12
+ ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
13
+
14
+ # VideoMeta
15
+ class VideoMeta(models.Model):
16
+ processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
17
+ endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
18
+ center = models.ForeignKey('Center', on_delete=models.CASCADE)
19
+ import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
20
+ ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
21
+
22
+ def __str__(self):
23
+
24
+ processor_name = self.processor.name if self.processor is not None else "None"
25
+ endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
26
+ center_name = self.center.name if self.center is not None else "None"
27
+
28
+ result_html = ""
29
+
30
+ result_html += f"Processor: {processor_name}<br>"
31
+ result_html += f"Endoscope: {endoscope_name}<br>"
32
+ result_html += f"Center: {center_name}<br>"
33
+
34
+ return result_html
35
+
36
+ # import meta should be created when video meta is created
37
+ def save(self, *args, **kwargs):
38
+ if self.import_meta is None:
39
+ self.import_meta = VideoImportMeta.objects.create()
40
+ super(VideoMeta, self).save(*args, **kwargs)
41
+
42
+ def initialize_ffmpeg_meta(self, file_path):
43
+ """Initializes FFMpeg metadata for the video file if not already done."""
44
+ self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
45
+ self.save()
46
+
47
+ def update_meta(self, file_path):
48
+ """Updates the video metadata from the file."""
49
+ self.initialize_ffmpeg_meta(file_path)
50
+ self.save()
51
+
52
+ def get_endo_roi(self):
53
+ endo_roi = self.processor.get_roi_endoscope_image()
54
+ return endo_roi
55
+
56
+ def get_fps(self):
57
+ if not self.ffmpeg_meta:
58
+ return None
59
+
60
+ return self.ffmpeg_meta.frame_rate
61
+
62
+
63
+ class FFMpegMeta(models.Model):
64
+ # Existing fields
65
+ duration = models.FloatField(blank=True, null=True)
66
+ width = models.IntegerField(blank=True, null=True)
67
+ height = models.IntegerField(blank=True, null=True)
68
+ frame_rate = models.FloatField(blank=True, null=True)
69
+
70
+ # New fields for comprehensive information
71
+ video_codec = models.CharField(max_length=50, blank=True, null=True)
72
+ audio_codec = models.CharField(max_length=50, blank=True, null=True)
73
+ audio_channels = models.IntegerField(blank=True, null=True)
74
+ audio_sample_rate = models.IntegerField(blank=True, null=True)
75
+
76
+ # Existing __str__ method can be updated to include new fields
77
+
78
+ @classmethod
79
+ def create_from_file(cls, file_path: Path):
80
+ """Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
81
+ try:
82
+ probe = ffmpeg.probe(str(file_path))
83
+ except ffmpeg.Error as e:
84
+ print(e.stderr)
85
+ return None
86
+
87
+ video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
88
+ audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
89
+
90
+ # Check for the existence of a video stream
91
+ if video_stream is None:
92
+ print(f"No video stream found in {file_path}")
93
+ return None
94
+
95
+ # Extract and store video metadata
96
+ metadata = {
97
+ 'duration': float(video_stream.get('duration', 0)),
98
+ 'width': int(video_stream.get('width', 0)),
99
+ 'height': int(video_stream.get('height', 0)),
100
+ 'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
101
+ 'video_codec': video_stream.get('codec_name', ''),
102
+ }
103
+
104
+ # If there are audio streams, extract and store audio metadata from the first stream
105
+ if audio_streams:
106
+ first_audio_stream = audio_streams[0]
107
+ metadata.update({
108
+ 'audio_codec': first_audio_stream.get('codec_name', ''),
109
+ 'audio_channels': int(first_audio_stream.get('channels', 0)),
110
+ 'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
111
+ })
112
+
113
+ # Create and return the FFMpegMeta instance
114
+ return cls.objects.create(**metadata)
115
+
116
+ class VideoImportMeta(models.Model):
117
+
118
+ video_anonymized = models.BooleanField(default=False)
119
+ video_patient_data_detected = models.BooleanField(default=False)
120
+ outside_detected = models.BooleanField(default=False)
121
+ patient_data_removed = models.BooleanField(default=False)
122
+ outside_removed = models.BooleanField(default=False)
123
+
124
+ def __str__(self):
125
+ result_html = ""
126
+
127
+ result_html += f"Video anonymized: {self.video_anonymized}<br>"
128
+ result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
129
+ result_html += f"Outside detected: {self.outside_detected}<br>"
130
+ result_html += f"Patient data removed: {self.patient_data_removed}<br>"
131
+ result_html += f"Outside removed: {self.outside_removed}<br>"
132
+ return result_html