endoreg-db 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- endoreg_db/data/__init__.py +3 -1
- endoreg_db/data/center/data.yaml +45 -0
- endoreg_db/data/pdf_type/data.yaml +28 -0
- endoreg_db/data/report_reader_flag/ukw-examination-generic.yaml +26 -0
- endoreg_db/data/report_reader_flag/ukw-histology-generic.yaml +19 -0
- endoreg_db/management/commands/load_base_db_data.py +12 -3
- endoreg_db/management/commands/load_center_data.py +3 -3
- endoreg_db/management/commands/load_pdf_type_data.py +61 -0
- endoreg_db/management/commands/load_report_reader_flag.py +46 -0
- endoreg_db/management/commands/reset_celery_schedule.py +9 -0
- endoreg_db/migrations/0013_rawpdffile.py +31 -0
- endoreg_db/migrations/0014_pdftype_alter_rawpdffile_file_pdfmeta.py +38 -0
- endoreg_db/migrations/0015_rename_report_processed_rawpdffile_state_report_processed_and_more.py +31 -0
- endoreg_db/migrations/0016_rawpdffile_state_report_processing_required.py +18 -0
- endoreg_db/migrations/0017_firstname_lastname_center_first_names_and_more.py +37 -0
- endoreg_db/migrations/0018_reportreaderflag_reportreaderconfig.py +37 -0
- endoreg_db/migrations/0019_pdftype_cut_off_above_lines_and_more.py +42 -0
- endoreg_db/migrations/0020_rename_endoscopy_info_line_pdftype_endoscope_info_line.py +18 -0
- endoreg_db/migrations/0021_alter_pdftype_endoscope_info_line.py +19 -0
- endoreg_db/migrations/0022_alter_pdftype_endoscope_info_line.py +19 -0
- endoreg_db/models/__init__.py +2 -0
- endoreg_db/models/center.py +6 -0
- endoreg_db/models/data_file/__init__.py +2 -3
- endoreg_db/models/data_file/import_classes/__init__.py +1 -0
- endoreg_db/models/data_file/import_classes/processing_functions/__init__.py +35 -0
- endoreg_db/models/data_file/import_classes/processing_functions/pdf.py +28 -0
- endoreg_db/models/data_file/import_classes/{processing_functions.py → processing_functions/video.py} +6 -7
- endoreg_db/models/data_file/import_classes/raw_pdf.py +185 -0
- endoreg_db/models/data_file/import_classes/raw_video.py +7 -5
- endoreg_db/models/data_file/metadata/__init__.py +2 -132
- endoreg_db/models/data_file/metadata/pdf_meta.py +70 -0
- endoreg_db/models/data_file/metadata/sensitive_meta.py +19 -1
- endoreg_db/models/data_file/metadata/video_meta.py +132 -0
- endoreg_db/models/data_file/report_file.py +1 -0
- endoreg_db/models/persons/__init__.py +3 -1
- endoreg_db/models/persons/first_name.py +18 -0
- endoreg_db/models/persons/last_name.py +20 -0
- endoreg_db/models/report_reader/__init__.py +2 -0
- endoreg_db/models/report_reader/report_reader_config.py +53 -0
- endoreg_db/models/report_reader/report_reader_flag.py +20 -0
- endoreg_db/utils/dataloader.py +172 -56
- endoreg_db/utils/hashs.py +18 -0
- {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/METADATA +2 -1
- {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/RECORD +46 -20
- {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/LICENSE +0 -0
- {endoreg_db-0.3.1.dist-info → endoreg_db-0.3.3.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from .video import (
|
|
2
|
+
get_videos_scheduled_for_frame_extraction,
|
|
3
|
+
extract_frames_from_video,
|
|
4
|
+
extract_frames_from_videos,
|
|
5
|
+
|
|
6
|
+
get_videos_scheduled_for_ocr,
|
|
7
|
+
videos_scheduled_for_ocr_preflight,
|
|
8
|
+
perform_ocr_on_video,
|
|
9
|
+
perform_ocr_on_videos,
|
|
10
|
+
|
|
11
|
+
videos_scheduled_for_initial_prediction_preflight,
|
|
12
|
+
get_videos_scheduled_for_initial_prediction,
|
|
13
|
+
get_multilabel_model,
|
|
14
|
+
get_multilabel_classifier,
|
|
15
|
+
get_crops,
|
|
16
|
+
perform_initial_prediction_on_video,
|
|
17
|
+
perform_initial_prediction_on_videos,
|
|
18
|
+
|
|
19
|
+
videos_scheduled_for_prediction_import_preflight,
|
|
20
|
+
get_videos_scheduled_for_prediction_import,
|
|
21
|
+
import_predictions_for_video,
|
|
22
|
+
import_predictions_for_videos,
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
delete_frames_preflight,
|
|
26
|
+
get_videos_scheduled_for_frame_deletion,
|
|
27
|
+
delete_frames_for_video,
|
|
28
|
+
delete_frames,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
from .pdf import (
|
|
32
|
+
get_pdf_files_scheduled_for_processing,
|
|
33
|
+
process_pdf_file,
|
|
34
|
+
process_pdf_files,
|
|
35
|
+
)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from ..raw_pdf import RawPdfFile
|
|
2
|
+
from logging import getLogger
|
|
3
|
+
#
|
|
4
|
+
# # setup logging to pdf_import.log
|
|
5
|
+
logger = getLogger('examination_pdf_import')
|
|
6
|
+
|
|
7
|
+
def get_pdf_files_scheduled_for_processing():
|
|
8
|
+
reports = RawPdfFile.objects.filter(
|
|
9
|
+
state_report_processing_required=True
|
|
10
|
+
)
|
|
11
|
+
return reports
|
|
12
|
+
|
|
13
|
+
def process_pdf_file(report:RawPdfFile):
|
|
14
|
+
if report.update(save=True, verbose=True):
|
|
15
|
+
logger.info(f"Report {report} processed successfully")
|
|
16
|
+
return True
|
|
17
|
+
else:
|
|
18
|
+
logger.error(f"Report {report} processing failed")
|
|
19
|
+
return False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def process_pdf_files():
|
|
23
|
+
reports = get_pdf_files_scheduled_for_processing()
|
|
24
|
+
for report in reports:
|
|
25
|
+
process_pdf_file(report)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
endoreg_db/models/data_file/import_classes/{processing_functions.py → processing_functions/video.py}
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from
|
|
1
|
+
from ..raw_video import RawVideoFile
|
|
2
2
|
import json
|
|
3
3
|
|
|
4
4
|
# # Starting point
|
|
@@ -201,8 +201,8 @@ def videos_scheduled_for_prediction_import_preflight():
|
|
|
201
201
|
|
|
202
202
|
def get_videos_scheduled_for_prediction_import():
|
|
203
203
|
return RawVideoFile.objects.filter(
|
|
204
|
-
|
|
205
|
-
|
|
204
|
+
state_initial_prediction_import_required=True,
|
|
205
|
+
state_initial_prediction_import_completed=False,
|
|
206
206
|
state_initial_prediction_completed=True
|
|
207
207
|
)
|
|
208
208
|
|
|
@@ -211,9 +211,9 @@ def import_predictions_for_video(video:RawVideoFile):
|
|
|
211
211
|
pass
|
|
212
212
|
|
|
213
213
|
# update state_prediction_import_completed
|
|
214
|
-
video.
|
|
215
|
-
video.
|
|
216
|
-
video.save()
|
|
214
|
+
# video.state_initial_prediction_import_required = False
|
|
215
|
+
# video.state_initial_prediction_import_completed = True
|
|
216
|
+
# video.save()
|
|
217
217
|
|
|
218
218
|
return video
|
|
219
219
|
|
|
@@ -222,7 +222,6 @@ def import_predictions_for_videos():
|
|
|
222
222
|
for video in videos:
|
|
223
223
|
import_predictions_for_video(video)
|
|
224
224
|
|
|
225
|
-
|
|
226
225
|
# # Step 4 - Delete Frames if not needed anymore
|
|
227
226
|
# function to query for videos scheduled for frame deletion,
|
|
228
227
|
# first we need to set state_frames_required = False for videos with:
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# models/data_file/import_classes/raw_pdf.py
|
|
2
|
+
# django db model "RawPdf"
|
|
3
|
+
# Class to store raw pdf file using django file field
|
|
4
|
+
# Class contains classmethod to create object from pdf file
|
|
5
|
+
# objects contains methods to extract text, extract metadata from text and anonymize text from pdf file uzing agl_report_reader.ReportReader class
|
|
6
|
+
# ------------------------------------------------------------------------------
|
|
7
|
+
|
|
8
|
+
from django.db import models
|
|
9
|
+
from django.core.files.storage import FileSystemStorage
|
|
10
|
+
from django.core.files import File
|
|
11
|
+
from django.conf import settings
|
|
12
|
+
from django.utils import timezone
|
|
13
|
+
from django.core.exceptions import ValidationError
|
|
14
|
+
from django.core.validators import FileExtensionValidator
|
|
15
|
+
from endoreg_db.utils.file_operations import get_uuid_filename
|
|
16
|
+
|
|
17
|
+
from agl_report_reader.report_reader import ReportReader
|
|
18
|
+
|
|
19
|
+
from endoreg_db.utils.hashs import get_pdf_hash
|
|
20
|
+
from ..metadata import SensitiveMeta
|
|
21
|
+
|
|
22
|
+
# setup logging to pdf_import.log
|
|
23
|
+
import logging
|
|
24
|
+
logger = logging.getLogger('pdf_import')
|
|
25
|
+
|
|
26
|
+
import shutil
|
|
27
|
+
|
|
28
|
+
class RawPdfFile(models.Model):
|
|
29
|
+
file = models.FileField(
|
|
30
|
+
upload_to='raw_pdf/',
|
|
31
|
+
validators=[FileExtensionValidator(allowed_extensions=['pdf'])],
|
|
32
|
+
storage=FileSystemStorage(location=settings.PSEUDO_DIR_RAW_PDF.resolve().as_posix()),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
pdf_hash = models.CharField(max_length=255, unique=True)
|
|
36
|
+
pdf_type = models.ForeignKey('PdfType', on_delete=models.CASCADE)
|
|
37
|
+
center = models.ForeignKey('Center', on_delete=models.CASCADE)
|
|
38
|
+
|
|
39
|
+
state_report_processing_required = models.BooleanField(default = True)
|
|
40
|
+
state_report_processed = models.BooleanField(default=False)
|
|
41
|
+
|
|
42
|
+
# report_file = models.OneToOneField("ReportFile", on_delete=models.CASCADE, null=True, blank=True)
|
|
43
|
+
sensitive_meta = models.OneToOneField(
|
|
44
|
+
'SensitiveMeta',
|
|
45
|
+
on_delete=models.CASCADE,
|
|
46
|
+
related_name='raw_pdf_file',
|
|
47
|
+
null=True,
|
|
48
|
+
blank=True,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
text = models.TextField(blank=True, null=True)
|
|
52
|
+
anonymized_text = models.TextField(blank=True, null=True)
|
|
53
|
+
|
|
54
|
+
raw_meta = models.JSONField(blank=True, null=True)
|
|
55
|
+
|
|
56
|
+
created_at = models.DateTimeField(auto_now_add=True)
|
|
57
|
+
|
|
58
|
+
def __str__(self):
|
|
59
|
+
str_repr = f"RawPdfFile: {self.file.name}"
|
|
60
|
+
return str_repr
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def create_from_file(
|
|
64
|
+
cls,
|
|
65
|
+
file_path,
|
|
66
|
+
center_name,
|
|
67
|
+
pdf_type_name,
|
|
68
|
+
destination_dir,
|
|
69
|
+
save=True,
|
|
70
|
+
):
|
|
71
|
+
from endoreg_db.models import PdfType, Center
|
|
72
|
+
logger.info(f"Creating RawPdfFile object from file: {file_path}")
|
|
73
|
+
original_file_name = file_path.name
|
|
74
|
+
|
|
75
|
+
new_file_name, uuid = get_uuid_filename(file_path)
|
|
76
|
+
|
|
77
|
+
if not destination_dir.exists():
|
|
78
|
+
destination_dir.mkdir(parents=True)
|
|
79
|
+
|
|
80
|
+
pdf_hash = get_pdf_hash(file_path)
|
|
81
|
+
|
|
82
|
+
# check if pdf file already exists
|
|
83
|
+
if cls.objects.filter(pdf_hash=pdf_hash).exists():
|
|
84
|
+
logger.warning(f"RawPdfFile with hash {pdf_hash} already exists")
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
assert pdf_type_name is not None, "pdf_type_name is required"
|
|
88
|
+
assert center_name is not None, "center_name is required"
|
|
89
|
+
|
|
90
|
+
pdf_type = PdfType.objects.get(name=pdf_type_name)
|
|
91
|
+
center = Center.objects.get(name=center_name)
|
|
92
|
+
|
|
93
|
+
new_file_path = destination_dir / new_file_name
|
|
94
|
+
|
|
95
|
+
logger.info(f"Copying file to {new_file_path}")
|
|
96
|
+
success = shutil.copy(file_path, new_file_path)
|
|
97
|
+
|
|
98
|
+
# validate copy operation by comparing hashs
|
|
99
|
+
assert get_pdf_hash(new_file_path) == pdf_hash, "Copy operation failed"
|
|
100
|
+
|
|
101
|
+
raw_pdf = cls(
|
|
102
|
+
file=new_file_path.resolve().as_posix(),
|
|
103
|
+
pdf_hash=pdf_hash,
|
|
104
|
+
pdf_type=pdf_type,
|
|
105
|
+
center=center,
|
|
106
|
+
)
|
|
107
|
+
logger.info(f"RawPdfFile object created: {raw_pdf}")
|
|
108
|
+
|
|
109
|
+
# remove source file
|
|
110
|
+
file_path.unlink()
|
|
111
|
+
logger.info(f"Source file removed: {file_path}")
|
|
112
|
+
|
|
113
|
+
if save:
|
|
114
|
+
raw_pdf.save()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
return raw_pdf
|
|
118
|
+
|
|
119
|
+
def process_file(self, verbose = False):
|
|
120
|
+
|
|
121
|
+
pdf_path = self.file.path
|
|
122
|
+
rr_config = self.get_report_reader_config()
|
|
123
|
+
|
|
124
|
+
rr = ReportReader(**rr_config) #FIXME In future we need to pass a configuration file
|
|
125
|
+
# This configuration file should be associated with pdf type
|
|
126
|
+
|
|
127
|
+
text, anonymized_text, report_meta = rr.process_report(pdf_path, verbose=verbose)
|
|
128
|
+
if not self.sensitive_meta:
|
|
129
|
+
sensitive_meta = SensitiveMeta.create_from_dict(report_meta)
|
|
130
|
+
sensitive_meta.save()
|
|
131
|
+
self.sensitive_meta = sensitive_meta
|
|
132
|
+
|
|
133
|
+
else:
|
|
134
|
+
# update existing sensitive meta
|
|
135
|
+
sensitive_meta = self.sensitive_meta
|
|
136
|
+
sensitive_meta.update_from_dict(report_meta)
|
|
137
|
+
|
|
138
|
+
return text, anonymized_text, report_meta
|
|
139
|
+
|
|
140
|
+
def update(self, save=True, verbose = True):
|
|
141
|
+
try:
|
|
142
|
+
self.text, self.anonymized_text, self.raw_meta = self.process_file(verbose = verbose)
|
|
143
|
+
self.state_report_processed = True
|
|
144
|
+
self.state_report_processing_required = False
|
|
145
|
+
|
|
146
|
+
if save:
|
|
147
|
+
|
|
148
|
+
self.save()
|
|
149
|
+
|
|
150
|
+
return True
|
|
151
|
+
|
|
152
|
+
except:
|
|
153
|
+
logger.error(f"Error processing file: {self.file.path}")
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
def save(self, *args, **kwargs):
|
|
157
|
+
if not self.file.name.endswith('.pdf'):
|
|
158
|
+
raise ValidationError('Only PDF files are allowed')
|
|
159
|
+
|
|
160
|
+
if not self.pdf_hash:
|
|
161
|
+
self.pdf_hash = get_pdf_hash(self.file.path)
|
|
162
|
+
|
|
163
|
+
super().save(*args, **kwargs)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def get_report_reader_config(self):
|
|
167
|
+
if self.pdf_type.endoscope_info_line:
|
|
168
|
+
endoscope_info_line = self.pdf_type.endoscope_info_line.value
|
|
169
|
+
else:
|
|
170
|
+
endoscope_info_line = None
|
|
171
|
+
settings_dict = {
|
|
172
|
+
"locale": "de_DE",
|
|
173
|
+
"employee_first_names": [_.name for _ in self.center.first_names.all()],
|
|
174
|
+
"employee_last_names": [_.name for _ in self.center.last_names.all()],
|
|
175
|
+
"text_date_format":'%d.%m.%Y',
|
|
176
|
+
"flags": {
|
|
177
|
+
"patient_info_line": self.pdf_type.patient_info_line.value,
|
|
178
|
+
"endoscope_info_line": endoscope_info_line,
|
|
179
|
+
"examiner_info_line": self.pdf_type.examiner_info_line.value,
|
|
180
|
+
"cut_off_below": [_.value for _ in self.pdf_type.cut_off_below_lines.all()],
|
|
181
|
+
"cut_off_above": [_.value for _ in self.pdf_type.cut_off_above_lines.all()],
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return settings_dict
|
|
@@ -15,9 +15,11 @@ from ..metadata import VideoMeta, SensitiveMeta
|
|
|
15
15
|
class RawVideoFile(models.Model):
|
|
16
16
|
uuid = models.UUIDField()
|
|
17
17
|
file = models.FileField(upload_to="raw_data/")
|
|
18
|
+
|
|
18
19
|
sensitive_meta = models.OneToOneField(
|
|
19
20
|
"SensitiveMeta", on_delete=models.CASCADE, blank=True, null=True
|
|
20
|
-
)
|
|
21
|
+
)
|
|
22
|
+
|
|
21
23
|
center = models.ForeignKey("Center", on_delete=models.CASCADE)
|
|
22
24
|
processor = models.ForeignKey(
|
|
23
25
|
"EndoscopyProcessor", on_delete=models.CASCADE, blank=True, null=True
|
|
@@ -69,7 +71,7 @@ class RawVideoFile(models.Model):
|
|
|
69
71
|
def create_from_file(
|
|
70
72
|
cls,
|
|
71
73
|
file_path: Path,
|
|
72
|
-
|
|
74
|
+
video_dir: Path,
|
|
73
75
|
center_name: str,
|
|
74
76
|
processor_name: str,
|
|
75
77
|
frame_dir_parent: Path,
|
|
@@ -87,8 +89,8 @@ class RawVideoFile(models.Model):
|
|
|
87
89
|
if not framedir.exists():
|
|
88
90
|
framedir.mkdir(parents=True, exist_ok=True)
|
|
89
91
|
|
|
90
|
-
if not
|
|
91
|
-
|
|
92
|
+
if not video_dir.exists():
|
|
93
|
+
video_dir.mkdir(parents=True, exist_ok=True)
|
|
92
94
|
|
|
93
95
|
video_hash = get_video_hash(file_path)
|
|
94
96
|
|
|
@@ -98,7 +100,7 @@ class RawVideoFile(models.Model):
|
|
|
98
100
|
processor = EndoscopyProcessor.objects.get(name=processor_name)
|
|
99
101
|
assert processor is not None, "Processor must exist"
|
|
100
102
|
|
|
101
|
-
new_filepath =
|
|
103
|
+
new_filepath = video_dir / new_file_name
|
|
102
104
|
|
|
103
105
|
print(f"Moving {file_path} to {new_filepath}")
|
|
104
106
|
shutil.move(file_path.resolve().as_posix(), new_filepath.resolve().as_posix())
|
|
@@ -1,133 +1,3 @@
|
|
|
1
|
-
from django.db import models
|
|
2
|
-
import ffmpeg
|
|
3
|
-
from pathlib import Path
|
|
4
1
|
from .sensitive_meta import SensitiveMeta
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
from django.conf import settings
|
|
8
|
-
|
|
9
|
-
# check if endoreg_center_id is set
|
|
10
|
-
if not hasattr(settings, 'ENDOREG_CENTER_ID'):
|
|
11
|
-
ENDOREG_CENTER_ID = 9999
|
|
12
|
-
else:
|
|
13
|
-
ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
|
|
14
|
-
|
|
15
|
-
# VideoMeta
|
|
16
|
-
class VideoMeta(models.Model):
|
|
17
|
-
processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
|
|
18
|
-
endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
|
|
19
|
-
center = models.ForeignKey('Center', on_delete=models.CASCADE)
|
|
20
|
-
import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
|
|
21
|
-
ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
|
|
22
|
-
|
|
23
|
-
def __str__(self):
|
|
24
|
-
|
|
25
|
-
processor_name = self.processor.name if self.processor is not None else "None"
|
|
26
|
-
endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
|
|
27
|
-
center_name = self.center.name if self.center is not None else "None"
|
|
28
|
-
|
|
29
|
-
result_html = ""
|
|
30
|
-
|
|
31
|
-
result_html += f"Processor: {processor_name}<br>"
|
|
32
|
-
result_html += f"Endoscope: {endoscope_name}<br>"
|
|
33
|
-
result_html += f"Center: {center_name}<br>"
|
|
34
|
-
|
|
35
|
-
return result_html
|
|
36
|
-
|
|
37
|
-
# import meta should be created when video meta is created
|
|
38
|
-
def save(self, *args, **kwargs):
|
|
39
|
-
if self.import_meta is None:
|
|
40
|
-
self.import_meta = VideoImportMeta.objects.create()
|
|
41
|
-
super(VideoMeta, self).save(*args, **kwargs)
|
|
42
|
-
|
|
43
|
-
def initialize_ffmpeg_meta(self, file_path):
|
|
44
|
-
"""Initializes FFMpeg metadata for the video file if not already done."""
|
|
45
|
-
self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
|
|
46
|
-
self.save()
|
|
47
|
-
|
|
48
|
-
def update_meta(self, file_path):
|
|
49
|
-
"""Updates the video metadata from the file."""
|
|
50
|
-
self.initialize_ffmpeg_meta(file_path)
|
|
51
|
-
self.save()
|
|
52
|
-
|
|
53
|
-
def get_endo_roi(self):
|
|
54
|
-
endo_roi = self.processor.get_roi_endoscope_image()
|
|
55
|
-
return endo_roi
|
|
56
|
-
|
|
57
|
-
def get_fps(self):
|
|
58
|
-
if not self.ffmpeg_meta:
|
|
59
|
-
return None
|
|
60
|
-
|
|
61
|
-
return self.ffmpeg_meta.frame_rate
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
class FFMpegMeta(models.Model):
|
|
65
|
-
# Existing fields
|
|
66
|
-
duration = models.FloatField(blank=True, null=True)
|
|
67
|
-
width = models.IntegerField(blank=True, null=True)
|
|
68
|
-
height = models.IntegerField(blank=True, null=True)
|
|
69
|
-
frame_rate = models.FloatField(blank=True, null=True)
|
|
70
|
-
|
|
71
|
-
# New fields for comprehensive information
|
|
72
|
-
video_codec = models.CharField(max_length=50, blank=True, null=True)
|
|
73
|
-
audio_codec = models.CharField(max_length=50, blank=True, null=True)
|
|
74
|
-
audio_channels = models.IntegerField(blank=True, null=True)
|
|
75
|
-
audio_sample_rate = models.IntegerField(blank=True, null=True)
|
|
76
|
-
|
|
77
|
-
# Existing __str__ method can be updated to include new fields
|
|
78
|
-
|
|
79
|
-
@classmethod
|
|
80
|
-
def create_from_file(cls, file_path: Path):
|
|
81
|
-
"""Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
|
|
82
|
-
try:
|
|
83
|
-
probe = ffmpeg.probe(str(file_path))
|
|
84
|
-
except ffmpeg.Error as e:
|
|
85
|
-
print(e.stderr)
|
|
86
|
-
return None
|
|
87
|
-
|
|
88
|
-
video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
|
|
89
|
-
audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
|
|
90
|
-
|
|
91
|
-
# Check for the existence of a video stream
|
|
92
|
-
if video_stream is None:
|
|
93
|
-
print(f"No video stream found in {file_path}")
|
|
94
|
-
return None
|
|
95
|
-
|
|
96
|
-
# Extract and store video metadata
|
|
97
|
-
metadata = {
|
|
98
|
-
'duration': float(video_stream.get('duration', 0)),
|
|
99
|
-
'width': int(video_stream.get('width', 0)),
|
|
100
|
-
'height': int(video_stream.get('height', 0)),
|
|
101
|
-
'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
|
|
102
|
-
'video_codec': video_stream.get('codec_name', ''),
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
# If there are audio streams, extract and store audio metadata from the first stream
|
|
106
|
-
if audio_streams:
|
|
107
|
-
first_audio_stream = audio_streams[0]
|
|
108
|
-
metadata.update({
|
|
109
|
-
'audio_codec': first_audio_stream.get('codec_name', ''),
|
|
110
|
-
'audio_channels': int(first_audio_stream.get('channels', 0)),
|
|
111
|
-
'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
|
|
112
|
-
})
|
|
113
|
-
|
|
114
|
-
# Create and return the FFMpegMeta instance
|
|
115
|
-
return cls.objects.create(**metadata)
|
|
116
|
-
|
|
117
|
-
class VideoImportMeta(models.Model):
|
|
118
|
-
|
|
119
|
-
video_anonymized = models.BooleanField(default=False)
|
|
120
|
-
video_patient_data_detected = models.BooleanField(default=False)
|
|
121
|
-
outside_detected = models.BooleanField(default=False)
|
|
122
|
-
patient_data_removed = models.BooleanField(default=False)
|
|
123
|
-
outside_removed = models.BooleanField(default=False)
|
|
124
|
-
|
|
125
|
-
def __str__(self):
|
|
126
|
-
result_html = ""
|
|
127
|
-
|
|
128
|
-
result_html += f"Video anonymized: {self.video_anonymized}<br>"
|
|
129
|
-
result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
|
|
130
|
-
result_html += f"Outside detected: {self.outside_detected}<br>"
|
|
131
|
-
result_html += f"Patient data removed: {self.patient_data_removed}<br>"
|
|
132
|
-
result_html += f"Outside removed: {self.outside_removed}<br>"
|
|
133
|
-
return result_html
|
|
2
|
+
from .pdf_meta import PdfMeta, PdfType
|
|
3
|
+
from .video_meta import VideoMeta, FFMpegMeta, VideoImportMeta
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from django.db import models
|
|
2
|
+
|
|
3
|
+
# import endoreg_center_id from django settings
|
|
4
|
+
from django.conf import settings
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
# import File class
|
|
8
|
+
from django.core.files import File
|
|
9
|
+
|
|
10
|
+
# # check if endoreg_center_id is set
|
|
11
|
+
# if not hasattr(settings, 'ENDOREG_CENTER_ID'):
|
|
12
|
+
# ENDOREG_CENTER_ID = 9999
|
|
13
|
+
# else:
|
|
14
|
+
# ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
|
|
15
|
+
|
|
16
|
+
class PdfType(models.Model):
|
|
17
|
+
name = models.CharField(max_length=255)
|
|
18
|
+
|
|
19
|
+
patient_info_line = models.ForeignKey(
|
|
20
|
+
"ReportReaderFlag",
|
|
21
|
+
related_name="pdf_type_patient_info_line",
|
|
22
|
+
on_delete=models.CASCADE
|
|
23
|
+
)
|
|
24
|
+
endoscope_info_line = models.ForeignKey(
|
|
25
|
+
"ReportReaderFlag",
|
|
26
|
+
related_name="pdf_type_endoscopy_info_line",
|
|
27
|
+
on_delete=models.CASCADE,
|
|
28
|
+
)
|
|
29
|
+
examiner_info_line = models.ForeignKey(
|
|
30
|
+
"ReportReaderFlag",
|
|
31
|
+
related_name="pdf_type_examiner_info_line",
|
|
32
|
+
on_delete=models.CASCADE
|
|
33
|
+
)
|
|
34
|
+
cut_off_above_lines = models.ManyToManyField(
|
|
35
|
+
"ReportReaderFlag",
|
|
36
|
+
related_name="pdf_type_cut_off_above_lines",
|
|
37
|
+
)
|
|
38
|
+
cut_off_below_lines = models.ManyToManyField(
|
|
39
|
+
"ReportReaderFlag",
|
|
40
|
+
related_name="pdf_type_cut_off_below_lines",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def __str__(self):
|
|
45
|
+
summary = f"{self.name}"
|
|
46
|
+
# add lines to summary
|
|
47
|
+
summary += f"\nPatient Info Line: {self.patient_info_line.value}"
|
|
48
|
+
summary += f"\nEndoscope Info Line: {self.endoscope_info_line.value}"
|
|
49
|
+
summary += f"\nExaminer Info Line: {self.examiner_info_line.value}"
|
|
50
|
+
summary += f"\nCut Off Above Lines: {[_.value for _ in self.cut_off_above_lines.all()]}"
|
|
51
|
+
summary += f"\nCut Off Below Lines: {[_.value for _ in self.cut_off_below_lines.all()]}"
|
|
52
|
+
|
|
53
|
+
return summary
|
|
54
|
+
|
|
55
|
+
class PdfMeta(models.Model):
|
|
56
|
+
pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
|
|
57
|
+
date = models.DateField()
|
|
58
|
+
time = models.TimeField()
|
|
59
|
+
pdf_hash = models.CharField(max_length=255, unique=True)
|
|
60
|
+
|
|
61
|
+
def __str__(self):
|
|
62
|
+
return self.pdf_hash
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def create_from_file(cls, pdf_file):
|
|
66
|
+
pdf_file = File(pdf_file)
|
|
67
|
+
pdf_meta = cls(file=pdf_file)
|
|
68
|
+
pdf_meta.save()
|
|
69
|
+
return pdf_meta
|
|
70
|
+
|
|
@@ -10,4 +10,22 @@ class SensitiveMeta(models.Model):
|
|
|
10
10
|
|
|
11
11
|
@classmethod
|
|
12
12
|
def create_from_dict(cls, data: dict):
|
|
13
|
-
|
|
13
|
+
# data can contain more fields than the model has
|
|
14
|
+
field_names = [_.name for _ in cls._meta.fields]
|
|
15
|
+
selected_data = {k: v for k, v in data.items() if k in field_names}
|
|
16
|
+
|
|
17
|
+
return cls.objects.create(**selected_data)
|
|
18
|
+
|
|
19
|
+
def update_from_dict(self, data: dict):
|
|
20
|
+
# data can contain more fields than the model has
|
|
21
|
+
field_names = [_.name for _ in self._meta.fields]
|
|
22
|
+
selected_data = {k: v for k, v in data.items() if k in field_names}
|
|
23
|
+
|
|
24
|
+
for k, v in selected_data.items():
|
|
25
|
+
setattr(self, k, v)
|
|
26
|
+
|
|
27
|
+
self.save()
|
|
28
|
+
|
|
29
|
+
def __str__(self):
|
|
30
|
+
return f"SensitiveMeta: {self.examination_date} {self.patient_first_name} {self.patient_last_name} (*{self.patient_dob})"
|
|
31
|
+
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
from django.db import models
|
|
2
|
+
import ffmpeg
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
# import endoreg_center_id from django settings
|
|
6
|
+
from django.conf import settings
|
|
7
|
+
|
|
8
|
+
# check if endoreg_center_id is set
|
|
9
|
+
if not hasattr(settings, 'ENDOREG_CENTER_ID'):
|
|
10
|
+
ENDOREG_CENTER_ID = 9999
|
|
11
|
+
else:
|
|
12
|
+
ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
|
|
13
|
+
|
|
14
|
+
# VideoMeta
|
|
15
|
+
class VideoMeta(models.Model):
|
|
16
|
+
processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
|
|
17
|
+
endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
|
|
18
|
+
center = models.ForeignKey('Center', on_delete=models.CASCADE)
|
|
19
|
+
import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
|
|
20
|
+
ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
|
|
21
|
+
|
|
22
|
+
def __str__(self):
|
|
23
|
+
|
|
24
|
+
processor_name = self.processor.name if self.processor is not None else "None"
|
|
25
|
+
endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
|
|
26
|
+
center_name = self.center.name if self.center is not None else "None"
|
|
27
|
+
|
|
28
|
+
result_html = ""
|
|
29
|
+
|
|
30
|
+
result_html += f"Processor: {processor_name}<br>"
|
|
31
|
+
result_html += f"Endoscope: {endoscope_name}<br>"
|
|
32
|
+
result_html += f"Center: {center_name}<br>"
|
|
33
|
+
|
|
34
|
+
return result_html
|
|
35
|
+
|
|
36
|
+
# import meta should be created when video meta is created
|
|
37
|
+
def save(self, *args, **kwargs):
|
|
38
|
+
if self.import_meta is None:
|
|
39
|
+
self.import_meta = VideoImportMeta.objects.create()
|
|
40
|
+
super(VideoMeta, self).save(*args, **kwargs)
|
|
41
|
+
|
|
42
|
+
def initialize_ffmpeg_meta(self, file_path):
|
|
43
|
+
"""Initializes FFMpeg metadata for the video file if not already done."""
|
|
44
|
+
self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
|
|
45
|
+
self.save()
|
|
46
|
+
|
|
47
|
+
def update_meta(self, file_path):
|
|
48
|
+
"""Updates the video metadata from the file."""
|
|
49
|
+
self.initialize_ffmpeg_meta(file_path)
|
|
50
|
+
self.save()
|
|
51
|
+
|
|
52
|
+
def get_endo_roi(self):
|
|
53
|
+
endo_roi = self.processor.get_roi_endoscope_image()
|
|
54
|
+
return endo_roi
|
|
55
|
+
|
|
56
|
+
def get_fps(self):
|
|
57
|
+
if not self.ffmpeg_meta:
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
return self.ffmpeg_meta.frame_rate
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class FFMpegMeta(models.Model):
|
|
64
|
+
# Existing fields
|
|
65
|
+
duration = models.FloatField(blank=True, null=True)
|
|
66
|
+
width = models.IntegerField(blank=True, null=True)
|
|
67
|
+
height = models.IntegerField(blank=True, null=True)
|
|
68
|
+
frame_rate = models.FloatField(blank=True, null=True)
|
|
69
|
+
|
|
70
|
+
# New fields for comprehensive information
|
|
71
|
+
video_codec = models.CharField(max_length=50, blank=True, null=True)
|
|
72
|
+
audio_codec = models.CharField(max_length=50, blank=True, null=True)
|
|
73
|
+
audio_channels = models.IntegerField(blank=True, null=True)
|
|
74
|
+
audio_sample_rate = models.IntegerField(blank=True, null=True)
|
|
75
|
+
|
|
76
|
+
# Existing __str__ method can be updated to include new fields
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def create_from_file(cls, file_path: Path):
|
|
80
|
+
"""Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
|
|
81
|
+
try:
|
|
82
|
+
probe = ffmpeg.probe(str(file_path))
|
|
83
|
+
except ffmpeg.Error as e:
|
|
84
|
+
print(e.stderr)
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
|
|
88
|
+
audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
|
|
89
|
+
|
|
90
|
+
# Check for the existence of a video stream
|
|
91
|
+
if video_stream is None:
|
|
92
|
+
print(f"No video stream found in {file_path}")
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
# Extract and store video metadata
|
|
96
|
+
metadata = {
|
|
97
|
+
'duration': float(video_stream.get('duration', 0)),
|
|
98
|
+
'width': int(video_stream.get('width', 0)),
|
|
99
|
+
'height': int(video_stream.get('height', 0)),
|
|
100
|
+
'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
|
|
101
|
+
'video_codec': video_stream.get('codec_name', ''),
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
# If there are audio streams, extract and store audio metadata from the first stream
|
|
105
|
+
if audio_streams:
|
|
106
|
+
first_audio_stream = audio_streams[0]
|
|
107
|
+
metadata.update({
|
|
108
|
+
'audio_codec': first_audio_stream.get('codec_name', ''),
|
|
109
|
+
'audio_channels': int(first_audio_stream.get('channels', 0)),
|
|
110
|
+
'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
# Create and return the FFMpegMeta instance
|
|
114
|
+
return cls.objects.create(**metadata)
|
|
115
|
+
|
|
116
|
+
class VideoImportMeta(models.Model):
|
|
117
|
+
|
|
118
|
+
video_anonymized = models.BooleanField(default=False)
|
|
119
|
+
video_patient_data_detected = models.BooleanField(default=False)
|
|
120
|
+
outside_detected = models.BooleanField(default=False)
|
|
121
|
+
patient_data_removed = models.BooleanField(default=False)
|
|
122
|
+
outside_removed = models.BooleanField(default=False)
|
|
123
|
+
|
|
124
|
+
def __str__(self):
|
|
125
|
+
result_html = ""
|
|
126
|
+
|
|
127
|
+
result_html += f"Video anonymized: {self.video_anonymized}<br>"
|
|
128
|
+
result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
|
|
129
|
+
result_html += f"Outside detected: {self.outside_detected}<br>"
|
|
130
|
+
result_html += f"Patient data removed: {self.patient_data_removed}<br>"
|
|
131
|
+
result_html += f"Outside removed: {self.outside_removed}<br>"
|
|
132
|
+
return result_html
|