endoreg-db 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

endoreg_db/models/data_file/metadata/video_meta.py

@@ -0,0 +1,132 @@
+from django.db import models
+import ffmpeg
+from pathlib import Path
+
+# import endoreg_center_id from django settings
+from django.conf import settings
+
+# check if endoreg_center_id is set
+if not hasattr(settings, 'ENDOREG_CENTER_ID'):
+    ENDOREG_CENTER_ID = 9999
+else:
+    ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
+
+# VideoMeta
+class VideoMeta(models.Model):
+    processor = models.ForeignKey('EndoscopyProcessor', on_delete=models.CASCADE, blank=True, null=True)
+    endoscope = models.ForeignKey('Endoscope', on_delete=models.CASCADE, blank=True, null=True)
+    center = models.ForeignKey('Center', on_delete=models.CASCADE)
+    import_meta = models.OneToOneField('VideoImportMeta', on_delete=models.CASCADE, blank=True, null=True)
+    ffmpeg_meta = models.OneToOneField('FFMpegMeta', on_delete=models.CASCADE, blank=True, null=True)
+
+    def __str__(self):
+
+        processor_name = self.processor.name if self.processor is not None else "None"
+        endoscope_name = self.endoscope.name if self.endoscope is not None else "None"
+        center_name = self.center.name if self.center is not None else "None"
+
+        result_html = ""
+
+        result_html += f"Processor: {processor_name}<br>"
+        result_html += f"Endoscope: {endoscope_name}<br>"
+        result_html += f"Center: {center_name}<br>"
+
+        return result_html
+    
+    # import meta should be created when video meta is created
+    def save(self, *args, **kwargs):
+        if self.import_meta is None:
+            self.import_meta = VideoImportMeta.objects.create()
+        super(VideoMeta, self).save(*args, **kwargs)
+
+    def initialize_ffmpeg_meta(self, file_path):
+        """Initializes FFMpeg metadata for the video file if not already done."""
+        self.ffmpeg_meta = FFMpegMeta.create_from_file(Path(file_path))
+        self.save()
+
+    def update_meta(self, file_path):
+        """Updates the video metadata from the file."""
+        self.initialize_ffmpeg_meta(file_path)
+        self.save()
+
+    def get_endo_roi(self):
+        endo_roi = self.processor.get_roi_endoscope_image()
+        return endo_roi
+
+    def get_fps(self):
+        if not self.ffmpeg_meta:
+            return None
+    
+        return self.ffmpeg_meta.frame_rate
+ 
+
+class FFMpegMeta(models.Model):
+    # Existing fields
+    duration = models.FloatField(blank=True, null=True)
+    width = models.IntegerField(blank=True, null=True)
+    height = models.IntegerField(blank=True, null=True)
+    frame_rate = models.FloatField(blank=True, null=True)
+    
+    # New fields for comprehensive information
+    video_codec = models.CharField(max_length=50, blank=True, null=True)
+    audio_codec = models.CharField(max_length=50, blank=True, null=True)
+    audio_channels = models.IntegerField(blank=True, null=True)
+    audio_sample_rate = models.IntegerField(blank=True, null=True)
+
+    # Existing __str__ method can be updated to include new fields
+
+    @classmethod
+    def create_from_file(cls, file_path: Path):
+        """Creates an FFMpegMeta instance from a video file using ffmpeg probe."""
+        try:
+            probe = ffmpeg.probe(str(file_path))
+        except ffmpeg.Error as e:
+            print(e.stderr)
+            return None
+
+        video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
+        audio_streams = [stream for stream in probe['streams'] if stream['codec_type'] == 'audio']
+
+        # Check for the existence of a video stream
+        if video_stream is None:
+            print(f"No video stream found in {file_path}")
+            return None
+
+        # Extract and store video metadata
+        metadata = {
+            'duration': float(video_stream.get('duration', 0)),
+            'width': int(video_stream.get('width', 0)),
+            'height': int(video_stream.get('height', 0)),
+            'frame_rate': float(next(iter(video_stream.get('avg_frame_rate', '').split('/')), 0)),
+            'video_codec': video_stream.get('codec_name', ''),
+        }
+
+        # If there are audio streams, extract and store audio metadata from the first stream
+        if audio_streams:
+            first_audio_stream = audio_streams[0]
+            metadata.update({
+                'audio_codec': first_audio_stream.get('codec_name', ''),
+                'audio_channels': int(first_audio_stream.get('channels', 0)),
+                'audio_sample_rate': int(first_audio_stream.get('sample_rate', 0)),
+            })
+
+        # Create and return the FFMpegMeta instance
+        return cls.objects.create(**metadata)
+
+class VideoImportMeta(models.Model):
+
+    video_anonymized = models.BooleanField(default=False)
+    video_patient_data_detected = models.BooleanField(default=False)
+    outside_detected = models.BooleanField(default=False)
+    patient_data_removed = models.BooleanField(default=False)
+    outside_removed = models.BooleanField(default=False)
+
+    def __str__(self):
+        result_html = ""
+
+        result_html += f"Video anonymized: {self.video_anonymized}<br>"
+        result_html += f"Video patient data detected: {self.video_patient_data_detected}<br>"
+        result_html += f"Outside detected: {self.outside_detected}<br>"
+        result_html += f"Patient data removed: {self.patient_data_removed}<br>"
+        result_html += f"Outside removed: {self.outside_removed}<br>"
+        return result_html

endoreg_db/models/data_file/report_file.py

@@ -16,6 +16,7 @@ class ReportFile(models.Model):
     time = models.TimeField(blank=True, null=True)
 
     def get_pdf_hash(self):
+        #FIXME should use endoreg_db.utils.get_pdf_hash in future
         pdf = self.pdf
         pdf_hash = None
 

endoreg_db/models/persons/__init__.py

@@ -1,4 +1,6 @@
 from .person import Person
 from .patient import Patient, PatientForm
 from .examiner import Examiner, ExaminerSerializer
-from .portal_user_information import PortalUserInfo, Profession
+from .portal_user_information import PortalUserInfo, Profession
+from .first_name import FirstName
+from .last_name import LastName

endoreg_db/models/persons/first_name.py

@@ -0,0 +1,18 @@
+# class to represent unique first-names
+# name attribute is natural key
+
+from django.db import models
+
+class FirstNameManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+    
+class FirstName(models.Model):
+    objects = FirstNameManager()
+    name = models.CharField(max_length=255, unique=True)
+    
+    def natural_key(self):
+        return (self.name,)
+    
+    def __str__(self):
+        return self.name

endoreg_db/models/persons/last_name.py

@@ -0,0 +1,20 @@
+# class to represent unique last_names
+# name attribute is natural key
+from django.db import models
+
+class LastNameManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+
+class LastName(models.Model):
+    objects = LastNameManager()
+    name = models.CharField(max_length=255, unique=True)
+
+    def natural_key(self):
+        return (self.name,)
+
+    def __str__(self):
+        return self.name
+    
+# Path: endoreg_db/models/persons/first_name.py
+        

endoreg_db/models/report_reader/__init__.py

@@ -0,0 +1,2 @@
+from .report_reader_config import ReportReaderConfig
+from .report_reader_flag import ReportReaderFlag

endoreg_db/models/report_reader/report_reader_config.py

@@ -0,0 +1,53 @@
+# ReportReaderConfig Class
+# Description: This class is used to store the configuration of the ReportReader
+
+# PATIENT_INFO_LINE_FLAG = "Patient: "
+# ENDOSCOPE_INFO_LINE_FLAG = "Gerät: "
+# EXAMINER_INFO_LINE_FLAG = "1. Unters.:"
+# CUT_OFF_BELOW_LINE_FLAG = "________________"
+
+
+# CUT_OFF_ABOVE_LINE_FLAGS = [
+#     ENDOSCOPE_INFO_LINE_FLAG,
+#     EXAMINER_INFO_LINE_FLAG,
+# ]
+
+# CUT_OFF_BELOW_LINE_FLAGS = [
+#         CUT_OFF_BELOW_LINE_FLAG
+#     ]
+
+from django.db import models
+from .report_reader_flag import ReportReaderFlag
+from ..center import Center
+from ..data_file import PdfType
+
+class ReportReaderConfig(models.Model):
+
+    locale = models.CharField(default="de_DE", max_length=10)
+    first_names = models.ManyToManyField('FirstName', related_name='report_reader_configs')
+    last_names = models.ManyToManyField('LastName', related_name='report_reader_configs')
+    text_date_format = models.CharField(default = "%d.%m.%Y", max_length=10)
+    patient_info_line_flag = models.ForeignKey(ReportReaderFlag, related_name='report_reader_configs_patient_info_line', on_delete=models.CASCADE)
+    endoscope_info_line_flag = models.ForeignKey(ReportReaderFlag, related_name='report_reader_configs_endoscope_info_line', on_delete=models.CASCADE)
+    examiner_info_line_flag = models.ForeignKey(ReportReaderFlag, related_name='report_reader_configs_examiner_info_line', on_delete=models.CASCADE)
+    cut_off_below = models.ManyToManyField(ReportReaderFlag, related_name='report_reader_configs_cut_off_below')
+    cut_off_above = models.ManyToManyField(ReportReaderFlag, related_name='report_reader_configs_cut_off_above')
+    
+    def __str__(self):
+        return self.locale
+    
+    def update_names_by_center(self, center:Center, save = True):
+        self.first_names.set(center.first_names.all())
+        self.last_names.set(center.last_names.all())
+        if save:
+            self.save()
+
+    def update_flags_by_pdf_type(self, pdf_type:PdfType, save = True):
+        self.patient_info_line_flag = pdf_type.patient_info_line_flag
+        self.endoscope_info_line_flag = pdf_type.endoscope_info_line_flag
+        self.examiner_info_line_flag = pdf_type.examiner_info_line_flag
+        self.cut_off_below.set(pdf_type.cut_off_below.all())
+        self.cut_off_above.set(pdf_type.cut_off_above.all())
+        if save:
+            self.save()
+

endoreg_db/models/report_reader/report_reader_flag.py

@@ -0,0 +1,20 @@
+# Django model for the report reader flag
+# have name and value
+# name is natural key
+
+from django.db import models
+
+class ReportReaderFlagManager(models.Manager):
+    def get_by_natural_key(self, name):
+        return self.get(name=name)
+    
+class ReportReaderFlag(models.Model):
+    objects = ReportReaderFlagManager()
+    name = models.CharField(max_length=255, unique=True)
+    value = models.CharField(max_length=255)
+    
+    def natural_key(self):
+        return (self.name,)
+    
+    def __str__(self):
+        return self.name

endoreg_db/utils/dataloader.py

@@ -1,69 +1,185 @@
 import os
 import yaml
+from django.core.exceptions import ObjectDoesNotExist
 
-def load_model_data_from_yaml(
-        command,
-        model_name,
-        metadata,
-        verbose,
-    ):
-
+def load_model_data_from_yaml(command, model_name, metadata, verbose):
+    """
+    Load model data from YAML files.
+    
+    Args:
+        command: Command object for stdout writing.
+        model_name: Name of the model being loaded.
+        metadata: Metadata including directory and foreign key information.
+        verbose: Boolean indicating whether to print verbose output.
+    """
     if verbose:
-        command.stdout.write(f"Start Loading {model_name}")
+        command.stdout.write(f"Start loading {model_name}")
     model = metadata["model"]
-    dir = metadata["dir"]
+    dir_path = metadata["dir"]
     foreign_keys = metadata["foreign_keys"]
     foreign_key_models = metadata["foreign_key_models"]
 
-    for file in [f for f in os.listdir(dir) if f.endswith('.yaml')]:
-        with open(os.path.join(dir, file), 'r') as f:
-            yaml_data = yaml.safe_load(f)
+    for file in [f for f in os.listdir(dir_path) if f.endswith('.yaml')]:
+        with open(os.path.join(dir_path, file), 'r') as file:
+            yaml_data = yaml.safe_load(file)
         
-        load_data_with_foreign_keys(
-            command,
-            model,
-            yaml_data,
-            foreign_keys,
-            foreign_key_models,
-            verbose
-        )
+        load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose)
 
 def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose):
-            # Since pathology types is a ManyToMany field, we need to hack arount
-            for entry in yaml_data:
-                fields = entry.get('fields', {})
-                name = fields.pop('name', None)
-                many_to_many_tuples = []
-                foreign_key_tuples = zip(foreign_keys, foreign_key_models)
-                for foreign_key, foreign_key_model in foreign_key_tuples:
-                    target_natural_key = fields.pop(foreign_key, None)
+    """
+    Load data handling foreign keys and many-to-many relationships.
+    
+    Args:
+        command: Command object for stdout writing.
+        model: The Django model for the data.
+        yaml_data: Data loaded from YAML.
+        foreign_keys: List of foreign keys.
+        foreign_key_models: Corresponding models for the foreign keys.
+        verbose: Boolean indicating whether to print verbose output.
+    """
+    for entry in yaml_data:
+        fields = entry.get('fields', {})
+        name = fields.pop('name', None)
+        m2m_relationships = {}  # Store many-to-many relationships
+
+        # Handle foreign keys and many-to-many relationships
+        for fk_field, fk_model in zip(foreign_keys, foreign_key_models):
+            target_keys = fields.pop(fk_field, None)
+            
+            # Ensure the foreign key exists
+            if target_keys is None:
+                if verbose:
+                    command.stdout.write(command.style.WARNING(f"Foreign key {fk_field} not found in fields"))
+                continue  # Skip if no foreign key provided
+
+            # Process many-to-many fields or foreign keys
+            if isinstance(target_keys, list):  # Assume many-to-many relationship
+                related_objects = []
+                for key in target_keys:
+                    obj, created = fk_model.objects.get_or_create(name=key)
+                    if created and verbose:
+                        command.stdout.write(command.style.SUCCESS(f"Created {fk_model.__name__} {key}"))
+                    related_objects.append(obj)
+                m2m_relationships[fk_field] = related_objects
+            else:  # Single foreign key relationship
+                try:
+                    obj = fk_model.objects.get_by_natural_key(target_keys)
+                except ObjectDoesNotExist:
+                    if verbose:
+                        command.stdout.write(command.style.WARNING(f"{fk_model.__name__} with key {target_keys} not found"))
+                    continue
+                fields[fk_field] = obj
+
+        # Create or update the main object
+        obj, created = model.objects.update_or_create(defaults=fields, name=name)
+        if created and verbose:
+            command.stdout.write(command.style.SUCCESS(f'Created {model.__name__} {name}'))
+        elif verbose:
+            command.stdout.write(command.style.WARNING(f'Skipped {model.__name__} {name}, already exists'))
+
+        # Set many-to-many relationships
+        for field_name, related_objs in m2m_relationships.items():
+            getattr(obj, field_name).set(related_objs)
+
+
+# def load_model_data_from_yaml(
+#         command,
+#         model_name,
+#         metadata,
+#         verbose,
+#     ):
+
+#     if verbose:
+#         command.stdout.write(f"Start Loading {model_name}")
+#     model = metadata["model"]
+#     dir = metadata["dir"]
+#     foreign_keys = metadata["foreign_keys"]
+#     foreign_key_models = metadata["foreign_key_models"]
+
+#     for file in [f for f in os.listdir(dir) if f.endswith('.yaml')]:
+#         with open(os.path.join(dir, file), 'r') as f:
+#             yaml_data = yaml.safe_load(f)
+        
+#         load_data_with_foreign_keys(
+#             command,
+#             model,
+#             yaml_data,
+#             foreign_keys,
+#             foreign_key_models,
+#             verbose
+#         )
+
+# def load_data_with_foreign_keys(command, model, yaml_data, foreign_keys, foreign_key_models, verbose):
+#             # Since pathology types is a ManyToMany field, we need to hack arount
+#             for entry in yaml_data:
+#                 fields = entry.get('fields', {})
+#                 name = fields.pop('name', None)
+#                 many_to_many_tuples = []
+#                 foreign_key_tuples = zip(foreign_keys, foreign_key_models)
+#                 for foreign_key, foreign_key_model in foreign_key_tuples:
+#                     # target_natural_key = fields.pop(foreign_key, None)
+#                     # get the target natural key, if it exists, should not alter fields
+#                     target_natural_key = fields.get(foreign_key, None)
+#                     assert target_natural_key, f"Foreign Key {foreign_key} not found in fields {fields}"
    
+#                     if (foreign_key == "first_names") or (foreign_key == "last_names"):
+#                         if isinstance(target_natural_key, list):
+#                             # For first_names and last_names, the field is a Many to Many field
+#                             # if names dont exist yet, we create them
+#                             fk_objects = []
+#                             for __name in target_natural_key:
+#                                 obj, created = foreign_key_model.objects.get_or_create(name=__name)
+#                                 if created:
+#                                     command.stdout.write(command.style.SUCCESS(f'Created {foreign_key_model.__name__} {__name}'))
+#                                 # fk_objects.append(obj)
+
+#                             fk_tuple = (foreign_key, fk_objects)
+#                             many_to_many_tuples.append(fk_tuple)
+#                         continue
+
+#                     if isinstance(target_natural_key, list):
+#                         # the field is a Many to X field.
+#                         fk_objects = [foreign_key_model.objects.get_by_natural_key(_) for _ in target_natural_key]
+#                         fk_tuple = (foreign_key, fk_objects)
+#                         many_to_many_tuples.append(fk_tuple)
+#                         continue
+#                     # Use the natural key to look up the related object
+#                     try:
+#                         obj = foreign_key_model.objects.get_by_natural_key(target_natural_key)
+#                     except:
+#                         # commandline log that the object was not found
+#                         command.stdout.write(command.style.WARNING(f'Object {foreign_key_model.__name__} {target_natural_key} not found'))
+#                         # commandline log entry
+#                         command.stdout.write(command.style.WARNING(_log))
+#                         # try to create by name if name is available
+#                         # create defaults dict from fields using the models fields
+#                         _field_names = [_.name for _ in foreign_key_model._meta.fields]
+#                         _defaults = {k: v for k, v in fields.items() if (k in _field_names) and v}
+                        
+#                         if target_natural_key:
+#                         # commandlie log
+#                             command.stdout.write(command.style.SUCCESS(f'Creating {foreign_key_model.__name__} {name} with defaults {_defaults}'))
+#                             obj, created = model.objects.get_or_create(
+#                                 name = target_natural_key,
+#                                 defaults=_defaults
+#                             )
+#                     # Assign the related object to the field
+#                     fields[foreign_key] = obj
+
+#                 if name:
+#                     try:
+#                         obj, created = model.objects.get_or_create(name=name, defaults=fields)
+#                     except:
+#                         # commandlinelog to print name, fields, target foreign key
+#                         command.stdout.write(command.style.WARNING(f'Object {model.__name__} {name} not found'))
+#                 else:
+#                     obj, created = model.objects.get_or_create(**fields)
+#                 if many_to_many_tuples:
+
+#                     for fk, fk_objects in many_to_many_tuples:
+#                         getattr(obj, fk).set(fk_objects)
 
-                    if isinstance(target_natural_key, list):
-                        # the field is a Many to X field.
-                        fk_objects = [foreign_key_model.objects.get_by_natural_key(_) for _ in target_natural_key]
-                        fk_tuple = (foreign_key, fk_objects)
-                        many_to_many_tuples.append(fk_tuple)
-                        continue
-                    # Use the natural key to look up the related object
-                    try:
-                        obj = foreign_key_model.objects.get_by_natural_key(target_natural_key)
-                    except model.DoesNotExist:
-                        command.stderr.write(command.style.ERROR(f'{model.__name__} with natural key {target_natural_key} does not exist. Skipping {name}.'))
-                        raise Exception(f'{model.__name__} with natural key {target_natural_key} does not exist. Skipping {name}.')
-
-                    # Assign the related object to the field
-                    fields[foreign_key] = obj
-
-                if name:
-                    obj, created = model.objects.get_or_create(name=name, defaults=fields)
-                else:
-                    obj, created = model.objects.get_or_create(**fields)
-                if many_to_many_tuples:
-                    for fk, fk_objects in many_to_many_tuples:
-                        getattr(obj, fk).set(fk_objects)
-
-                if created and verbose:
-                    command.stdout.write(command.style.SUCCESS(f'Created {model.__name__} {name}'))
-                elif verbose:
-                    command.stdout.write(command.style.WARNING(f'Skipped {model.__name__} {name}, already exists'))
+#                 if created and verbose:
+#                     command.stdout.write(command.style.SUCCESS(f'Created {model.__name__} {name}'))
+#                 elif verbose:
+#                     command.stdout.write(command.style.WARNING(f'Skipped {model.__name__} {name}, already exists'))

endoreg_db/utils/hashs.py

@@ -1,4 +1,5 @@
 import hashlib
+from pathlib import Path
 
 def get_video_hash(video_path):
     """
@@ -14,3 +15,20 @@ def get_video_hash(video_path):
 
     return video_hash
 
+def get_pdf_hash(pdf_path:Path):
+    """
+    Get the hash of a pdf file.
+    """
+    pdf_hash = None
+
+    # Open the file in binary mode and read its contents
+    with open(pdf_path, 'rb') as f:
+        pdf_contents = f.read()
+        # Create a hash object using SHA-256 algorithm
+
+    hash_object = hashlib.sha256(pdf_contents, usedforsecurity=False)
+    # Get the hexadecimal representation of the hash
+    pdf_hash = hash_object.hexdigest()
+    assert len(pdf_hash) <= 255, "Hash length exceeds 255 characters"
+
+    return pdf_hash

{endoreg_db-0.3.0.dist-info → endoreg_db-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: endoreg-db
-Version: 0.3.0
+Version: 0.3.2
 Summary: EndoReg Db Django App
 License: GNU3
 Author: Thomas J. Lux
@@ -9,6 +9,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: agl-report-reader (>=0.2.0,<0.3.0)
 Requires-Dist: django (>=4.2,<4.3)
 Requires-Dist: django-bootstrap5 (>=23.4,<24.0)
 Requires-Dist: djangorestframework (>=3.14.0,<4.0.0)