endoreg-db 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

endoreg_db/management/commands/import_legacy_images.py

@@ -0,0 +1,94 @@
+# command expects a directory as an argument
+# directory should contain a img_dicts.jsonl file, a settings.json and an "images" directory
+# images should be in jpeg format
+# reads the img_dicts.jsonl file and creates a new database entry for each image
+# img_dicts contains a line for each image in the following format:
+# {"id": 30604292, "path": "images/30604292.jpg", "labels": ["appendix", "polyp"]}
+
+# import_settings.json example:
+# {
+#     "annotator": "legacy",
+#     "labelset_name": "multilabel_classification",
+#     "labelset_version": 0
+# }
+
+import os
+import json
+from tqdm import tqdm
+from django.core.management.base import BaseCommand
+from endoreg_db.models import LegacyImage, LabelSet, Label, ImageClassificationAnnotation
+from django.core.files import File
+from uuid import uuid4
+
+class Command(BaseCommand):
+    """
+    Imports images from a directory into the database
+
+    Usage:
+        python manage.py import_legacy_images <directory>
+    """
+    help = 'Imports images from a directory into the database'
+
+    def add_arguments(self, parser):
+        parser.add_argument('directory', type=str)
+
+    def handle(self, *args, **options):
+        directory = options['directory']
+        if not os.path.isdir(directory):
+            raise Exception(f"Directory {directory} does not exist")
+
+        # read settings.json
+        with open(os.path.join(directory, 'import_settings.json'), 'r') as f:
+            settings = json.load(f)
+
+        # read img_dicts.jsonl
+        with open(os.path.join(directory, 'img_dicts.jsonl'), 'r') as f:
+            img_dicts = [json.loads(line) for line in f.readlines()]
+
+        # get labelset
+        labelset_name = settings['labelset_name']
+        labelset_version = settings['labelset_version']
+        try:
+            labelset = LabelSet.objects.get(name=labelset_name, version=labelset_version)
+        except LabelSet.DoesNotExist:
+            raise Exception(f"No labelset found with the name {labelset_name} and version {labelset_version}")
+
+
+        # get labels in dict to lookup by name
+        labels = {label.name: label for label in labelset.labels.all()}
+
+        # create images and image_classification_annotations
+        for img_dict in tqdm(img_dicts):
+
+            # Open the image file
+            with open(os.path.join(directory, img_dict['path']), 'rb') as f:
+                # Create the Django File object
+                django_file = File(f)
+                
+                # Extract only the filename
+                filename = os.path.basename(img_dict['path'])
+                img_suffix = os.path.splitext(filename)[1]
+                # assign uuid as new filename
+                filename = str(uuid4()) + img_suffix
+                
+
+                # Create a new LegacyImage instance and save the image
+                image = LegacyImage(image=filename, suffix=img_suffix)
+                image.image.save(filename, django_file, save=True)
+
+                image_annotations = []
+                for label_name, label in labels.items():
+                    if label_name in img_dict['labels']:
+                        value = True
+                    else:
+                        value = False
+
+                    image_annotations.append(ImageClassificationAnnotation(
+                        legacy_image=image,
+                        label=label,
+                        value=value,
+                        annotator=settings['annotator']
+                    ))
+
+                ImageClassificationAnnotation.objects.bulk_create(image_annotations)
+

endoreg_db/management/commands/import_legacy_videos.py

@@ -0,0 +1,76 @@
+from endoreg_db.utils.hashs import get_video_hash
+import subprocess
+from django.core.files import File
+from django.core.management.base import BaseCommand
+from endoreg_db.models.data_file.video import LegacyVideo
+import os
+from tqdm import tqdm
+import cv2
+
+def convert_mkv_to_mp4(source_path, target_path):
+    """
+    Convert a .mkv file to a .mp4 file using FFmpeg.
+    
+    Parameters:
+        mov_file_path (str): The file path of the input .MOV file.
+        mp4_file_path (str): The file path where the output .mp4 file should be saved.
+        
+    Returns:
+        None
+    """
+    cmd = ["ffmpeg", "-y", "-i", source_path, "-vcodec", "h264", "-acodec", "aac", target_path]
+    subprocess.run(cmd)
+
+
+class Command(BaseCommand):
+    """
+    Imports videos from a directory into the database
+
+    Usage:
+        python manage.py import_legacy_videos <directory>
+    """
+    help = 'Imports videos from a directory into the database'
+
+    def add_arguments(self, parser):
+        parser.add_argument('directory', type=str)
+
+    def handle(self, *args, **options):
+        directory = options['directory']
+        if not os.path.isdir(directory):
+            raise Exception(f"Directory {directory} does not exist")
+
+        # iterate over all subdirectories and gather all .mkv files
+        mkv_files = []
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                if file.endswith(".mkv"):
+                    mkv_files.append(os.path.join(root, file))
+
+        # create videos
+        for mkv_file in tqdm(mkv_files):
+            # get the name of the video
+            video_name = os.path.basename(mkv_file).replace(".mkv", "")
+
+            # convert the .mkv file to a .mp4 file
+            mp4_file = os.path.join(directory, video_name + ".mp4")
+            if not os.path.isfile(mp4_file):
+                convert_mkv_to_mp4(mkv_file, mp4_file)
+
+            # create the video object
+            video_hash = get_video_hash(mp4_file)
+
+            # check if the video already exists
+            if LegacyVideo.objects.filter(video_hash=video_hash).exists():
+                print(f"Video with hash {video_hash} already exists. Skipping...")
+                continue
+
+            video = LegacyVideo.objects.create(video_hash=video_hash, suffix = "mp4")
+
+            # open the .mp4 file
+            with open(mp4_file, 'rb') as f:
+                # create the Django File object
+                django_file = File(f)
+                # save the file to the video
+                video.file.save(video_name + ".mp4", django_file)
+            raise Exception("Stop here: NEED TO FIX OPENCV DEPENDENCY")
+            video.initialize_video_specs(cv2.VideoCapture(video.file.path))

endoreg_db/management/commands/load_active_model_data.py

@@ -0,0 +1,45 @@
+from django.conf import settings
+import os
+from django.core.management.base import BaseCommand
+from ...models import ActiveModel
+from ...utils import load_model_data_from_yaml
+from ...data import ACTIVE_MODEL_DATA_DIR
+
+SOURCE_DIR = ACTIVE_MODEL_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
+
+MODEL_0 = ActiveModel
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    MODEL_0.__name__,
+]
+
+IMPORT_METADATA = {
+    MODEL_0.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": MODEL_0, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    }
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_ai_model_data.py

@@ -0,0 +1,45 @@
+from django.conf import settings
+import os
+from django.core.management.base import BaseCommand
+from ...models import ModelType
+from ...utils import load_model_data_from_yaml
+from ...data import MODEL_TYPE_DATA_DIR
+
+SOURCE_DIR = MODEL_TYPE_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
+
+MODEL_0 = ModelType
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    MODEL_0.__name__,
+]
+
+IMPORT_METADATA = {
+    MODEL_0.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": MODEL_0, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    }
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_base_db_data.py

@@ -0,0 +1,62 @@
+from django.core.management import BaseCommand, call_command
+
+class Command(BaseCommand):
+    help = 'Run all data loading commands in the correct order'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output for all commands',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+
+        self.stdout.write(self.style.SUCCESS("Populating base db models with data..."))
+
+        # Run the load_profession_data command
+        self.stdout.write(self.style.SUCCESS("Running load_profession_data..."))
+        call_command('load_profession_data', verbose=verbose)
+
+        # rund the load_user_groups command
+        self.stdout.write(self.style.SUCCESS("Running load_user_groups..."))
+        call_command('load_user_groups', verbose=verbose)
+
+        # Run the load_endoscopy_processor_data command
+        self.stdout.write(self.style.SUCCESS("Running load_endoscopy_processor_data..."))
+        call_command('load_endoscopy_processor_data', verbose=verbose)
+
+        # Run the load_endoscope_type_data command
+        self.stdout.write(self.style.SUCCESS("Running load_endoscope_type_data..."))
+        call_command('load_endoscope_type_data', verbose=verbose)
+
+        # Run the load_unit_data command
+        self.stdout.write(self.style.SUCCESS("Running load_unit_data..."))
+        call_command('load_unit_data', verbose=verbose)
+
+        # Run the load_information_source command
+        self.stdout.write(self.style.SUCCESS("Running load_information_source..."))
+        call_command('load_information_source', verbose=verbose)
+
+        # Run the load_center_data command
+        self.stdout.write(self.style.SUCCESS("Running load_center_data..."))
+        call_command('load_center_data', verbose=verbose)
+
+        # Run the load_examination_data command
+        self.stdout.write(self.style.SUCCESS("Running load_examination_data..."))
+        call_command('load_examination_data', verbose=verbose)
+
+        # Run the load_label_data command
+        self.stdout.write(self.style.SUCCESS("Running load_label_data..."))
+        call_command('load_label_data', verbose=verbose)
+
+        # Run the load_ai_model_data command
+        self.stdout.write(self.style.SUCCESS("Running load_ai_model_data..."))
+        call_command('load_ai_model_data', verbose=verbose)
+
+        # Run the load_active_model_data command
+        self.stdout.write(self.style.SUCCESS("Running load_active_model_data..."))
+        call_command('load_active_model_data', verbose=verbose)
+        
+        self.stdout.write(self.style.SUCCESS("All data loading commands executed successfully."))

endoreg_db/management/commands/load_center_data.py

@@ -0,0 +1,43 @@
+from django.core.management.base import BaseCommand
+from ...utils import load_model_data_from_yaml
+from ...models import Center
+from ...data import CENTER_DATA_DIR
+
+SOURCE_DIR = CENTER_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
+
+MODEL_0 = Center
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    MODEL_0.__name__,
+]
+
+IMPORT_METADATA = {
+    MODEL_0.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": MODEL_0, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    }
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_endoscope_type_data.py

@@ -0,0 +1,45 @@
+from django.conf import settings
+import os
+from django.core.management.base import BaseCommand
+from ...models import EndoscopeType
+from ...utils import load_model_data_from_yaml
+from ...data import ENDOSCOPE_TYPE_DATA_DIR
+
+SOURCE_DIR = ENDOSCOPE_TYPE_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
+
+MODEL_0 = EndoscopeType
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    MODEL_0.__name__,
+]
+
+IMPORT_METADATA = {
+    MODEL_0.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": MODEL_0, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    }
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_endoscopy_processor_data.py

@@ -0,0 +1,45 @@
+from django.conf import settings
+import os
+from django.core.management.base import BaseCommand
+from ...models import EndoscopyProcessor
+from ...utils import load_model_data_from_yaml
+from ...data import ENDOSCOPY_PROCESSOR_DATA_DIR
+
+SOURCE_DIR = ENDOSCOPY_PROCESSOR_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
+
+MODEL_0 = EndoscopyProcessor
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    MODEL_0.__name__,
+]
+
+IMPORT_METADATA = {
+    MODEL_0.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": MODEL_0, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    }
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_examination_data.py

@@ -0,0 +1,75 @@
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from ...models import (
+    Examination,
+    ExaminationType,  # Replace 'your_app_name' with the actual app name
+    ExaminationTime,
+    ExaminationTimeType,
+)
+import os
+from ...utils import load_model_data_from_yaml
+from ...data import EXAMINATION_DATA_DIR
+
+SOURCE_DIR = EXAMINATION_DATA_DIR 
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    "ExaminationType",
+    "Examination",
+    "ExaminationTimeType",
+    "ExaminationTime"
+]
+
+IMPORT_METADATA = {
+    # "": { # same as model name in "import models", e.g. "Intervention"
+    #     "subdir": os.path.join(SOURCE_DIR,""), # e.g. "interventions"
+    #     "model": None, # e.g. Intervention
+    #     "foreign_keys": [], # e.g. ["intervention_types"]
+    #     "foreign_key_models": [] # e.g. [InterventionType]
+    # },
+    "ExaminationType": {
+        "dir": os.path.join(SOURCE_DIR,"type"), # e.g. "interventions"
+        "model": ExaminationType, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    },
+    "Examination": {
+        "dir": os.path.join(SOURCE_DIR,"examinations"), # e.g. "interventions"
+        "model": Examination, # e.g. Intervention
+        "foreign_keys": ["examination_types"], # e.g. ["intervention_types"]
+        "foreign_key_models": [ExaminationType] # e.g. [InterventionType]
+    },
+    "ExaminationTimeType": {
+        "dir": os.path.join(SOURCE_DIR,"time-type"), # e.g. "interventions"
+        "model": ExaminationTimeType, # e.g. Intervention
+        "foreign_keys": ["examinations"], # e.g. ["intervention_types"]
+        "foreign_key_models": [Examination] # e.g. [InterventionType]
+    },
+    "ExaminationTime": {
+        "dir": os.path.join(SOURCE_DIR,"time"), # e.g. "interventions"
+        "model": ExaminationTime, # e.g. Intervention
+        "foreign_keys": ["time_types"], # e.g. ["intervention_types"]
+        "foreign_key_models": [ExaminationTimeType] # e.g. [InterventionType]
+    },
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_information_source.py

@@ -0,0 +1,45 @@
+from django.conf import settings
+import os
+from django.core.management.base import BaseCommand
+from ...models import InformationSource
+from ...utils import load_model_data_from_yaml
+from ...data import INFORMATION_DATA_DIR
+
+SOURCE_DIR = INFORMATION_DATA_DIR # e.g. settings.DATA_DIR_INTERVENTION
+
+MODEL_0 = InformationSource
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    MODEL_0.__name__,
+]
+
+IMPORT_METADATA = {
+    MODEL_0.__name__: {
+        "dir": SOURCE_DIR, # e.g. "interventions"
+        "model": MODEL_0, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    }
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )

endoreg_db/management/commands/load_label_data.py

@@ -0,0 +1,67 @@
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from ...models import (
+    Label,
+    LabelType,
+    LabelSet
+)
+import os
+from ...utils import load_model_data_from_yaml
+from ...data import LABEL_DATA_DIR
+
+SOURCE_DIR = LABEL_DATA_DIR 
+
+IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
+    "LabelType",
+    "Label",
+    "LabelSet",
+]
+
+IMPORT_METADATA = {
+    # "": { # same as model name in "import models", e.g. "Intervention"
+    #     "subdir": os.path.join(SOURCE_DIR,""), # e.g. "interventions"
+    #     "model": None, # e.g. Intervention
+    #     "foreign_keys": [], # e.g. ["intervention_types"]
+    #     "foreign_key_models": [] # e.g. [InterventionType]
+    # },
+    "LabelType": {
+        "dir": os.path.join(SOURCE_DIR,"label-type"), # e.g. "interventions"
+        "model": LabelType, # e.g. Intervention
+        "foreign_keys": [], # e.g. ["intervention_types"]
+        "foreign_key_models": [] # e.g. [InterventionType]
+    },
+    "Label": {
+        "dir": os.path.join(SOURCE_DIR,"label"), # e.g. "interventions"
+        "model": Label, # e.g. Intervention
+        "foreign_keys": ["label_type"], # e.g. ["intervention_types"]
+        "foreign_key_models": [LabelType] # e.g. [InterventionType]
+    },
+    "LabelSet": {
+        "dir": os.path.join(SOURCE_DIR,"label-set"), # e.g. "interventions"
+        "model": LabelSet, # e.g. Intervention
+        "foreign_keys": ["labels"], # e.g. ["intervention_types"]
+        "foreign_key_models": [Label] # e.g. [InterventionType]
+    },
+}
+
+class Command(BaseCommand):
+    help = """Load all .yaml files in the data/intervention directory
+    into the Intervention and InterventionType model"""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+        for model_name in IMPORT_MODELS:
+            _metadata = IMPORT_METADATA[model_name]
+            load_model_data_from_yaml(
+                self,
+                model_name,
+                _metadata,
+                verbose
+            )