endoreg-db 0.5.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

endoreg_db/management/commands/delete_legacy_images.py

@@ -1,19 +0,0 @@
-# django command to delete all LegacyImage objects
-
-from django.core.management.base import BaseCommand, CommandError
-from endoreg_db.models import LegacyImage
-from tqdm import tqdm
-
-class Command(BaseCommand):
-    help = 'Deletes all LegacyImage objects including the linked files (legacy_image.image)'
-
-    def handle(self, *args, **options):
-        legacy_images = LegacyImage.objects.all()
-
-        for legacy_image in tqdm(legacy_images):
-            legacy_image.image.delete()
-            legacy_image.delete()
-
-        self.stdout.write(self.style.SUCCESS('Successfully deleted all LegacyImage objects and linked image files'))
-
-

endoreg_db/management/commands/delete_legacy_videos.py

@@ -1,17 +0,0 @@
-# Django command to fetch all LegacyVideo objects, delete the associated file and delete the object from the database
-
-from endoreg_db.models.data_file.video import LegacyVideo
-from tqdm import tqdm
-from django.core.management.base import BaseCommand
-
-class Command(BaseCommand):
-    """
-    Deletes all LegacyVideos in the database.
-    """
-    help = 'Deletes all LegacyVideos in the database.'
-
-    def handle(self, *args, **options):
-        videos = LegacyVideo.objects.all()
-        for video in tqdm(videos):
-            video.file.delete()
-            video.delete()

endoreg_db/management/commands/extract_legacy_video_frames.py

@@ -1,18 +0,0 @@
-# Django Command to:
-# fetch all LegacyVideos and extract frames from them 
-# track progress for all videos and for each video with tqdm
-
-from endoreg_db.models.data_file.video import LegacyVideo
-from tqdm import tqdm
-from django.core.management.base import BaseCommand
-
-class Command(BaseCommand):
-    """
-    Extracts frames from all LegacyVideos in the database.
-    """
-    help = 'Extracts frames from all LegacyVideos in the database.'
-
-    def handle(self, *args, **options):
-        videos = LegacyVideo.objects.all()
-        for video in tqdm(videos):
-            video.extract_all_frames()

endoreg_db/management/commands/import_legacy_images.py

@@ -1,94 +0,0 @@
-# command expects a directory as an argument
-# directory should contain a img_dicts.jsonl file, a settings.json and an "images" directory
-# images should be in jpeg format
-# reads the img_dicts.jsonl file and creates a new database entry for each image
-# img_dicts contains a line for each image in the following format:
-# {"id": 30604292, "path": "images/30604292.jpg", "labels": ["appendix", "polyp"]}
-
-# import_settings.json example:
-# {
-#     "annotator": "legacy",
-#     "labelset_name": "multilabel_classification",
-#     "labelset_version": 0
-# }
-
-import os
-import json
-from tqdm import tqdm
-from django.core.management.base import BaseCommand
-from endoreg_db.models import LegacyImage, LabelSet, Label, ImageClassificationAnnotation
-from django.core.files import File
-from uuid import uuid4
-
-class Command(BaseCommand):
-    """
-    Imports images from a directory into the database
-
-    Usage:
-        python manage.py import_legacy_images <directory>
-    """
-    help = 'Imports images from a directory into the database'
-
-    def add_arguments(self, parser):
-        parser.add_argument('directory', type=str)
-
-    def handle(self, *args, **options):
-        directory = options['directory']
-        if not os.path.isdir(directory):
-            raise Exception(f"Directory {directory} does not exist")
-
-        # read settings.json
-        with open(os.path.join(directory, 'import_settings.json'), 'r') as f:
-            settings = json.load(f)
-
-        # read img_dicts.jsonl
-        with open(os.path.join(directory, 'img_dicts.jsonl'), 'r') as f:
-            img_dicts = [json.loads(line) for line in f.readlines()]
-
-        # get labelset
-        labelset_name = settings['labelset_name']
-        labelset_version = settings['labelset_version']
-        try:
-            labelset = LabelSet.objects.get(name=labelset_name, version=labelset_version)
-        except LabelSet.DoesNotExist:
-            raise Exception(f"No labelset found with the name {labelset_name} and version {labelset_version}")
-
-
-        # get labels in dict to lookup by name
-        labels = {label.name: label for label in labelset.labels.all()}
-
-        # create images and image_classification_annotations
-        for img_dict in tqdm(img_dicts):
-
-            # Open the image file
-            with open(os.path.join(directory, img_dict['path']), 'rb') as f:
-                # Create the Django File object
-                django_file = File(f)
-                
-                # Extract only the filename
-                filename = os.path.basename(img_dict['path'])
-                img_suffix = os.path.splitext(filename)[1]
-                # assign uuid as new filename
-                filename = str(uuid4()) + img_suffix
-                
-
-                # Create a new LegacyImage instance and save the image
-                image = LegacyImage(image=filename, suffix=img_suffix)
-                image.image.save(filename, django_file, save=True)
-
-                image_annotations = []
-                for label_name, label in labels.items():
-                    if label_name in img_dict['labels']:
-                        value = True
-                    else:
-                        value = False
-
-                    image_annotations.append(ImageClassificationAnnotation(
-                        legacy_image=image,
-                        label=label,
-                        value=value,
-                        annotator=settings['annotator']
-                    ))
-
-                ImageClassificationAnnotation.objects.bulk_create(image_annotations)
-

endoreg_db/management/commands/import_legacy_videos.py

@@ -1,76 +0,0 @@
-from endoreg_db.utils.hashs import get_video_hash
-import subprocess
-from django.core.files import File
-from django.core.management.base import BaseCommand
-from endoreg_db.models.data_file.video import LegacyVideo
-import os
-from tqdm import tqdm
-# import cv2
-
-def convert_mkv_to_mp4(source_path, target_path):
-    """
-    Convert a .mkv file to a .mp4 file using FFmpeg.
-    
-    Parameters:
-        mov_file_path (str): The file path of the input .MOV file.
-        mp4_file_path (str): The file path where the output .mp4 file should be saved.
-        
-    Returns:
-        None
-    """
-    cmd = ["ffmpeg", "-y", "-i", source_path, "-vcodec", "h264", "-acodec", "aac", target_path]
-    subprocess.run(cmd)
-
-
-class Command(BaseCommand):
-    """
-    Imports videos from a directory into the database
-
-    Usage:
-        python manage.py import_legacy_videos <directory>
-    """
-    help = 'Imports videos from a directory into the database'
-
-    def add_arguments(self, parser):
-        parser.add_argument('directory', type=str)
-
-    def handle(self, *args, **options):
-        directory = options['directory']
-        if not os.path.isdir(directory):
-            raise Exception(f"Directory {directory} does not exist")
-
-        # iterate over all subdirectories and gather all .mkv files
-        mkv_files = []
-        for root, dirs, files in os.walk(directory):
-            for file in files:
-                if file.endswith(".mkv"):
-                    mkv_files.append(os.path.join(root, file))
-
-        # create videos
-        for mkv_file in tqdm(mkv_files):
-            # get the name of the video
-            video_name = os.path.basename(mkv_file).replace(".mkv", "")
-
-            # convert the .mkv file to a .mp4 file
-            mp4_file = os.path.join(directory, video_name + ".mp4")
-            if not os.path.isfile(mp4_file):
-                convert_mkv_to_mp4(mkv_file, mp4_file)
-
-            # create the video object
-            video_hash = get_video_hash(mp4_file)
-
-            # check if the video already exists
-            if LegacyVideo.objects.filter(video_hash=video_hash).exists():
-                print(f"Video with hash {video_hash} already exists. Skipping...")
-                continue
-
-            video = LegacyVideo.objects.create(video_hash=video_hash, suffix = "mp4")
-
-            # open the .mp4 file
-            with open(mp4_file, 'rb') as f:
-                # create the Django File object
-                django_file = File(f)
-                # save the file to the video
-                video.file.save(video_name + ".mp4", django_file)
-            raise Exception("Stop here: NEED TO FIX OPENCV DEPENDENCY")
-            video.initialize_video_specs(cv2.VideoCapture(video.file.path))

endoreg_db/management/commands/load_label_data.py

@@ -1,67 +0,0 @@
-from django.conf import settings
-from django.core.management.base import BaseCommand
-from ...models import (
-    Label,
-    LabelType,
-    LabelSet
-)
-import os
-from ...utils import load_model_data_from_yaml
-from ...data import LABEL_DATA_DIR
-
-SOURCE_DIR = LABEL_DATA_DIR 
-
-IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
-    "LabelType",
-    "Label",
-    "LabelSet",
-]
-
-IMPORT_METADATA = {
-    # "": { # same as model name in "import models", e.g. "Intervention"
-    #     "subdir": os.path.join(SOURCE_DIR,""), # e.g. "interventions"
-    #     "model": None, # e.g. Intervention
-    #     "foreign_keys": [], # e.g. ["intervention_types"]
-    #     "foreign_key_models": [] # e.g. [InterventionType]
-    # },
-    "LabelType": {
-        "dir": os.path.join(SOURCE_DIR,"label-type"), # e.g. "interventions"
-        "model": LabelType, # e.g. Intervention
-        "foreign_keys": [], # e.g. ["intervention_types"]
-        "foreign_key_models": [] # e.g. [InterventionType]
-    },
-    "Label": {
-        "dir": os.path.join(SOURCE_DIR,"label"), # e.g. "interventions"
-        "model": Label, # e.g. Intervention
-        "foreign_keys": ["label_type"], # e.g. ["intervention_types"]
-        "foreign_key_models": [LabelType] # e.g. [InterventionType]
-    },
-    "LabelSet": {
-        "dir": os.path.join(SOURCE_DIR,"label-set"), # e.g. "interventions"
-        "model": LabelSet, # e.g. Intervention
-        "foreign_keys": ["labels"], # e.g. ["intervention_types"]
-        "foreign_key_models": [Label] # e.g. [InterventionType]
-    },
-}
-
-class Command(BaseCommand):
-    help = """Load all .yaml files in the data/intervention directory
-    into the Intervention and InterventionType model"""
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            '--verbose',
-            action='store_true',
-            help='Display verbose output',
-        )
-
-    def handle(self, *args, **options):
-        verbose = options['verbose']
-        for model_name in IMPORT_MODELS:
-            _metadata = IMPORT_METADATA[model_name]
-            load_model_data_from_yaml(
-                self,
-                model_name,
-                _metadata,
-                verbose
-            )

endoreg_db/migrations/0002_anonymizedimagelabel_anonymousimageannotation_and_more.py

@@ -1,55 +0,0 @@
-# Generated by Django 4.2.15 on 2024-08-28 11:05
-
-import django.core.files.storage
-import django.core.validators
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='AnonymizedImageLabel',
-            fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.CharField(max_length=255, unique=True)),
-                ('description', models.TextField(blank=True, null=True)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='AnonymousImageAnnotation',
-            fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('image_name', models.CharField(max_length=255)),
-                ('slug', models.SlugField(blank=True, null=True, unique=True)),
-                ('polyp_count', models.IntegerField()),
-                ('comments', models.CharField(max_length=255)),
-                ('gender', models.CharField(max_length=255)),
-                ('name_image_url', models.CharField(max_length=255)),
-                ('date_created', models.DateTimeField(auto_now_add=True)),
-                ('label', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.label')),
-            ],
-        ),
-        migrations.AlterField(
-            model_name='rawpdffile',
-            name='file',
-            field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/home/agl-admin/agl-validator-backend/agl_validator_backend/erc_data/raw_pdf'), upload_to='raw_pdf/', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['pdf'])]),
-        ),
-        migrations.CreateModel(
-            name='DroppedName',
-            fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.CharField(max_length=255)),
-                ('gender', models.CharField(max_length=255)),
-                ('x', models.FloatField()),
-                ('y', models.FloatField()),
-                ('name_image_url', models.CharField(max_length=255)),
-                ('annotation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='dropped_names', to='endoreg_db.anonymousimageannotation')),
-            ],
-        ),
-    ]

endoreg_db/migrations/0003_anonymousimageannotation_original_image_url_and_more.py

@@ -1,39 +0,0 @@
-# Generated by Django 4.2.15 on 2024-08-29 07:15
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0002_anonymizedimagelabel_anonymousimageannotation_and_more'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='anonymousimageannotation',
-            name='original_image_url',
-            field=models.CharField(default='https://example.com/placeholder.jpg', max_length=255),
-        ),
-        migrations.AddField(
-            model_name='anonymousimageannotation',
-            name='processed',
-            field=models.BooleanField(default=False),
-        ),
-        migrations.AddField(
-            model_name='droppedname',
-            name='box_coordinates',
-            field=models.CharField(blank=True, max_length=255, null=True),
-        ),
-        migrations.AlterField(
-            model_name='anonymousimageannotation',
-            name='comments',
-            field=models.TextField(blank=True, null=True),
-        ),
-        migrations.AlterField(
-            model_name='anonymousimageannotation',
-            name='label',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.anonymizedimagelabel'),
-        ),
-    ]

endoreg_db/migrations/0004_alter_rawpdffile_file.py

@@ -1,20 +0,0 @@
-# Generated by Django 4.2.14 on 2024-09-04 19:48
-
-import django.core.files.storage
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0003_anonymousimageannotation_original_image_url_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='rawpdffile',
-            name='file',
-            field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/home/agl-admin/agl_anonymizer/agl_anonymizer/erc_data/raw_pdf'), upload_to='raw_pdf/', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['pdf'])]),
-        ),
-    ]

endoreg_db/migrations/0005_uploadedfile_alter_rawpdffile_file_anonymizedfile.py

@@ -1,40 +0,0 @@
-# Generated by Django 4.2.15 on 2024-09-17 11:46
-
-import django.core.files.storage
-import django.core.validators
-from django.db import migrations, models
-import django.db.models.deletion
-import django.utils.timezone
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0004_alter_rawpdffile_file'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='UploadedFile',
-            fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('original_file', models.FileField(upload_to='uploads/original/')),
-                ('upload_date', models.DateTimeField(default=django.utils.timezone.now)),
-                ('description', models.TextField(blank=True, null=True)),
-            ],
-        ),
-        migrations.AlterField(
-            model_name='rawpdffile',
-            name='file',
-            field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/mnt/hdd-sensitive/Pseudo/import/pdf'), upload_to='raw_pdf/', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['pdf'])]),
-        ),
-        migrations.CreateModel(
-            name='AnonymizedFile',
-            fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('anonymized_file', models.FileField(upload_to='uploads/anonymized/')),
-                ('anonymization_date', models.DateTimeField(default=django.utils.timezone.now)),
-                ('original_file', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='anonymized_file', to='endoreg_db.uploadedfile')),
-            ],
-        ),
-    ]

endoreg_db/migrations/0006_alter_rawpdffile_file.py

@@ -1,20 +0,0 @@
-# Generated by Django 5.1.1 on 2024-10-01 16:31
-
-import django.core.files.storage
-import django.core.validators
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0005_uploadedfile_alter_rawpdffile_file_anonymizedfile'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='rawpdffile',
-            name='file',
-            field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/home/agl-admin/dev/agl-monitor-flake/agl_monitor/erc_data/raw_pdf'), upload_to='raw_pdf/', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['pdf'])]),
-        ),
-    ]

endoreg_db/migrations/0007_networkdevicelogentry_datetime_and_more.py

@@ -1,43 +0,0 @@
-# Generated by Django 5.1.1 on 2024-10-12 15:28
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0006_alter_rawpdffile_file'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='datetime',
-            field=models.DateTimeField(blank=True, default=None, null=True),
-        ),
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='ping_vpn',
-            field=models.BooleanField(default=False),
-        ),
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='ping_www',
-            field=models.BooleanField(default=False),
-        ),
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='vpn_service_restart_attempt',
-            field=models.BooleanField(default=False),
-        ),
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='vpn_service_restart_success',
-            field=models.BooleanField(blank=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='vpn_service_status',
-            field=models.BooleanField(default=False),
-        ),
-    ]

endoreg_db/migrations/0008_networkdevicelogentry_aglnet_ip_and_more.py

@@ -1,28 +0,0 @@
-# Generated by Django 5.1.1 on 2024-10-12 16:13
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0007_networkdevicelogentry_datetime_and_more'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='aglnet_ip',
-            field=models.GenericIPAddressField(blank=True, null=True),
-        ),
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='hostname',
-            field=models.CharField(blank=True, max_length=255, null=True),
-        ),
-        migrations.AddField(
-            model_name='networkdevicelogentry',
-            name='transferred_to_host',
-            field=models.BooleanField(default=False),
-        ),
-    ]

endoreg_db/migrations/0009_alter_networkdevicelogentry_vpn_service_status.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.1.1 on 2024-10-12 16:51
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0008_networkdevicelogentry_aglnet_ip_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='networkdevicelogentry',
-            name='vpn_service_status',
-            field=models.CharField(blank=True, max_length=255, null=True),
-        ),
-    ]

endoreg_db/migrations/0010_remove_networkdevicelogentry_hostname.py

@@ -1,17 +0,0 @@
-# Generated by Django 5.1.1 on 2024-10-12 16:54
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('endoreg_db', '0009_alter_networkdevicelogentry_vpn_service_status'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='networkdevicelogentry',
-            name='hostname',
-        ),
-    ]

endoreg_db/models/legacy_data/__init__.py

@@ -1,3 +0,0 @@
-from ..data_file.frame import LegacyFrame
-from .image import LegacyImage
-from ..data_file.video import LegacyVideo

endoreg_db/models/legacy_data/image.py

@@ -1,34 +0,0 @@
-from django.db import models
-from ..label.label import Label
-from ..annotation.image_classification import ImageClassificationAnnotation
-
-class LegacyImage(models.Model):
-    # Add any other fields you need to store frame-related information
-    image = models.ImageField(upload_to="legacy_images")  # Or some other field type, depending on how you're storing the frame
-    suffix = models.CharField(max_length=255)
-    # ImageClassificationAnnotation has a foreign key to this model (related name: image_classification_annotations)
-
-    def get_classification_annotations(self):
-        """
-        Get all image classification annotations for this image.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_image=self)
-    
-    def get_classification_annotations_by_label(self, label:Label):
-        """
-        Get all image classification annotations for this image with the given label.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_image=self, label=label)
-    
-    def get_classification_annotations_by_value(self, value:bool):
-        """
-        Get all image classification annotations for this image with the given value.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_image=self, value=value)
-    
-    def get_classification_annotations_by_label_and_value(self, label:Label, value:bool):
-        """
-        Get all image classification annotations for this image with the given label and value.
-        """
-        return ImageClassificationAnnotation.objects.filter(legacy_image=self, label=label, value=value)
-    

endoreg_db/models/patient_examination/__init__.py

@@ -1,35 +0,0 @@
-from django.db import models
-
-# Serializer located in serializers/examination.py
-class PatientExamination(models.Model):
-    patient = models.ForeignKey('Patient', on_delete=models.CASCADE, related_name='patient_examinations')
-    examination = models.ForeignKey('Examination', on_delete=models.CASCADE, null = True, blank = True)
-    video = models.OneToOneField('Video', on_delete=models.CASCADE, null = True, blank = True, related_name='patient_examination')
-    report_file = models.OneToOneField('ReportFile', on_delete=models.CASCADE, null = True, blank = True, related_name='patient_examination')
-
-    class Meta:
-        verbose_name = 'Patient Examination'
-        verbose_name_plural = 'Patient Examinations'
-        ordering = ['patient', 'examination']
-
-    def __str__(self):
-        return f"{self.patient} - {self.report_file}"
-    
-    def find_matching_video_from_patient(self):
-        """
-        Finds a video for this patient examination based on the patient's videos.
-        For this, the videos date must be the same as the report file's date.
-        #TODO add more criteria for matching: Examination type
-        """
-        videos = self.patient.video_set.filter(date=self.report_file.date, patient_examination__isnull=True)
-        if videos:
-            if len(videos) > 1:
-                print(f"Warning: Found more than one video for patient {self.patient} on date {self.report_file.date}. Choosing the first one.")
-            return videos[0]
-        else:
-            videos = self.patient.video_set.filter(patient_examination__isnull=True)
-            if len(videos)==1:
-                return videos[0]
-            
-        return None
-