endoreg-db 0.2.2__tar.gz → 0.3.0__tar.gz

{endoreg_db-0.2.2 → endoreg_db-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: endoreg-db
-Version: 0.2.2
+Version: 0.3.0
 Summary: EndoReg Db Django App
 License: GNU3
 Author: Thomas J. Lux
@@ -12,8 +12,11 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: django (>=4.2,<4.3)
 Requires-Dist: django-bootstrap5 (>=23.4,<24.0)
 Requires-Dist: djangorestframework (>=3.14.0,<4.0.0)
+Requires-Dist: ffmpeg-python (>=0.2.0,<0.3.0)
 Requires-Dist: icecream (>=2.1.3,<3.0.0)
+Requires-Dist: opencv-python (>=4.9.0.80,<5.0.0.0)
 Requires-Dist: pillow (>=10.2.0,<11.0.0)
+Requires-Dist: pytesseract (>=0.3.10,<0.4.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
 Requires-Dist: tqdm (>=4.66.2,<5.0.0)
 Description-Content-Type: text/markdown
@@ -24,3 +27,6 @@ endoreg-db
 
 ## Poetry add pypi token
 poetry config pypi-token.pypi your-api-token
+
+
+

{endoreg_db-0.2.2 → endoreg_db-0.3.0}/README.md

@@ -3,4 +3,6 @@ endoreg-db
 
 
 ## Poetry add pypi token
-poetry config pypi-token.pypi your-api-token
+poetry config pypi-token.pypi your-api-token
+
+

endoreg_db-0.3.0/endoreg_db/management/commands/delete_all.py

@@ -0,0 +1,18 @@
+from django.core.management.base import BaseCommand
+from django.apps import apps
+from django.db import transaction
+
+class Command(BaseCommand):
+    help = 'Deletes all objects from all models in the "endoreg_db" app.'
+
+    def handle(self, *args, **options):
+        app_models = apps.get_models()
+        app_models = [model for model in app_models if "endoreg_db" in model._meta.app_label]  # Filter models by the "endoreg_db" app
+        for model in app_models:
+            with transaction.atomic():  # Use a separate transaction for each model
+                try:
+                    model.objects.all().delete()
+                    self.stdout.write(self.style.SUCCESS(f'Successfully deleted all objects in {model._meta.db_table}'))
+                except Exception as e:
+                    # This catches exceptions and continues with the next model
+                    self.stdout.write(self.style.ERROR(f'Error deleting objects in {model._meta.db_table}: {e}'))

endoreg_db-0.3.0/endoreg_db/management/commands/fix_auth_permission.py

@@ -0,0 +1,20 @@
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import Permission
+from django.contrib.contenttypes.models import ContentType
+from django.apps import apps
+
+class Command(BaseCommand):
+    help = 'Sets up default permissions for each model in the app.'
+
+    def handle(self, *args, **options):
+        for model in apps.get_models():
+            content_type = ContentType.objects.get_for_model(model)
+            permissions = ['view', 'add', 'change', 'delete']
+            for perm in permissions:
+                codename = f'{perm}_{model._meta.model_name}'
+                name = f'Can {perm} {model._meta.verbose_name}'
+                Permission.objects.get_or_create(
+                    codename=codename,
+                    defaults={'name': name, 'content_type': content_type},
+                )
+            self.stdout.write(self.style.SUCCESS(f'Set up permissions for {model._meta.verbose_name}'))

endoreg_db-0.3.0/endoreg_db/management/commands/load_user_groups.py

@@ -0,0 +1,28 @@
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import Group, Permission
+from django.contrib.contenttypes.models import ContentType
+from django.apps import apps
+
+
+class Command(BaseCommand):
+    help = "Create additional user groups and permissions for all models in 'endoreg_db' app."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Display verbose output',
+        )
+
+    def handle(self, *args, **options):
+        verbose = options['verbose']
+
+        # Create groups
+        groups = ["demo", "verified", "agl", "endo_reg_user", "g_play_user", "ukw_user"]
+        for group_name in groups:
+            _group, created = Group.objects.get_or_create(name=group_name)
+            if verbose and created:
+                self.stdout.write(self.style.SUCCESS(f"Created group {group_name}"))
+
+        if verbose:
+            self.stdout.write(self.style.SUCCESS("All groups processed successfully."))

{endoreg_db-0.2.2 → endoreg_db-0.3.0}/endoreg_db/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2 on 2024-02-13 12:43
+# Generated by Django 4.2.10 on 2024-02-13 22:19
 
 from django.conf import settings
 from django.db import migrations, models

endoreg_db-0.3.0/endoreg_db/migrations/0002_rawvideofile.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.11 on 2024-03-09 17:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RawVideoFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField()),
+                ('file', models.FileField(upload_to='raw_data/')),
+                ('original_file_name', models.CharField(max_length=255)),
+                ('hash', models.CharField(max_length=255, unique=True)),
+                ('uploaded_at', models.DateTimeField(auto_now_add=True)),
+                ('state_frames_extracted', models.BooleanField(default=False)),
+                ('frame_dir', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0003_rawvideofile_frames_required.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-03-09 20:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0002_rawvideofile'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='frames_required',
+            field=models.BooleanField(default=True),
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0004_rename_hash_rawvideofile_video_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-03-10 12:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0003_rawvideofile_frames_required'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='rawvideofile',
+            old_name='hash',
+            new_name='video_hash',
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0005_ffmpegmeta_remove_videoimportmeta_center_and_more.py

@@ -0,0 +1,56 @@
+# Generated by Django 4.2.11 on 2024-03-10 13:07
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0004_rename_hash_rawvideofile_video_hash'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FFMpegMeta',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('duration', models.FloatField(blank=True, null=True)),
+                ('width', models.IntegerField(blank=True, null=True)),
+                ('height', models.IntegerField(blank=True, null=True)),
+                ('frame_rate', models.FloatField(blank=True, null=True)),
+                ('video_codec', models.CharField(blank=True, max_length=50, null=True)),
+                ('audio_codec', models.CharField(blank=True, max_length=50, null=True)),
+                ('audio_channels', models.IntegerField(blank=True, null=True)),
+                ('audio_sample_rate', models.IntegerField(blank=True, null=True)),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='videoimportmeta',
+            name='center',
+        ),
+        migrations.RemoveField(
+            model_name='videoimportmeta',
+            name='endoscope',
+        ),
+        migrations.RemoveField(
+            model_name='videoimportmeta',
+            name='processor',
+        ),
+        migrations.CreateModel(
+            name='VideoMeta',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('center', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.center')),
+                ('endoscope', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.endoscope')),
+                ('ffmpeg_meta', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.ffmpegmeta')),
+                ('import_meta', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.videoimportmeta')),
+                ('processor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.endoscopyprocessor')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='video_meta',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.videometa'),
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0006_rawvideofile_center_alter_videometa_processor.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.11 on 2024-03-10 14:46
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0005_ffmpegmeta_remove_videoimportmeta_center_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='center',
+            field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.center'),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='videometa',
+            name='processor',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.endoscopyprocessor'),
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0007_rawvideofile_processor.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.11 on 2024-03-10 15:05
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0006_rawvideofile_center_alter_videometa_processor'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='processor',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.endoscopyprocessor'),
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0008_rename_frames_required_rawvideofile_state_frames_required.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-03-10 15:45
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0007_rawvideofile_processor'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='rawvideofile',
+            old_name='frames_required',
+            new_name='state_frames_required',
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0009_sensitivemeta_rawvideofile_sensitive_meta.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.11 on 2024-03-10 17:44
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0008_rename_frames_required_rawvideofile_state_frames_required'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SensitiveMeta',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('examination_date', models.DateField(blank=True, null=True)),
+                ('patient_first_name', models.CharField(blank=True, max_length=255, null=True)),
+                ('patient_last_name', models.CharField(blank=True, max_length=255, null=True)),
+                ('patient_dob', models.DateField(blank=True, null=True)),
+                ('endoscope_type', models.CharField(blank=True, max_length=255, null=True)),
+                ('endoscope_serial_number', models.CharField(blank=True, max_length=255, null=True)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='sensitive_meta',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='endoreg_db.sensitivemeta'),
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0010_rename_endoscope_serial_number_sensitivemeta_endoscope_sn.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-03-10 17:50
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0009_sensitivemeta_rawvideofile_sensitive_meta'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='sensitivemeta',
+            old_name='endoscope_serial_number',
+            new_name='endoscope_sn',
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0011_rawvideofile_state_sensitive_data_retrieved.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-03-10 18:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0010_rename_endoscope_serial_number_sensitivemeta_endoscope_sn'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_sensitive_data_retrieved',
+            field=models.BooleanField(default=False),
+        ),
+    ]

endoreg_db-0.3.0/endoreg_db/migrations/0012_rawvideofile_prediction_dir_and_more.py

@@ -0,0 +1,109 @@
+# Generated by Django 4.2.11 on 2024-03-18 23:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('endoreg_db', '0011_rawvideofile_state_sensitive_data_retrieved'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='prediction_dir',
+            field=models.CharField(default='.', max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_anonym_video_performed',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_anonym_video_required',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_dataset_complete',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_finalized',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_follow_up_intervention_available',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_follow_up_intervention_required',
+            field=models.BooleanField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_histology_available',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_histology_required',
+            field=models.BooleanField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_initial_prediction_completed',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_initial_prediction_import_completed',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_initial_prediction_import_required',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_initial_prediction_required',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_ocr_completed',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_ocr_required',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_ocr_result_validated',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_original_reports_deleted',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_original_video_deleted',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='rawvideofile',
+            name='state_outside_validated',
+            field=models.BooleanField(default=False),
+        ),
+    ]

{endoreg_db-0.2.2 → endoreg_db-0.3.0}/endoreg_db/models/data_file/__init__.py

@@ -1,4 +1,7 @@
 from .frame import Frame, LegacyFrame
 from .report_file import ReportFile
 from .video import Video, LegacyVideo, VideoImportMeta
-from .video_segment import LegacyLabelVideoSegment, LabelVideoSegment
+from .video_segment import LegacyLabelVideoSegment, LabelVideoSegment
+from .metadata import FFMpegMeta, SensitiveMeta
+
+from .import_classes import RawVideoFile

{endoreg_db-0.2.2 → endoreg_db-0.3.0}/endoreg_db/models/data_file/base_classes/__init__.py

@@ -1,3 +1,2 @@
 from .abstract_frame import AbstractFrame
 from .abstract_video import AbstractVideo
-

{endoreg_db-0.2.2 → endoreg_db-0.3.0}/endoreg_db/models/data_file/base_classes/abstract_video.py

@@ -3,6 +3,7 @@ from PIL import Image
 from django.core.files.base import ContentFile
 from django.db import models, transaction
 from tqdm import tqdm
+# import cv2
 import io
 from datetime import date
 

endoreg_db-0.3.0/endoreg_db/models/data_file/import_classes/__init__.py

@@ -0,0 +1,31 @@
+import os
+from pathlib import Path
+
+from .raw_video import RawVideoFile
+
+# FileImporter class
+# This class is used to import data from a file into the database.
+# Expects a directory containing files to import.
+# creates correct import file object depending on file type
+
+# FileImporter class
+# This class is used to import data from a file into the database by creating objects for the files.
+# main method is import_files which expects a path to a directory containing files to import.
+# creates correct import file object depending on file type by checking the file extension
+
+class FileImporter:
+    def __init__(self, directory):
+        self.directory = directory
+
+    def import_files(self):
+        directory_path = Path(self.directory)
+        for file in directory_path.iterdir():
+            if file.is_file():
+                if file.suffix.lower() in ['.mov', '.mp4']:
+                    RawVideoFile.create_from_file(file)
+                else:
+                    raise ValueError(f"File type {file.suffix} not supported")
+            else:
+                raise ValueError(f"{file} is not a file")
+            
+