endoreg-db 0.6.3__py3-none-any.whl → 0.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/data/__init__.py +14 -0
- endoreg_db/data/disease_classification/chronic_kidney_disease.yaml +2 -2
- endoreg_db/data/disease_classification_choice/chronic_kidney_disease.yaml +6 -6
- endoreg_db/data/examination/examinations/data.yaml +22 -21
- endoreg_db/data/examination/type/data.yaml +12 -0
- endoreg_db/data/examination_indication/endoscopy.yaml +417 -1
- endoreg_db/data/examination_indication_classification/endoscopy.yaml +157 -5
- endoreg_db/data/finding/data.yaml +18 -11
- endoreg_db/data/finding_intervention/endoscopy.yaml +26 -121
- endoreg_db/data/finding_intervention/endoscopy_colonoscopy.yaml +163 -0
- endoreg_db/data/finding_intervention/endoscopy_egd.yaml +128 -0
- endoreg_db/data/finding_intervention/endoscopy_ercp.yaml +32 -0
- endoreg_db/data/finding_intervention/endoscopy_eus_lower.yaml +9 -0
- endoreg_db/data/finding_intervention/endoscopy_eus_upper.yaml +36 -0
- endoreg_db/data/information_source/endoscopy_guidelines.yaml +7 -0
- endoreg_db/data/medication_indication/anticoagulation.yaml +4 -4
- endoreg_db/data/pdf_type/data.yaml +9 -16
- endoreg_db/data/requirement/colonoscopy_indications.yaml +56 -0
- endoreg_db/data/requirement/disease_cardiovascular.yaml +79 -0
- endoreg_db/data/requirement/disease_classification_choice_cardiovascular.yaml +38 -0
- endoreg_db/data/requirement/disease_hepatology.yaml +12 -0
- endoreg_db/data/requirement/disease_misc.yaml +12 -0
- endoreg_db/data/requirement/disease_renal.yaml +80 -0
- endoreg_db/data/requirement/event_cardiology.yaml +251 -0
- endoreg_db/data/requirement/lab_value.yaml +120 -0
- endoreg_db/data/requirement_operator/lab_operators.yaml +128 -0
- endoreg_db/data/requirement_operator/model_operators.yaml +90 -0
- endoreg_db/data/requirement_set/endoscopy_bleeding_risk.yaml +12 -0
- endoreg_db/data/requirement_set_type/data.yaml +20 -0
- endoreg_db/data/requirement_type/requirement_types.yaml +83 -0
- endoreg_db/data/risk/bleeding.yaml +26 -0
- endoreg_db/data/risk/thrombosis.yaml +37 -0
- endoreg_db/data/risk_type/data.yaml +27 -0
- endoreg_db/data/unit/time.yaml +36 -1
- endoreg_db/management/commands/load_base_db_data.py +14 -1
- endoreg_db/management/commands/load_center_data.py +46 -21
- endoreg_db/management/commands/load_examination_indication_data.py +49 -27
- endoreg_db/management/commands/load_requirement_data.py +156 -0
- endoreg_db/management/commands/load_risk_data.py +56 -0
- endoreg_db/migrations/0009_requirementoperator_requirementsettype_and_more.py +154 -0
- endoreg_db/models/__init__.py +20 -0
- endoreg_db/models/ai_model/ai_model.py +0 -13
- endoreg_db/models/ai_model/model_meta.py +2 -12
- endoreg_db/models/data_file/base_classes/abstract_frame.py +0 -2
- endoreg_db/models/data_file/base_classes/abstract_pdf.py +0 -9
- endoreg_db/models/data_file/base_classes/abstract_video.py +7 -8
- endoreg_db/models/data_file/base_classes/utils.py +0 -22
- endoreg_db/models/data_file/frame.py +1 -1
- endoreg_db/models/data_file/import_classes/raw_pdf.py +5 -11
- endoreg_db/models/data_file/import_classes/raw_video.py +6 -4
- endoreg_db/models/data_file/video/video.py +3 -3
- endoreg_db/models/disease.py +88 -19
- endoreg_db/models/event.py +108 -21
- endoreg_db/models/examination/examination_indication.py +108 -29
- endoreg_db/models/examination/examination_type.py +20 -6
- endoreg_db/models/information_source.py +37 -1
- endoreg_db/models/laboratory/lab_value.py +83 -32
- endoreg_db/models/requirement/__init__.py +11 -0
- endoreg_db/models/requirement/requirement.py +325 -0
- endoreg_db/models/requirement/requirement_evaluation/__init__.py +134 -0
- endoreg_db/models/requirement/requirement_evaluation/requirement_type_parser.py +102 -0
- endoreg_db/models/requirement/requirement_operator.py +58 -0
- endoreg_db/models/requirement/requirement_set.py +127 -0
- endoreg_db/models/risk/__init__.py +7 -0
- endoreg_db/models/risk/risk.py +72 -0
- endoreg_db/models/risk/risk_type.py +55 -0
- endoreg_db/serializers/video_segmentation.py +12 -25
- endoreg_db/urls.py +17 -3
- endoreg_db/utils/__init__.py +43 -0
- endoreg_db/utils/dataloader.py +38 -19
- endoreg_db/utils/hashs.py +1 -0
- endoreg_db/utils/paths.py +86 -0
- endoreg_db/views/views.py +107 -0
- {endoreg_db-0.6.3.dist-info → endoreg_db-0.6.4.dist-info}/METADATA +1 -1
- {endoreg_db-0.6.3.dist-info → endoreg_db-0.6.4.dist-info}/RECORD +77 -42
- endoreg_db/management/commands/load_name_data.py +0 -37
- {endoreg_db-0.6.3.dist-info → endoreg_db-0.6.4.dist-info}/WHEEL +0 -0
- {endoreg_db-0.6.3.dist-info → endoreg_db-0.6.4.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# Generated by Django 5.1.3 on 2025-04-06 23:05
|
|
2
|
+
|
|
3
|
+
import django.core.files.storage
|
|
4
|
+
import django.core.validators
|
|
5
|
+
import django.db.models.deletion
|
|
6
|
+
from django.db import migrations, models
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Migration(migrations.Migration):
|
|
10
|
+
|
|
11
|
+
dependencies = [
|
|
12
|
+
('endoreg_db', '0008_remove_reportfile_patient_examination_and_more'),
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
operations = [
|
|
16
|
+
migrations.CreateModel(
|
|
17
|
+
name='RequirementOperator',
|
|
18
|
+
fields=[
|
|
19
|
+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
20
|
+
('name', models.CharField(max_length=100, unique=True)),
|
|
21
|
+
('name_de', models.CharField(blank=True, max_length=100, null=True)),
|
|
22
|
+
('name_en', models.CharField(blank=True, max_length=100, null=True)),
|
|
23
|
+
('description', models.TextField(blank=True, null=True)),
|
|
24
|
+
],
|
|
25
|
+
),
|
|
26
|
+
migrations.CreateModel(
|
|
27
|
+
name='RequirementSetType',
|
|
28
|
+
fields=[
|
|
29
|
+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
30
|
+
('name', models.CharField(max_length=100, unique=True)),
|
|
31
|
+
('name_de', models.CharField(blank=True, max_length=100, null=True)),
|
|
32
|
+
('name_en', models.CharField(blank=True, max_length=100, null=True)),
|
|
33
|
+
('description', models.TextField(blank=True, null=True)),
|
|
34
|
+
],
|
|
35
|
+
),
|
|
36
|
+
migrations.CreateModel(
|
|
37
|
+
name='RequirementType',
|
|
38
|
+
fields=[
|
|
39
|
+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
40
|
+
('name', models.CharField(max_length=100, unique=True)),
|
|
41
|
+
('name_de', models.CharField(blank=True, max_length=100, null=True)),
|
|
42
|
+
('name_en', models.CharField(blank=True, max_length=100, null=True)),
|
|
43
|
+
('description', models.TextField(blank=True, null=True)),
|
|
44
|
+
],
|
|
45
|
+
),
|
|
46
|
+
migrations.CreateModel(
|
|
47
|
+
name='RiskType',
|
|
48
|
+
fields=[
|
|
49
|
+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
50
|
+
('name', models.CharField(max_length=100, unique=True)),
|
|
51
|
+
('name_de', models.CharField(blank=True, max_length=100, null=True)),
|
|
52
|
+
('name_en', models.CharField(blank=True, max_length=100, null=True)),
|
|
53
|
+
('description', models.TextField(blank=True, null=True)),
|
|
54
|
+
],
|
|
55
|
+
),
|
|
56
|
+
migrations.RemoveField(
|
|
57
|
+
model_name='examinationindication',
|
|
58
|
+
name='classification',
|
|
59
|
+
),
|
|
60
|
+
migrations.AddField(
|
|
61
|
+
model_name='examinationindication',
|
|
62
|
+
name='classifications',
|
|
63
|
+
field=models.ManyToManyField(blank=True, related_name='indications', to='endoreg_db.examinationindicationclassification'),
|
|
64
|
+
),
|
|
65
|
+
migrations.AddField(
|
|
66
|
+
model_name='examinationindication',
|
|
67
|
+
name='description',
|
|
68
|
+
field=models.TextField(blank=True, null=True),
|
|
69
|
+
),
|
|
70
|
+
migrations.AddField(
|
|
71
|
+
model_name='examinationindication',
|
|
72
|
+
name='expected_interventions',
|
|
73
|
+
field=models.ManyToManyField(blank=True, related_name='indications', to='endoreg_db.findingintervention'),
|
|
74
|
+
),
|
|
75
|
+
migrations.AddField(
|
|
76
|
+
model_name='examinationindicationclassification',
|
|
77
|
+
name='description',
|
|
78
|
+
field=models.TextField(blank=True, null=True),
|
|
79
|
+
),
|
|
80
|
+
migrations.AddField(
|
|
81
|
+
model_name='examinationindicationclassification',
|
|
82
|
+
name='examination',
|
|
83
|
+
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='indication_classifications', to='endoreg_db.examination'),
|
|
84
|
+
),
|
|
85
|
+
migrations.AlterField(
|
|
86
|
+
model_name='rawpdffile',
|
|
87
|
+
name='file',
|
|
88
|
+
field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/home/admin/dev/endoreg-db/erc_data'), upload_to='raw_pdf/', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['pdf'])]),
|
|
89
|
+
),
|
|
90
|
+
migrations.AlterField(
|
|
91
|
+
model_name='rawvideofile',
|
|
92
|
+
name='file',
|
|
93
|
+
field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/home/admin/dev/endoreg-db/erc_data'), upload_to='db_raw_videos', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['mp4'])]),
|
|
94
|
+
),
|
|
95
|
+
migrations.AlterField(
|
|
96
|
+
model_name='video',
|
|
97
|
+
name='file',
|
|
98
|
+
field=models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/home/admin/dev/endoreg-db/erc_data'), upload_to='db_videos', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['mp4'])]),
|
|
99
|
+
),
|
|
100
|
+
migrations.CreateModel(
|
|
101
|
+
name='Requirement',
|
|
102
|
+
fields=[
|
|
103
|
+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
104
|
+
('name', models.CharField(max_length=100, unique=True)),
|
|
105
|
+
('name_de', models.CharField(blank=True, max_length=100, null=True)),
|
|
106
|
+
('name_en', models.CharField(blank=True, max_length=100, null=True)),
|
|
107
|
+
('description', models.TextField(blank=True, null=True)),
|
|
108
|
+
('numeric_value', models.FloatField(blank=True, help_text='Numeric value for the requirement. If not set, the requirement is not used in calculations.', null=True)),
|
|
109
|
+
('numeric_value_min', models.FloatField(blank=True, help_text='Minimum numeric value for the requirement. If not set, the requirement is not used in calculations.', null=True)),
|
|
110
|
+
('numeric_value_max', models.FloatField(blank=True, help_text='Maximum numeric value for the requirement. If not set, the requirement is not used in calculations.', null=True)),
|
|
111
|
+
('string_value', models.CharField(blank=True, help_text='String value for the requirement. If not set, the requirement is not used in calculations.', max_length=100, null=True)),
|
|
112
|
+
('string_values', models.TextField(blank=True, help_text=" ','-separated list of string values for the requirement.If not set, the requirement is not used in calculations.", null=True)),
|
|
113
|
+
('disease_classification_choices', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.diseaseclassificationchoice')),
|
|
114
|
+
('diseases', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.disease')),
|
|
115
|
+
('events', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.event')),
|
|
116
|
+
('examination_indications', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.examinationindication')),
|
|
117
|
+
('examinations', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.examination')),
|
|
118
|
+
('finding_interventions', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.findingintervention')),
|
|
119
|
+
('finding_location_classification_choices', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.findinglocationclassificationchoice')),
|
|
120
|
+
('finding_morphology_classification_choices', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.findingmorphologyclassificationchoice')),
|
|
121
|
+
('findings', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.finding')),
|
|
122
|
+
('lab_values', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.labvalue')),
|
|
123
|
+
('unit', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='required_in', to='endoreg_db.unit')),
|
|
124
|
+
('operators', models.ManyToManyField(blank=True, related_name='required_in', to='endoreg_db.requirementoperator')),
|
|
125
|
+
('requirement_types', models.ManyToManyField(blank=True, related_name='linked_requirements', to='endoreg_db.requirementtype')),
|
|
126
|
+
],
|
|
127
|
+
),
|
|
128
|
+
migrations.CreateModel(
|
|
129
|
+
name='RequirementSet',
|
|
130
|
+
fields=[
|
|
131
|
+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
132
|
+
('name', models.CharField(max_length=100, unique=True)),
|
|
133
|
+
('name_de', models.CharField(blank=True, max_length=100, null=True)),
|
|
134
|
+
('name_en', models.CharField(blank=True, max_length=100, null=True)),
|
|
135
|
+
('description', models.TextField(blank=True, null=True)),
|
|
136
|
+
('information_sources', models.ManyToManyField(blank=True, related_name='requirement_sets', to='endoreg_db.informationsource')),
|
|
137
|
+
('links_to_sets', models.ManyToManyField(blank=True, related_name='links_from_sets', to='endoreg_db.requirementset')),
|
|
138
|
+
('requirements', models.ManyToManyField(blank=True, related_name='requirement_sets', to='endoreg_db.requirement')),
|
|
139
|
+
('requirement_set_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='requirement_sets', to='endoreg_db.requirementsettype')),
|
|
140
|
+
],
|
|
141
|
+
),
|
|
142
|
+
migrations.CreateModel(
|
|
143
|
+
name='Risk',
|
|
144
|
+
fields=[
|
|
145
|
+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
146
|
+
('name', models.CharField(max_length=100, unique=True)),
|
|
147
|
+
('name_de', models.CharField(blank=True, max_length=100, null=True)),
|
|
148
|
+
('name_en', models.CharField(blank=True, max_length=100, null=True)),
|
|
149
|
+
('description', models.TextField(blank=True, null=True)),
|
|
150
|
+
('risk_value', models.FloatField(blank=True, help_text='Risk value for the risk. If not set, the risk is not used in calculations.', null=True)),
|
|
151
|
+
('risk_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='risks', to='endoreg_db.risktype')),
|
|
152
|
+
],
|
|
153
|
+
),
|
|
154
|
+
]
|
endoreg_db/models/__init__.py
CHANGED
|
@@ -97,6 +97,19 @@ from .case_template import (
|
|
|
97
97
|
CaseTemplateRuleValueType,
|
|
98
98
|
)
|
|
99
99
|
|
|
100
|
+
from .risk import (
|
|
101
|
+
Risk,
|
|
102
|
+
RiskType,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
from .requirement import (
|
|
106
|
+
Requirement,
|
|
107
|
+
RequirementSet,
|
|
108
|
+
RequirementSetType,
|
|
109
|
+
RequirementOperator,
|
|
110
|
+
RequirementType,
|
|
111
|
+
)
|
|
112
|
+
|
|
100
113
|
# Rules
|
|
101
114
|
from .rules import (
|
|
102
115
|
Rule,
|
|
@@ -373,4 +386,11 @@ __all__ = [
|
|
|
373
386
|
"RawPdfFile",
|
|
374
387
|
"RawVideoFile",
|
|
375
388
|
"VideoSegmentationLabelSet",
|
|
389
|
+
"RequirementSet",
|
|
390
|
+
"RequirementSetType",
|
|
391
|
+
"Requirement",
|
|
392
|
+
"RequirementOperator",
|
|
393
|
+
"RequirementType",
|
|
394
|
+
"Risk",
|
|
395
|
+
"RiskType",
|
|
376
396
|
]
|
|
@@ -1,22 +1,9 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Django model for AI models.
|
|
3
3
|
"""
|
|
4
|
-
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
import os
|
|
7
4
|
from django.db import models
|
|
8
5
|
from icecream import ic
|
|
9
6
|
|
|
10
|
-
PSEUDO_DIR = Path(os.environ.get("DJANGO_PSEUDO_DIR", Path("./erc_data"))).expanduser()
|
|
11
|
-
|
|
12
|
-
STORAGE_LOCATION = PSEUDO_DIR
|
|
13
|
-
WEIGHTS_DIR_NAME = "db_model_weights"
|
|
14
|
-
WEIGHTS_DIR = STORAGE_LOCATION / WEIGHTS_DIR_NAME
|
|
15
|
-
|
|
16
|
-
if not WEIGHTS_DIR.exists():
|
|
17
|
-
WEIGHTS_DIR.mkdir(parents=True)
|
|
18
|
-
|
|
19
|
-
|
|
20
7
|
class AiModelManager(models.Manager):
|
|
21
8
|
"""
|
|
22
9
|
Manager for AI models with custom query methods.
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
This module defines the ModelMeta and ModelMetaManager classes for managing AI model metadata.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
import os
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from typing import Optional, TYPE_CHECKING
|
|
8
7
|
import shutil
|
|
@@ -10,20 +9,11 @@ import shutil
|
|
|
10
9
|
from django.db import models
|
|
11
10
|
from django.core.validators import FileExtensionValidator
|
|
12
11
|
from icecream import ic
|
|
12
|
+
from ...utils import WEIGHTS_DIR
|
|
13
13
|
|
|
14
14
|
if TYPE_CHECKING:
|
|
15
15
|
from endoreg_db.models import LabelSet, AiModel # pylint: disable=import-outside-toplevel
|
|
16
16
|
|
|
17
|
-
PSEUDO_DIR = Path(os.environ.get("DJANGO_PSEUDO_DIR", Path("./erc_data"))).expanduser()
|
|
18
|
-
|
|
19
|
-
STORAGE_LOCATION = PSEUDO_DIR
|
|
20
|
-
WEIGHTS_DIR_NAME = "db_model_weights"
|
|
21
|
-
WEIGHTS_DIR = STORAGE_LOCATION / WEIGHTS_DIR_NAME
|
|
22
|
-
|
|
23
|
-
if not WEIGHTS_DIR.exists():
|
|
24
|
-
WEIGHTS_DIR.mkdir(parents=True)
|
|
25
|
-
|
|
26
|
-
|
|
27
17
|
class ModelMetaManager(models.Manager):
|
|
28
18
|
"""
|
|
29
19
|
Custom manager for ModelMeta with additional query methods.
|
|
@@ -57,7 +47,7 @@ class ModelMeta(models.Model):
|
|
|
57
47
|
activation = models.CharField(max_length=255, default="sigmoid")
|
|
58
48
|
|
|
59
49
|
weights = models.FileField(
|
|
60
|
-
upload_to=
|
|
50
|
+
upload_to=WEIGHTS_DIR.name,
|
|
61
51
|
validators=[FileExtensionValidator(allowed_extensions=["ckpt"])],
|
|
62
52
|
# storage=FileSystemStorage(location=STORAGE_LOCATION.resolve().as_posix()),
|
|
63
53
|
null=True,
|
|
@@ -5,12 +5,10 @@ from endoreg_db.models.annotation.image_classification import (
|
|
|
5
5
|
from endoreg_db.models.label.label import Label
|
|
6
6
|
|
|
7
7
|
from django.db import models
|
|
8
|
-
from .utils import FRAME_DIR_NAME
|
|
9
8
|
|
|
10
9
|
if TYPE_CHECKING:
|
|
11
10
|
from endoreg_db.models import Video, RawVideoFile
|
|
12
11
|
|
|
13
|
-
|
|
14
12
|
class AbstractFrame(models.Model):
|
|
15
13
|
frame_number = models.IntegerField()
|
|
16
14
|
# Add any other fields you need to store frame-related information
|
|
@@ -26,21 +26,12 @@ import logging
|
|
|
26
26
|
from pathlib import Path
|
|
27
27
|
from typing import TYPE_CHECKING
|
|
28
28
|
|
|
29
|
-
from ..base_classes.utils import (
|
|
30
|
-
STORAGE_LOCATION,
|
|
31
|
-
)
|
|
32
29
|
|
|
33
30
|
if TYPE_CHECKING:
|
|
34
31
|
from endoreg_db.models import SensitiveMeta
|
|
35
32
|
|
|
36
33
|
logger = logging.getLogger("pdf_import")
|
|
37
34
|
|
|
38
|
-
RAW_PDF_DIR_NAME = "raw_pdf"
|
|
39
|
-
RAW_PDF_DIR = STORAGE_LOCATION / RAW_PDF_DIR_NAME
|
|
40
|
-
|
|
41
|
-
if not RAW_PDF_DIR.exists():
|
|
42
|
-
RAW_PDF_DIR.mkdir(parents=True)
|
|
43
|
-
|
|
44
35
|
|
|
45
36
|
class AbstractPdfFile(models.Model):
|
|
46
37
|
pdf_hash = models.CharField(max_length=255, unique=True)
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
""" """
|
|
2
2
|
|
|
3
|
+
from django.db import models
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
from collections import defaultdict, Counter
|
|
5
6
|
import shutil
|
|
6
7
|
import os
|
|
7
|
-
import subprocess
|
|
8
8
|
from typing import Optional, List, TYPE_CHECKING, Union
|
|
9
|
-
from django.db import models, transaction
|
|
10
9
|
from icecream import ic
|
|
11
|
-
from tqdm import tqdm
|
|
12
10
|
from endoreg_db.utils.hashs import get_video_hash
|
|
13
11
|
from endoreg_db.utils.file_operations import get_uuid_filename
|
|
14
12
|
from endoreg_db.utils.ocr import extract_text_from_rois
|
|
@@ -21,12 +19,11 @@ from ....utils.video import (
|
|
|
21
19
|
)
|
|
22
20
|
|
|
23
21
|
from ..metadata import VideoMeta, SensitiveMeta
|
|
24
|
-
from
|
|
25
|
-
|
|
22
|
+
from ....utils import (
|
|
23
|
+
STORAGE_DIR,
|
|
26
24
|
VIDEO_DIR,
|
|
27
25
|
FRAME_DIR,
|
|
28
26
|
)
|
|
29
|
-
from .prepare_bulk_frames import prepare_bulk_frames
|
|
30
27
|
|
|
31
28
|
if TYPE_CHECKING:
|
|
32
29
|
from endoreg_db.models import (
|
|
@@ -180,10 +177,10 @@ class AbstractVideoFile(models.Model):
|
|
|
180
177
|
ic(f"No existing DB entry found, creating new with UUID {uuid}")
|
|
181
178
|
|
|
182
179
|
try:
|
|
183
|
-
relative_path = transcoded_file_path.relative_to(
|
|
180
|
+
relative_path = transcoded_file_path.relative_to(STORAGE_DIR)
|
|
184
181
|
except ValueError as e:
|
|
185
182
|
raise Exception(
|
|
186
|
-
f"{transcoded_file_path} is outside
|
|
183
|
+
f"{transcoded_file_path} is outside STORAGE_DIR {STORAGE_DIR}"
|
|
187
184
|
) from e
|
|
188
185
|
|
|
189
186
|
video = cls(
|
|
@@ -325,6 +322,8 @@ class AbstractVideoFile(models.Model):
|
|
|
325
322
|
ModelMeta,
|
|
326
323
|
AiModel,
|
|
327
324
|
) # pylint: disable=import-outside-toplevel
|
|
325
|
+
#TODO Create issue to movie this function to the endo-ai module
|
|
326
|
+
#endoreg-db is our "base" module, so it should not depend "upstream" models
|
|
328
327
|
from endo_ai.predictor.inference_dataset import InferenceDataset # pylint: disable=import-outside-toplevel
|
|
329
328
|
from endo_ai.predictor.model_loader import MultiLabelClassificationNet
|
|
330
329
|
from endo_ai.predictor.predict import Classifier
|
|
@@ -7,28 +7,6 @@ from pathlib import Path
|
|
|
7
7
|
import cv2
|
|
8
8
|
import numpy as np
|
|
9
9
|
|
|
10
|
-
DJANGO_NAME_SALT = os.environ.get("DJANGO_NAME_SALT", "default_salt")
|
|
11
|
-
|
|
12
|
-
# Directory stuff
|
|
13
|
-
PSEUDO_DIR = Path(os.environ.get("DJANGO_PSEUDO_DIR", Path("./erc_data"))).expanduser()
|
|
14
|
-
STORAGE_LOCATION = PSEUDO_DIR
|
|
15
|
-
FRAME_DIR_NAME = os.environ.get("DJANGO_FRAME_DIR_NAME", "db_frames")
|
|
16
|
-
RAW_FRAME_DIR_NAME = os.environ.get("DJANGO_RAW_FRAME_DIR_NAME", "db_raw_frames")
|
|
17
|
-
VIDEO_DIR_NAME = os.environ.get("DJANGO_VIDEO_DIR_NAME", "db_videos")
|
|
18
|
-
RAW_VIDEO_DIR_NAME = os.environ.get("DJANGO_RAW_VIDEO_DIR_NAME", "db_raw_videos")
|
|
19
|
-
|
|
20
|
-
FRAME_DIR = STORAGE_LOCATION / FRAME_DIR_NAME
|
|
21
|
-
VIDEO_DIR = STORAGE_LOCATION / VIDEO_DIR_NAME
|
|
22
|
-
RAW_VIDEO_DIR = STORAGE_LOCATION / RAW_VIDEO_DIR_NAME
|
|
23
|
-
|
|
24
|
-
TEST_RUN = os.environ.get("TEST_RUN", False)
|
|
25
|
-
TEST_RUN_FRAME_NUMBER = os.environ.get("TEST_RUN_FRAME_NUMBER", 1000)
|
|
26
|
-
|
|
27
|
-
VIDEO_DIR.mkdir(parents=True, exist_ok=True)
|
|
28
|
-
RAW_VIDEO_DIR.mkdir(parents=True, exist_ok=True)
|
|
29
|
-
|
|
30
|
-
# AI Stuff
|
|
31
|
-
FRAME_PROCESSING_BATCH_SIZE = os.environ.get("DJANGO_FRAME_PROCESSING_BATCH_SIZE", 10)
|
|
32
10
|
|
|
33
11
|
|
|
34
12
|
def anonymize_frame(
|
|
@@ -4,7 +4,7 @@ from endoreg_db.models.label.label import Label
|
|
|
4
4
|
from .base_classes import AbstractFrame
|
|
5
5
|
from django.db import models
|
|
6
6
|
|
|
7
|
-
from
|
|
7
|
+
from ...utils import FRAME_DIR_NAME, RAW_FRAME_DIR_NAME
|
|
8
8
|
|
|
9
9
|
if TYPE_CHECKING:
|
|
10
10
|
from endoreg_db.models import RawVideoFile, Video
|
|
@@ -7,7 +7,6 @@
|
|
|
7
7
|
|
|
8
8
|
from django.db import models
|
|
9
9
|
from django.core.files.storage import FileSystemStorage
|
|
10
|
-
from django.conf import settings
|
|
11
10
|
from django.core.exceptions import ValidationError
|
|
12
11
|
from django.core.validators import FileExtensionValidator
|
|
13
12
|
from endoreg_db.utils.file_operations import get_uuid_filename
|
|
@@ -25,24 +24,19 @@ import logging
|
|
|
25
24
|
import shutil
|
|
26
25
|
from pathlib import Path
|
|
27
26
|
|
|
28
|
-
from
|
|
29
|
-
|
|
27
|
+
from ....utils import (
|
|
28
|
+
STORAGE_DIR,
|
|
29
|
+
RAW_PDF_DIR_NAME,
|
|
30
|
+
RAW_PDF_DIR,
|
|
30
31
|
)
|
|
31
32
|
|
|
32
33
|
logger = logging.getLogger("pdf_import")
|
|
33
34
|
|
|
34
|
-
RAW_PDF_DIR_NAME = "raw_pdf"
|
|
35
|
-
RAW_PDF_DIR = STORAGE_LOCATION / RAW_PDF_DIR_NAME
|
|
36
|
-
|
|
37
|
-
if not RAW_PDF_DIR.exists():
|
|
38
|
-
RAW_PDF_DIR.mkdir(parents=True)
|
|
39
|
-
|
|
40
|
-
|
|
41
35
|
class RawPdfFile(AbstractPdfFile):
|
|
42
36
|
file = models.FileField(
|
|
43
37
|
upload_to=f"{RAW_PDF_DIR_NAME}/",
|
|
44
38
|
validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
|
|
45
|
-
storage=FileSystemStorage(location=
|
|
39
|
+
storage=FileSystemStorage(location=STORAGE_DIR.resolve().as_posix()),
|
|
46
40
|
)
|
|
47
41
|
|
|
48
42
|
patient = models.ForeignKey(
|
|
@@ -13,12 +13,14 @@ from django.core.files.storage import FileSystemStorage
|
|
|
13
13
|
from endoreg_db.utils.validate_endo_roi import validate_endo_roi
|
|
14
14
|
from ..base_classes.utils import (
|
|
15
15
|
anonymize_frame,
|
|
16
|
-
RAW_VIDEO_DIR_NAME,
|
|
17
|
-
VIDEO_DIR,
|
|
18
|
-
STORAGE_LOCATION,
|
|
19
16
|
)
|
|
20
17
|
from ..base_classes.abstract_video import AbstractVideoFile
|
|
18
|
+
from ....utils import (
|
|
21
19
|
|
|
20
|
+
RAW_VIDEO_DIR_NAME,
|
|
21
|
+
VIDEO_DIR,
|
|
22
|
+
STORAGE_DIR,
|
|
23
|
+
)
|
|
22
24
|
if TYPE_CHECKING:
|
|
23
25
|
# import Queryset
|
|
24
26
|
from django.db.models import QuerySet
|
|
@@ -35,7 +37,7 @@ class RawVideoFile(AbstractVideoFile):
|
|
|
35
37
|
file = models.FileField(
|
|
36
38
|
upload_to=RAW_VIDEO_DIR_NAME,
|
|
37
39
|
validators=[FileExtensionValidator(allowed_extensions=["mp4"])], # FIXME
|
|
38
|
-
storage=FileSystemStorage(location=
|
|
40
|
+
storage=FileSystemStorage(location=STORAGE_DIR.resolve().as_posix()),
|
|
39
41
|
)
|
|
40
42
|
|
|
41
43
|
patient = models.ForeignKey(
|
|
@@ -5,9 +5,9 @@ from django.core.files.storage import FileSystemStorage
|
|
|
5
5
|
from typing import TYPE_CHECKING
|
|
6
6
|
import cv2
|
|
7
7
|
from ..base_classes import AbstractVideoFile
|
|
8
|
-
from
|
|
8
|
+
from ....utils import (
|
|
9
9
|
VIDEO_DIR_NAME,
|
|
10
|
-
|
|
10
|
+
STORAGE_DIR,
|
|
11
11
|
FRAME_PROCESSING_BATCH_SIZE,
|
|
12
12
|
)
|
|
13
13
|
|
|
@@ -27,7 +27,7 @@ class Video(AbstractVideoFile):
|
|
|
27
27
|
file = models.FileField(
|
|
28
28
|
upload_to=VIDEO_DIR_NAME,
|
|
29
29
|
validators=[FileExtensionValidator(allowed_extensions=["mp4"])], # FIXME
|
|
30
|
-
storage=FileSystemStorage(location=
|
|
30
|
+
storage=FileSystemStorage(location=STORAGE_DIR.resolve().as_posix()),
|
|
31
31
|
)
|
|
32
32
|
|
|
33
33
|
patient = models.ForeignKey(
|
endoreg_db/models/disease.py
CHANGED
|
@@ -1,10 +1,21 @@
|
|
|
1
1
|
from django.db import models
|
|
2
2
|
from typing import List
|
|
3
3
|
|
|
4
|
+
|
|
4
5
|
class DiseaseManager(models.Manager):
|
|
5
6
|
def get_by_natural_key(self, name):
|
|
7
|
+
"""
|
|
8
|
+
Retrieve a model instance by its natural key.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
name: The natural key value used to match the model's 'name' field.
|
|
12
|
+
|
|
13
|
+
Returns:
|
|
14
|
+
The model instance corresponding to the provided natural key.
|
|
15
|
+
"""
|
|
6
16
|
return self.get(name=name)
|
|
7
17
|
|
|
18
|
+
|
|
8
19
|
class Disease(models.Model):
|
|
9
20
|
name = models.CharField(max_length=255, unique=True)
|
|
10
21
|
name_de = models.CharField(max_length=255, blank=True, null=True)
|
|
@@ -15,65 +26,123 @@ class Disease(models.Model):
|
|
|
15
26
|
objects = DiseaseManager()
|
|
16
27
|
|
|
17
28
|
def natural_key(self):
|
|
29
|
+
"""
|
|
30
|
+
Return the natural key for this model instance.
|
|
31
|
+
|
|
32
|
+
The natural key is defined as a tuple containing the instance's name.
|
|
33
|
+
"""
|
|
18
34
|
return (self.name,)
|
|
19
35
|
|
|
20
36
|
def __str__(self):
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
37
|
+
"""
|
|
38
|
+
Return the string representation of the instance using its name attribute.
|
|
39
|
+
"""
|
|
40
|
+
return str(self.name)
|
|
41
|
+
|
|
42
|
+
def get_classifications(self) -> List["DiseaseClassification"]:
|
|
43
|
+
"""
|
|
44
|
+
Retrieves all classifications associated with this disease.
|
|
25
45
|
|
|
46
|
+
Returns:
|
|
47
|
+
List[DiseaseClassification]: A list of related disease classification objects.
|
|
48
|
+
"""
|
|
49
|
+
classifications: List[DiseaseClassification] = [
|
|
50
|
+
_ for _ in self.disease_classifications.all()
|
|
51
|
+
]
|
|
52
|
+
return classifications
|
|
26
53
|
|
|
27
54
|
|
|
28
|
-
|
|
29
|
-
def get_classifications(self)->List['DiseaseClassification']:
|
|
30
|
-
classifications: List[DiseaseClassification] = [_ for _ in self.disease_classifications.all()]
|
|
31
|
-
return classifications
|
|
32
|
-
|
|
33
55
|
class DiseaseClassificationManager(models.Manager):
|
|
34
56
|
def get_by_natural_key(self, name):
|
|
57
|
+
"""
|
|
58
|
+
Retrieves a model instance using its natural key.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
name: A unique identifier representing the natural key of the instance.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
The model instance corresponding to the given natural key.
|
|
65
|
+
"""
|
|
35
66
|
return self.get(name=name)
|
|
36
67
|
|
|
68
|
+
|
|
37
69
|
class DiseaseClassification(models.Model):
|
|
38
70
|
name = models.CharField(max_length=255, unique=True)
|
|
39
71
|
name_de = models.CharField(max_length=255, blank=True, null=True)
|
|
40
72
|
name_en = models.CharField(max_length=255, blank=True, null=True)
|
|
41
73
|
|
|
42
74
|
disease = models.ForeignKey(
|
|
43
|
-
Disease, on_delete=models.CASCADE,
|
|
44
|
-
related_name='disease_classifications'
|
|
75
|
+
Disease, on_delete=models.CASCADE, related_name="disease_classifications"
|
|
45
76
|
)
|
|
46
77
|
|
|
47
78
|
objects = DiseaseClassificationManager()
|
|
48
79
|
|
|
49
80
|
def natural_key(self):
|
|
81
|
+
"""
|
|
82
|
+
Return the natural key for the instance.
|
|
83
|
+
|
|
84
|
+
Returns a single-element tuple containing the object's name, which is used as a
|
|
85
|
+
unique identifier for natural key-based lookups.
|
|
86
|
+
"""
|
|
50
87
|
return (self.name,)
|
|
51
88
|
|
|
52
89
|
def __str__(self):
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
90
|
+
"""
|
|
91
|
+
Return the string representation of the model instance.
|
|
92
|
+
|
|
93
|
+
This method converts the instance's name attribute to a string, ensuring a clear
|
|
94
|
+
and consistent display across Django interfaces.
|
|
95
|
+
"""
|
|
96
|
+
return str(self.name)
|
|
97
|
+
|
|
98
|
+
def get_choices(self) -> List["DiseaseClassificationChoice"]:
|
|
99
|
+
"""
|
|
100
|
+
Retrieves all choices associated with this disease classification.
|
|
101
|
+
|
|
102
|
+
Returns:
|
|
103
|
+
List[DiseaseClassificationChoice]: A list of related disease classification choices.
|
|
104
|
+
"""
|
|
105
|
+
choices: List[DiseaseClassificationChoice] = [
|
|
106
|
+
_ for _ in self.disease_classification_choices.all()
|
|
107
|
+
]
|
|
57
108
|
return choices
|
|
58
|
-
|
|
109
|
+
|
|
110
|
+
|
|
59
111
|
class DiseaseClassificationChoiceManager(models.Manager):
|
|
60
112
|
def get_by_natural_key(self, name):
|
|
113
|
+
"""Retrieve an object by its natural key.
|
|
114
|
+
|
|
115
|
+
Queries for and returns the instance whose 'name' attribute matches the provided key.
|
|
116
|
+
"""
|
|
61
117
|
return self.get(name=name)
|
|
62
118
|
|
|
119
|
+
|
|
63
120
|
class DiseaseClassificationChoice(models.Model):
|
|
64
121
|
name = models.CharField(max_length=255, unique=True)
|
|
65
122
|
name_de = models.CharField(max_length=255, blank=True, null=True)
|
|
66
123
|
name_en = models.CharField(max_length=255, blank=True, null=True)
|
|
67
124
|
|
|
68
125
|
disease_classification = models.ForeignKey(
|
|
69
|
-
DiseaseClassification,
|
|
70
|
-
|
|
126
|
+
DiseaseClassification,
|
|
127
|
+
on_delete=models.CASCADE,
|
|
128
|
+
related_name="disease_classification_choices",
|
|
71
129
|
)
|
|
72
130
|
|
|
73
131
|
objects = DiseaseClassificationChoiceManager()
|
|
74
132
|
|
|
75
133
|
def natural_key(self):
|
|
134
|
+
"""
|
|
135
|
+
Return a tuple representing the natural key for the instance.
|
|
136
|
+
|
|
137
|
+
The tuple contains the unique name of the model instance, which is used to
|
|
138
|
+
identify it naturally.
|
|
139
|
+
"""
|
|
76
140
|
return (self.name,)
|
|
77
141
|
|
|
78
142
|
def __str__(self):
|
|
79
|
-
|
|
143
|
+
"""
|
|
144
|
+
Return the string representation of the object's name.
|
|
145
|
+
|
|
146
|
+
This method converts the model's 'name' attribute to a string for a human-readable display.
|
|
147
|
+
"""
|
|
148
|
+
return str(self.name)
|