endoreg-db 0.8.4.2__py3-none-any.whl → 0.8.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of endoreg-db might be problematic. Click here for more details.
- endoreg_db/data/ai_model_meta/default_multilabel_classification.yaml +23 -1
- endoreg_db/data/setup_config.yaml +38 -0
- endoreg_db/management/commands/load_ai_model_data.py +17 -15
- endoreg_db/management/commands/setup_endoreg_db.py +207 -39
- endoreg_db/models/medical/hardware/endoscopy_processor.py +10 -1
- endoreg_db/models/metadata/model_meta_logic.py +20 -52
- endoreg_db/models/metadata/sensitive_meta_logic.py +48 -143
- endoreg_db/services/video_import.py +173 -175
- endoreg_db/utils/setup_config.py +177 -0
- {endoreg_db-0.8.4.2.dist-info → endoreg_db-0.8.4.4.dist-info}/METADATA +1 -2
- {endoreg_db-0.8.4.2.dist-info → endoreg_db-0.8.4.4.dist-info}/RECORD +13 -11
- {endoreg_db-0.8.4.2.dist-info → endoreg_db-0.8.4.4.dist-info}/WHEEL +0 -0
- {endoreg_db-0.8.4.2.dist-info → endoreg_db-0.8.4.4.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,5 +1,27 @@
|
|
|
1
|
+
# Model metadata configuration with setup hints
|
|
1
2
|
- model: endoreg_db.model_meta
|
|
2
3
|
fields:
|
|
3
4
|
name: "image_multilabel_classification_colonoscopy_default"
|
|
4
|
-
version: 1
|
|
5
|
+
version: "1"
|
|
6
|
+
model: "image_multilabel_classification_colonoscopy_default"
|
|
5
7
|
labelset: "multilabel_classification_colonoscopy_default"
|
|
8
|
+
activation: "sigmoid"
|
|
9
|
+
mean: "0.485,0.456,0.406" # ImageNet defaults
|
|
10
|
+
std: "0.229,0.224,0.225" # ImageNet defaults
|
|
11
|
+
size_x: 224
|
|
12
|
+
size_y: 224
|
|
13
|
+
axes: "CHW"
|
|
14
|
+
batchsize: 32
|
|
15
|
+
num_workers: 4
|
|
16
|
+
description: "Default colonoscopy image multilabel classification model from YAML"
|
|
17
|
+
# Note: weights field intentionally left empty - will be set by setup command or model meta logic
|
|
18
|
+
|
|
19
|
+
# Setup configuration hints for this model
|
|
20
|
+
setup_config:
|
|
21
|
+
is_primary_model: true # Mark this as the primary model for setup
|
|
22
|
+
weight_filenames: # Specific weight file patterns for this model
|
|
23
|
+
- "colo_segmentation_RegNetX800MF_6.ckpt"
|
|
24
|
+
- "image_multilabel_classification_colonoscopy_default_v1_*.ckpt"
|
|
25
|
+
huggingface_fallback: # HF configuration for this model
|
|
26
|
+
repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
|
|
27
|
+
filename: "colo_segmentation_RegNetX800MF_base.ckpt"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# EndoReg DB Setup Configuration
|
|
2
|
+
# This file defines which models should be used for setup and their fallback configurations
|
|
3
|
+
|
|
4
|
+
default_models:
|
|
5
|
+
primary_classification_model: "image_multilabel_classification_colonoscopy_default"
|
|
6
|
+
primary_labelset: "multilabel_classification_colonoscopy_default"
|
|
7
|
+
|
|
8
|
+
# HuggingFace fallback configuration
|
|
9
|
+
huggingface_fallback:
|
|
10
|
+
enabled: true
|
|
11
|
+
repo_id: "wg-lux/colo_segmentation_RegNetX800MF_base"
|
|
12
|
+
filename: "colo_segmentation_RegNetX800MF_base.ckpt"
|
|
13
|
+
labelset_name: "multilabel_classification_colonoscopy_default"
|
|
14
|
+
|
|
15
|
+
# Local weights search patterns
|
|
16
|
+
weights_search_patterns:
|
|
17
|
+
# Primary model weight filename patterns (supports wildcards)
|
|
18
|
+
- "colo_segmentation_RegNetX800MF_*.ckpt"
|
|
19
|
+
- "image_multilabel_classification_colonoscopy_default_*.ckpt"
|
|
20
|
+
- "*_colonoscopy_*.ckpt"
|
|
21
|
+
|
|
22
|
+
# Search directories for local weights (relative to project root)
|
|
23
|
+
weights_search_dirs:
|
|
24
|
+
- "tests/assets"
|
|
25
|
+
- "assets"
|
|
26
|
+
- "data/storage/model_weights"
|
|
27
|
+
- "${STORAGE_DIR}/model_weights" # Environment variable substitution
|
|
28
|
+
|
|
29
|
+
# Auto-generation defaults when creating metadata
|
|
30
|
+
auto_generation_defaults:
|
|
31
|
+
activation: "sigmoid"
|
|
32
|
+
mean: "0.485,0.456,0.406" # ImageNet defaults
|
|
33
|
+
std: "0.229,0.224,0.225" # ImageNet defaults
|
|
34
|
+
size_x: 224
|
|
35
|
+
size_y: 224
|
|
36
|
+
axes: "CHW"
|
|
37
|
+
batchsize: 32
|
|
38
|
+
num_workers: 4
|
|
@@ -1,25 +1,27 @@
|
|
|
1
1
|
from django.core.management.base import BaseCommand
|
|
2
|
+
|
|
3
|
+
from ...data import (
|
|
4
|
+
AI_MODEL_DATA_DIR,
|
|
5
|
+
AI_MODEL_META_DATA_DIR, # Add this import
|
|
6
|
+
MODEL_TYPE_DATA_DIR,
|
|
7
|
+
VIDEO_SEGMENTATION_LABEL_DATA_DIR,
|
|
8
|
+
VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
|
|
9
|
+
)
|
|
2
10
|
from ...models import (
|
|
11
|
+
AiModel,
|
|
12
|
+
ModelMeta, # Add ModelMeta back to imports
|
|
3
13
|
ModelType,
|
|
4
14
|
VideoSegmentationLabel,
|
|
5
15
|
VideoSegmentationLabelSet,
|
|
6
|
-
AiModel,
|
|
7
16
|
)
|
|
8
17
|
from ...utils import load_model_data_from_yaml
|
|
9
|
-
from ...data import (
|
|
10
|
-
MODEL_TYPE_DATA_DIR,
|
|
11
|
-
VIDEO_SEGMENTATION_LABEL_DATA_DIR,
|
|
12
|
-
VIDEO_SEGMENTATION_LABELSET_DATA_DIR,
|
|
13
|
-
AI_MODEL_DATA_DIR,
|
|
14
|
-
)
|
|
15
|
-
|
|
16
18
|
|
|
17
19
|
IMPORT_MODELS = [ # string as model key, serves as key in IMPORT_METADATA
|
|
18
20
|
ModelType.__name__,
|
|
19
|
-
# ModelMeta.__name__,
|
|
20
21
|
VideoSegmentationLabel.__name__,
|
|
21
22
|
VideoSegmentationLabelSet.__name__,
|
|
22
23
|
AiModel.__name__,
|
|
24
|
+
ModelMeta.__name__, # Re-enable ModelMeta loading
|
|
23
25
|
]
|
|
24
26
|
|
|
25
27
|
IMPORT_METADATA = {
|
|
@@ -29,12 +31,12 @@ IMPORT_METADATA = {
|
|
|
29
31
|
"foreign_keys": [], # e.g. ["interventions"]
|
|
30
32
|
"foreign_key_models": [], # e.g. [Intervention]
|
|
31
33
|
},
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
ModelMeta.__name__: {
|
|
35
|
+
"dir": AI_MODEL_META_DATA_DIR, # e.g. "ai_model_meta"
|
|
36
|
+
"model": ModelMeta, # e.g. ModelMeta
|
|
37
|
+
"foreign_keys": ["labelset", "model"], # Foreign key relationships
|
|
38
|
+
"foreign_key_models": [VideoSegmentationLabelSet, AiModel], # Actual model classes
|
|
39
|
+
},
|
|
38
40
|
VideoSegmentationLabel.__name__: {
|
|
39
41
|
"dir": VIDEO_SEGMENTATION_LABEL_DATA_DIR, # e.g. "interventions"
|
|
40
42
|
"model": VideoSegmentationLabel, # e.g. Intervention
|
|
@@ -3,21 +3,21 @@ Django management command to perform complete setup for EndoReg DB when used as
|
|
|
3
3
|
This command ensures all necessary data and configurations are initialized.
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
-
import os
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
6
|
from django.core.management import call_command
|
|
10
7
|
from django.core.management.base import BaseCommand
|
|
8
|
+
|
|
11
9
|
from endoreg_db.models import ModelMeta
|
|
12
10
|
|
|
11
|
+
|
|
13
12
|
class Command(BaseCommand):
|
|
14
13
|
help = """
|
|
15
14
|
Complete setup for EndoReg DB when used as an embedded app.
|
|
16
15
|
This command performs all necessary initialization steps:
|
|
17
16
|
1. Loads base database data
|
|
18
|
-
2. Sets up
|
|
19
|
-
3.
|
|
20
|
-
4.
|
|
17
|
+
2. Sets up caching (if using db cache)
|
|
18
|
+
3. Loads default models from setup configuration file (setup_config.yaml)
|
|
19
|
+
4. Loads models according to fallback chain (Local Files -> HuggingFace -> graceful failure)
|
|
20
|
+
5. Initializes model metadata
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
23
|
def add_arguments(self, parser):
|
|
@@ -31,13 +31,22 @@ class Command(BaseCommand):
|
|
|
31
31
|
action="store_true",
|
|
32
32
|
help="Force recreation of AI model metadata even if it exists",
|
|
33
33
|
)
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"--yaml-only",
|
|
36
|
+
action="store_true",
|
|
37
|
+
help="Only use YAML-defined models, don't auto-generate missing metadata",
|
|
38
|
+
)
|
|
34
39
|
|
|
35
40
|
def handle(self, *args, **options):
|
|
36
41
|
skip_ai = options.get("skip_ai_setup", False)
|
|
37
42
|
force_recreate = options.get("force_recreate", False)
|
|
43
|
+
yaml_only = options.get("yaml_only", False)
|
|
38
44
|
|
|
39
45
|
self.stdout.write(self.style.SUCCESS("🚀 Starting EndoReg DB embedded app setup..."))
|
|
40
46
|
|
|
47
|
+
if yaml_only:
|
|
48
|
+
self.stdout.write(self.style.WARNING("📋 YAML-only mode: Will not auto-generate missing metadata"))
|
|
49
|
+
|
|
41
50
|
# Step 1: Load base database data
|
|
42
51
|
self.stdout.write("\n📊 Step 1: Loading base database data...")
|
|
43
52
|
try:
|
|
@@ -47,7 +56,7 @@ class Command(BaseCommand):
|
|
|
47
56
|
self.stdout.write(self.style.ERROR(f"❌ Failed to load base data: {e}"))
|
|
48
57
|
return
|
|
49
58
|
|
|
50
|
-
|
|
59
|
+
# Step 2: Create cache table (only if using database caching)
|
|
51
60
|
self.stdout.write("\n💾 Step 2: Setting up caching...")
|
|
52
61
|
from django.conf import settings
|
|
53
62
|
|
|
@@ -87,10 +96,16 @@ class Command(BaseCommand):
|
|
|
87
96
|
# Step 5: Create model metadata
|
|
88
97
|
self.stdout.write("\n📋 Step 5: Creating AI model metadata...")
|
|
89
98
|
try:
|
|
99
|
+
# Load setup configuration
|
|
100
|
+
from endoreg_db.utils.setup_config import setup_config
|
|
101
|
+
|
|
102
|
+
# Get primary model from configuration
|
|
103
|
+
default_model_name = setup_config.get_primary_model_name()
|
|
104
|
+
primary_labelset = setup_config.get_primary_labelset_name()
|
|
105
|
+
|
|
90
106
|
# Check if model metadata already exists
|
|
91
107
|
from endoreg_db.models import AiModel
|
|
92
108
|
|
|
93
|
-
default_model_name = "image_multilabel_classification_colonoscopy_default"
|
|
94
109
|
ai_model = AiModel.objects.filter(name=default_model_name).first()
|
|
95
110
|
|
|
96
111
|
if not ai_model:
|
|
@@ -101,14 +116,14 @@ class Command(BaseCommand):
|
|
|
101
116
|
if existing_meta and not force_recreate:
|
|
102
117
|
self.stdout.write(self.style.SUCCESS("✅ Model metadata already exists (use --force-recreate to recreate)"))
|
|
103
118
|
else:
|
|
104
|
-
# Try to create model metadata
|
|
119
|
+
# Try to create model metadata using configurable approach
|
|
105
120
|
model_path = self._find_model_weights_file()
|
|
106
121
|
if model_path:
|
|
107
122
|
call_command(
|
|
108
123
|
"create_multilabel_model_meta",
|
|
109
124
|
model_name=default_model_name,
|
|
110
125
|
model_meta_version=1,
|
|
111
|
-
image_classification_labelset_name=
|
|
126
|
+
image_classification_labelset_name=primary_labelset,
|
|
112
127
|
model_path=str(model_path),
|
|
113
128
|
)
|
|
114
129
|
self.stdout.write(self.style.SUCCESS("✅ AI model metadata created successfully"))
|
|
@@ -119,6 +134,15 @@ class Command(BaseCommand):
|
|
|
119
134
|
self.stdout.write(self.style.ERROR(f"❌ Failed to create AI model metadata: {e}"))
|
|
120
135
|
return
|
|
121
136
|
|
|
137
|
+
# Step 5.5: Validate and fix AI model active metadata
|
|
138
|
+
self.stdout.write("\n🔧 Step 5.5: Validating AI model active metadata...")
|
|
139
|
+
try:
|
|
140
|
+
self._validate_and_fix_ai_model_metadata(yaml_only)
|
|
141
|
+
self.stdout.write(self.style.SUCCESS("✅ AI model metadata validation completed"))
|
|
142
|
+
except Exception as e:
|
|
143
|
+
self.stdout.write(self.style.ERROR(f"❌ Failed to validate AI model metadata: {e}"))
|
|
144
|
+
return
|
|
145
|
+
|
|
122
146
|
# Step 6: Verification
|
|
123
147
|
self.stdout.write("\n🔍 Step 6: Verifying setup...")
|
|
124
148
|
try:
|
|
@@ -135,37 +159,38 @@ class Command(BaseCommand):
|
|
|
135
159
|
self.stdout.write("3. Start development server: python manage.py runserver")
|
|
136
160
|
|
|
137
161
|
def _find_model_weights_file(self):
|
|
138
|
-
"""Find the model weights file
|
|
139
|
-
#
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
162
|
+
"""Find the model weights file using configurable search patterns and directories."""
|
|
163
|
+
# Load setup configuration
|
|
164
|
+
from endoreg_db.utils.setup_config import setup_config
|
|
165
|
+
|
|
166
|
+
# First try to find weights using configured patterns
|
|
167
|
+
found_files = setup_config.find_model_weights_files()
|
|
168
|
+
if found_files:
|
|
169
|
+
self.stdout.write(f"Found model weights at: {found_files[0]}")
|
|
170
|
+
return found_files[0]
|
|
171
|
+
|
|
172
|
+
# If no local weights found and HuggingFace fallback is enabled
|
|
173
|
+
hf_config = setup_config.get_huggingface_config()
|
|
174
|
+
if hf_config.get("enabled", True):
|
|
175
|
+
self.stdout.write("📦 No local model weights found — attempting HuggingFace download...")
|
|
176
|
+
try:
|
|
177
|
+
if not ModelMeta.objects.exists():
|
|
178
|
+
ModelMeta.setup_default_from_huggingface(
|
|
179
|
+
hf_config.get("repo_id", "wg-lux/colo_segmentation_RegNetX800MF_base"),
|
|
180
|
+
labelset_name=hf_config.get("labelset_name", "multilabel_classification_colonoscopy_default"),
|
|
181
|
+
)
|
|
182
|
+
self.stdout.write("✅ Default ModelMeta created from HuggingFace.")
|
|
183
|
+
|
|
184
|
+
# Try to find the downloaded weights
|
|
185
|
+
found_files = setup_config.find_model_weights_files()
|
|
186
|
+
if found_files:
|
|
187
|
+
return found_files[0]
|
|
158
188
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
self.stdout.write(f"Found model weights at: {path}")
|
|
162
|
-
return path
|
|
189
|
+
except Exception as e:
|
|
190
|
+
self.stdout.write(f"⚠️ HuggingFace download failed: {e}")
|
|
163
191
|
|
|
164
|
-
self.stdout.write("Model weights file not found in
|
|
165
|
-
|
|
192
|
+
self.stdout.write("Model weights file not found in configured locations")
|
|
166
193
|
return None
|
|
167
|
-
|
|
168
|
-
|
|
169
194
|
|
|
170
195
|
def _verify_setup(self):
|
|
171
196
|
"""Verify that the setup was successful."""
|
|
@@ -205,5 +230,148 @@ class Command(BaseCommand):
|
|
|
205
230
|
self.stdout.write(f"Found {meta_count} model metadata record(s)")
|
|
206
231
|
|
|
207
232
|
self.stdout.write("Setup verification passed")
|
|
208
|
-
|
|
209
233
|
|
|
234
|
+
def _validate_and_fix_ai_model_metadata(self, yaml_only=False):
|
|
235
|
+
"""
|
|
236
|
+
Validate that all AI models have proper active metadata and fix if necessary.
|
|
237
|
+
This addresses the "No model metadata found for this model" error.
|
|
238
|
+
|
|
239
|
+
Args:
|
|
240
|
+
yaml_only (bool): If True, only set active metadata but don't create new metadata
|
|
241
|
+
"""
|
|
242
|
+
from endoreg_db.models import AiModel, LabelSet, ModelMeta
|
|
243
|
+
from endoreg_db.utils.setup_config import setup_config
|
|
244
|
+
|
|
245
|
+
all_models = AiModel.objects.all()
|
|
246
|
+
fixed_count = 0
|
|
247
|
+
|
|
248
|
+
# Get configurable defaults
|
|
249
|
+
defaults = setup_config.get_auto_generation_defaults()
|
|
250
|
+
primary_labelset_name = setup_config.get_primary_labelset_name()
|
|
251
|
+
|
|
252
|
+
for model in all_models:
|
|
253
|
+
self.stdout.write(f"Checking model: {model.name}")
|
|
254
|
+
|
|
255
|
+
# Check if model has metadata versions
|
|
256
|
+
metadata_count = model.metadata_versions.count()
|
|
257
|
+
self.stdout.write(f" Metadata versions: {metadata_count}")
|
|
258
|
+
|
|
259
|
+
if metadata_count == 0:
|
|
260
|
+
if yaml_only:
|
|
261
|
+
self.stdout.write(f" ⚠️ YAML-only mode: Skipping auto-generation for {model.name}")
|
|
262
|
+
continue
|
|
263
|
+
|
|
264
|
+
# Create metadata for models that don't have any
|
|
265
|
+
self.stdout.write(f" Creating metadata for {model.name}...")
|
|
266
|
+
|
|
267
|
+
# Use configured labelset or create default
|
|
268
|
+
labelset = None
|
|
269
|
+
try:
|
|
270
|
+
labelset = LabelSet.objects.get(name=primary_labelset_name)
|
|
271
|
+
except LabelSet.DoesNotExist:
|
|
272
|
+
labelset = LabelSet.objects.first()
|
|
273
|
+
if not labelset:
|
|
274
|
+
labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
|
|
275
|
+
|
|
276
|
+
# Create basic metadata WITH weights if available
|
|
277
|
+
weights_file = self._find_model_weights_file()
|
|
278
|
+
weights_path = ""
|
|
279
|
+
if weights_file:
|
|
280
|
+
# If we have weights, set up the relative path
|
|
281
|
+
from pathlib import Path
|
|
282
|
+
|
|
283
|
+
from endoreg_db.utils.paths import STORAGE_DIR
|
|
284
|
+
|
|
285
|
+
try:
|
|
286
|
+
weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
|
|
287
|
+
except ValueError:
|
|
288
|
+
# If file is not in storage dir, copy it there
|
|
289
|
+
import shutil
|
|
290
|
+
|
|
291
|
+
weights_dir = STORAGE_DIR / "model_weights"
|
|
292
|
+
weights_dir.mkdir(parents=True, exist_ok=True)
|
|
293
|
+
dest_path = weights_dir / Path(weights_file).name
|
|
294
|
+
shutil.copy2(weights_file, dest_path)
|
|
295
|
+
weights_path = str(dest_path.relative_to(STORAGE_DIR))
|
|
296
|
+
self.stdout.write(f" Copied weights to: {dest_path}")
|
|
297
|
+
|
|
298
|
+
# Create basic metadata using configurable defaults
|
|
299
|
+
meta = ModelMeta.objects.create(
|
|
300
|
+
name=model.name,
|
|
301
|
+
version="1.0",
|
|
302
|
+
model=model,
|
|
303
|
+
labelset=labelset,
|
|
304
|
+
weights=weights_path, # Set weights if available
|
|
305
|
+
activation=defaults.get("activation", "sigmoid"),
|
|
306
|
+
mean=defaults.get("mean", "0.485,0.456,0.406"),
|
|
307
|
+
std=defaults.get("std", "0.229,0.224,0.225"),
|
|
308
|
+
size_x=defaults.get("size_x", 224),
|
|
309
|
+
size_y=defaults.get("size_y", 224),
|
|
310
|
+
axes=defaults.get("axes", "CHW"),
|
|
311
|
+
batchsize=defaults.get("batchsize", 32),
|
|
312
|
+
num_workers=defaults.get("num_workers", 4),
|
|
313
|
+
description=f"Auto-generated metadata for {model.name}",
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
model.active_meta = meta
|
|
317
|
+
model.save()
|
|
318
|
+
fixed_count += 1
|
|
319
|
+
self.stdout.write(f" ✅ Created and set metadata for {model.name}")
|
|
320
|
+
|
|
321
|
+
elif not model.active_meta:
|
|
322
|
+
# Model has metadata but no active meta set
|
|
323
|
+
first_meta = model.metadata_versions.first()
|
|
324
|
+
if first_meta:
|
|
325
|
+
self.stdout.write(f" Setting active metadata for {model.name}...")
|
|
326
|
+
|
|
327
|
+
# Check if the metadata has weights - if not, try to assign them
|
|
328
|
+
if not first_meta.weights:
|
|
329
|
+
self.stdout.write(" Metadata exists but no weights assigned, attempting to add weights...")
|
|
330
|
+
weights_file = self._find_model_weights_file()
|
|
331
|
+
if weights_file:
|
|
332
|
+
from pathlib import Path
|
|
333
|
+
|
|
334
|
+
from endoreg_db.utils.paths import STORAGE_DIR
|
|
335
|
+
|
|
336
|
+
try:
|
|
337
|
+
weights_path = str(Path(weights_file).relative_to(STORAGE_DIR))
|
|
338
|
+
except ValueError:
|
|
339
|
+
# Copy weights to storage if not already there
|
|
340
|
+
import shutil
|
|
341
|
+
|
|
342
|
+
weights_dir = STORAGE_DIR / "model_weights"
|
|
343
|
+
weights_dir.mkdir(parents=True, exist_ok=True)
|
|
344
|
+
dest_path = weights_dir / Path(weights_file).name
|
|
345
|
+
shutil.copy2(weights_file, dest_path)
|
|
346
|
+
weights_path = str(dest_path.relative_to(STORAGE_DIR))
|
|
347
|
+
self.stdout.write(f" Copied weights to: {dest_path}")
|
|
348
|
+
|
|
349
|
+
# Assign the relative path to the FileField
|
|
350
|
+
first_meta.weights.name = weights_path
|
|
351
|
+
first_meta.save(update_fields=["weights"])
|
|
352
|
+
self.stdout.write(f" Added weights to existing metadata: {weights_path}")
|
|
353
|
+
|
|
354
|
+
model.active_meta = first_meta
|
|
355
|
+
model.save()
|
|
356
|
+
fixed_count += 1
|
|
357
|
+
self.stdout.write(f" ✅ Set active metadata: {first_meta.name} v{first_meta.version}")
|
|
358
|
+
else:
|
|
359
|
+
self.stdout.write(f" ⚠️ No metadata versions available for {model.name}")
|
|
360
|
+
|
|
361
|
+
else:
|
|
362
|
+
self.stdout.write(f" ✅ Model {model.name} has active metadata: {model.active_meta}")
|
|
363
|
+
|
|
364
|
+
# Verify all models can get latest version
|
|
365
|
+
self.stdout.write("\nTesting model metadata access...")
|
|
366
|
+
for model in all_models:
|
|
367
|
+
try:
|
|
368
|
+
latest = model.get_latest_version()
|
|
369
|
+
self.stdout.write(f" ✅ {model.name}: {latest}")
|
|
370
|
+
except Exception as e:
|
|
371
|
+
self.stdout.write(f" ❌ {model.name}: {e}")
|
|
372
|
+
raise Exception(f"Model {model.name} still has metadata issues: {e}")
|
|
373
|
+
|
|
374
|
+
if fixed_count > 0:
|
|
375
|
+
self.stdout.write(f"Fixed metadata for {fixed_count} model(s)")
|
|
376
|
+
else:
|
|
377
|
+
self.stdout.write("All models already had proper metadata")
|
|
@@ -160,7 +160,7 @@ class EndoscopyProcessor(models.Model):
|
|
|
160
160
|
"height": self.endoscope_sn_height,
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
-
def get_rois(self) -> dict[
|
|
163
|
+
def get_rois(self) -> dict[str, dict[str, int | None] | None]:
|
|
164
164
|
return {
|
|
165
165
|
"endoscope_image": self.get_roi_endoscope_image(),
|
|
166
166
|
"examination_date": self.get_roi_examination_date(),
|
|
@@ -171,3 +171,12 @@ class EndoscopyProcessor(models.Model):
|
|
|
171
171
|
"endoscope_type": self.get_roi_endoscope_type(),
|
|
172
172
|
"endoscope_sn": self.get_roi_endoscopy_sn(),
|
|
173
173
|
}
|
|
174
|
+
|
|
175
|
+
def get_sensitive_rois(self) -> dict[str, dict[str, int | None] | None]:
|
|
176
|
+
return {
|
|
177
|
+
"examination_date": self.get_roi_examination_date(),
|
|
178
|
+
"examination_time": self.get_roi_examination_time(),
|
|
179
|
+
"patient_first_name": self.get_roi_patient_first_name(),
|
|
180
|
+
"patient_last_name": self.get_roi_patient_last_name(),
|
|
181
|
+
"patient_dob": self.get_roi_patient_dob(),
|
|
182
|
+
}
|
|
@@ -2,6 +2,7 @@ import shutil
|
|
|
2
2
|
from logging import getLogger
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import TYPE_CHECKING, Any, Optional, Type
|
|
5
|
+
|
|
5
6
|
from django.core.files import File
|
|
6
7
|
from django.db import transaction
|
|
7
8
|
from huggingface_hub import hf_hub_download
|
|
@@ -18,18 +19,14 @@ if TYPE_CHECKING:
|
|
|
18
19
|
from .model_meta import ModelMeta # Import ModelMeta for type hinting
|
|
19
20
|
|
|
20
21
|
|
|
21
|
-
def get_latest_version_number_logic(
|
|
22
|
-
cls: Type["ModelMeta"], meta_name: str, model_name: str
|
|
23
|
-
) -> int:
|
|
22
|
+
def get_latest_version_number_logic(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> int:
|
|
24
23
|
"""
|
|
25
24
|
Finds the highest numerical version for a given meta_name and model_name.
|
|
26
25
|
Iterates through all versions, attempts to parse them as integers,
|
|
27
26
|
and returns the maximum integer found. If no numeric versions are found,
|
|
28
27
|
returns 0.
|
|
29
28
|
"""
|
|
30
|
-
versions_qs = cls.objects.filter(
|
|
31
|
-
name=meta_name, model__name=model_name
|
|
32
|
-
).values_list("version", flat=True)
|
|
29
|
+
versions_qs = cls.objects.filter(name=meta_name, model__name=model_name).values_list("version", flat=True)
|
|
33
30
|
|
|
34
31
|
max_v = 0
|
|
35
32
|
found_numeric_version = False
|
|
@@ -84,24 +81,17 @@ def create_from_file_logic(
|
|
|
84
81
|
|
|
85
82
|
if requested_version:
|
|
86
83
|
target_version = str(requested_version)
|
|
87
|
-
existing = cls.objects.filter(
|
|
88
|
-
name=meta_name, model=ai_model, version=target_version
|
|
89
|
-
).first()
|
|
84
|
+
existing = cls.objects.filter(name=meta_name, model=ai_model, version=target_version).first()
|
|
90
85
|
if existing and not bump_if_exists:
|
|
91
86
|
raise ValueError(
|
|
92
|
-
f"ModelMeta '{meta_name}' version '{target_version}' for model '{model_name}' "
|
|
93
|
-
f"already exists. Use bump_if_exists=True to increment."
|
|
87
|
+
f"ModelMeta '{meta_name}' version '{target_version}' for model '{model_name}' already exists. Use bump_if_exists=True to increment."
|
|
94
88
|
)
|
|
95
89
|
elif existing and bump_if_exists:
|
|
96
90
|
target_version = str(latest_version_num + 1)
|
|
97
|
-
logger.info(
|
|
98
|
-
f"Bumping version for {meta_name}/{model_name} to {target_version}"
|
|
99
|
-
)
|
|
91
|
+
logger.info(f"Bumping version for {meta_name}/{model_name} to {target_version}")
|
|
100
92
|
else:
|
|
101
93
|
target_version = str(latest_version_num + 1)
|
|
102
|
-
logger.info(
|
|
103
|
-
f"Setting next version for {meta_name}/{model_name} to {target_version}"
|
|
104
|
-
)
|
|
94
|
+
logger.info(f"Setting next version for {meta_name}/{model_name} to {target_version}")
|
|
105
95
|
|
|
106
96
|
# --- Prepare Weights File ---
|
|
107
97
|
source_weights_path = Path(weights_file).resolve()
|
|
@@ -111,10 +101,7 @@ def create_from_file_logic(
|
|
|
111
101
|
# Construct destination path within MEDIA_ROOT/WEIGHTS_DIR
|
|
112
102
|
weights_filename = source_weights_path.name
|
|
113
103
|
# Relative path for the FileField upload_to
|
|
114
|
-
relative_dest_path = (
|
|
115
|
-
Path(WEIGHTS_DIR.relative_to(STORAGE_DIR))
|
|
116
|
-
/ f"{meta_name}_v{target_version}_{weights_filename}"
|
|
117
|
-
)
|
|
104
|
+
relative_dest_path = Path(WEIGHTS_DIR.relative_to(STORAGE_DIR)) / f"{meta_name}_v{target_version}_{weights_filename}"
|
|
118
105
|
# Full path for shutil.copy
|
|
119
106
|
full_dest_path = STORAGE_DIR / relative_dest_path
|
|
120
107
|
|
|
@@ -127,8 +114,6 @@ def create_from_file_logic(
|
|
|
127
114
|
logger.info(f"Copied weights from {source_weights_path} to {full_dest_path}")
|
|
128
115
|
except Exception as e:
|
|
129
116
|
raise IOError(f"Failed to copy weights file: {e}") from e
|
|
130
|
-
|
|
131
|
-
|
|
132
117
|
|
|
133
118
|
# --- Create/Update ModelMeta Instance ---
|
|
134
119
|
defaults = {
|
|
@@ -146,11 +131,6 @@ def create_from_file_logic(
|
|
|
146
131
|
version=target_version,
|
|
147
132
|
defaults=defaults,
|
|
148
133
|
)
|
|
149
|
-
|
|
150
|
-
with open(full_dest_path, "rb") as f:
|
|
151
|
-
model_meta.weights.save(relative_dest_path.name, File(f), save=False)
|
|
152
|
-
model_meta.save()
|
|
153
|
-
|
|
154
134
|
|
|
155
135
|
if created:
|
|
156
136
|
logger.info(f"Created new ModelMeta: {model_meta}")
|
|
@@ -160,8 +140,8 @@ def create_from_file_logic(
|
|
|
160
140
|
# --- Optionally update AiModel's active_meta ---
|
|
161
141
|
# You might want to add logic here to automatically set the newly created/updated
|
|
162
142
|
# meta as the active one for the AiModel, e.g.:
|
|
163
|
-
|
|
164
|
-
|
|
143
|
+
ai_model.active_meta = model_meta
|
|
144
|
+
ai_model.save()
|
|
165
145
|
|
|
166
146
|
return model_meta
|
|
167
147
|
|
|
@@ -240,22 +220,14 @@ def get_model_meta_by_name_version_logic(
|
|
|
240
220
|
try:
|
|
241
221
|
return cls.objects.get(name=meta_name, model=ai_model, version=version)
|
|
242
222
|
except Exception as exc:
|
|
243
|
-
raise cls.DoesNotExist(
|
|
244
|
-
f"ModelMeta '{meta_name}' version '{version}' for model '{model_name}' not found."
|
|
245
|
-
) from exc
|
|
223
|
+
raise cls.DoesNotExist(f"ModelMeta '{meta_name}' version '{version}' for model '{model_name}' not found.") from exc
|
|
246
224
|
else:
|
|
247
225
|
# Get latest version
|
|
248
|
-
latest = (
|
|
249
|
-
cls.objects.filter(name=meta_name, model=ai_model)
|
|
250
|
-
.order_by("-date_created")
|
|
251
|
-
.first()
|
|
252
|
-
)
|
|
226
|
+
latest = cls.objects.filter(name=meta_name, model=ai_model).order_by("-date_created").first()
|
|
253
227
|
if latest:
|
|
254
228
|
return latest
|
|
255
229
|
else:
|
|
256
|
-
raise cls.DoesNotExist(
|
|
257
|
-
f"No ModelMeta found for '{meta_name}' and model '{model_name}'."
|
|
258
|
-
)
|
|
230
|
+
raise cls.DoesNotExist(f"No ModelMeta found for '{meta_name}' and model '{model_name}'.")
|
|
259
231
|
|
|
260
232
|
|
|
261
233
|
import re
|
|
@@ -273,9 +245,7 @@ def infer_default_model_meta_from_hf(model_id: str) -> dict[str, Any]:
|
|
|
273
245
|
"""
|
|
274
246
|
|
|
275
247
|
if not (info := model_info(model_id)):
|
|
276
|
-
logger.info(
|
|
277
|
-
f"Could not retrieve model info for {model_id}, using ColoReg segmentation defaults."
|
|
278
|
-
)
|
|
248
|
+
logger.info(f"Could not retrieve model info for {model_id}, using ColoReg segmentation defaults.")
|
|
279
249
|
return {
|
|
280
250
|
"name": "wg-lux/colo_segmentation_RegNetX800MF_base",
|
|
281
251
|
"activation": "sigmoid",
|
|
@@ -324,9 +294,7 @@ def infer_default_model_meta_from_hf(model_id: str) -> dict[str, Any]:
|
|
|
324
294
|
}
|
|
325
295
|
|
|
326
296
|
|
|
327
|
-
def setup_default_from_huggingface_logic(
|
|
328
|
-
cls, model_id: str, labelset_name: str | None = None
|
|
329
|
-
):
|
|
297
|
+
def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str | None = None):
|
|
330
298
|
"""
|
|
331
299
|
Downloads model weights from Hugging Face and auto-fills ModelMeta fields.
|
|
332
300
|
"""
|
|
@@ -340,11 +308,11 @@ def setup_default_from_huggingface_logic(
|
|
|
340
308
|
)
|
|
341
309
|
|
|
342
310
|
ai_model, _ = AiModel.objects.get_or_create(name=meta["name"])
|
|
343
|
-
labelset = (
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
311
|
+
labelset = LabelSet.objects.first() if not labelset_name else LabelSet.objects.get(name=labelset_name)
|
|
312
|
+
model_meta = ModelMeta.objects.filter(name=meta["name"], model=ai_model).first()
|
|
313
|
+
if model_meta:
|
|
314
|
+
logger.info(f"ModelMeta {meta['name']} for model {ai_model.name} already exists. Skipping creation.")
|
|
315
|
+
return model_meta
|
|
348
316
|
|
|
349
317
|
return create_from_file_logic(
|
|
350
318
|
cls,
|