endoreg-db 0.8.4.3__tar.gz → 0.8.4.5__tar.gz

{endoreg_db-0.8.4.3 → endoreg_db-0.8.4.5}/CONFIGURATION_GUIDE.md

@@ -65,33 +65,6 @@ The command automatically detects your cache configuration:
 
 Use `--skip-ai-setup` if AI video processing features are not needed, or `--force-recreate` to recreate AI metadata.
 
-### AI Model Validation and Troubleshooting
-
-If you encounter "No model metadata found for this model" errors during video processing, use the dedicated validation command:
-
-```bash
-# Check AI model metadata status without making changes
-python manage.py validate_ai_models --dry-run
-
-# Fix AI model metadata issues automatically  
-python manage.py validate_ai_models
-
-# View detailed help
-python manage.py validate_ai_models --help
-```
-
-The validation command automatically:
-- Checks all AI models for proper metadata configuration
-- Creates missing metadata with sensible defaults (ImageNet normalization, etc.)
-- Sets active metadata for models that don't have it assigned
-- Validates that all models can access their latest versions
-
-Common scenarios this fixes:
-- **"No model metadata found"** errors in pipe_1 video processing
-- AI models exist but have no `active_meta` set
-- Models have metadata versions but none are marked as active
-- Missing ModelMeta records for classification/segmentation models
-
 This repo standalone (local):
 - Development server: DJANGO_SETTINGS_MODULE=config.settings.dev python manage.py runserver
 - Tests (persistent test DB): pytest --reuse-db --create-db

{endoreg_db-0.8.4.3 → endoreg_db-0.8.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: endoreg-db
-Version: 0.8.4.3
+Version: 0.8.4.5
 Summary: EndoReg Db Django App
 Project-URL: Homepage, https://info.coloreg.de
 Project-URL: Repository, https://github.com/wg-lux/endoreg-db
@@ -29,7 +29,6 @@ Requires-Dist: dotenv>=0.9.9
 Requires-Dist: faker>=37.6.0
 Requires-Dist: flake8>=7.3.0
 Requires-Dist: gunicorn>=23.0.0
-Requires-Dist: huggingface-hub>=0.35.3
 Requires-Dist: icecream>=2.1.4
 Requires-Dist: librosa==0.11.0
 Requires-Dist: llvmlite>=0.44.0

{endoreg_db-0.8.4.3 → endoreg_db-0.8.4.5}/endoreg_db/models/media/video/pipe_1.py

@@ -1,20 +1,23 @@
 import logging
-from typing import TYPE_CHECKING, Optional, Dict, List, Tuple
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+
 from django.db import transaction
+
 from endoreg_db.helpers.download_segmentation_model import download_segmentation_model
 
 # Added imports
 
 # Configure logging
-logger = logging.getLogger(__name__) # Changed from "video_file"
+logger = logging.getLogger(__name__)  # Changed from "video_file"
 
 if TYPE_CHECKING:
     from endoreg_db.models import VideoFile
 
-
     # --- Pipeline 1 ---
+
+
 def _pipe_1(
-    video_file:"VideoFile",
+    video_file: "VideoFile",
     model_name: str,
     model_meta_version: Optional[int] = None,
     delete_frames_after: bool = False,
@@ -28,11 +31,12 @@ def _pipe_1(
     """
     Pipeline 1: Extract frames, text, predict, create segments, optionally delete frames.
     """
-    success = False # Initialize success flag
-    from .video_file_segments import _convert_sequences_to_db_segments # Added import
-    from ...metadata import ModelMeta, VideoPredictionMeta
+    success = False  # Initialize success flag
     from endoreg_db.models import AiModel, LabelVideoSegment
 
+    from ...metadata import ModelMeta, VideoPredictionMeta
+    from .video_file_segments import _convert_sequences_to_db_segments  # Added import
+
     video_file.refresh_from_db()
     video_file.update_video_meta()
 
@@ -43,16 +47,12 @@ def _pipe_1(
         video_file.extract_frames(overwrite=False)  # Avoid overwriting if already extracted
 
         logger.info("Pipe 1: Extracting text metadata...")
-        video_file.update_text_metadata(
-            ocr_frame_fraction=ocr_frame_fraction, cap=ocr_cap, overwrite=False
-        )
+        video_file.update_text_metadata(ocr_frame_fraction=ocr_frame_fraction, cap=ocr_cap, overwrite=False)
         with transaction.atomic():
             state = video_file.get_or_create_state()
             if not state.frames_extracted:
                 logger.error("Pipe 1 failed: Frame extraction did not complete successfully.")
                 return False
-            
-            
 
             # 3. Perform Initial Prediction
             logger.info(f"Pipe 1: Performing prediction with model '{model_name}'...")
@@ -74,7 +74,7 @@ def _pipe_1(
                 except AiModel.DoesNotExist:
                     logger.error(f"Pipe 1 failed: Model '{model_name}' not found.")
                     return False
-                
+
             except ModelMeta.DoesNotExist:
                 try:
                     model_name = download_segmentation_model()
@@ -82,11 +82,9 @@ def _pipe_1(
                     if model_meta_version is not None:
                         model_meta = ai_model_obj.metadata_versions.get(version=model_meta_version)
                     else:
-                        model_meta = ModelMeta.setup_default_from_huggingface()
+                        model_meta = ai_model_obj.get_latest_version()
                 except ModelMeta.DoesNotExist:
-                    logger.error(
-                        f"Pipe 1 failed: ModelMeta version {model_meta_version} for model '{model_name}' not found."
-                    )
+                    logger.error(f"Pipe 1 failed: ModelMeta version {model_meta_version} for model '{model_name}' not found.")
                     return False
             try:
                 sequences: Optional[Dict[str, List[Tuple[int, int]]]] = video_file.predict_video(
@@ -107,7 +105,7 @@ def _pipe_1(
 
             # --- Set and Save State ---
             state.initial_prediction_completed = True
-            state.save(update_fields=['initial_prediction_completed'])
+            state.save(update_fields=["initial_prediction_completed"])
             logger.info("Pipe 1: Set initial_prediction_completed state to True.")
 
             logger.info(f"Pipe 1: Sequences returned from prediction: {sequences}")
@@ -117,9 +115,7 @@ def _pipe_1(
             # 4. Create LabelVideoSegments
             logger.info("Pipe 1: Creating LabelVideoSegments from predictions...")
             try:
-                video_prediction_meta = VideoPredictionMeta.objects.get(
-                    video_file=video_file, model_meta=model_meta
-                )
+                video_prediction_meta = VideoPredictionMeta.objects.get(video_file=video_file, model_meta=model_meta)
                 logger.info(f"Pipe 1: Calling _convert_sequences_to_db_segments for video {video_file.uuid} with prediction meta {video_prediction_meta.pk}")
                 _convert_sequences_to_db_segments(
                     video=video_file,
@@ -127,9 +123,9 @@ def _pipe_1(
                     video_prediction_meta=video_prediction_meta,
                 )
                 video_file.sequences = sequences
-                video_file.save(update_fields=['sequences'])
+                video_file.save(update_fields=["sequences"])
                 state.lvs_created = True
-                state.save(update_fields=['lvs_created'])
+                state.save(update_fields=["lvs_created"])
                 logger.info("Pipe 1: Set lvs_created state to True.")
                 logger.info("Pipe 1: LabelVideoSegment creation complete.")
                 lvs_count_after = LabelVideoSegment.objects.filter(video_file=video_file).count()
@@ -139,7 +135,7 @@ def _pipe_1(
                 raise
 
         logger.info(f"Pipe 1 completed successfully for video {video_file.uuid}")
-        success = True # Set success flag
+        success = True  # Set success flag
         return True
 
     except Exception as e:
@@ -147,7 +143,7 @@ def _pipe_1(
         return False
     finally:
         # 5. Optionally delete frames
-        if delete_frames_after and success: # Check success flag
+        if delete_frames_after and success:  # Check success flag
             logger.info("Pipe 1: Deleting frames after processing...")
             try:
                 video_file.delete_frames()
@@ -156,15 +152,16 @@ def _pipe_1(
                 logger.error(f"Pipe 1 failed during frame deletion: {e}", exc_info=True)
         else:
             logger.info("Pipe 1: Frame deletion skipped.")
-            
+
+
 # --- Test after Pipe 1 ---
-def _test_after_pipe_1(video_file:"VideoFile", start_frame: int = 0, end_frame: int = 100) -> bool:
+def _test_after_pipe_1(video_file: "VideoFile", start_frame: int = 0, end_frame: int = 100) -> bool:
     """
     Simulates human annotation validation after Pipe 1.
     Creates 'outside' segments and marks sensitive meta as verified.
     """
-    from ...label import LabelVideoSegment, Label
-    
+    from ...label import Label, LabelVideoSegment
+
     logger.info(f"Starting _test_after_pipe_1 for video {video_file.uuid}")
     try:
         # 1. Create 'outside' LabelVideoSegments
@@ -172,15 +169,15 @@ def _test_after_pipe_1(video_file:"VideoFile", start_frame: int = 0, end_frame:
             outside_label = Label.objects.get(name__iexact="outside")
             logger.info(f"Creating 'outside' annotation segment [{start_frame}-{end_frame}]")
             # Create a segment - assuming custom_create handles saving
-            outside_segment = LabelVideoSegment.objects.create( # Assign to variable
+            outside_segment = LabelVideoSegment.objects.create(  # Assign to variable
                 video_file=video_file,
                 label=outside_label,
                 start_frame_number=start_frame,
                 end_frame_number=end_frame,
-                prediction_meta=None, 
+                prediction_meta=None,
             )
             # Ensure the segment has a state and mark it as validated
-            segment_state, created = outside_segment.get_or_create_state() # Unpack the tuple
+            segment_state, created = outside_segment.get_or_create_state()  # Unpack the tuple
             segment_state.is_validated = True
             segment_state.save()
             logger.info(f"Marked 'outside' segment {outside_segment.pk} as validated. Created: {created}")
@@ -202,7 +199,7 @@ def _test_after_pipe_1(video_file:"VideoFile", start_frame: int = 0, end_frame:
             # Example: using a boolean field
             video_file.sensitive_meta.state.dob_verified = True
             video_file.sensitive_meta.state.names_verified = True
-            video_file.sensitive_meta.state.save() # Save the SensitiveMeta instance
+            video_file.sensitive_meta.state.save()  # Save the SensitiveMeta instance
             logger.info("Sensitive meta state updated.")
 
         else:

{endoreg_db-0.8.4.3 → endoreg_db-0.8.4.5}/endoreg_db/models/metadata/model_meta_logic.py

@@ -3,6 +3,7 @@ from logging import getLogger
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, Type
 
+from django.core.files import File
 from django.db import transaction
 from huggingface_hub import hf_hub_download
 
@@ -18,6 +19,13 @@ if TYPE_CHECKING:
     from .model_meta import ModelMeta  # Import ModelMeta for type hinting
 
 
+def _get_model_meta_class():
+    """Lazy import to avoid circular imports"""
+    from .model_meta import ModelMeta
+
+    return ModelMeta
+
+
 def get_latest_version_number_logic(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> int:
     """
     Finds the highest numerical version for a given meta_name and model_name.
@@ -307,7 +315,14 @@ def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str
     )
 
     ai_model, _ = AiModel.objects.get_or_create(name=meta["name"])
-    labelset = LabelSet.objects.first() if not labelset_name else LabelSet.objects.get(name=labelset_name)
+    if not labelset_name:
+        labelset = LabelSet.objects.first()
+        if not labelset:
+            raise ValueError("No labelset found and no labelset_name provided")
+    else:
+        labelset = LabelSet.objects.get(name=labelset_name)
+
+    ModelMeta = _get_model_meta_class()
     model_meta = ModelMeta.objects.filter(name=meta["name"], model=ai_model).first()
     if model_meta:
         logger.info(f"ModelMeta {meta['name']} for model {ai_model.name} already exists. Skipping creation.")
@@ -326,99 +341,3 @@ def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str
         size_y=meta["size_y"],
         description=meta["description"],
     )
-
-
-def validate_and_fix_ai_model_metadata_logic():
-    """
-    Validates that all AI models have proper active metadata and fixes any issues.
-    This prevents the "No model metadata found for this model" error.
-
-    Returns:
-        dict: Summary of fixes applied
-    """
-    from ..administration.ai.ai_model import AiModel
-    from ..label.label_set import LabelSet
-
-    summary = {"models_checked": 0, "models_fixed": 0, "metadata_created": 0, "active_meta_set": 0, "errors": []}
-
-    try:
-        all_models = AiModel.objects.all()
-        summary["models_checked"] = all_models.count()
-
-        for model in all_models:
-            logger.info(f"Validating model: {model.name}")
-
-            # Check if model has metadata versions
-            metadata_count = model.metadata_versions.count()
-
-            if metadata_count == 0:
-                # Create metadata for models that don't have any
-                logger.info(f"Creating metadata for {model.name}")
-
-                # Use existing labelset or create default
-                labelset = LabelSet.objects.first()
-                if not labelset:
-                    labelset = LabelSet.objects.create(name="default_colonoscopy_labels", description="Default colonoscopy classification labels")
-
-                # Import here to avoid circular imports
-                from .model_meta import ModelMeta
-
-                # Create basic metadata
-                meta = ModelMeta.objects.create(
-                    name=model.name,
-                    version="1.0",
-                    model=model,
-                    labelset=labelset,
-                    activation="sigmoid" if "classification" in model.name else "sigmoid",
-                    mean="0.485,0.456,0.406",  # ImageNet defaults
-                    std="0.229,0.224,0.225",  # ImageNet defaults
-                    size_x=224,
-                    size_y=224,
-                    axes="CHW",
-                    batchsize=32,
-                    num_workers=4,
-                    description=f"Auto-generated metadata for {model.name}",
-                )
-
-                model.active_meta = meta
-                model.save()
-                summary["models_fixed"] += 1
-                summary["metadata_created"] += 1
-                logger.info(f"Created and set metadata for {model.name}")
-
-            elif not model.active_meta:
-                # Model has metadata but no active meta set
-                first_meta = model.metadata_versions.first()
-                if first_meta:
-                    logger.info(f"Setting active metadata for {model.name}")
-                    model.active_meta = first_meta
-                    model.save()
-                    summary["models_fixed"] += 1
-                    summary["active_meta_set"] += 1
-                    logger.info(f"Set active metadata: {first_meta.name} v{first_meta.version}")
-                else:
-                    error_msg = f"No metadata versions available for {model.name}"
-                    logger.warning(error_msg)
-                    summary["errors"].append(error_msg)
-
-            else:
-                logger.info(f"Model {model.name} has valid active metadata: {model.active_meta}")
-
-        # Verify all models can get latest version
-        logger.info("Testing model metadata access...")
-        for model in all_models:
-            try:
-                latest = model.get_latest_version()
-                logger.info(f"✅ {model.name}: {latest}")
-            except Exception as e:
-                error_msg = f"Model {model.name} metadata test failed: {e}"
-                logger.error(error_msg)
-                summary["errors"].append(error_msg)
-
-        return summary
-
-    except Exception as e:
-        error_msg = f"Validation failed: {e}"
-        logger.error(error_msg)
-        summary["errors"].append(error_msg)
-        return summary