endoreg-db 0.4.5__py3-none-any.whl → 0.8.6.3__py3-none-any.whl

endoreg_db/models/metadata/__init__.py

@@ -0,0 +1,19 @@
+"""Models for storing metadata associated with other data entities."""
+
+from .sensitive_meta import SensitiveMeta
+from .pdf_meta import PdfMeta, PdfType
+from .video_meta import VideoMeta, FFMpegMeta, VideoImportMeta
+from .model_meta import ModelMeta
+from .video_prediction_meta import VideoPredictionMeta
+
+
+__all__ = [
+    "SensitiveMeta",
+    "PdfMeta",
+    "PdfType",
+    "VideoMeta",
+    "FFMpegMeta",
+    "VideoImportMeta",
+    "ModelMeta",
+    "VideoPredictionMeta",
+]

endoreg_db/models/metadata/model_meta.py

@@ -0,0 +1,206 @@
+"""
+Defines the ModelMeta model and its manager for storing metadata related to AI models,
+including versioning, configuration, and associated weights files.
+Logic is primarily handled in model_meta_logic.py.
+"""
+
+from typing import Optional, TYPE_CHECKING, Tuple, Dict, Any, Type
+# Removed shutil import, now in logic
+
+from django.db import models
+from django.core.validators import FileExtensionValidator
+# Removed torch import, now in logic
+# from torch import nn
+
+from ..utils import WEIGHTS_DIR, STORAGE_DIR
+# Import logic functions
+from . import model_meta_logic as logic
+
+if TYPE_CHECKING:
+    from endoreg_db.models import LabelSet, AiModel  # pylint: disable=import-outside-toplevel
+
+
+class ModelMetaManager(models.Manager):
+    """
+    Custom manager for the ModelMeta model.
+
+    Provides methods for retrieving ModelMeta instances using natural keys.
+    """
+    # ... existing code ...
+
+
+class ModelMeta(models.Model):
+    """
+    Represents the metadata for a specific version of an AI model.
+    Logic for creation, querying, and configuration is in model_meta_logic.py.
+    """
+
+    # ... (fields remain the same: name, version, model, labelset, activation, weights, mean, std, etc.) ...
+    # --- Model Identification ---
+    name = models.CharField(
+        max_length=255,
+        help_text="User-defined name for this specific model metadata set (e.g., 'segmentation_base').",
+    )
+    version = models.CharField(
+        max_length=255,
+        help_text="Version identifier for this metadata set (e.g., '1', '2a').",
+    )
+    model = models.ForeignKey(
+        "AiModel",
+        on_delete=models.CASCADE,
+        related_name="metadata_versions",
+        help_text="The base AI model architecture this metadata belongs to.",
+
+    )
+
+    # --- Model Configuration ---
+    labelset = models.ForeignKey(
+        "LabelSet",
+        on_delete=models.CASCADE,
+        related_name="model_metadata",
+        help_text="The set of labels this model version predicts.",
+    )
+    activation = models.CharField(
+        max_length=50, default="sigmoid", help_text="Output activation function (e.g., 'sigmoid', 'softmax', 'none')."
+    )
+    weights = models.FileField(
+        upload_to=WEIGHTS_DIR.name,  # Use .name for relative path
+        validators=[FileExtensionValidator(allowed_extensions=["ckpt"])],
+        null=True,
+        blank=True,
+        help_text="Path to the model weights file (.ckpt), relative to MEDIA_ROOT.",
+    )
+
+    # --- Normalization and Input Shape ---
+    mean = models.CharField(
+        max_length=255,
+        default="0.45211223,0.27139644,0.19264949",
+        help_text="Comma-separated mean values for input normalization.",
+    )
+    std = models.CharField(
+        max_length=255,
+        default="0.31418097,0.21088019,0.16059452",
+        help_text="Comma-separated standard deviation values for input normalization.",
+    )
+    size_x = models.IntegerField(default=716, help_text="Expected input image width.")
+    size_y = models.IntegerField(default=716, help_text="Expected input image height.")
+    axes = models.CharField(
+        max_length=10, default="2,0,1", help_text="Comma-separated target axis order (e.g., '2,0,1' for CHW)."
+    )
+
+    # --- Inference Parameters ---
+    batchsize = models.IntegerField(default=16, help_text="Default batch size for inference.")
+    num_workers = models.IntegerField(default=0, help_text="Default number of workers for data loading.")
+
+    # --- Metadata ---
+    description = models.TextField(blank=True, null=True, help_text="Optional description.")
+    date_created = models.DateTimeField(auto_now_add=True)
+
+
+    objects = ModelMetaManager()
+
+    # --- Type Hinting for Related Fields ---
+    if TYPE_CHECKING:
+        labelset: "LabelSet"
+        model: "AiModel"  # Corrected from ai_model to match field name
+
+    class Meta:
+        """Metadata options for the ModelMeta model."""
+        # ... existing code ...
+
+    @classmethod
+    def create_from_file(
+        cls: Type["ModelMeta"],
+        meta_name: str,
+        model_name: str,
+        labelset_name: str,
+        weights_file: str,
+        requested_version: Optional[str] = None,
+        bump_if_exists: bool = False,
+        **kwargs: Any,
+    ) -> "ModelMeta":
+        """
+        Creates or updates a ModelMeta instance using external logic.
+        """
+        # Delegate to logic function, passing the class (cls)
+        return logic.create_from_file_logic(
+            cls, meta_name, model_name, labelset_name, weights_file,
+            requested_version, bump_if_exists, **kwargs
+        )
+        
+    @classmethod
+    def setup_default_from_huggingface(
+        cls: Type["ModelMeta"],
+        model_id: str = "wg-lux/colo_segmentation_RegNetX800MF_base",
+        labelset_name: Optional[str] = None,
+    ) -> "ModelMeta":
+        """
+        Downloads a pretrained model from Hugging Face and initializes ModelMeta automatically.
+        """
+        # If labelset_name is not provided, handle default logic here if needed
+        return logic.setup_default_from_huggingface_logic(cls, model_id, labelset_name)
+
+
+
+    @classmethod
+    def get_latest_version_number(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> int:
+        """
+        Gets the latest version *number* using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_latest_version_number_logic(cls, meta_name, model_name)
+
+
+    @staticmethod
+    def get_activation_function(activation_name: str) -> "TorchModule":
+        """
+        Retrieves a PyTorch activation function using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_activation_function_logic(activation_name)
+
+    def get_inference_dataset_config(self) -> Dict[str, Any]:
+        """
+        Generates an inference dataset configuration using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_inference_dataset_config_logic(self)
+
+    def natural_key(self) -> Tuple[str, str]:
+        """
+        Returns the natural key for serialization.
+        """
+        # Assuming natural key is based on name and version, linked to model name
+        return (self.name, self.version, self.model.natural_key())
+
+    def __str__(self) -> str:
+        """String representation of the ModelMeta instance."""
+        # Ensure a string is always returned
+        model_name = self.model.name if self.model else "UnknownModel"
+        return f"ModelMeta: {self.name} (v{self.version}) for {model_name}"
+
+    def get_config_dict(self) -> Dict[str, Any]:
+        """
+        Returns the model's configuration parameters using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_config_dict_logic(self)
+
+    @classmethod
+    def get_by_name_version(
+        cls: Type["ModelMeta"],
+        meta_name: str,
+        model_name: str,
+        version: Optional[str] = None,
+    ) -> "ModelMeta":
+        """
+        Retrieves a ModelMeta instance by name, model name, and optionally version using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_model_meta_by_name_version_logic(cls, meta_name, model_name, version)
+
+    @classmethod
+    def get_latest(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> "ModelMeta":
+        """Alias for get_by_name_version(meta_name, model_name, version=None) using external logic."""
+        # Delegate directly to the specific logic function
+        return logic.get_model_meta_by_name_version_logic(cls, meta_name, model_name, version=None)

endoreg_db/models/metadata/model_meta_logic.py

@@ -0,0 +1,343 @@
+import shutil
+from logging import getLogger
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional, Type
+
+from django.core.files import File
+from django.db import transaction
+from huggingface_hub import hf_hub_download
+
+# Assuming ModelMeta, AiModel, LabelSet are importable from the correct locations
+# Adjust imports based on your project structure if necessary
+from ..administration.ai.ai_model import AiModel
+from ..label.label_set import LabelSet
+from ..utils import STORAGE_DIR, WEIGHTS_DIR
+
+logger = getLogger("ai_model")
+
+if TYPE_CHECKING:
+    from .model_meta import ModelMeta  # Import ModelMeta for type hinting
+
+
+def _get_model_meta_class():
+    """Lazy import to avoid circular imports"""
+    from .model_meta import ModelMeta
+
+    return ModelMeta
+
+
+def get_latest_version_number_logic(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> int:
+    """
+    Finds the highest numerical version for a given meta_name and model_name.
+    Iterates through all versions, attempts to parse them as integers,
+    and returns the maximum integer found. If no numeric versions are found,
+    returns 0.
+    """
+    versions_qs = cls.objects.filter(name=meta_name, model__name=model_name).values_list("version", flat=True)
+
+    max_v = 0
+    found_numeric_version = False
+
+    for v_str in versions_qs:
+        if v_str is None:  # Skip None versions
+            continue
+        try:
+            v_int = int(v_str)
+            if not found_numeric_version or v_int > max_v:
+                max_v = v_int
+            found_numeric_version = True
+        except ValueError:
+            logger.warning(
+                f"Warning: Could not parse version string '{v_str}' as an integer for "
+                f"meta_name='{meta_name}', model_name='{model_name}' while determining the max version."
+            )
+
+    return max_v if found_numeric_version else 0
+
+
+@transaction.atomic
+def create_from_file_logic(
+    cls: Type["ModelMeta"],  # cls is ModelMeta
+    meta_name: str,
+    model_name: str,
+    labelset_name: str,
+    weights_file: str,
+    requested_version: Optional[str] = None,
+    bump_if_exists: bool = False,
+    **kwargs: Any,
+) -> "ModelMeta":
+    """
+    Logic to create or update a ModelMeta instance from a weights file.
+
+    Handles finding related objects, versioning, file copying, and saving.
+    """
+    # --- Find Related Objects ---
+    try:
+        ai_model = AiModel.objects.get(name=model_name)
+    except AiModel.DoesNotExist as exc:
+        raise ValueError(f"AiModel with name '{model_name}' not found.") from exc
+
+    try:
+        label_set = LabelSet.objects.get(name=labelset_name)
+    except LabelSet.DoesNotExist as exc:
+        raise ValueError(f"LabelSet with name '{labelset_name}' not found.") from exc
+
+    # --- Determine Version ---
+    target_version: str
+    latest_version_num = get_latest_version_number_logic(cls, meta_name, model_name)
+
+    if requested_version:
+        target_version = str(requested_version)
+        existing = cls.objects.filter(name=meta_name, model=ai_model, version=target_version).first()
+        if existing and not bump_if_exists:
+            raise ValueError(
+                f"ModelMeta '{meta_name}' version '{target_version}' for model '{model_name}' already exists. Use bump_if_exists=True to increment."
+            )
+        elif existing and bump_if_exists:
+            target_version = str(latest_version_num + 1)
+            logger.info(f"Bumping version for {meta_name}/{model_name} to {target_version}")
+    else:
+        target_version = str(latest_version_num + 1)
+        logger.info(f"Setting next version for {meta_name}/{model_name} to {target_version}")
+
+    # --- Prepare Weights File ---
+    source_weights_path = Path(weights_file).resolve()
+    if not source_weights_path.exists():
+        raise FileNotFoundError(f"Weights file not found: {source_weights_path}")
+
+    # Construct destination path within MEDIA_ROOT/WEIGHTS_DIR
+    weights_filename = source_weights_path.name
+    # Relative path for the FileField upload_to
+    relative_dest_path = Path(WEIGHTS_DIR.relative_to(STORAGE_DIR)) / f"{meta_name}_v{target_version}_{weights_filename}"
+    # Full path for shutil.copy
+    full_dest_path = STORAGE_DIR / relative_dest_path
+
+    # Ensure the destination directory exists
+    full_dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Copy the file
+    try:
+        shutil.copy(source_weights_path, full_dest_path)
+        logger.info(f"Copied weights from {source_weights_path} to {full_dest_path}")
+    except Exception as e:
+        raise IOError(f"Failed to copy weights file: {e}") from e
+
+    # --- Create/Update ModelMeta Instance ---
+    defaults = {
+        "labelset": label_set,
+        "weights": relative_dest_path.as_posix(),  # Store relative path for FileField
+        **kwargs,  # Pass through other fields like activation, mean, std, etc.
+    }
+
+    # Remove None values from defaults to avoid overriding model defaults unnecessarily
+    defaults = {k: v for k, v in defaults.items() if v is not None}
+
+    model_meta, created = cls.objects.update_or_create(
+        name=meta_name,
+        model=ai_model,
+        version=target_version,
+        defaults=defaults,
+    )
+
+    if created:
+        logger.info(f"Created new ModelMeta: {model_meta}")
+    else:
+        logger.info(f"Updated existing ModelMeta: {model_meta}")
+
+    # --- Optionally update AiModel's active_meta ---
+    # You might want to add logic here to automatically set the newly created/updated
+    # meta as the active one for the AiModel, e.g.:
+    ai_model.active_meta = model_meta
+    ai_model.save()
+
+    return model_meta
+
+
+# --- Add other logic functions referenced by ModelMeta here ---
+# (get_latest_version_number_logic, get_activation_function_logic, etc.)
+# Placeholder for get_activation_function_logic
+def get_activation_function_logic(activation_name: str):
+    import torch.nn as nn  # Import locally as it's specific to this function
+
+    if activation_name.lower() == "sigmoid":
+        return nn.Sigmoid()
+    elif activation_name.lower() == "softmax":
+        # Note: Softmax usually requires specifying the dimension
+        return nn.Softmax(dim=1)  # Assuming dim=1 (channels) is common
+    elif activation_name.lower() == "none":
+        return nn.Identity()
+    else:
+        # Consider adding more activations or raising an error
+        raise ValueError(f"Unsupported activation function: {activation_name}")
+
+
+# Placeholder for get_inference_dataset_config_logic
+def get_inference_dataset_config_logic(model_meta: "ModelMeta") -> dict:
+    # This would typically extract relevant fields from model_meta
+    # for configuring a dataset during inference
+    return {
+        "mean": [float(x) for x in model_meta.mean.split(",")],
+        "std": [float(x) for x in model_meta.std.split(",")],
+        "size_y": model_meta.size_y,  # Add size_y key
+        "size_x": model_meta.size_x,  # Add size_x key
+        "axes": [int(x) for x in model_meta.axes.split(",")],
+        # Add other relevant config like normalization type, etc.
+    }
+
+
+# Placeholder for get_config_dict_logic
+def get_config_dict_logic(model_meta: "ModelMeta") -> dict:
+    # Returns a dictionary representation of the model's configuration
+    return {
+        "name": model_meta.name,
+        "version": model_meta.version,
+        "model_name": model_meta.model.name,
+        "labelset_name": model_meta.labelset.name,
+        "activation": model_meta.activation,
+        "weights_path": model_meta.weights.path if model_meta.weights else None,
+        "mean": model_meta.mean,
+        "std": model_meta.std,
+        "size_x": model_meta.size_x,
+        "size_y": model_meta.size_y,
+        "axes": model_meta.axes,
+        "batchsize": model_meta.batchsize,
+        "num_workers": model_meta.num_workers,
+        "description": model_meta.description,
+        # Add any other relevant fields
+    }
+
+
+# Placeholder for get_model_meta_by_name_version_logic
+def get_model_meta_by_name_version_logic(
+    cls: Type["ModelMeta"],
+    meta_name: str,
+    model_name: str,
+    version: Optional[str] = None,
+) -> "ModelMeta":
+    """
+    Retrieves a ModelMeta instance by name, model name, and optionally version.
+    If version is None, retrieves the latest version based on date_created.
+    """
+    try:
+        ai_model = AiModel.objects.get(name=model_name)
+    except AiModel.DoesNotExist as exc:
+        raise cls.DoesNotExist(f"AiModel with name '{model_name}' not found.") from exc
+
+    if version:
+        try:
+            return cls.objects.get(name=meta_name, model=ai_model, version=version)
+        except Exception as exc:
+            raise cls.DoesNotExist(f"ModelMeta '{meta_name}' version '{version}' for model '{model_name}' not found.") from exc
+    else:
+        # Get latest version
+        latest = cls.objects.filter(name=meta_name, model=ai_model).order_by("-date_created").first()
+        if latest:
+            return latest
+        else:
+            raise cls.DoesNotExist(f"No ModelMeta found for '{meta_name}' and model '{model_name}'.")
+
+
+import re
+
+from huggingface_hub import model_info
+
+
+def infer_default_model_meta_from_hf(model_id: str) -> dict[str, Any]:
+    """
+    Infers default model metadata (activation, normalization, input size)
+    from a Hugging Face model_id using its tags and architecture.
+
+    Returns:
+        A dict with fields: name, activation, mean, std, size_x, size_y
+    """
+
+    if not (info := model_info(model_id)):
+        logger.info(f"Could not retrieve model info for {model_id}, using ColoReg segmentation defaults.")
+        return {
+            "name": "wg-lux/colo_segmentation_RegNetX800MF_base",
+            "activation": "sigmoid",
+            "mean": (0.45211223, 0.27139644, 0.19264949),
+            "std": (0.31418097, 0.21088019, 0.16059452),
+            "size_x": 716,
+            "size_y": 716,
+            "description": f"Defaults for unknown model {model_id}",
+        }
+
+    # Extract architecture from tags or model_id ---
+    tags = info.tags or []
+    model_name = model_id.split("/")[-1].lower()
+
+    # Heuristics for architecture and task
+    architecture = next((t for t in tags if t.startswith("architecture:")), None)
+    task = next((t for t in tags if t.startswith("task:")), None)
+
+    # Default values
+    activation = "sigmoid"
+    size_x = size_y = 716
+    mean = (0.45211223, 0.27139644, 0.19264949)
+    std = (0.31418097, 0.21088019, 0.16059452)
+
+    # --- 2. Task-based inference ---
+    if task:
+        if "segmentation" in task or "detection" in task:
+            activation = "sigmoid"
+        elif any(k in task for k in ["classification"]):
+            activation = "softmax"
+
+    # --- 3. Architecture-based inference ---
+    if architecture:
+        arch = architecture.replace("architecture:", "")
+    else:
+        arch = re.sub(r"[^a-z0-9]+", "_", model_name)
+
+    return {
+        "name": arch,
+        "activation": activation,
+        "mean": mean,
+        "std": std,
+        "size_x": size_x,
+        "size_y": size_y,
+        "description": f"Inferred defaults for {model_id}",
+    }
+
+
+def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str | None = None):
+    """
+    Downloads model weights from Hugging Face and auto-fills ModelMeta fields.
+    """
+    meta = infer_default_model_meta_from_hf(model_id)
+
+    # Download weights
+    weights_path = hf_hub_download(
+        repo_id=model_id,
+        filename="colo_segmentation_RegNetX800MF_base.ckpt",
+        local_dir=WEIGHTS_DIR,
+    )
+
+    ai_model, _ = AiModel.objects.get_or_create(name=meta["name"])
+    if not labelset_name:
+        labelset = LabelSet.objects.first()
+        if not labelset:
+            raise ValueError("No labelset found and no labelset_name provided")
+    else:
+        labelset = LabelSet.objects.get(name=labelset_name)
+
+    ModelMeta = _get_model_meta_class()
+    model_meta = ModelMeta.objects.filter(name=meta["name"], model=ai_model).first()
+    if model_meta:
+        logger.info(f"ModelMeta {meta['name']} for model {ai_model.name} already exists. Skipping creation.")
+        return model_meta
+
+    return create_from_file_logic(
+        cls,
+        meta_name=meta["name"],
+        model_name=ai_model.name,
+        labelset_name=labelset.name,
+        weights_file=weights_path,
+        activation=meta["activation"],
+        mean=meta["mean"],
+        std=meta["std"],
+        size_x=meta["size_x"],
+        size_y=meta["size_y"],
+        description=meta["description"],
+    )

endoreg_db/models/{data_file/metadata → metadata}/pdf_meta.py

@@ -1,19 +1,16 @@
 from django.db import models
-
-# import endoreg_center_id from django settings
-from django.conf import settings
-
-
-# import File class
 from django.core.files import File
+from typing import TYPE_CHECKING
 
-# # check if endoreg_center_id is set
-# if not hasattr(settings, 'ENDOREG_CENTER_ID'):
-#     ENDOREG_CENTER_ID = 9999
-# else:
-#     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
+if TYPE_CHECKING:
+    from ..media.pdf.report_reader.report_reader_flag import ReportReaderFlag
 
 class PdfType(models.Model):
+    """
+    Defines a specific type or format of PDF report, linking to flags used for parsing.
+
+    Used to configure how different PDF report layouts are processed.
+    """
     name = models.CharField(max_length=255)
 
     patient_info_line = models.ForeignKey(
@@ -40,8 +37,15 @@ class PdfType(models.Model):
         related_name="pdf_type_cut_off_below_lines", 
     )
 
+    if TYPE_CHECKING:
+        patient_info_line: "ReportReaderFlag"
+        endoscope_info_line: "ReportReaderFlag"
+        examiner_info_line: "ReportReaderFlag"
+        cut_off_above_lines: models.QuerySet["ReportReaderFlag"]
+        cut_off_below_lines: models.QuerySet["ReportReaderFlag"]
 
     def __str__(self):
+        """Returns a string summary of the PDF type and its associated flags."""
         summary = f"{self.name}"
         # add lines to summary
         summary += f"\nPatient Info Line: {self.patient_info_line.value}"
@@ -52,19 +56,34 @@ class PdfType(models.Model):
     
         return summary
 
+    @classmethod
+    def default_pdf_type(cls):
+        """Returns a default PdfType instance, typically used as a fallback."""
+        return PdfType.objects.get(name="ukw-endoscopy-examination-report-generic")
+
 class PdfMeta(models.Model):
+    """
+    Stores metadata associated with a specific PDF document file.
+    """
     pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
     date = models.DateField()
     time = models.TimeField()
     pdf_hash = models.CharField(max_length=255, unique=True)
 
+    if TYPE_CHECKING:
+        pdf_type: "PdfType"
+
     def __str__(self):
-        return self.pdf_hash
+        """Returns the PDF hash as its string representation."""
+        return str(self.pdf_hash)
 
     @classmethod
     def create_from_file(cls, pdf_file):
+        """
+        Creates a PdfMeta instance from a PDF file object.
+        Note: This implementation seems incomplete; it doesn't extract hash, date, time, or type.
+        """
         pdf_file = File(pdf_file)
         pdf_meta = cls(file=pdf_file)
         pdf_meta.save()
         return pdf_meta
-