endoreg-db 0.8.4.4__py3-none-any.whl → 0.8.8.0__py3-none-any.whl

endoreg_db/models/metadata/model_meta.py

@@ -4,20 +4,24 @@ including versioning, configuration, and associated weights files.
 Logic is primarily handled in model_meta_logic.py.
 """
 
-from typing import Optional, TYPE_CHECKING, Tuple, Dict, Any, Type
-# Removed shutil import, now in logic
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, cast
 
-from django.db import models
 from django.core.validators import FileExtensionValidator
+
+# Removed shutil import, now in logic
+from django.db import models
+
 # Removed torch import, now in logic
 # from torch import nn
+from ..utils import WEIGHTS_DIR
 
-from ..utils import WEIGHTS_DIR, STORAGE_DIR
 # Import logic functions
 from . import model_meta_logic as logic
 
 if TYPE_CHECKING:
-    from endoreg_db.models import LabelSet, AiModel  # pylint: disable=import-outside-toplevel
+    from django.db.models.fields.files import FieldFile
+
+    from endoreg_db.models import AiModel, LabelSet  # pylint: disable=import-outside-toplevel
 
 
 class ModelMetaManager(models.Manager):
@@ -26,7 +30,24 @@ class ModelMetaManager(models.Manager):
 
     Provides methods for retrieving ModelMeta instances using natural keys.
     """
-    # ... existing code ...
+
+    def get_by_natural_key(self, name: str, version: str, model_name: str) -> "ModelMeta":
+        """
+        Retrieves a ModelMeta instance using its natural key.
+
+        This method returns the ModelMeta whose name, version, and associated model's name
+        match the provided natural key. It is primarily used to support Django's natural key
+        serialization during data import/export and deserialization processes.
+
+        Args:
+            name: The name of the ModelMeta.
+            version: The version identifier of the ModelMeta.
+            model_name: The name of the associated AiModel.
+
+        Returns:
+            The ModelMeta object corresponding to the given natural key.
+        """
+        return self.get(name=name, version=version, model__name=model_name)
 
 
 class ModelMeta(models.Model):
@@ -50,7 +71,6 @@ class ModelMeta(models.Model):
         on_delete=models.CASCADE,
         related_name="metadata_versions",
         help_text="The base AI model architecture this metadata belongs to.",
-
     )
 
     # --- Model Configuration ---
@@ -60,15 +80,13 @@ class ModelMeta(models.Model):
         related_name="model_metadata",
         help_text="The set of labels this model version predicts.",
     )
-    activation = models.CharField(
-        max_length=50, default="sigmoid", help_text="Output activation function (e.g., 'sigmoid', 'softmax', 'none')."
-    )
+    activation = models.CharField(max_length=50, default="sigmoid", help_text="Output activation function (e.g., 'sigmoid', 'softmax', 'none').")
     weights = models.FileField(
         upload_to=WEIGHTS_DIR.name,  # Use .name for relative path
-        validators=[FileExtensionValidator(allowed_extensions=["ckpt"])],
+        validators=[FileExtensionValidator(allowed_extensions=["safetensors", "pth", "pt"])],
         null=True,
         blank=True,
-        help_text="Path to the model weights file (.ckpt), relative to MEDIA_ROOT.",
+        help_text="Path to the model weights file (.safetensors), relative to MEDIA_ROOT.",
     )
 
     # --- Normalization and Input Shape ---
@@ -84,9 +102,7 @@ class ModelMeta(models.Model):
     )
     size_x = models.IntegerField(default=716, help_text="Expected input image width.")
     size_y = models.IntegerField(default=716, help_text="Expected input image height.")
-    axes = models.CharField(
-        max_length=10, default="2,0,1", help_text="Comma-separated target axis order (e.g., '2,0,1' for CHW)."
-    )
+    axes = models.CharField(max_length=10, default="2,0,1", help_text="Comma-separated target axis order (e.g., '2,0,1' for CHW).")
 
     # --- Inference Parameters ---
     batchsize = models.IntegerField(default=16, help_text="Default batch size for inference.")
@@ -96,17 +112,21 @@ class ModelMeta(models.Model):
     description = models.TextField(blank=True, null=True, help_text="Optional description.")
     date_created = models.DateTimeField(auto_now_add=True)
 
-
     objects = ModelMetaManager()
 
     # --- Type Hinting for Related Fields ---
     if TYPE_CHECKING:
-        labelset: "LabelSet"
-        model: "AiModel"  # Corrected from ai_model to match field name
+        labelset: models.ForeignKey["LabelSet"]
+        model: models.ForeignKey["AiModel"]  # Corrected from ai_model to match field name
+        weights = cast(FieldFile, weights)
 
     class Meta:
         """Metadata options for the ModelMeta model."""
-        # ... existing code ...
+
+        unique_together = (("name", "version", "model"),)
+        ordering = ["-date_created"]
+        verbose_name = "Model Metadata"
+        verbose_name_plural = "Model Metadata"
 
     @classmethod
     def create_from_file(
@@ -115,6 +135,7 @@ class ModelMeta(models.Model):
         model_name: str,
         labelset_name: str,
         weights_file: str,
+        labelset_version: Optional[int | str] = None,
         requested_version: Optional[str] = None,
         bump_if_exists: bool = False,
         **kwargs: Any,
@@ -124,23 +145,34 @@ class ModelMeta(models.Model):
         """
         # Delegate to logic function, passing the class (cls)
         return logic.create_from_file_logic(
-            cls, meta_name, model_name, labelset_name, weights_file,
-            requested_version, bump_if_exists, **kwargs
+            cls,
+            meta_name,
+            model_name,
+            labelset_name,
+            weights_file,
+            labelset_version=labelset_version,
+            requested_version=requested_version,
+            bump_if_exists=bump_if_exists,
+            **kwargs,
         )
-        
+
     @classmethod
     def setup_default_from_huggingface(
         cls: Type["ModelMeta"],
         model_id: str = "wg-lux/colo_segmentation_RegNetX800MF_base",
         labelset_name: Optional[str] = None,
+        labelset_version: Optional[int | str] = None,
     ) -> "ModelMeta":
         """
         Downloads a pretrained model from Hugging Face and initializes ModelMeta automatically.
         """
         # If labelset_name is not provided, handle default logic here if needed
-        return logic.setup_default_from_huggingface_logic(cls, model_id, labelset_name)
-
-
+        return logic.setup_default_from_huggingface_logic(
+            cls,
+            model_id=model_id,
+            labelset_name=labelset_name,
+            labelset_version=labelset_version,
+        )
 
     @classmethod
     def get_latest_version_number(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> int:
@@ -150,9 +182,8 @@ class ModelMeta(models.Model):
         # Delegate to logic function
         return logic.get_latest_version_number_logic(cls, meta_name, model_name)
 
-
     @staticmethod
-    def get_activation_function(activation_name: str) -> "TorchModule":
+    def get_activation_function(activation_name: str):
         """
         Retrieves a PyTorch activation function using external logic.
         """
@@ -166,12 +197,12 @@ class ModelMeta(models.Model):
         # Delegate to logic function
         return logic.get_inference_dataset_config_logic(self)
 
-    def natural_key(self) -> Tuple[str, str]:
+    def natural_key(self) -> Tuple[str, str, str]:
         """
         Returns the natural key for serialization.
         """
         # Assuming natural key is based on name and version, linked to model name
-        return (self.name, self.version, self.model.natural_key())
+        return (self.name, self.version, self.model.natural_key()[0])
 
     def __str__(self) -> str:
         """String representation of the ModelMeta instance."""

endoreg_db/models/metadata/model_meta_logic.py

@@ -1,9 +1,8 @@
 import shutil
 from logging import getLogger
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Type
+from typing import TYPE_CHECKING, Any, Iterable, Optional, Type
 
-from django.core.files import File
 from django.db import transaction
 from huggingface_hub import hf_hub_download
 
@@ -19,6 +18,13 @@ if TYPE_CHECKING:
     from .model_meta import ModelMeta  # Import ModelMeta for type hinting
 
 
+def _get_model_meta_class():
+    """Lazy import to avoid circular imports"""
+    from .model_meta import ModelMeta
+
+    return ModelMeta
+
+
 def get_latest_version_number_logic(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> int:
     """
     Finds the highest numerical version for a given meta_name and model_name.
@@ -55,6 +61,7 @@ def create_from_file_logic(
     model_name: str,
     labelset_name: str,
     weights_file: str,
+    labelset_version: Optional[int | str] = None,
     requested_version: Optional[str] = None,
     bump_if_exists: bool = False,
     **kwargs: Any,
@@ -70,10 +77,23 @@ def create_from_file_logic(
     except AiModel.DoesNotExist as exc:
         raise ValueError(f"AiModel with name '{model_name}' not found.") from exc
 
+    labelset_qs = LabelSet.objects.filter(name=labelset_name)
+    if labelset_version not in (None, "", -1):
+        try:
+            version_value = int(labelset_version)
+        except (TypeError, ValueError):
+            version_value = labelset_version
+        labelset_qs = labelset_qs.filter(version=version_value)
+
     try:
-        label_set = LabelSet.objects.get(name=labelset_name)
+        label_set = labelset_qs.get()
     except LabelSet.DoesNotExist as exc:
-        raise ValueError(f"LabelSet with name '{labelset_name}' not found.") from exc
+        raise ValueError(f"LabelSet '{labelset_name}' with version '{labelset_version}' not found.") from exc
+    except LabelSet.MultipleObjectsReturned:
+        # Prefer the highest version when duplicates remain and no explicit version requested
+        label_set = labelset_qs.order_by("-version").first()
+        if not label_set:
+            raise ValueError(f"LabelSet '{labelset_name}' could not be resolved.")
 
     # --- Determine Version ---
     target_version: str
@@ -165,16 +185,92 @@ def get_activation_function_logic(activation_name: str):
 
 
 # Placeholder for get_inference_dataset_config_logic
+def _normalise_scalar_sequence(value: Any) -> str | None:
+    """Serialise sequence-like values into the canonical comma-separated form."""
+
+    if value in (None, ""):
+        return None
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (list, tuple)):
+        return ",".join(str(item) for item in value)
+    return str(value)
+
+
+def _parse_float_sequence(value: Any, *, fallback: Iterable[float]) -> list[float]:
+    """Coerce stored normalisation strings/iterables to float lists."""
+
+    if value in (None, ""):
+        return list(fallback)
+
+    tokens: Iterable[Any]
+    if isinstance(value, str):
+        tokens = [tok.strip() for tok in value.split(",") if tok.strip()]
+    elif isinstance(value, (list, tuple)):
+        tokens = value
+    else:
+        tokens = [value]
+
+    parsed: list[float] = []
+    for token in tokens:
+        try:
+            parsed.append(float(token))
+        except (TypeError, ValueError):
+            logger.warning("Failed to parse normalisation value %r; using fallback", token)
+            return list(fallback)
+
+    return parsed or list(fallback)
+
+
+def _parse_axes(axes_value: str | Iterable[Any]) -> list[int]:
+    """Normalise stored axis notation to integer indices.
+
+    Historic metadata mixes numeric strings ("2,0,1") with symbolic tokens
+    ("CHW", "HWC"). The inference code ultimately expects a list of integers,
+    so we coerce common symbolic forms to their numeric equivalents and fall
+    back to sequential indices when the format is unknown.
+    """
+
+    if not axes_value:
+        return [0, 1, 2]
+
+    if isinstance(axes_value, str):
+        token_source = axes_value.strip()
+        if "," in token_source:
+            tokens = [token.strip() for token in token_source.split(",") if token.strip()]
+        else:
+            tokens = [char for char in token_source if char.strip()]
+    else:
+        tokens = [str(token).strip() for token in axes_value if str(token).strip()]
+
+    symbol_map = {"C": 0, "H": 1, "W": 2}
+    parsed_axes: list[int] = []
+
+    for idx, token in enumerate(tokens):
+        normalised = token.upper()
+        if normalised.lstrip("+-").isdigit():
+            parsed_axes.append(int(normalised))
+        elif normalised in symbol_map:
+            parsed_axes.append(symbol_map[normalised])
+        else:
+            logger.warning("Unknown axes token '%s'; defaulting to index order", token)
+            parsed_axes.append(idx)
+
+    return parsed_axes or [0, 1, 2]
+
+
 def get_inference_dataset_config_logic(model_meta: "ModelMeta") -> dict:
     # This would typically extract relevant fields from model_meta
     # for configuring a dataset during inference
+    DEFAULT_MEAN = (0.45211223, 0.27139644, 0.19264949)
+    DEFAULT_STD = (0.31418097, 0.21088019, 0.16059452)
+
     return {
-        "mean": [float(x) for x in model_meta.mean.split(",")],
-        "std": [float(x) for x in model_meta.std.split(",")],
-        "size_y": model_meta.size_y,  # Add size_y key
-        "size_x": model_meta.size_x,  # Add size_x key
-        "axes": [int(x) for x in model_meta.axes.split(",")],
-        # Add other relevant config like normalization type, etc.
+        "mean": _parse_float_sequence(model_meta.mean, fallback=DEFAULT_MEAN),
+        "std": _parse_float_sequence(model_meta.std, fallback=DEFAULT_STD),
+        "size_y": int(model_meta.size_y) if model_meta.size_y else 716,
+        "size_x": int(model_meta.size_x) if model_meta.size_x else 716,
+        "axes": _parse_axes(model_meta.axes),
     }
 
 
@@ -294,21 +390,45 @@ def infer_default_model_meta_from_hf(model_id: str) -> dict[str, Any]:
     }
 
 
-def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str | None = None):
+def setup_default_from_huggingface_logic(
+    cls,
+    model_id: str,
+    labelset_name: str | None = None,
+    labelset_version: Optional[int | str] = None,
+):
     """
     Downloads model weights from Hugging Face and auto-fills ModelMeta fields.
     """
     meta = infer_default_model_meta_from_hf(model_id)
 
-    # Download weights
-    weights_path = hf_hub_download(
-        repo_id=model_id,
-        filename="colo_segmentation_RegNetX800MF_base.ckpt",
-        local_dir=WEIGHTS_DIR,
-    )
+    # Download safetensor weights; raise a clear error if unavailable
+    try:
+        weights_path = hf_hub_download(
+            repo_id=model_id,
+            filename="colo_segmentation_RegNetX800MF_base.safetensors",
+            local_dir=WEIGHTS_DIR,
+        )
+    except Exception as exc:  # pragma: no cover - network errors
+        raise RuntimeError("Failed to download safetensor weights from Hugging Face; ensure the repository provides a .safetensors artifact.") from exc
 
     ai_model, _ = AiModel.objects.get_or_create(name=meta["name"])
-    labelset = LabelSet.objects.first() if not labelset_name else LabelSet.objects.get(name=labelset_name)
+    if not labelset_name:
+        labelset = LabelSet.objects.first()
+        if not labelset:
+            raise ValueError("No labelset found and no labelset_name provided")
+    else:
+        labelset_qs = LabelSet.objects.filter(name=labelset_name)
+        if labelset_version not in (None, "", -1):
+            try:
+                version_value = int(labelset_version)
+            except (TypeError, ValueError):
+                version_value = labelset_version
+            labelset_qs = labelset_qs.filter(version=version_value)
+        labelset = labelset_qs.order_by("-version").first()
+        if not labelset:
+            raise ValueError(f"LabelSet '{labelset_name}' with version '{labelset_version}' not found.")
+
+    ModelMeta = _get_model_meta_class()
     model_meta = ModelMeta.objects.filter(name=meta["name"], model=ai_model).first()
     if model_meta:
         logger.info(f"ModelMeta {meta['name']} for model {ai_model.name} already exists. Skipping creation.")
@@ -319,6 +439,7 @@ def setup_default_from_huggingface_logic(cls, model_id: str, labelset_name: str
         meta_name=meta["name"],
         model_name=ai_model.name,
         labelset_name=labelset.name,
+        labelset_version=labelset.version,
         weights_file=weights_path,
         activation=meta["activation"],
         mean=meta["mean"],

endoreg_db/models/metadata/pdf_meta.py

@@ -1,48 +1,44 @@
-from django.db import models
+from typing import TYPE_CHECKING, cast
+
 from django.core.files import File
-from typing import TYPE_CHECKING
+from django.db import models
 
 if TYPE_CHECKING:
     from ..media.pdf.report_reader.report_reader_flag import ReportReaderFlag
 
+
 class PdfType(models.Model):
     """
     Defines a specific type or format of PDF report, linking to flags used for parsing.
 
     Used to configure how different PDF report layouts are processed.
     """
+
     name = models.CharField(max_length=255)
 
-    patient_info_line = models.ForeignKey(
-        "ReportReaderFlag",
-        related_name="pdf_type_patient_info_line", 
-        on_delete=models.CASCADE
-    )
+    patient_info_line = models.ForeignKey("ReportReaderFlag", related_name="pdf_type_patient_info_line", on_delete=models.CASCADE)
     endoscope_info_line = models.ForeignKey(
         "ReportReaderFlag",
-        related_name="pdf_type_endoscopy_info_line", 
+        related_name="pdf_type_endoscopy_info_line",
         on_delete=models.CASCADE,
     )
-    examiner_info_line = models.ForeignKey(
-        "ReportReaderFlag",
-        related_name="pdf_type_examiner_info_line", 
-        on_delete=models.CASCADE
-    )
+    examiner_info_line = models.ForeignKey("ReportReaderFlag", related_name="pdf_type_examiner_info_line", on_delete=models.CASCADE)
     cut_off_above_lines = models.ManyToManyField(
         "ReportReaderFlag",
-        related_name="pdf_type_cut_off_above_lines", 
+        related_name="pdf_type_cut_off_above_lines",
     )
     cut_off_below_lines = models.ManyToManyField(
         "ReportReaderFlag",
-        related_name="pdf_type_cut_off_below_lines", 
+        related_name="pdf_type_cut_off_below_lines",
     )
 
     if TYPE_CHECKING:
-        patient_info_line: "ReportReaderFlag"
-        endoscope_info_line: "ReportReaderFlag"
-        examiner_info_line: "ReportReaderFlag"
-        cut_off_above_lines: models.QuerySet["ReportReaderFlag"]
-        cut_off_below_lines: models.QuerySet["ReportReaderFlag"]
+        patient_info_line: models.ForeignKey["ReportReaderFlag"]
+        endoscope_info_line: models.ForeignKey["ReportReaderFlag"]
+        examiner_info_line: models.ForeignKey["ReportReaderFlag"]
+
+        cut_off_above_lines = cast(models.manager.RelatedManager["ReportReaderFlag"], cut_off_above_lines)
+        cut_off_below_lines = cast(models.manager.RelatedManager["ReportReaderFlag"], cut_off_below_lines)
 
     def __str__(self):
         """Returns a string summary of the PDF type and its associated flags."""
@@ -53,7 +49,7 @@ class PdfType(models.Model):
         summary += f"\nExaminer Info Line: {self.examiner_info_line.value}"
         summary += f"\nCut Off Above Lines: {[_.value for _ in self.cut_off_above_lines.all()]}"
         summary += f"\nCut Off Below Lines: {[_.value for _ in self.cut_off_below_lines.all()]}"
-    
+
         return summary
 
     @classmethod
@@ -61,18 +57,17 @@ class PdfType(models.Model):
         """Returns a default PdfType instance, typically used as a fallback."""
         return PdfType.objects.get(name="ukw-endoscopy-examination-report-generic")
 
+
 class PdfMeta(models.Model):
     """
     Stores metadata associated with a specific PDF document file.
     """
+
     pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
     date = models.DateField()
     time = models.TimeField()
     pdf_hash = models.CharField(max_length=255, unique=True)
 
-    if TYPE_CHECKING:
-        pdf_type: "PdfType"
-
     def __str__(self):
         """Returns the PDF hash as its string representation."""
         return str(self.pdf_hash)