endoreg-db 0.6.4__py3-none-any.whl → 0.8.2__py3-none-any.whl

endoreg_db/models/metadata/model_meta.py

@@ -0,0 +1,193 @@
+"""
+Defines the ModelMeta model and its manager for storing metadata related to AI models,
+including versioning, configuration, and associated weights files.
+Logic is primarily handled in model_meta_logic.py.
+"""
+
+from typing import Optional, TYPE_CHECKING, Tuple, Dict, Any, Type
+# Removed shutil import, now in logic
+
+from django.db import models
+from django.core.validators import FileExtensionValidator
+# Removed torch import, now in logic
+# from torch import nn
+
+from ..utils import WEIGHTS_DIR, STORAGE_DIR
+# Import logic functions
+from . import model_meta_logic as logic
+
+if TYPE_CHECKING:
+    from endoreg_db.models import LabelSet, AiModel  # pylint: disable=import-outside-toplevel
+    from torch.nn import Module as TorchModule
+
+
+class ModelMetaManager(models.Manager):
+    """
+    Custom manager for the ModelMeta model.
+
+    Provides methods for retrieving ModelMeta instances using natural keys.
+    """
+    # ... existing code ...
+
+
+class ModelMeta(models.Model):
+    """
+    Represents the metadata for a specific version of an AI model.
+    Logic for creation, querying, and configuration is in model_meta_logic.py.
+    """
+
+    # ... (fields remain the same: name, version, model, labelset, activation, weights, mean, std, etc.) ...
+    # --- Model Identification ---
+    name = models.CharField(
+        max_length=255,
+        help_text="User-defined name for this specific model metadata set (e.g., 'segmentation_base').",
+    )
+    version = models.CharField(
+        max_length=255,
+        help_text="Version identifier for this metadata set (e.g., '1', '2a').",
+    )
+    model = models.ForeignKey(
+        "AiModel",
+        on_delete=models.CASCADE,
+        related_name="metadata_versions",
+        help_text="The base AI model architecture this metadata belongs to.",
+
+    )
+
+    # --- Model Configuration ---
+    labelset = models.ForeignKey(
+        "LabelSet",
+        on_delete=models.CASCADE,
+        related_name="model_metadata",
+        help_text="The set of labels this model version predicts.",
+    )
+    activation = models.CharField(
+        max_length=50, default="sigmoid", help_text="Output activation function (e.g., 'sigmoid', 'softmax', 'none')."
+    )
+    weights = models.FileField(
+        upload_to=WEIGHTS_DIR.name,  # Use .name for relative path
+        validators=[FileExtensionValidator(allowed_extensions=["ckpt"])],
+        null=True,
+        blank=True,
+        help_text="Path to the model weights file (.ckpt), relative to MEDIA_ROOT.",
+    )
+
+    # --- Normalization and Input Shape ---
+    mean = models.CharField(
+        max_length=255,
+        default="0.45211223,0.27139644,0.19264949",
+        help_text="Comma-separated mean values for input normalization.",
+    )
+    std = models.CharField(
+        max_length=255,
+        default="0.31418097,0.21088019,0.16059452",
+        help_text="Comma-separated standard deviation values for input normalization.",
+    )
+    size_x = models.IntegerField(default=716, help_text="Expected input image width.")
+    size_y = models.IntegerField(default=716, help_text="Expected input image height.")
+    axes = models.CharField(
+        max_length=10, default="2,0,1", help_text="Comma-separated target axis order (e.g., '2,0,1' for CHW)."
+    )
+
+    # --- Inference Parameters ---
+    batchsize = models.IntegerField(default=16, help_text="Default batch size for inference.")
+    num_workers = models.IntegerField(default=0, help_text="Default number of workers for data loading.")
+
+    # --- Metadata ---
+    description = models.TextField(blank=True, null=True, help_text="Optional description.")
+    date_created = models.DateTimeField(auto_now_add=True)
+
+
+    objects = ModelMetaManager()
+
+    # --- Type Hinting for Related Fields ---
+    if TYPE_CHECKING:
+        labelset: "LabelSet"
+        model: "AiModel"  # Corrected from ai_model to match field name
+
+    class Meta:
+        """Metadata options for the ModelMeta model."""
+        # ... existing code ...
+
+    @classmethod
+    def create_from_file(
+        cls: Type["ModelMeta"],
+        meta_name: str,
+        model_name: str,
+        labelset_name: str,
+        weights_file: str,
+        requested_version: Optional[str] = None,
+        bump_if_exists: bool = False,
+        **kwargs: Any,
+    ) -> "ModelMeta":
+        """
+        Creates or updates a ModelMeta instance using external logic.
+        """
+        # Delegate to logic function, passing the class (cls)
+        return logic.create_from_file_logic(
+            cls, meta_name, model_name, labelset_name, weights_file,
+            requested_version, bump_if_exists, **kwargs
+        )
+
+    @classmethod
+    def get_latest_version_number(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> int:
+        """
+        Gets the latest version *number* using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_latest_version_number_logic(cls, meta_name, model_name)
+
+
+    @staticmethod
+    def get_activation_function(activation_name: str) -> "TorchModule":
+        """
+        Retrieves a PyTorch activation function using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_activation_function_logic(activation_name)
+
+    def get_inference_dataset_config(self) -> Dict[str, Any]:
+        """
+        Generates an inference dataset configuration using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_inference_dataset_config_logic(self)
+
+    def natural_key(self) -> Tuple[str, str]:
+        """
+        Returns the natural key for serialization.
+        """
+        # Assuming natural key is based on name and version, linked to model name
+        return (self.name, self.version, self.model.natural_key())
+
+    def __str__(self) -> str:
+        """String representation of the ModelMeta instance."""
+        # Ensure a string is always returned
+        model_name = self.model.name if self.model else "UnknownModel"
+        return f"ModelMeta: {self.name} (v{self.version}) for {model_name}"
+
+    def get_config_dict(self) -> Dict[str, Any]:
+        """
+        Returns the model's configuration parameters using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_config_dict_logic(self)
+
+    @classmethod
+    def get_by_name_version(
+        cls: Type["ModelMeta"],
+        meta_name: str,
+        model_name: str,
+        version: Optional[str] = None,
+    ) -> "ModelMeta":
+        """
+        Retrieves a ModelMeta instance by name, model name, and optionally version using external logic.
+        """
+        # Delegate to logic function
+        return logic.get_model_meta_by_name_version_logic(cls, meta_name, model_name, version)
+
+    @classmethod
+    def get_latest(cls: Type["ModelMeta"], meta_name: str, model_name: str) -> "ModelMeta":
+        """Alias for get_by_name_version(meta_name, model_name, version=None) using external logic."""
+        # Delegate directly to the specific logic function
+        return logic.get_model_meta_by_name_version_logic(cls, meta_name, model_name, version=None)

endoreg_db/models/metadata/model_meta_logic.py

@@ -0,0 +1,236 @@
+import shutil
+from pathlib import Path
+from typing import Optional, TYPE_CHECKING, Any, Type
+
+from django.db import transaction
+
+# Assuming ModelMeta, AiModel, LabelSet are importable from the correct locations
+# Adjust imports based on your project structure if necessary
+from ..administration.ai.ai_model import AiModel
+from ..label.label_set import LabelSet
+from ..utils import WEIGHTS_DIR, STORAGE_DIR
+
+from logging import getLogger
+
+logger = getLogger("ai_model")
+
+if TYPE_CHECKING:
+    from .model_meta import ModelMeta # Import ModelMeta for type hinting
+
+
+def get_latest_version_number_logic(
+    cls: Type["ModelMeta"], meta_name: str, model_name: str
+) -> int:
+    """
+    Finds the highest numerical version for a given meta_name and model_name.
+    Iterates through all versions, attempts to parse them as integers,
+    and returns the maximum integer found. If no numeric versions are found,
+    returns 0.
+    """
+    versions_qs = cls.objects.filter(
+        name=meta_name, model__name=model_name
+    ).values_list('version', flat=True)
+
+    max_v = 0
+    found_numeric_version = False
+
+    for v_str in versions_qs:
+        if v_str is None: # Skip None versions
+            continue
+        try:
+            v_int = int(v_str)
+            if not found_numeric_version or v_int > max_v:
+                max_v = v_int
+            found_numeric_version = True
+        except ValueError:
+            logger.warning(
+                f"Warning: Could not parse version string '{v_str}' as an integer for "
+                f"meta_name='{meta_name}', model_name='{model_name}' while determining the max version."
+            )
+    
+    return max_v if found_numeric_version else 0
+
+
+@transaction.atomic
+def create_from_file_logic(
+    cls: Type["ModelMeta"], # cls is ModelMeta
+    meta_name: str,
+    model_name: str,
+    labelset_name: str,
+    weights_file: str,
+    requested_version: Optional[str] = None,
+    bump_if_exists: bool = False,
+    **kwargs: Any,
+) -> "ModelMeta":
+    """
+    Logic to create or update a ModelMeta instance from a weights file.
+
+    Handles finding related objects, versioning, file copying, and saving.
+    """
+    # --- Find Related Objects ---
+    try:
+        ai_model = AiModel.objects.get(name=model_name)
+    except AiModel.DoesNotExist as exc:
+        raise ValueError(f"AiModel with name '{model_name}' not found.") from exc
+
+    try:
+        label_set = LabelSet.objects.get(name=labelset_name)
+    except LabelSet.DoesNotExist as exc:
+        raise ValueError(f"LabelSet with name '{labelset_name}' not found.") from exc
+
+    # --- Determine Version ---
+    target_version: str
+    latest_version_num = get_latest_version_number_logic(cls, meta_name, model_name)
+
+    if requested_version:
+        target_version = str(requested_version)
+        existing = cls.objects.filter(
+            name=meta_name, model=ai_model, version=target_version
+        ).first()
+        if existing and not bump_if_exists:
+            raise ValueError(
+                f"ModelMeta '{meta_name}' version '{target_version}' for model '{model_name}' "
+                f"already exists. Use bump_if_exists=True to increment."
+            )
+        elif existing and bump_if_exists:
+            target_version = str(latest_version_num + 1)
+            logger.info(f"Bumping version for {meta_name}/{model_name} to {target_version}")
+    else:
+        target_version = str(latest_version_num + 1)
+        logger.info(f"Setting next version for {meta_name}/{model_name} to {target_version}")
+
+
+    # --- Prepare Weights File ---
+    source_weights_path = Path(weights_file).resolve()
+    if not source_weights_path.exists():
+        raise FileNotFoundError(f"Weights file not found: {source_weights_path}")
+
+    # Construct destination path within MEDIA_ROOT/WEIGHTS_DIR
+    weights_filename = source_weights_path.name
+    # Relative path for the FileField upload_to
+    relative_dest_path = Path(WEIGHTS_DIR.relative_to(STORAGE_DIR)) / f"{meta_name}_v{target_version}_{weights_filename}"
+    # Full path for shutil.copy
+    full_dest_path = STORAGE_DIR / relative_dest_path
+
+    # Ensure the destination directory exists
+    full_dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Copy the file
+    try:
+        shutil.copy(source_weights_path, full_dest_path)
+        logger.info(f"Copied weights from {source_weights_path} to {full_dest_path}")
+    except Exception as e:
+        raise IOError(f"Failed to copy weights file: {e}") from e
+
+    # --- Create/Update ModelMeta Instance ---
+    defaults = {
+        "labelset": label_set,
+        "weights": relative_dest_path.as_posix(), # Store relative path for FileField
+        **kwargs, # Pass through other fields like activation, mean, std, etc.
+    }
+
+    # Remove None values from defaults to avoid overriding model defaults unnecessarily
+    defaults = {k: v for k, v in defaults.items() if v is not None}
+
+    model_meta, created = cls.objects.update_or_create(
+        name=meta_name,
+        model=ai_model,
+        version=target_version,
+        defaults=defaults,
+    )
+
+    if created:
+        logger.info(f"Created new ModelMeta: {model_meta}")
+    else:
+        logger.info(f"Updated existing ModelMeta: {model_meta}")
+
+    # --- Optionally update AiModel's active_meta ---
+    # You might want to add logic here to automatically set the newly created/updated
+    # meta as the active one for the AiModel, e.g.:
+    # ai_model.active_meta = model_meta
+    # ai_model.save()
+
+    return model_meta
+
+# --- Add other logic functions referenced by ModelMeta here ---
+# (get_latest_version_number_logic, get_activation_function_logic, etc.)
+# Placeholder for get_activation_function_logic
+def get_activation_function_logic(activation_name: str):
+    import torch.nn as nn # Import locally as it's specific to this function
+    if activation_name.lower() == "sigmoid":
+        return nn.Sigmoid()
+    elif activation_name.lower() == "softmax":
+        # Note: Softmax usually requires specifying the dimension
+        return nn.Softmax(dim=1) # Assuming dim=1 (channels) is common
+    elif activation_name.lower() == "none":
+        return nn.Identity()
+    else:
+        # Consider adding more activations or raising an error
+        raise ValueError(f"Unsupported activation function: {activation_name}")
+
+# Placeholder for get_inference_dataset_config_logic
+def get_inference_dataset_config_logic(model_meta: "ModelMeta") -> dict:
+    # This would typically extract relevant fields from model_meta
+    # for configuring a dataset during inference
+    return {
+        "mean": [float(x) for x in model_meta.mean.split(',')],
+        "std": [float(x) for x in model_meta.std.split(',')],
+        "size_y": model_meta.size_y, # Add size_y key
+        "size_x": model_meta.size_x, # Add size_x key
+        "axes": [int(x) for x in model_meta.axes.split(',')],
+        # Add other relevant config like normalization type, etc.
+    }
+
+# Placeholder for get_config_dict_logic
+def get_config_dict_logic(model_meta: "ModelMeta") -> dict:
+    # Returns a dictionary representation of the model's configuration
+    return {
+        "name": model_meta.name,
+        "version": model_meta.version,
+        "model_name": model_meta.model.name,
+        "labelset_name": model_meta.labelset.name,
+        "activation": model_meta.activation,
+        "weights_path": model_meta.weights.path if model_meta.weights else None,
+        "mean": model_meta.mean,
+        "std": model_meta.std,
+        "size_x": model_meta.size_x,
+        "size_y": model_meta.size_y,
+        "axes": model_meta.axes,
+        "batchsize": model_meta.batchsize,
+        "num_workers": model_meta.num_workers,
+        "description": model_meta.description,
+        # Add any other relevant fields
+    }
+
+# Placeholder for get_model_meta_by_name_version_logic
+def get_model_meta_by_name_version_logic(
+    cls: Type["ModelMeta"],
+    meta_name: str,
+    model_name: str,
+    version: Optional[str] = None,
+) -> "ModelMeta":
+    """
+    Retrieves a ModelMeta instance by name, model name, and optionally version.
+    If version is None, retrieves the latest version based on date_created.
+    """
+    try:
+        ai_model = AiModel.objects.get(name=model_name)
+    except AiModel.DoesNotExist as exc:
+        raise cls.DoesNotExist(f"AiModel with name '{model_name}' not found.") from exc
+
+    if version:
+        try:
+            return cls.objects.get(name=meta_name, model=ai_model, version=version)
+        except Exception as exc:
+            raise cls.DoesNotExist(
+                f"ModelMeta '{meta_name}' version '{version}' for model '{model_name}' not found."
+            ) from exc
+    else:
+        # Get latest version
+        latest = cls.objects.filter(name=meta_name, model=ai_model).order_by("-date_created").first()
+        if latest:
+            return latest
+        else:
+            raise cls.DoesNotExist(
+                f"No ModelMeta found for '{meta_name}' and model '{model_name}'."
+            )

endoreg_db/models/{data_file/metadata → metadata}/pdf_meta.py

@@ -1,19 +1,16 @@
 from django.db import models
-
-# import endoreg_center_id from django settings
-from django.conf import settings
-
-
-# import File class
 from django.core.files import File
+from typing import TYPE_CHECKING
 
-# # check if endoreg_center_id is set
-# if not hasattr(settings, 'ENDOREG_CENTER_ID'):
-#     ENDOREG_CENTER_ID = 9999
-# else:
-#     ENDOREG_CENTER_ID = settings.ENDOREG_CENTER_ID
+if TYPE_CHECKING:
+    from ..media.pdf.report_reader.report_reader_flag import ReportReaderFlag
 
 class PdfType(models.Model):
+    """
+    Defines a specific type or format of PDF report, linking to flags used for parsing.
+
+    Used to configure how different PDF report layouts are processed.
+    """
     name = models.CharField(max_length=255)
 
     patient_info_line = models.ForeignKey(
@@ -40,8 +37,15 @@ class PdfType(models.Model):
         related_name="pdf_type_cut_off_below_lines", 
     )
 
+    if TYPE_CHECKING:
+        patient_info_line: "ReportReaderFlag"
+        endoscope_info_line: "ReportReaderFlag"
+        examiner_info_line: "ReportReaderFlag"
+        cut_off_above_lines: models.QuerySet["ReportReaderFlag"]
+        cut_off_below_lines: models.QuerySet["ReportReaderFlag"]
 
     def __str__(self):
+        """Returns a string summary of the PDF type and its associated flags."""
         summary = f"{self.name}"
         # add lines to summary
         summary += f"\nPatient Info Line: {self.patient_info_line.value}"
@@ -54,21 +58,32 @@ class PdfType(models.Model):
 
     @classmethod
     def default_pdf_type(cls):
+        """Returns a default PdfType instance, typically used as a fallback."""
         return PdfType.objects.get(name="ukw-endoscopy-examination-report-generic")
 
 class PdfMeta(models.Model):
+    """
+    Stores metadata associated with a specific PDF document file.
+    """
     pdf_type = models.ForeignKey(PdfType, on_delete=models.CASCADE)
     date = models.DateField()
     time = models.TimeField()
     pdf_hash = models.CharField(max_length=255, unique=True)
 
+    if TYPE_CHECKING:
+        pdf_type: "PdfType"
+
     def __str__(self):
-        return self.pdf_hash
+        """Returns the PDF hash as its string representation."""
+        return str(self.pdf_hash)
 
     @classmethod
     def create_from_file(cls, pdf_file):
+        """
+        Creates a PdfMeta instance from a PDF file object.
+        Note: This implementation seems incomplete; it doesn't extract hash, date, time, or type.
+        """
         pdf_file = File(pdf_file)
         pdf_meta = cls(file=pdf_file)
         pdf_meta.save()
         return pdf_meta
-