mblt-vision-python 0.0.0__py3-none-any.whl

mblt_vision/utils/evaluation/__init__.py

@@ -0,0 +1,51 @@
+"""
+Evaluation scripts for various datasets.
+"""
+
+from __future__ import annotations
+
+from ._result import EvaluationResult
+from .eval_ade20k import (
+    ADE20KResult,
+    SemanticMetricAccumulator,
+    SemanticSegmentationResult,
+    calculate_semantic_metrics,
+    eval_ade20k,
+    eval_semantic_segmentation,
+)
+from .eval_cityscapes import eval_cityscapes
+from .eval_coco import COCOResult, eval_coco, eval_coco_metrics
+from .eval_dota import DOTAResult, eval_dota
+from .eval_imagenet import ImageNetResult, eval_imagenet, eval_imagenet_metrics
+from .eval_nyu_depth import (
+    NYUDepthMetricAccumulator,
+    NYUDepthResult,
+    calculate_nyu_depth_metrics,
+    eval_nyu_depth,
+)
+from .eval_widerface import WiderFaceResult, eval_widerface
+
+__all__: list[str] = [
+    "eval_coco",
+    "eval_coco_metrics",
+    "COCOResult",
+    "EvaluationResult",
+    "ADE20KResult",
+    "SemanticMetricAccumulator",
+    "SemanticSegmentationResult",
+    "calculate_semantic_metrics",
+    "eval_ade20k",
+    "eval_cityscapes",
+    "eval_semantic_segmentation",
+    "DOTAResult",
+    "eval_dota",
+    "ImageNetResult",
+    "eval_imagenet",
+    "eval_imagenet_metrics",
+    "NYUDepthResult",
+    "NYUDepthMetricAccumulator",
+    "calculate_nyu_depth_metrics",
+    "eval_nyu_depth",
+    "WiderFaceResult",
+    "eval_widerface",
+]

mblt_vision/utils/evaluation/_result.py

@@ -0,0 +1,22 @@
+"""Shared structural contract for Vision evaluator results."""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class EvaluationResult(Protocol):
+    """Result object exposing the primary and secondary benchmark scores."""
+
+    @property
+    def primary_score(self) -> float:
+        """Return the evaluator's primary score."""
+
+        ...
+
+    @property
+    def secondary_score(self) -> float:
+        """Return the evaluator's secondary score."""
+
+        ...

mblt_vision/utils/evaluation/eval_ade20k.py

@@ -0,0 +1,302 @@
+"""ADE20K evaluation for semantic-segmentation models."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, NamedTuple
+
+import numpy as np
+import torch
+from tqdm import tqdm
+
+from ..datasets import (
+    CustomADE20K,
+    CustomCityscapes,
+    get_ade20k_loader,
+    get_cityscapes_loader,
+)
+
+if TYPE_CHECKING:
+    from ...wrapper import MBLT_Engine
+
+
+class SemanticSegmentationResult(NamedTuple):
+    """Generic semantic metrics ordered from primary to secondary."""
+
+    miou: float
+    pixel_accuracy: float
+
+    @property
+    def primary_score(self) -> float:
+        """Return mean intersection-over-union."""
+
+        return self.miou
+
+    @property
+    def secondary_score(self) -> float:
+        """Return overall valid-pixel accuracy."""
+
+        return self.pixel_accuracy
+
+
+ADE20KResult = SemanticSegmentationResult
+
+
+class SemanticMetricAccumulator:
+    """Accumulate an ignore-aware semantic confusion matrix."""
+
+    def __init__(self, nc: int, ignore_label: int = 255) -> None:
+        """Initialize an empty confusion matrix.
+
+        Args:
+            nc: Number of semantic classes.
+            ignore_label: Target label excluded from metrics.
+        """
+
+        self.nc = nc
+        self.ignore_label = ignore_label
+        self.matrix = np.zeros((nc, nc), dtype=np.int64)
+
+    def update(self, prediction: np.ndarray, target: np.ndarray) -> None:
+        """Accumulate one or more predicted and target class maps.
+
+        Args:
+            prediction: Predicted class maps.
+            target: Target class maps with optional ignore labels.
+
+        Raises:
+            ValueError: If prediction and target shapes differ.
+        """
+
+        prediction = np.asarray(prediction)
+        target = np.asarray(target)
+        if prediction.shape != target.shape:
+            raise ValueError(
+                f"Semantic prediction and target shapes must match, got {prediction.shape} and {target.shape}."
+            )
+        valid_target = (
+            np.isfinite(target)
+            & (target >= 0)
+            & (target < self.nc)
+            & (target == np.floor(target))
+        )
+        allowed_target = valid_target | (target == self.ignore_label)
+        if not bool(allowed_target.all()):
+            invalid_values = np.asarray(np.unique(target[~allowed_target]))
+            raise ValueError(
+                f"Semantic targets must be finite class IDs in [0, {self.nc - 1}] "
+                f"or ignore label {self.ignore_label}; got {invalid_values.tolist()}."
+            )
+        valid_prediction = (
+            np.isfinite(prediction)
+            & (prediction >= 0)
+            & (prediction < self.nc)
+            & (prediction == np.floor(prediction))
+        )
+        invalid_prediction = valid_target & ~valid_prediction
+        if invalid_prediction.any():
+            invalid_values = np.asarray(np.unique(prediction[invalid_prediction]))
+            raise ValueError(
+                f"Semantic predictions at valid target pixels must be finite class IDs in [0, {self.nc - 1}], "
+                f"got {invalid_values.tolist()}."
+            )
+        if valid_target.any():
+            histogram = np.bincount(
+                self.nc * target[valid_target].astype(np.int64)
+                + prediction[valid_target].astype(np.int64),
+                minlength=self.nc**2,
+            )
+            self.matrix += histogram.reshape(self.nc, self.nc)
+
+    def result(self) -> SemanticSegmentationResult:
+        """Compute mIoU over present classes and overall pixel accuracy.
+
+        Returns:
+            Pooled semantic-segmentation metrics.
+
+        Raises:
+            ValueError: If no valid target pixels were accumulated.
+        """
+
+        ground_truth = self.matrix.sum(axis=1)
+        predicted = self.matrix.sum(axis=0)
+        intersection = np.diag(self.matrix)
+        union = ground_truth + predicted - intersection
+        present = ground_truth > 0
+        if not present.any():
+            raise ValueError("Semantic evaluation received no valid target pixels.")
+        iou = np.divide(
+            intersection,
+            union,
+            out=np.zeros(self.nc, dtype=np.float64),
+            where=union > 0,
+        )
+        total = int(self.matrix.sum())
+        return SemanticSegmentationResult(
+            miou=float(iou[present].mean()),
+            pixel_accuracy=float(intersection.sum() / total),
+        )
+
+
+def calculate_semantic_metrics(
+    prediction: np.ndarray,
+    target: np.ndarray,
+    nc: int = 150,
+    ignore_label: int = 255,
+) -> ADE20KResult:
+    """Calculate semantic metrics for one batch of class maps.
+
+    Args:
+        prediction: Predicted class maps.
+        target: Target class maps.
+        nc: Number of semantic classes.
+        ignore_label: Target label excluded from metrics.
+
+    Returns:
+        Semantic metrics exposed through the ADE20K compatibility alias.
+
+    Raises:
+        ValueError: If shapes differ or no valid target pixels are present.
+    """
+
+    accumulator = SemanticMetricAccumulator(nc=nc, ignore_label=ignore_label)
+    accumulator.update(prediction, target)
+    return accumulator.result()
+
+
+def _evaluate_semantic_loader(
+    model: MBLT_Engine,
+    loader: torch.utils.data.DataLoader,
+    nc: int,
+    description: str = "Evaluating semantic segmentation",
+) -> SemanticSegmentationResult:
+    """Evaluate input-space semantic maps from an organized validation loader.
+
+    Args:
+        model: Initialized semantic-segmentation engine.
+        loader: Organized validation data loader.
+        nc: Number of semantic classes.
+        description: Progress-bar description.
+
+    Returns:
+        Pooled semantic-segmentation metrics.
+
+    Raises:
+        ValueError: If postprocessing returns no class maps or no valid targets exist.
+    """
+
+    accumulator = SemanticMetricAccumulator(nc=nc)
+    for inputs, targets, _shapes, _ratio_pads, _ in tqdm(loader, desc=description):
+        # TODO: Restore logits to original geometry using shapes and ratio_pads when
+        # Ultralytics adopts native-geometry semantic validation metrics.
+        result = model.postprocess(model(inputs))
+        semantic_mask = result.semantic_mask
+        if semantic_mask is None:
+            raise ValueError("Semantic postprocessor returned no class maps.")
+        prediction = (
+            semantic_mask.detach().cpu().numpy()
+            if isinstance(semantic_mask, torch.Tensor)
+            else semantic_mask
+        )
+        accumulator.update(np.asarray(prediction), targets)
+    return accumulator.result()
+
+
+def eval_semantic_segmentation(
+    model: MBLT_Engine,
+    data_path: str,
+    batch_size: int,
+    dataset: str | None = None,
+) -> SemanticSegmentationResult:
+    """Evaluate a semantic model with the loader for its configured taxonomy.
+
+    Args:
+        model: Initialized semantic-segmentation engine.
+        data_path: Organized dataset root.
+        batch_size: Number of validation samples per inference batch.
+        dataset: Optional taxonomy override. Defaults to ``model.post_cfg.dataset``.
+
+    Returns:
+        Generic mIoU and pixel-accuracy result.
+
+    Raises:
+        ValueError: If preprocessing metadata, image size, taxonomy, predictions, or targets are invalid.
+    """
+
+    configured_dataset = model.post_cfg.get("dataset")
+    if not isinstance(configured_dataset, str) or not configured_dataset:
+        raise ValueError(
+            "Semantic validation requires model.post_cfg.dataset to declare the model taxonomy."
+        )
+    configured_taxonomy = configured_dataset.lower()
+    if dataset is not None:
+        if not isinstance(dataset, str) or not dataset:
+            raise ValueError(
+                "Semantic validation dataset overrides must be non-empty strings."
+            )
+        requested_taxonomy = dataset.lower()
+        if requested_taxonomy != configured_taxonomy:
+            raise ValueError(
+                f"Requested semantic validation taxonomy {requested_taxonomy!r} conflicts with "
+                f"the model's configured taxonomy {configured_taxonomy!r}."
+            )
+    taxonomy = configured_taxonomy
+
+    letterbox_cfg = model.pre_cfg.get("LetterBox")
+    if not isinstance(letterbox_cfg, dict) or "img_size" not in letterbox_cfg:
+        raise ValueError(
+            "Semantic validation requires a LetterBox img_size in the model preprocessing config."
+        )
+    image_size = letterbox_cfg["img_size"]
+    if not isinstance(image_size, list) or len(image_size) != 2:
+        raise ValueError(
+            "Semantic validation img_size must be a two-item [height, width] list."
+        )
+
+    image_size_tuple = (int(image_size[0]), int(image_size[1]))
+    if taxonomy == "ade20k":
+        validation_dataset = CustomADE20K(data_path)
+        loader = get_ade20k_loader(
+            validation_dataset,
+            batch_size,
+            model.preprocess_with_metadata,
+            image_size=image_size_tuple,
+        )
+        default_nc = 150
+        description = "Evaluating ADE20K"
+    elif taxonomy == "cityscapes":
+        validation_dataset = CustomCityscapes(data_path)
+        loader = get_cityscapes_loader(
+            validation_dataset,
+            batch_size,
+            model.preprocess_with_metadata,
+            image_size=image_size_tuple,
+        )
+        default_nc = 19
+        description = "Evaluating Cityscapes"
+    else:
+        raise ValueError(f"Unsupported semantic validation dataset: {taxonomy!r}.")
+    nc = int(getattr(model.postprocessor, "nc", default_nc))
+    return _evaluate_semantic_loader(model, loader, nc, description=description)
+
+
+def eval_ade20k(model: MBLT_Engine, data_path: str, batch_size: int) -> ADE20KResult:
+    """Evaluate a semantic-segmentation model on ADE20K validation masks.
+
+    Args:
+        model: Initialized semantic-segmentation engine.
+        data_path: Organized ADE20K dataset root.
+        batch_size: Number of validation samples per inference batch.
+
+    Returns:
+        ADE20K metrics through the generic semantic result type.
+
+    Raises:
+        ValueError: If the dataset, preprocessing metadata, predictions, or targets are invalid.
+    """
+
+    return eval_semantic_segmentation(
+        model,
+        data_path,
+        batch_size,
+        dataset="ade20k",
+    )

mblt_vision/utils/evaluation/eval_cityscapes.py

@@ -0,0 +1,32 @@
+"""Cityscapes evaluation for semantic-segmentation models."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .eval_ade20k import SemanticSegmentationResult, eval_semantic_segmentation
+
+if TYPE_CHECKING:
+    from ...wrapper import MBLT_Engine
+
+
+def eval_cityscapes(
+    model: MBLT_Engine, data_path: str, batch_size: int
+) -> SemanticSegmentationResult:
+    """Evaluate a semantic-segmentation model on Cityscapes validation masks.
+
+    Args:
+        model: Initialized semantic-segmentation engine.
+        data_path: Organized Cityscapes dataset root.
+        batch_size: Number of validation samples per inference batch.
+
+    Returns:
+        Cityscapes mIoU and pixel-accuracy metrics.
+
+    Raises:
+        ValueError: If the dataset, preprocessing metadata, predictions, or targets are invalid.
+    """
+
+    return eval_semantic_segmentation(
+        model, data_path, batch_size, dataset="cityscapes"
+    )