mblt-vision-python 0.0.0__py3-none-any.whl

mblt_vision/utils/evaluation/eval_imagenet.py

@@ -0,0 +1,151 @@
+"""
+Evaluation script for ImageNet dataset.
+"""
+
+from __future__ import annotations
+
+import math
+from time import time
+from typing import TYPE_CHECKING, NamedTuple
+
+import numpy as np
+import torch
+from tqdm import tqdm
+
+from ..datasets import CustomImageFolder, get_imagenet_loader
+from ..datasets.readiness import IMAGENET_SYNSET_ORDER, IMAGENET_SYNSETS
+
+if TYPE_CHECKING:
+    from ...wrapper import MBLT_Engine
+
+
+class ImageNetResult(NamedTuple):
+    """ImageNet metrics ordered from primary to secondary."""
+
+    top1: float
+    top5: float
+
+    @property
+    def primary_score(self) -> float:
+        """Return the primary ImageNet validation metric."""
+        return self.top1
+
+    @property
+    def secondary_score(self) -> float:
+        """Return the secondary ImageNet validation metric."""
+        return self.top5
+
+
+def eval_imagenet_metrics(
+    model: MBLT_Engine, data_path: str, batch_size: int
+) -> ImageNetResult:
+    """Evaluates a classification model on the ImageNet validation set.
+
+    Computes Top-1 and Top-5 accuracy and inference speed (FPS) on the NPU.
+
+    Args:
+        model (MBLT_Engine): The vision engine to evaluate.
+        data_path (str): Path to the ImageNet validation images.
+        batch_size (int): Number of images per inference batch.
+
+    Returns:
+        ImageNetResult: Top-1 primary accuracy and Top-5 secondary accuracy.
+    """
+    dataset_name = model.post_cfg.get("dataset")
+    if not isinstance(dataset_name, str) or dataset_name.lower() != "imagenet":
+        raise ValueError(
+            "ImageNet evaluation requires model post_cfg.dataset to be 'imagenet', "
+            f"got {dataset_name!r}."
+        )
+    dataset = CustomImageFolder(data_path)
+    unknown_synsets = set(dataset.classes) - IMAGENET_SYNSETS
+    if unknown_synsets:
+        raise ValueError(
+            "ImageNet evaluation found non-canonical synset directories: "
+            f"{', '.join(sorted(unknown_synsets)[:5])}."
+        )
+    dataset.class_to_idx = {
+        synset: index
+        for index, synset in enumerate(IMAGENET_SYNSET_ORDER)
+        if synset in dataset.class_to_idx
+    }
+    dataset.make_dataset()
+    num_data = len(dataset)
+    if num_data == 0:
+        raise ValueError(
+            f"ImageNet evaluation dataset contains no supported images: {data_path}."
+        )
+    dataloader = get_imagenet_loader(dataset, batch_size, model.preprocess)
+    total_iter = math.ceil(num_data / batch_size)
+    pbar = tqdm(dataloader, total=total_iter, desc="Evaluating ImageNet")
+    inference_time = 0.0
+    cum_num_data = 0
+    cum_top1_correct = 0
+    cum_top5_correct = 0
+    top1_acc = 0.0
+    top5_acc = 0.0
+    for input_npu, label in pbar:
+        cum_num_data += len(label)
+        tic = time()
+        out_npu = model(input_npu)
+        inference_time += time() - tic
+        result = model.postprocess(out_npu)
+        output = result.output
+        label_array = np.asarray(label)
+        if not isinstance(output, (np.ndarray, torch.Tensor)):
+            raise TypeError(
+                f"Expected classification output to be a tensor or ndarray, got {type(output)}."
+            )
+        if output.ndim != 2:
+            raise ValueError(
+                f"ImageNet classification output must have shape [B, C], got {tuple(output.shape)}."
+            )
+
+        output_batch_size = output.shape[0]
+        if output_batch_size != len(label_array):
+            raise ValueError(
+                "ImageNet classification output batch size does not match labels: "
+                f"got {output_batch_size} outputs for {len(label_array)} labels."
+            )
+        if isinstance(output, np.ndarray):
+            prediction = output.argmax(-1)
+        else:
+            prediction = output.argmax(-1).cpu().numpy()
+        top_k = min(5, output.shape[-1])
+        if isinstance(output, torch.Tensor):
+            top5_prediction = output.topk(top_k, dim=-1).indices.cpu().numpy()
+        else:
+            top5_prediction = np.argpartition(output, -top_k, axis=-1)[:, -top_k:]
+        cum_top1_correct += (prediction == label_array).sum().item()
+        cum_top5_correct += (
+            np.any(top5_prediction == label_array[:, np.newaxis], axis=-1).sum().item()
+        )
+        top1_acc = cum_top1_correct / cum_num_data
+        top5_acc = cum_top5_correct / cum_num_data
+        pbar.set_postfix_str(
+            f"Top 1 Acc.: {100 * top1_acc:.3f}%, Top 5 Acc.: {100 * top5_acc:.3f}%, "
+            f"NPU FPS: {cum_num_data / inference_time:.3f}"
+        )
+    pbar.close()
+    print("ImageNet evaluation completed")
+    print(
+        f"Top 1 Acc.: {100 * top1_acc:.3f}%, "
+        f"Top 5 Acc.: {100 * top5_acc:.3f}%, "
+        f"NPU FPS: {cum_num_data / inference_time:.3f}"
+    )
+    return ImageNetResult(top1=top1_acc, top5=top5_acc)
+
+
+def eval_imagenet(model: MBLT_Engine, data_path: str, batch_size: int) -> float:
+    """Evaluate ImageNet and return Top-1 accuracy for numeric API compatibility.
+
+    Args:
+        model: Vision engine to evaluate.
+        data_path: Path to the ImageNet validation images.
+        batch_size: Number of images per inference batch.
+
+    Returns:
+        Top-1 accuracy in the range 0.0 to 1.0.
+    """
+
+    return eval_imagenet_metrics(model, data_path, batch_size).top1

mblt_vision/utils/evaluation/eval_nyu_depth.py

@@ -0,0 +1,189 @@
+"""NYU Depth V2 evaluation for monocular depth-estimation models."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import numpy as np
+from mblt_vision.utils.preprocess import build_preprocess
+from tqdm import tqdm
+
+from ..datasets import CustomNYUDepth, get_nyu_depth_loader
+
+if TYPE_CHECKING:
+    from ...wrapper import MBLT_Engine
+
+
+@dataclass(frozen=True)
+class NYUDepthResult:
+    """Median-aligned NYU Depth V2 metrics."""
+
+    delta1: float
+    abs_rel: float
+    rmse: float
+
+    @property
+    def primary_score(self) -> float:
+        """Return the primary NYU Depth validation metric."""
+
+        return self.delta1
+
+    @property
+    def secondary_score(self) -> float:
+        """Return the secondary NYU Depth validation metric."""
+
+        return self.abs_rel
+
+
+class NYUDepthMetricAccumulator:
+    """Accumulate median-aligned metrics over every valid NYU depth pixel."""
+
+    MIN_DEPTH = 0.001
+    MAX_DEPTH = 100.0
+
+    def __init__(self) -> None:
+        """Initialize zero-valued pixel sums."""
+
+        self.delta1_sum = 0.0
+        self.abs_rel_sum = 0.0
+        self.squared_error_sum = 0.0
+        self.valid_pixel_count = 0
+
+    def update(self, prediction: np.ndarray, target: np.ndarray) -> None:
+        """Median-align one prediction and add its valid-pixel statistics."""
+
+        prediction = _as_real_float32(prediction, "prediction")
+        target = _as_real_float32(target, "target")
+        if prediction.shape != target.shape:
+            raise ValueError(
+                f"NYU Depth prediction and target shapes must match, got {prediction.shape} and {target.shape}."
+            )
+        if not np.isfinite(target).all():
+            raise ValueError("NYU Depth target contains non-finite values.")
+        if (target < 0).any():
+            raise ValueError("NYU Depth target contains negative values.")
+        valid = (
+            np.isfinite(target) & (target > self.MIN_DEPTH) & (target < self.MAX_DEPTH)
+        )
+        valid_pixel_count = int(valid.sum())
+        if valid_pixel_count == 0:
+            raise ValueError(
+                "NYU Depth sample has no valid pixels in the (0.001, 100.0) range."
+            )
+
+        predicted, actual = prediction[valid], target[valid]
+        invalid_prediction_count = int((~np.isfinite(predicted)).sum())
+        if invalid_prediction_count:
+            raise ValueError(
+                f"NYU Depth prediction contains {invalid_prediction_count} non-finite value(s) at valid target pixels."
+            )
+        median_prediction = np.median(np.maximum(predicted, self.MIN_DEPTH))
+        median_target = np.median(actual)
+        aligned = predicted * (median_target / median_prediction)
+        aligned = np.clip(aligned, self.MIN_DEPTH, self.MAX_DEPTH)
+        ratio = np.maximum(actual / aligned, aligned / actual)
+        self.delta1_sum += float(np.sum(ratio < 1.25))
+        self.abs_rel_sum += float(np.sum(np.abs(actual - aligned) / actual))
+        self.squared_error_sum += float(np.sum((actual - aligned) ** 2))
+        self.valid_pixel_count += valid_pixel_count
+
+    def result(self) -> NYUDepthResult:
+        """Return metrics pooled over all accumulated valid pixels."""
+
+        if self.valid_pixel_count == 0:
+            raise ValueError("NYU Depth evaluation received no valid pixels.")
+        return NYUDepthResult(
+            delta1=self.delta1_sum / self.valid_pixel_count,
+            abs_rel=self.abs_rel_sum / self.valid_pixel_count,
+            rmse=float(np.sqrt(self.squared_error_sum / self.valid_pixel_count)),
+        )
+
+
+def _as_real_float32(values: np.ndarray, name: str) -> np.ndarray:
+    """Validate metric input dtype before converting it to float32."""
+
+    array = np.asarray(values)
+    if not np.issubdtype(array.dtype, np.number) or np.issubdtype(
+        array.dtype, np.complexfloating
+    ):
+        raise ValueError(
+            f"NYU Depth {name} must use a real numeric dtype, got {array.dtype}."
+        )
+    return np.asarray(array, dtype=np.float32)
+
+
+def calculate_nyu_depth_metrics(
+    prediction: np.ndarray, target: np.ndarray
+) -> NYUDepthResult:
+    """Calculate official pooled metrics for one median-aligned NYU sample."""
+
+    accumulator = NYUDepthMetricAccumulator()
+    accumulator.update(prediction, target)
+    return accumulator.result()
+
+
+def eval_nyu_depth(
+    model: MBLT_Engine, data_path: str, batch_size: int
+) -> NYUDepthResult:
+    """Evaluate a depth model on paired NYU validation images and depth maps."""
+
+    dataset_name = model.post_cfg.get("dataset")
+    if not isinstance(dataset_name, str) or dataset_name.lower() != "nyu-depth":
+        raise ValueError(
+            "NYU Depth evaluation requires model post_cfg.dataset to be 'nyu-depth', "
+            f"got {dataset_name!r}."
+        )
+    dataset = CustomNYUDepth(data_path)
+    letterbox_cfg = model.pre_cfg.get("LetterBox")
+    if not isinstance(letterbox_cfg, dict) or "img_size" not in letterbox_cfg:
+        raise ValueError(
+            "NYU Depth validation requires a LetterBox img_size in the model preprocessing config."
+        )
+    image_size = letterbox_cfg["img_size"]
+    if not isinstance(image_size, list) or len(image_size) != 2:
+        raise ValueError(
+            "NYU Depth validation img_size must be a two-item [height, width] list."
+        )
+
+    validation_pre_cfg = {
+        name: config for name, config in model.pre_cfg.items() if name != "LetterBox"
+    }
+    validation_preprocessor = build_preprocess(validation_pre_cfg)
+    loader = get_nyu_depth_loader(
+        dataset,
+        batch_size,
+        validation_preprocessor,
+        image_size=(int(image_size[0]), int(image_size[1])),
+    )
+    accumulator = NYUDepthMetricAccumulator()
+    for inputs, targets, _, _, _ in tqdm(loader, desc="Evaluating NYU Depth"):
+        output = model(inputs)
+        result = model.postprocess(output)
+        depth = result.depth
+        if depth is None:
+            raise ValueError("Depth postprocessor returned no depth maps.")
+        if isinstance(depth, list):
+            maps = depth
+        elif len(targets) == 1 and depth.ndim == 2:
+            maps = [depth]
+        else:
+            if depth.ndim < 3 or depth.shape[0] != len(targets):
+                raise ValueError(
+                    "Depth postprocessor output batch length mismatch: "
+                    f"maps={depth.shape[0] if depth.ndim else 0}, "
+                    f"targets={len(targets)}."
+                )
+            maps = [depth[index] for index in range(len(targets))]
+        if len(maps) != len(targets):
+            raise ValueError(
+                f"Depth postprocessor returned {len(maps)} maps for {len(targets)} targets."
+            )
+        for prediction, target in zip(maps, targets):
+            array = (
+                prediction.detach().cpu().numpy()
+                if hasattr(prediction, "detach")
+                else np.asarray(prediction)
+            )
+            accumulator.update(array, target)
+    return accumulator.result()