evalsuite-python 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

evalsuite/__init__.py

@@ -5,7 +5,7 @@
 >>> print(result.summary())  # doctest: +SKIP
 """
 
-from . import calibration, classification, clinical, plot, regression, stats
+from . import calibration, classification, clinical, plot, regression, stats, vision
 from .api import evaluate
 from .calibration import (
     CalibrationReport,
@@ -111,8 +111,49 @@ from .stats import (
     wilcoxon_test,
 )
 from .version import __version__
+from .vision import (
+    DetectionReport,
+    SegmentationReport,
+    average_precision_detection,
+    average_surface_distance,
+    boundary_iou,
+    box_iou,
+    detection_pr_curve,
+    detection_report,
+    dice,
+    from_coco,
+    hausdorff_distance,
+    iou,
+    mean_average_precision,
+    mean_pixel_accuracy,
+    miou,
+    per_image_scores,
+    pixel_accuracy,
+    segmentation_confusion,
+    segmentation_report,
+)
 
 __all__ = [
+    "SegmentationReport",
+    "segmentation_report",
+    "detection_pr_curve",
+    "vision",
+    "DetectionReport",
+    "average_precision_detection",
+    "average_surface_distance",
+    "boundary_iou",
+    "box_iou",
+    "detection_report",
+    "dice",
+    "from_coco",
+    "hausdorff_distance",
+    "iou",
+    "mean_average_precision",
+    "mean_pixel_accuracy",
+    "miou",
+    "per_image_scores",
+    "pixel_accuracy",
+    "segmentation_confusion",
     "CalibrationReport",
     "calibration_report",
     "calibration",

evalsuite/api.py

@@ -149,6 +149,16 @@ def evaluate(
     >>> round(r["accuracy"], 2)
     0.75
     """
+    if isinstance(y_true, (list, tuple)) and y_true and isinstance(y_true[0], dict):
+        raise UnsupportedTaskError(
+            "y_true looks like object detection annotations (one dict per image); use "
+            "evalsuite.detection_report(y_true, y_pred) or evalsuite.mean_average_precision(...)."
+        )
+    if np.ndim(y_true) >= 3:
+        raise UnsupportedTaskError(
+            "y_true has 3 or more dimensions, which looks like segmentation masks (images first); use "
+            "evalsuite.segmentation_report(y_true, y_pred) or evalsuite.dice / evalsuite.iou."
+        )
     task = task or _infer_task(y_true, y_prob)
     if task == "classification":
         return _evaluate_classification(

evalsuite/benchmarks.py

@@ -6,7 +6,8 @@ quantities on the same data, and the largest absolute difference between their r
 speed is never shown for numbers that disagree.
 
 References: scikit-learn for classification and regression (``suite="core"``); scikit-learn, statsmodels
-and SciPy for the v0.2.0 clinical, calibration and statistics functions (``suite="clinical"``). A case
+and SciPy for the v0.2.0 clinical, calibration and statistics functions (``suite="clinical"``);
+scikit-learn, SciPy and pycocotools for segmentation and detection (``suite="vision"``). A case
 whose reference library is not installed is timed for EvalSuite only.
 
     >>> from evalsuite.benchmarks import run_benchmarks
@@ -120,6 +121,145 @@ def _core_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, sk_fn) for (name, ref, es_fn, _), sk_fn in zip(cases, sk)]
 
 
+def _vision_cases(n: int, rng: np.random.Generator) -> list[Case]:
+    """v0.3.0: segmentation overlap and surface distance (n = pixels) and COCO detection (n / 1000 images)."""
+    import evalsuite as es
+
+    side = 64
+    n_img = max(1, n // (side * side))
+    k = 5
+    yy, xx = np.ogrid[:side, :side]
+    true = np.zeros((n_img, side, side), dtype=np.int64)
+    for i in range(n_img):
+        for c in range(1, k):
+            cy, cx, r = rng.integers(8, side - 8), rng.integers(8, side - 8), rng.integers(4, 14)
+            true[i][(yy - cy) ** 2 + (xx - cx) ** 2 <= r * r] = c
+    pred = np.roll(true, 1, axis=2)
+    noise = rng.random(pred.shape) < 0.02
+    pred[noise] = rng.integers(0, k, int(noise.sum()))
+    labels = list(range(k))
+
+    def es_overlap() -> list[float]:
+        d = np.asarray(es.dice(true, pred, average=None).value)
+        j = np.asarray(es.iou(true, pred, average=None).value)
+        return [*d, *j]
+
+    n_hd = min(n_img, 50)
+
+    def es_hd() -> list[float]:
+        return [float(es.hausdorff_distance(true[i], pred[i], labels=[k - 1])) for i in range(n_hd)]
+
+    n_det = max(10, n // 1000)
+    y_true, y_pred = [], []
+    for _ in range(n_det):
+        m = int(rng.integers(1, 8))
+        xy = rng.uniform(0, 500, (m, 2))
+        wh = rng.uniform(8, 160, (m, 2))
+        boxes = np.column_stack([xy, xy + wh])
+        lab = rng.integers(1, 6, m)
+        y_true.append({"boxes": boxes, "labels": lab, "iscrowd": np.zeros(m, int)})
+        jitter = boxes + rng.normal(0, 4, boxes.shape)
+        jitter[:, 2:] = np.maximum(jitter[:, 2:], jitter[:, :2] + 1)
+        extra = rng.uniform(0, 500, (3, 2))
+        fp = np.column_stack([extra, extra + rng.uniform(10, 80, (3, 2))])
+        y_pred.append(
+            {
+                "boxes": np.vstack([jitter, fp]),
+                "labels": np.r_[lab, rng.integers(1, 6, 3)],
+                "scores": rng.random(m + 3),
+            }
+        )
+
+    def es_map() -> list[float]:
+        r = es.detection_report(y_true, y_pred)
+        return [r["map"], r["map_50"], r["map_75"], r["mar_100"]]
+
+    cases: list[Case] = [
+        ("segmentation: Dice and IoU per class (n = pixels)", "scikit-learn", es_overlap, None),
+        (f"segmentation: Hausdorff distance ({n_hd} image{'s' if n_hd != 1 else ''})", "SciPy", es_hd, None),
+        (f"detection: COCO evaluation ({n_det} images)", "pycocotools", es_map, None),
+    ]
+    refs: dict[str, Callable[[], Any]] = {}
+    try:
+        import sklearn.metrics as skm
+
+        def sk_overlap() -> list[float]:
+            ft, fp_ = true.ravel(), pred.ravel()
+            return [
+                *skm.f1_score(ft, fp_, labels=labels, average=None),
+                *skm.jaccard_score(ft, fp_, labels=labels, average=None),
+            ]
+
+        refs[cases[0][0]] = sk_overlap
+    except ImportError:
+        pass
+    from scipy import ndimage
+    from scipy.spatial.distance import directed_hausdorff
+
+    def scipy_hd() -> list[float]:
+        out = []
+        for i in range(n_hd):
+            pts = []
+            for m_ in (true[i] == k - 1, pred[i] == k - 1):
+                er = ndimage.binary_erosion(m_, structure=ndimage.generate_binary_structure(2, 1), border_value=0)
+                pts.append(np.argwhere(m_ & ~er).astype(float))
+            out.append(max(directed_hausdorff(pts[0], pts[1])[0], directed_hausdorff(pts[1], pts[0])[0]))
+        return out
+
+    refs[cases[1][0]] = scipy_hd
+    try:
+        import contextlib as _ctx
+        import io
+
+        from pycocotools.coco import COCO  # type: ignore[import-untyped]
+        from pycocotools.cocoeval import COCOeval  # type: ignore[import-untyped]
+
+        def coco_map() -> list[float]:
+            images, anns, dets, aid = [], [], [], 1
+            for i, (t, p) in enumerate(zip(y_true, y_pred)):
+                images.append({"id": i + 1})
+                for b, c in zip(t["boxes"], t["labels"]):
+                    w, h = b[2] - b[0], b[3] - b[1]
+                    anns.append(
+                        {
+                            "id": aid,
+                            "image_id": i + 1,
+                            "category_id": int(c),
+                            "bbox": [b[0], b[1], w, h],
+                            "area": w * h,
+                            "iscrowd": 0,
+                        }
+                    )
+                    aid += 1
+                for b, c, sc in zip(p["boxes"], p["labels"], p["scores"]):
+                    dets.append(
+                        {
+                            "image_id": i + 1,
+                            "category_id": int(c),
+                            "bbox": [b[0], b[1], b[2] - b[0], b[3] - b[1]],
+                            "score": float(sc),
+                        }
+                    )
+            with _ctx.redirect_stdout(io.StringIO()):
+                gt = COCO()
+                gt.dataset = {
+                    "images": images,
+                    "annotations": anns,
+                    "categories": [{"id": c} for c in range(1, 6)],
+                }
+                gt.createIndex()
+                ev = COCOeval(gt, gt.loadRes(dets), "bbox")
+                ev.evaluate()
+                ev.accumulate()
+                ev.summarize()
+            return [ev.stats[0], ev.stats[1], ev.stats[2], ev.stats[8]]
+
+        refs[cases[2][0]] = coco_map
+    except ImportError:
+        pass
+    return [(name, ref, es_fn, refs.get(name)) for name, ref, es_fn, _ in cases]
+
+
 def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     """v0.2.0: diagnostic accuracy, calibration, decision curves and statistical tests."""
     import evalsuite as es
@@ -286,6 +426,7 @@ class BenchmarkResult:
             + (f" | scikit-learn {env['sklearn']}" if env.get("sklearn") else "")
             + (f" | statsmodels {env['statsmodels']}" if env.get("statsmodels") else "")
             + (f" | SciPy {env['scipy']}" if env.get("scipy") else "")
+            + (f" | pycocotools {env['pycocotools']}" if env.get("pycocotools") else "")
             + f" | {env['machine']} | fastest of {env['repeat']} runs"
         )
         note = "Speed-up > 1 means EvalSuite is faster. Max |difference| compares EvalSuite with the reference."
@@ -354,7 +495,8 @@ def run_benchmarks(
     """Time and memory for evaluation workloads at each size, against a reference implementation.
 
     ``suite``: ``"core"`` (classification and regression vs scikit-learn), ``"clinical"`` (v0.2.0 clinical,
-    calibration and statistics vs scikit-learn, statsmodels, SciPy) or ``"all"`` (default).
+    calibration and statistics vs scikit-learn, statsmodels, SciPy), ``"vision"`` (segmentation and COCO
+    detection vs scikit-learn, SciPy, pycocotools) or ``"all"`` (default).
     ``compare_sklearn=False`` times EvalSuite alone. Rows keep ``sklearn_ms``/``sklearn_peak_mb`` for rows
     whose reference is scikit-learn, for compatibility with 0.1.x.
     """
@@ -364,16 +506,24 @@ def run_benchmarks(
 
     if repeat < 1:
         raise ValueError("repeat must be at least 1.")
-    if suite not in ("all", "core", "clinical"):
-        raise ValueError("suite must be 'all', 'core' or 'clinical'.")
+    if suite not in ("all", "core", "clinical", "vision"):
+        raise ValueError("suite must be 'all', 'core', 'clinical' or 'vision'.")
     rng = np.random.default_rng(random_state)
     rows: list[dict[str, Any]] = []
-    versions: dict[str, Optional[str]] = {"sklearn": None, "statsmodels": None}
+    versions: dict[str, Optional[str]] = {"sklearn": None, "statsmodels": None, "pycocotools": None}
     if compare_sklearn:
         for mod in versions:
             with contextlib.suppress(ImportError):
-                versions[mod] = __import__(mod).__version__
-    builders = {"core": [_core_cases], "clinical": [_clinical_cases], "all": [_core_cases, _clinical_cases]}[suite]
+                __import__(mod)
+                from importlib.metadata import version as _dist_version
+
+                versions[mod] = _dist_version("scikit-learn" if mod == "sklearn" else mod)
+    builders = {
+        "core": [_core_cases],
+        "clinical": [_clinical_cases],
+        "vision": [_vision_cases],
+        "all": [_core_cases, _clinical_cases, _vision_cases],
+    }[suite]
     for n in sizes:
         for build in builders:
             for name, ref_name, es_fn, ref_fn in build(int(n), rng):
@@ -410,6 +560,7 @@ def run_benchmarks(
         "scipy": scipy.__version__ if suite != "core" else None,
         "sklearn": versions["sklearn"],
         "statsmodels": versions["statsmodels"] if suite != "core" else None,
+        "pycocotools": versions["pycocotools"] if suite in ("all", "vision") else None,
         "machine": f"{platform.system()} {platform.machine()}",
         "repeat": repeat,
         "suite": suite,

evalsuite/calibration.py

@@ -128,7 +128,7 @@ def logistic_fit(
     beta = np.zeros(design.shape[1])
     for _ in range(max_iter):
         eta = design @ beta + off
-        mu = 1 / (1 + np.exp(-eta))
+        mu = 1 / (1 + np.exp(-np.clip(eta, -700, 700)))
         grad = design.T @ (w * (y - mu))
         hess = (design * (w * mu * (1 - mu))[:, None]).T @ design
         try:

evalsuite/cli/main.py

@@ -226,6 +226,77 @@ def cmd_calibration(args: argparse.Namespace) -> int:
     return 0
 
 
+def read_masks(path: str, name: str) -> Any:
+    """Masks from .npy / .npz (first array, or the array named like the file stem) or a folder of image
+    files (PNG, TIFF, ...; read with Pillow, sorted by file name)."""
+    import numpy as np
+
+    if not os.path.exists(path):
+        raise CLIError(f"{name}: no such file or folder: {path}")
+    if os.path.isdir(path):
+        try:
+            from PIL import Image
+        except ImportError as exc:
+            raise CLIError(f"{name}: reading image folders needs Pillow (pip install pillow).") from exc
+        files = sorted(
+            f for f in os.listdir(path) if f.lower().endswith((".png", ".tif", ".tiff", ".bmp", ".gif", ".jpg"))
+        )
+        if not files:
+            raise CLIError(f"{name}: no image files in {path}.")
+        return [np.asarray(Image.open(os.path.join(path, f))) for f in files]
+    ext = os.path.splitext(path)[1].lower()
+    if ext == ".npy":
+        return np.load(path, allow_pickle=False)
+    if ext == ".npz":
+        with np.load(path, allow_pickle=False) as data:
+            if not data.files:
+                raise CLIError(f"{name}: {path} contains no arrays.")
+            return data[data.files[0]]
+    raise CLIError(f"{name}: use a .npy or .npz file, or a folder of mask images.")
+
+
+def cmd_segmentation(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    spacing = tuple(float(x) for x in args.spacing.split(",")) if args.spacing else None
+    report = es.segmentation_report(
+        read_masks(args.y_true, "y_true"),
+        read_masks(args.y_pred, "y_pred"),
+        num_classes=args.num_classes,
+        ignore_index=args.ignore_index,
+        spacing=spacing,
+        aggregate=args.aggregate,
+        include_background=not args.no_background,
+    )
+    emit(report, args)
+    if args.plot:
+        _save_figure(lambda ax: es.plot.per_class(_report_iou(report), ax=ax), args.plot)
+    return 0
+
+
+def _report_iou(report: Any) -> Any:
+    import numpy as np
+
+    from ..core.result import MetricResult
+
+    return MetricResult(
+        "iou", "IoU", np.array([r["iou"] for r in report.rows]), labels=tuple(r["name"] for r in report.rows)
+    )
+
+
+def cmd_detection(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    y_true, y_pred = es.from_coco(args.ground_truth, args.results)
+    report = es.detection_report(y_true, y_pred, box_format="xywh", max_detections=args.max_detections)
+    emit(report, args)
+    if args.plot:
+        _save_figure(
+            lambda ax: es.plot.detection_pr(y_true, y_pred, ax=ax, box_format="xywh", iou_threshold=0.5), args.plot
+        )
+    return 0
+
+
 def cmd_compare(args: argparse.Namespace) -> int:
     import evalsuite as es
 
@@ -410,6 +481,31 @@ def build_parser() -> argparse.ArgumentParser:
     add_output(p)
     p.set_defaults(func=cmd_calibration)
 
+    p = sub.add_parser(
+        "segmentation",
+        help="segmentation masks: Dice, IoU, mIoU, Boundary IoU, HD95, ASSD per class",
+    )
+    p.add_argument("y_true", help="true masks: .npy/.npz (images on the first axis) or a folder of mask images")
+    p.add_argument("y_pred", help="predicted masks, same layout as y_true")
+    p.add_argument("--num-classes", type=int, help="number of classes (default: largest label + 1)")
+    p.add_argument("--ignore-index", type=int, help="label to ignore, e.g. 255")
+    p.add_argument("--spacing", help="pixel spacing per axis for surface distances, e.g. 0.8,0.8")
+    p.add_argument("--aggregate", choices=("dataset", "image"), default="dataset")
+    p.add_argument("--no-background", action="store_true", help="leave class 0 out")
+    p.add_argument("--plot", metavar="PATH", help="also save a per-class IoU bar chart")
+    add_output(p)
+    p.set_defaults(func=cmd_segmentation)
+
+    p = sub.add_parser(
+        "detection", help="object detection, COCO protocol: mAP, mAP@.50, mAP@.75, mAR, AP per class"
+    )
+    p.add_argument("ground_truth", help="COCO ground-truth JSON (images, annotations, categories)")
+    p.add_argument("results", help="COCO results JSON (image_id, category_id, bbox, score)")
+    p.add_argument("--max-detections", type=int, default=100)
+    p.add_argument("--plot", metavar="PATH", help="also save precision-recall curves per class (IoU 0.5)")
+    add_output(p, digits=3)
+    p.set_defaults(func=cmd_detection)
+
     p = sub.add_parser(
         "plot", help="save a ROC, PR, calibration, decision-curve, confusion-matrix or residual plot"
     )
@@ -433,9 +529,10 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--repeat", type=int, default=5, help="timed repetitions per case; the fastest is reported")
     p.add_argument(
         "--suite",
-        choices=("all", "core", "clinical"),
+        choices=("all", "core", "clinical", "vision"),
         default="all",
-        help="core: classification/regression; clinical: v0.2.0 clinical, calibration and tests (default all)",
+        help="core: classification/regression; clinical: clinical, calibration, tests; vision: segmentation, "
+        "detection (default all)",
     )
     p.add_argument("--no-sklearn", action="store_true", help="time EvalSuite only (skip reference libraries)")
     p.add_argument("--seed", type=int, default=0)

evalsuite/plot.py

@@ -21,7 +21,18 @@ if TYPE_CHECKING:
 
     from .stats.compare import ComparisonResult
 
-__all__ = ["calibration", "comparison", "confusion_matrix", "decision_curve", "pr", "residuals", "roc"]
+__all__ = [
+    "calibration",
+    "comparison",
+    "confusion_matrix",
+    "decision_curve",
+    "detection_pr",
+    "per_class",
+    "pr",
+    "residuals",
+    "roc",
+    "segmentation",
+]
 
 Scores = Union[ArrayLike, Mapping[str, ArrayLike]]
 _STYLES = ("-", "--", "-.", ":")
@@ -339,3 +350,156 @@ def decision_curve(
     )
     ax.legend(loc="upper right", frameon=False)
     return ax
+
+
+def per_class(
+    result: Any,
+    *,
+    ax: Optional[Axes] = None,
+    class_names: Optional[Mapping[Any, str]] = None,
+    sort: bool = False,
+    metric: str = "dice",
+) -> Axes:
+    """Horizontal bar chart of per-class scores, with the macro mean as a reference line.
+
+    ``result`` is any metric computed with ``average=None`` (e.g. ``es.iou(..., average=None)`` or
+    ``es.f1(..., average=None)``), a :class:`SegmentationReport` (``metric`` picks the column: ``"dice"``,
+    ``"iou"``, ``"boundary_iou"``, ``"hd95"`` or ``"assd"``) or a :class:`DetectionReport` (AP per class).
+    Undefined classes (NaN) are labelled "n/a"."""
+    rows = getattr(result, "rows", None)
+    if isinstance(rows, tuple) and rows and "class" in rows[0]:
+        if metric not in rows[0]:
+            raise InputValidationError(
+                f"Unknown metric {metric!r}; use one of dice, iou, boundary_iou, hd95, assd."
+            )
+        values = np.array([r[metric] for r in rows], dtype=np.float64)
+        labels: list[Any] = [r["class"] for r in rows]
+        default_names = [str(r["name"]) for r in rows]
+        title = {"iou": "IoU", "boundary_iou": "Boundary IoU", "hd95": "HD95", "assd": "ASSD"}.get(metric, "Dice")
+    elif isinstance(getattr(result, "per_class", None), Mapping):
+        labels = list(result.per_class)
+        values = np.array([result.per_class[c] for c in labels], dtype=np.float64)
+        default_names = [str(c) for c in labels]
+        title = "AP@[.50:.95]"
+    elif isinstance(getattr(result, "value", None), np.ndarray) and result.labels is not None:
+        values = np.asarray(result.value, dtype=np.float64)
+        labels = list(result.labels)
+        default_names = [str(c) for c in labels]
+        title = result.name
+    else:
+        raise InputValidationError(
+            "per_class() needs a per-class result (a metric computed with average=None) "
+            "or a segmentation or detection report."
+        )
+    names = [str(class_names.get(c, d)) if class_names else d for c, d in zip(labels, default_names)]
+    order = np.argsort(np.nan_to_num(values, nan=-np.inf)) if sort else np.arange(values.size)[::-1]
+    ax = _axes(ax, (5.0, max(2.4, 0.38 * values.size + 1.2)))
+    finite = np.isfinite(values)
+    ax.barh(np.arange(values.size), np.where(finite, values, 0)[order], color="#2a6fb0")
+    for y, i in enumerate(order):
+        ax.text(
+            values[i] if finite[i] else 0,
+            y,
+            f" {values[i]:.3f}" if finite[i] else " n/a",
+            va="center",
+            fontsize=8,
+        )
+    if finite.any():
+        ax.axvline(float(np.nanmean(values[finite])), color="0.3", linestyle="--", linewidth=1, label="macro mean")
+        ax.legend(loc="lower right", frameon=False)
+    ax.set(yticks=np.arange(values.size), yticklabels=[names[i] for i in order], xlabel=title)
+    ax.set_title(f"{title} per class")
+    upper = float(np.nanmax(values[finite])) if finite.any() else 1.0
+    ax.set_xlim(0, max(1.0, upper * 1.15) if upper <= 1 else upper * 1.15)
+    return ax
+
+
+def segmentation(
+    image: Optional[ArrayLike],
+    y_true: ArrayLike,
+    y_pred: ArrayLike,
+    *,
+    ax: Optional[Axes] = None,
+    alpha: float = 0.45,
+    class_names: Optional[Mapping[int, str]] = None,
+    ignore_index: Optional[int] = None,
+) -> Axes:
+    """One image with the predicted mask as a colour overlay and the true class boundaries as contours, so
+    over- and under-segmentation are visible at a glance. Dice and IoU (macro, excluding background) are in
+    the title. ``image`` may be None (masks only), grey-scale or RGB."""
+    from .vision.segmentation import dice as dice_fn
+    from .vision.segmentation import iou as iou_fn
+
+    plt = _plt()
+    t = np.asarray(y_true)
+    pr = np.asarray(y_pred)
+    if t.ndim != 2 or t.shape != pr.shape:
+        raise InputValidationError(
+            "segmentation() plots one 2-D image: y_true and y_pred must be equal 2-D masks."
+        )
+    ax = _axes(ax, (5.0, 5.0 * t.shape[0] / max(t.shape[1], 1)))
+    if image is not None:
+        img = np.asarray(image)
+        ax.imshow(img, cmap="gray" if img.ndim == 2 else None)
+    else:
+        ax.imshow(np.ones(t.shape), cmap="gray", vmin=0, vmax=1)
+    classes = sorted(
+        int(c) for c in np.unique(np.concatenate([t.ravel(), pr.ravel()])) if c != 0 and c != ignore_index
+    )
+    cmap = plt.get_cmap("tab10")
+    overlay = np.zeros((*t.shape, 4))
+    for i, c in enumerate(classes):
+        color = cmap(i % 10)
+        overlay[pr == c] = (*color[:3], alpha)
+        if np.any(t == c):
+            ax.contour((t == c).astype(float), levels=[0.5], colors=[color[:3]], linewidths=1.6)
+        ax.plot(
+            [],
+            [],
+            color=color[:3],
+            linewidth=6,
+            alpha=0.6,
+            label=str(class_names.get(c, c)) if class_names else str(c),
+        )
+    ax.imshow(overlay)
+    if classes:
+        d = float(dice_fn(t, pr, ignore_index=ignore_index, include_background=False))
+        j = float(iou_fn(t, pr, ignore_index=ignore_index, include_background=False))
+        ax.set_title(f"Prediction (fill) vs truth (outline): Dice {d:.3f}, IoU {j:.3f}", fontsize=9)
+        ax.legend(loc="upper right", fontsize=7, frameon=True, title="class", title_fontsize=7)
+    ax.set_axis_off()
+    return ax
+
+
+def detection_pr(
+    y_true: Any,
+    y_pred: Any,
+    *,
+    ax: Optional[Axes] = None,
+    iou_threshold: float = 0.5,
+    box_format: str = "xyxy",
+    class_names: Optional[Mapping[int, str]] = None,
+    max_classes: int = 10,
+) -> Axes:
+    """Precision-recall curves per class for object detection (COCO 101-point interpolation) at one IoU
+    threshold, with each class's AP in the legend; classes beyond ``max_classes`` (by AP) are omitted."""
+    from .vision.detection import detection_pr_curve
+
+    curves = detection_pr_curve(y_true, y_pred, iou_threshold=iou_threshold, box_format=box_format)  # type: ignore[arg-type]
+    if not curves:
+        raise InputValidationError("No class has ground-truth boxes; nothing to plot.")
+    ranked = sorted(curves.items(), key=lambda kv: -float(np.mean(kv[1][1])))[:max_classes]
+    ax = _axes(ax)
+    for i, (c, (rc, pr)) in enumerate(ranked):
+        name = class_names.get(c, str(c)) if class_names else str(c)
+        ax.plot(rc, pr, label=f"{name}: AP = {float(np.mean(pr)):.3f}", **_style(i))
+    m = float(np.mean([np.mean(p) for _, (_, p) in curves.items()]))
+    ax.set(
+        xlabel="Recall",
+        ylabel="Precision",
+        title=f"Detection PR curves @IoU {iou_threshold:g} (mAP = {m:.3f})",
+        xlim=(0, 1.01),
+        ylim=(0, 1.02),
+    )
+    ax.legend(loc="lower left", frameon=False, fontsize="small")
+    return ax

evalsuite/stats/_resolve.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import contextlib
 import inspect
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Any, Optional
 
 import numpy as np
@@ -33,7 +33,30 @@ def resolve_metric(metric: Any) -> tuple[MetricFn, str]:
     raise InputValidationError("metric must be a metric function (e.g. es.f1) or its name (e.g. 'f1').")
 
 
+def as_observations(x: Any, name: str) -> NDArray[Any]:
+    """Observations to resample along the first axis: 1-D/2-D arrays (labels, values, probabilities), stacks of
+    segmentation masks (3-D or more: images first), or per-image detection dicts / differently sized masks
+    (held in a 1-D object array)."""
+    if (
+        isinstance(x, (list, tuple))
+        and x
+        and (isinstance(x[0], Mapping) or (np.ndim(x[0]) >= 2 and len({np.shape(m) for m in x}) > 1))
+    ):
+        out = np.empty(len(x), dtype=object)
+        for i, v in enumerate(x):
+            out[i] = v
+        return out
+    arr = np.asarray(x.to_numpy() if hasattr(x, "to_numpy") else x)
+    if arr.ndim > 2:
+        if arr.shape[0] == 0:
+            raise InputValidationError(f"{name} is empty. Provide at least one observation.")
+        return arr
+    return to_numpy(x, name, allow_2d=True)
+
+
 def is_categorical(y: NDArray[Any]) -> bool:
+    if y.ndim > 2 or (y.dtype == object and y.size and not isinstance(y.flat[0], (str, bytes))):
+        return False
     try:
         return target_type(y) in ("binary", "multiclass", "multilabel")
     except Exception:
@@ -57,9 +80,9 @@ class MetricCall:
                 "Pass exactly one of y_pred (label/value metrics) or y_prob (probability metrics)."
             )
         self.fn = fn
-        self.y_true = to_numpy(y_true, "y_true", allow_2d=True)
+        self.y_true = as_observations(y_true, "y_true")
         second = y_pred if y_pred is not None else y_prob
-        self.second = to_numpy(second, "y_pred" if y_pred is not None else "y_prob", allow_2d=True)
+        self.second = as_observations(second, "y_pred" if y_pred is not None else "y_prob")
         if self.second.shape[0] != self.y_true.shape[0]:
             raise InputValidationError(
                 f"y_true and {'y_pred' if y_pred is not None else 'y_prob'} must contain the same number of "

evalsuite/stats/compare.py

@@ -15,8 +15,7 @@ from ..core.exceptions import InputValidationError
 from ..core.registry import metric_info
 from ..core.result import _json_safe, _latex_escape, _latex_table
 from ..core.types import ArrayLike
-from ..core.validation import to_numpy
-from ._resolve import MetricCall, is_categorical, resolve_metric
+from ._resolve import MetricCall, as_observations, is_categorical, resolve_metric
 from .effect import adjust_pvalues
 from .intervals import _check_level, _check_resamples, bootstrap_distribution, resample_indices
 from .paired import delong_test, mcnemar_test
@@ -296,10 +295,13 @@ def compare(
         raise InputValidationError("compare() needs at least two models (a dict of name -> predictions).")
     if baseline is not None and baseline not in names:
         raise InputValidationError(f"baseline={baseline!r} is not one of the models: {', '.join(names)}.")
-    yt = to_numpy(y_true, "y_true", allow_2d=True)
+    yt = as_observations(y_true, "y_true")
     categorical = is_categorical(yt)
     if metrics is None:
-        if categorical:
+        if yt.ndim > 2 or (yt.dtype == object and yt.size and not isinstance(yt[0], (str, bytes))):
+            is_detection = yt.dtype == object and isinstance(yt[0], Mapping)
+            metric_names = ["mean_average_precision"] if is_detection else ["dice", "iou"]
+        elif categorical:
             metric_names = list(_DEFAULT_CLF) if predictions else []
             if probabilities:
                 metric_names += _DEFAULT_CLF_PROB if np.unique(yt).shape[0] <= 2 and yt.ndim == 1 else ["roc_auc"]