evalsuite-python 0.2.1__py3-none-any.whl → 0.3.1__py3-none-any.whl

evalsuite/__init__.py

@@ -5,7 +5,7 @@
 >>> print(result.summary())  # doctest: +SKIP
 """
 
-from . import calibration, classification, clinical, plot, regression, stats
+from . import calibration, classification, clinical, plot, regression, stats, vision
 from .api import evaluate
 from .calibration import (
     CalibrationReport,
@@ -111,8 +111,49 @@ from .stats import (
     wilcoxon_test,
 )
 from .version import __version__
+from .vision import (
+    DetectionReport,
+    SegmentationReport,
+    average_precision_detection,
+    average_surface_distance,
+    boundary_iou,
+    box_iou,
+    detection_pr_curve,
+    detection_report,
+    dice,
+    from_coco,
+    hausdorff_distance,
+    iou,
+    mean_average_precision,
+    mean_pixel_accuracy,
+    miou,
+    per_image_scores,
+    pixel_accuracy,
+    segmentation_confusion,
+    segmentation_report,
+)
 
 __all__ = [
+    "SegmentationReport",
+    "segmentation_report",
+    "detection_pr_curve",
+    "vision",
+    "DetectionReport",
+    "average_precision_detection",
+    "average_surface_distance",
+    "boundary_iou",
+    "box_iou",
+    "detection_report",
+    "dice",
+    "from_coco",
+    "hausdorff_distance",
+    "iou",
+    "mean_average_precision",
+    "mean_pixel_accuracy",
+    "miou",
+    "per_image_scores",
+    "pixel_accuracy",
+    "segmentation_confusion",
     "CalibrationReport",
     "calibration_report",
     "calibration",

evalsuite/api.py

@@ -149,6 +149,16 @@ def evaluate(
     >>> round(r["accuracy"], 2)
     0.75
     """
+    if isinstance(y_true, (list, tuple)) and y_true and isinstance(y_true[0], dict):
+        raise UnsupportedTaskError(
+            "y_true looks like object detection annotations (one dict per image); use "
+            "evalsuite.detection_report(y_true, y_pred) or evalsuite.mean_average_precision(...)."
+        )
+    if np.ndim(y_true) >= 3:
+        raise UnsupportedTaskError(
+            "y_true has 3 or more dimensions, which looks like segmentation masks (images first); use "
+            "evalsuite.segmentation_report(y_true, y_pred) or evalsuite.dice / evalsuite.iou."
+        )
     task = task or _infer_task(y_true, y_prob)
     if task == "classification":
         return _evaluate_classification(

evalsuite/benchmarks.py

@@ -6,7 +6,8 @@ quantities on the same data, and the largest absolute difference between their r
 speed is never shown for numbers that disagree.
 
 References: scikit-learn for classification and regression (``suite="core"``); scikit-learn, statsmodels
-and SciPy for the v0.2.0 clinical, calibration and statistics functions (``suite="clinical"``). A case
+and SciPy for the v0.2.0 clinical, calibration and statistics functions (``suite="clinical"``);
+scikit-learn, SciPy and pycocotools for segmentation and detection (``suite="vision"``). A case
 whose reference library is not installed is timed for EvalSuite only.
 
     >>> from evalsuite.benchmarks import run_benchmarks
@@ -16,8 +17,10 @@ whose reference library is not installed is timed for EvalSuite only.
 from __future__ import annotations
 
 import contextlib
+import importlib
 import json
 import platform
+import re
 import time
 import tracemalloc
 import warnings
@@ -120,6 +123,145 @@ def _core_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, sk_fn) for (name, ref, es_fn, _), sk_fn in zip(cases, sk)]
 
 
+def _vision_cases(n: int, rng: np.random.Generator) -> list[Case]:
+    """v0.3.0: segmentation overlap and surface distance (n = pixels) and COCO detection (n / 1000 images)."""
+    import evalsuite as es
+
+    side = 64
+    n_img = max(1, n // (side * side))
+    k = 5
+    yy, xx = np.ogrid[:side, :side]
+    true = np.zeros((n_img, side, side), dtype=np.int64)
+    for i in range(n_img):
+        for c in range(1, k):
+            cy, cx, r = rng.integers(8, side - 8), rng.integers(8, side - 8), rng.integers(4, 14)
+            true[i][(yy - cy) ** 2 + (xx - cx) ** 2 <= r * r] = c
+    pred = np.roll(true, 1, axis=2)
+    noise = rng.random(pred.shape) < 0.02
+    pred[noise] = rng.integers(0, k, int(noise.sum()))
+    labels = list(range(k))
+
+    def es_overlap() -> list[float]:
+        d = np.asarray(es.dice(true, pred, average=None).value)
+        j = np.asarray(es.iou(true, pred, average=None).value)
+        return [*d, *j]
+
+    n_hd = min(n_img, 50)
+
+    def es_hd() -> list[float]:
+        return [float(es.hausdorff_distance(true[i], pred[i], labels=[k - 1])) for i in range(n_hd)]
+
+    n_det = max(10, n // 1000)
+    y_true, y_pred = [], []
+    for _ in range(n_det):
+        m = int(rng.integers(1, 8))
+        xy = rng.uniform(0, 500, (m, 2))
+        wh = rng.uniform(8, 160, (m, 2))
+        boxes = np.column_stack([xy, xy + wh])
+        lab = rng.integers(1, 6, m)
+        y_true.append({"boxes": boxes, "labels": lab, "iscrowd": np.zeros(m, int)})
+        jitter = boxes + rng.normal(0, 4, boxes.shape)
+        jitter[:, 2:] = np.maximum(jitter[:, 2:], jitter[:, :2] + 1)
+        extra = rng.uniform(0, 500, (3, 2))
+        fp = np.column_stack([extra, extra + rng.uniform(10, 80, (3, 2))])
+        y_pred.append(
+            {
+                "boxes": np.vstack([jitter, fp]),
+                "labels": np.r_[lab, rng.integers(1, 6, 3)],
+                "scores": rng.random(m + 3),
+            }
+        )
+
+    def es_map() -> list[float]:
+        r = es.detection_report(y_true, y_pred)
+        return [r["map"], r["map_50"], r["map_75"], r["mar_100"]]
+
+    cases: list[Case] = [
+        ("segmentation: Dice and IoU per class (n = pixels)", "scikit-learn", es_overlap, None),
+        (f"segmentation: Hausdorff distance ({n_hd} image{'s' if n_hd != 1 else ''})", "SciPy", es_hd, None),
+        (f"detection: COCO evaluation ({n_det} images)", "pycocotools", es_map, None),
+    ]
+    refs: dict[str, Callable[[], Any]] = {}
+    try:
+        import sklearn.metrics as skm
+
+        def sk_overlap() -> list[float]:
+            ft, fp_ = true.ravel(), pred.ravel()
+            return [
+                *skm.f1_score(ft, fp_, labels=labels, average=None),
+                *skm.jaccard_score(ft, fp_, labels=labels, average=None),
+            ]
+
+        refs[cases[0][0]] = sk_overlap
+    except ImportError:
+        pass
+    from scipy import ndimage
+    from scipy.spatial.distance import directed_hausdorff
+
+    def scipy_hd() -> list[float]:
+        out = []
+        for i in range(n_hd):
+            pts = []
+            for m_ in (true[i] == k - 1, pred[i] == k - 1):
+                er = ndimage.binary_erosion(m_, structure=ndimage.generate_binary_structure(2, 1), border_value=0)
+                pts.append(np.argwhere(m_ & ~er).astype(float))
+            out.append(max(directed_hausdorff(pts[0], pts[1])[0], directed_hausdorff(pts[1], pts[0])[0]))
+        return out
+
+    refs[cases[1][0]] = scipy_hd
+    try:
+        import contextlib as _ctx
+        import io
+
+        from pycocotools.coco import COCO  # type: ignore[import-untyped]
+        from pycocotools.cocoeval import COCOeval  # type: ignore[import-untyped]
+
+        def coco_map() -> list[float]:
+            images, anns, dets, aid = [], [], [], 1
+            for i, (t, p) in enumerate(zip(y_true, y_pred)):
+                images.append({"id": i + 1})
+                for b, c in zip(t["boxes"], t["labels"]):
+                    w, h = b[2] - b[0], b[3] - b[1]
+                    anns.append(
+                        {
+                            "id": aid,
+                            "image_id": i + 1,
+                            "category_id": int(c),
+                            "bbox": [b[0], b[1], w, h],
+                            "area": w * h,
+                            "iscrowd": 0,
+                        }
+                    )
+                    aid += 1
+                for b, c, sc in zip(p["boxes"], p["labels"], p["scores"]):
+                    dets.append(
+                        {
+                            "image_id": i + 1,
+                            "category_id": int(c),
+                            "bbox": [b[0], b[1], b[2] - b[0], b[3] - b[1]],
+                            "score": float(sc),
+                        }
+                    )
+            with _ctx.redirect_stdout(io.StringIO()):
+                gt = COCO()
+                gt.dataset = {
+                    "images": images,
+                    "annotations": anns,
+                    "categories": [{"id": c} for c in range(1, 6)],
+                }
+                gt.createIndex()
+                ev = COCOeval(gt, gt.loadRes(dets), "bbox")
+                ev.evaluate()
+                ev.accumulate()
+                ev.summarize()
+            return [ev.stats[0], ev.stats[1], ev.stats[2], ev.stats[8]]
+
+        refs[cases[2][0]] = coco_map
+    except ImportError:
+        pass
+    return [(name, ref, es_fn, refs.get(name)) for name, ref, es_fn, _ in cases]
+
+
 def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     """v0.2.0: diagnostic accuracy, calibration, decision curves and statistical tests."""
     import evalsuite as es
@@ -231,6 +373,28 @@ def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, own if own is not None else refs.get(name)) for name, ref, es_fn, own in cases]
 
 
+_GROUPS = (
+    ("Classification and regression", ("binary", "10 classes", "regression")),
+    ("Clinical, calibration and statistics", ("clinical", "calibration", "decision", "statistics", "multiple")),
+    ("Segmentation and object detection", ("segmentation", "detection")),
+)
+MATCH_TOLERANCE = 1e-9
+
+
+def benchmark_group(case: str) -> str:
+    """The suite a benchmark case belongs to, by its name."""
+    lowered = case.lower()
+    for title, prefixes in _GROUPS:
+        if lowered.startswith(prefixes):
+            return title
+    return "Other"
+
+
+def _geomean(values: Sequence[float]) -> Optional[float]:
+    vals = [v for v in values if v is not None and v > 0]
+    return float(np.exp(np.mean(np.log(vals)))) if vals else None
+
+
 @dataclass(frozen=True, eq=False)
 class BenchmarkResult:
     """Rows of timings (milliseconds), peak memory (MiB) and agreement, plus the environment they ran in."""
@@ -242,6 +406,60 @@ class BenchmarkResult:
         object.__setattr__(self, "rows", tuple(MappingProxyType(dict(r)) for r in self.rows))
         object.__setattr__(self, "environment", MappingProxyType(dict(self.environment)))
 
+    def sorted_rows(self) -> list[Any]:
+        """Rows in alphabetical order of case, then by size."""
+        return sorted(self.rows, key=lambda r: (r["case"].lower(), r["n"]))
+
+    def overall(self) -> list[dict[str, Any]]:
+        """One summary row per suite and one for everything: cases, how many agree with the reference
+        (max |difference| <= 1e-9), and the geometric-mean, smallest and largest speed-up."""
+
+        def summarise(name: str, rows: Sequence[Any]) -> dict[str, Any]:
+            compared = [r for r in rows if r["speedup"] is not None]
+            speedups = [r["speedup"] for r in compared]
+            return {
+                "group": name,
+                "cases": len({re.sub(r"\s*\(\d+ images?\)", "", r["case"]) for r in rows}),
+                "rows": len(rows),
+                "compared": len(compared),
+                "matching": sum(
+                    r["max_abs_diff"] is not None and r["max_abs_diff"] <= MATCH_TOLERANCE for r in rows
+                ),
+                "faster": sum(sp > 1 for sp in speedups),
+                "geomean_speedup": _geomean(speedups),
+                "min_speedup": min(speedups) if speedups else None,
+                "max_speedup": max(speedups) if speedups else None,
+            }
+
+        groups = sorted({benchmark_group(r["case"]) for r in self.rows})
+        out = [summarise(g, [r for r in self.rows if benchmark_group(r["case"]) == g]) for g in groups]
+        out.append(summarise("Overall", self.rows))
+        return out
+
+    _OVERALL_TITLES = ("Suite", "Cases", "Rows", "Match reference", "Faster", "Geo-mean speed-up", "Range")
+
+    def _overall_cells(self) -> list[list[str]]:
+        def x(v: Optional[float]) -> str:
+            return "–" if v is None else f"{v:.2f}×"
+
+        return [
+            [
+                o["group"],
+                str(o["cases"]),
+                str(o["rows"]),
+                f"{o['matching']}/{o['compared']}",
+                f"{o['faster']}/{o['compared']}",
+                x(o["geomean_speedup"]),
+                "–" if o["min_speedup"] is None else f"{x(o['min_speedup'])}–{x(o['max_speedup'])}",
+            ]
+            for o in self.overall()
+        ]
+
+    def overall_markdown(self) -> str:
+        """The overall summary as a Markdown table."""
+        lines = ["| " + " | ".join(self._OVERALL_TITLES) + " |", "| --- |" + " ---: |" * 6]
+        return "\n".join(lines + ["| " + " | ".join(c) + " |" for c in self._overall_cells()])
+
     def _cells(self, digits: int) -> list[list[str]]:
         def f(v: Any, d: int = digits) -> str:
             return "–" if v is None else f"{v:.{d}f}"
@@ -258,7 +476,7 @@ class BenchmarkResult:
                 f(r.get("reference_peak_mb"), 2),
                 "–" if r["max_abs_diff"] is None else f"{r['max_abs_diff']:.1e}",
             ]
-            for r in self.rows
+            for r in self.sorted_rows()
         ]
 
     _TITLES = (
@@ -286,10 +504,18 @@ class BenchmarkResult:
             + (f" | scikit-learn {env['sklearn']}" if env.get("sklearn") else "")
             + (f" | statsmodels {env['statsmodels']}" if env.get("statsmodels") else "")
             + (f" | SciPy {env['scipy']}" if env.get("scipy") else "")
+            + (f" | pycocotools {env['pycocotools']}" if env.get("pycocotools") else "")
             + f" | {env['machine']} | fastest of {env['repeat']} runs"
         )
         note = "Speed-up > 1 means EvalSuite is faster. Max |difference| compares EvalSuite with the reference."
-        return "\n".join([head, "", line(self._TITLES), *(line(c) for c in cells), "", note])
+        ocells = self._overall_cells()
+        owidths = [max(len(t), *(len(c[i]) for c in ocells)) for i, t in enumerate(self._OVERALL_TITLES)]
+
+        def oline(c: Sequence[str]) -> str:
+            return "  ".join(x.ljust(owidths[i]) if i == 0 else x.rjust(owidths[i]) for i, x in enumerate(c))
+
+        overall = ["Overall", oline(self._OVERALL_TITLES), *(oline(c) for c in ocells), ""]
+        return "\n".join([head, "", *overall, line(self._TITLES), *(line(c) for c in cells), "", note])
 
     def __repr__(self) -> str:
         return self.summary()
@@ -297,7 +523,13 @@ class BenchmarkResult:
     def to_dict(self) -> dict[str, Any]:
         return cast(
             "dict[str, Any]",
-            _json_safe({"environment": dict(self.environment), "rows": [dict(r) for r in self.rows]}),
+            _json_safe(
+                {
+                    "environment": dict(self.environment),
+                    "overall": self.overall(),
+                    "rows": [dict(r) for r in self.rows],
+                }
+            ),
         )
 
     def to_json(self, *, indent: Optional[int] = 2) -> str:
@@ -354,7 +586,8 @@ def run_benchmarks(
     """Time and memory for evaluation workloads at each size, against a reference implementation.
 
     ``suite``: ``"core"`` (classification and regression vs scikit-learn), ``"clinical"`` (v0.2.0 clinical,
-    calibration and statistics vs scikit-learn, statsmodels, SciPy) or ``"all"`` (default).
+    calibration and statistics vs scikit-learn, statsmodels, SciPy), ``"vision"`` (segmentation and COCO
+    detection vs scikit-learn, SciPy, pycocotools) or ``"all"`` (default).
     ``compare_sklearn=False`` times EvalSuite alone. Rows keep ``sklearn_ms``/``sklearn_peak_mb`` for rows
     whose reference is scikit-learn, for compatibility with 0.1.x.
     """
@@ -364,16 +597,24 @@ def run_benchmarks(
 
     if repeat < 1:
         raise ValueError("repeat must be at least 1.")
-    if suite not in ("all", "core", "clinical"):
-        raise ValueError("suite must be 'all', 'core' or 'clinical'.")
+    if suite not in ("all", "core", "clinical", "vision"):
+        raise ValueError("suite must be 'all', 'core', 'clinical' or 'vision'.")
     rng = np.random.default_rng(random_state)
     rows: list[dict[str, Any]] = []
-    versions: dict[str, Optional[str]] = {"sklearn": None, "statsmodels": None}
+    versions: dict[str, Optional[str]] = {"sklearn": None, "statsmodels": None, "pycocotools": None}
     if compare_sklearn:
         for mod in versions:
             with contextlib.suppress(ImportError):
-                versions[mod] = __import__(mod).__version__
-    builders = {"core": [_core_cases], "clinical": [_clinical_cases], "all": [_core_cases, _clinical_cases]}[suite]
+                importlib.import_module(mod)  # fixed names only: sklearn, statsmodels, pycocotools
+                from importlib.metadata import version as _dist_version
+
+                versions[mod] = _dist_version("scikit-learn" if mod == "sklearn" else mod)
+    builders = {
+        "core": [_core_cases],
+        "clinical": [_clinical_cases],
+        "vision": [_vision_cases],
+        "all": [_core_cases, _clinical_cases, _vision_cases],
+    }[suite]
     for n in sizes:
         for build in builders:
             for name, ref_name, es_fn, ref_fn in build(int(n), rng):
@@ -410,6 +651,7 @@ def run_benchmarks(
         "scipy": scipy.__version__ if suite != "core" else None,
         "sklearn": versions["sklearn"],
         "statsmodels": versions["statsmodels"] if suite != "core" else None,
+        "pycocotools": versions["pycocotools"] if suite in ("all", "vision") else None,
         "machine": f"{platform.system()} {platform.machine()}",
         "repeat": repeat,
         "suite": suite,

evalsuite/calibration.py

@@ -128,7 +128,7 @@ def logistic_fit(
     beta = np.zeros(design.shape[1])
     for _ in range(max_iter):
         eta = design @ beta + off
-        mu = 1 / (1 + np.exp(-eta))
+        mu = 1 / (1 + np.exp(-np.clip(eta, -700, 700)))
         grad = design.T @ (w * (y - mu))
         hess = (design * (w * mu * (1 - mu))[:, None]).T @ design
         try:

evalsuite/cli/main.py

@@ -226,6 +226,77 @@ def cmd_calibration(args: argparse.Namespace) -> int:
     return 0
 
 
+def read_masks(path: str, name: str) -> Any:
+    """Masks from .npy / .npz (first array, or the array named like the file stem) or a folder of image
+    files (PNG, TIFF, ...; read with Pillow, sorted by file name)."""
+    import numpy as np
+
+    if not os.path.exists(path):
+        raise CLIError(f"{name}: no such file or folder: {path}")
+    if os.path.isdir(path):
+        try:
+            from PIL import Image
+        except ImportError as exc:
+            raise CLIError(f"{name}: reading image folders needs Pillow (pip install pillow).") from exc
+        files = sorted(
+            f for f in os.listdir(path) if f.lower().endswith((".png", ".tif", ".tiff", ".bmp", ".gif", ".jpg"))
+        )
+        if not files:
+            raise CLIError(f"{name}: no image files in {path}.")
+        return [np.asarray(Image.open(os.path.join(path, f))) for f in files]
+    ext = os.path.splitext(path)[1].lower()
+    if ext == ".npy":
+        return np.load(path, allow_pickle=False)
+    if ext == ".npz":
+        with np.load(path, allow_pickle=False) as data:
+            if not data.files:
+                raise CLIError(f"{name}: {path} contains no arrays.")
+            return data[data.files[0]]
+    raise CLIError(f"{name}: use a .npy or .npz file, or a folder of mask images.")
+
+
+def cmd_segmentation(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    spacing = tuple(float(x) for x in args.spacing.split(",")) if args.spacing else None
+    report = es.segmentation_report(
+        read_masks(args.y_true, "y_true"),
+        read_masks(args.y_pred, "y_pred"),
+        num_classes=args.num_classes,
+        ignore_index=args.ignore_index,
+        spacing=spacing,
+        aggregate=args.aggregate,
+        include_background=not args.no_background,
+    )
+    emit(report, args)
+    if args.plot:
+        _save_figure(lambda ax: es.plot.per_class(_report_iou(report), ax=ax), args.plot)
+    return 0
+
+
+def _report_iou(report: Any) -> Any:
+    import numpy as np
+
+    from ..core.result import MetricResult
+
+    return MetricResult(
+        "iou", "IoU", np.array([r["iou"] for r in report.rows]), labels=tuple(r["name"] for r in report.rows)
+    )
+
+
+def cmd_detection(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    y_true, y_pred = es.from_coco(args.ground_truth, args.results)
+    report = es.detection_report(y_true, y_pred, box_format="xywh", max_detections=args.max_detections)
+    emit(report, args)
+    if args.plot:
+        _save_figure(
+            lambda ax: es.plot.detection_pr(y_true, y_pred, ax=ax, box_format="xywh", iou_threshold=0.5), args.plot
+        )
+    return 0
+
+
 def cmd_compare(args: argparse.Namespace) -> int:
     import evalsuite as es
 
@@ -410,6 +481,31 @@ def build_parser() -> argparse.ArgumentParser:
     add_output(p)
     p.set_defaults(func=cmd_calibration)
 
+    p = sub.add_parser(
+        "segmentation",
+        help="segmentation masks: Dice, IoU, mIoU, Boundary IoU, HD95, ASSD per class",
+    )
+    p.add_argument("y_true", help="true masks: .npy/.npz (images on the first axis) or a folder of mask images")
+    p.add_argument("y_pred", help="predicted masks, same layout as y_true")
+    p.add_argument("--num-classes", type=int, help="number of classes (default: largest label + 1)")
+    p.add_argument("--ignore-index", type=int, help="label to ignore, e.g. 255")
+    p.add_argument("--spacing", help="pixel spacing per axis for surface distances, e.g. 0.8,0.8")
+    p.add_argument("--aggregate", choices=("dataset", "image"), default="dataset")
+    p.add_argument("--no-background", action="store_true", help="leave class 0 out")
+    p.add_argument("--plot", metavar="PATH", help="also save a per-class IoU bar chart")
+    add_output(p)
+    p.set_defaults(func=cmd_segmentation)
+
+    p = sub.add_parser(
+        "detection", help="object detection, COCO protocol: mAP, mAP@.50, mAP@.75, mAR, AP per class"
+    )
+    p.add_argument("ground_truth", help="COCO ground-truth JSON (images, annotations, categories)")
+    p.add_argument("results", help="COCO results JSON (image_id, category_id, bbox, score)")
+    p.add_argument("--max-detections", type=int, default=100)
+    p.add_argument("--plot", metavar="PATH", help="also save precision-recall curves per class (IoU 0.5)")
+    add_output(p, digits=3)
+    p.set_defaults(func=cmd_detection)
+
     p = sub.add_parser(
         "plot", help="save a ROC, PR, calibration, decision-curve, confusion-matrix or residual plot"
     )
@@ -433,9 +529,10 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--repeat", type=int, default=5, help="timed repetitions per case; the fastest is reported")
     p.add_argument(
         "--suite",
-        choices=("all", "core", "clinical"),
+        choices=("all", "core", "clinical", "vision"),
         default="all",
-        help="core: classification/regression; clinical: v0.2.0 clinical, calibration and tests (default all)",
+        help="core: classification/regression; clinical: clinical, calibration, tests; vision: segmentation, "
+        "detection (default all)",
     )
     p.add_argument("--no-sklearn", action="store_true", help="time EvalSuite only (skip reference libraries)")
     p.add_argument("--seed", type=int, default=0)

evalsuite/core/validation.py

@@ -45,9 +45,25 @@ def to_numpy(x: ArrayLike, name: str, *, allow_2d: bool = False) -> NDArray[Any]
         raise InputValidationError(f"{name} must be {expected}; received an array with shape {arr.shape}.")
     if arr.shape[0] == 0:
         raise InputValidationError(f"{name} is empty. Provide at least one observation.")
+    if (
+        arr.ndim == 1
+        and arr.dtype.kind in "US"
+        and isinstance(x, (list, tuple))
+        and not all(isinstance(v, (str, bytes)) for v in x)
+    ):
+        # NumPy silently turns [0, "a"] into ["0", "a"]; a mix of numbers and strings is almost always a bug.
+        raise InputValidationError(f"{name} mixes numbers and strings; use one label type throughout.")
     return arr
 
 
+def _label_kind(arr: NDArray[Any]) -> str:
+    if arr.dtype.kind in "US":
+        return "string"
+    if arr.dtype.kind in "biuf":
+        return "number"
+    return "other"
+
+
 def check_finite(arr: NDArray[Any], name: str) -> None:
     if np.issubdtype(arr.dtype, np.number) and not np.all(np.isfinite(arr)):
         n_nan = int(np.isnan(arr).sum())
@@ -126,6 +142,11 @@ def resolve_labels(
         if np.unique(lab).shape[0] != lab.shape[0]:
             raise InputValidationError("labels contains duplicates.")
         return lab
+    if y_pred is not None and {_label_kind(y_true), _label_kind(y_pred)} == {"string", "number"}:
+        raise InputValidationError(
+            f"y_true has {_label_kind(y_true)} labels but y_pred has {_label_kind(y_pred)} labels; "
+            "use the same label type in both (for example map class names to integers first)."
+        )
     present = y_true if y_pred is None else np.concatenate([y_true, y_pred])
     try:
         return unique_labels(present)