evalsuite-python 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

evalsuite/__init__.py

@@ -5,7 +5,7 @@
 >>> print(result.summary())  # doctest: +SKIP
 """
 
-from . import calibration, classification, clinical, plot, regression, stats, vision
+from . import calibration, classification, clinical, plot, regression, stats, text, vision
 from .api import evaluate
 from .calibration import (
     CalibrationReport,
@@ -110,6 +110,8 @@ from .stats import (
     t_test,
     wilcoxon_test,
 )
+from .text import *  # noqa: F403  (every LLM-evaluation function, listed in text.__all__)
+from .text import __all__ as _text_all
 from .version import __version__
 from .vision import (
     DetectionReport,
@@ -134,6 +136,7 @@ from .vision import (
 )
 
 __all__ = [
+    "text",
     "SegmentationReport",
     "segmentation_report",
     "detection_pr_curve",
@@ -258,3 +261,4 @@ __all__ = [
     "specificity",
     "top_k_accuracy",
 ]
+__all__ += list(_text_all)

evalsuite/benchmarks.py

@@ -17,8 +17,10 @@ whose reference library is not installed is timed for EvalSuite only.
 from __future__ import annotations
 
 import contextlib
+import importlib
 import json
 import platform
+import re
 import time
 import tracemalloc
 import warnings
@@ -211,8 +213,8 @@ def _vision_cases(n: int, rng: np.random.Generator) -> list[Case]:
         import contextlib as _ctx
         import io
 
-        from pycocotools.coco import COCO  # type: ignore[import-untyped]
-        from pycocotools.cocoeval import COCOeval  # type: ignore[import-untyped]
+        from pycocotools.coco import COCO
+        from pycocotools.cocoeval import COCOeval
 
         def coco_map() -> list[float]:
             images, anns, dets, aid = [], [], [], 1
@@ -260,6 +262,155 @@ def _vision_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, refs.get(name)) for name, ref, es_fn, _ in cases]
 
 
+_LLM_VOCAB = (  # noqa: SIM905
+    "the a cat dog sat on mat quickly model models data run running ran 2.5 3,000 well-known state-of-the-art "
+    "( ) , . ! ? don't U.S. e-mail Hello WORLD naive cafe results show improves baseline evaluation"
+).split()
+
+
+def _llm_cases(n: int, rng: np.random.Generator) -> list[Case]:
+    """v0.4.0: text generation, retrieval, rater agreement and structured output (n / 100 examples)."""
+    import evalsuite as es
+
+    m = max(10, n // 100)
+    preds, refs_ = [], []
+    for _ in range(m):
+        base = list(rng.choice(_LLM_VOCAB, int(rng.integers(5, 25))))
+        keep = [w for w in base if rng.random() > 0.2] + list(rng.choice(_LLM_VOCAB, int(rng.integers(0, 4))))
+        preds.append(" ".join(keep))
+        refs_.append(" ".join(base))
+    queries = [
+        (set(rng.choice(200, int(rng.integers(1, 6)), replace=False).tolist()), rng.permutation(200)[:20].tolist())
+        for _ in range(m)
+    ]
+    rel, ret = [q[0] for q in queries], [q[1] for q in queries]
+    ratings = rng.integers(1, 6, (4, m)).astype(float)
+    ratings[rng.random(ratings.shape) < 0.1] = np.nan
+    schema = {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string", "minLength": 1},
+            "age": {"type": "integer", "minimum": 0},
+            "tags": {"type": "array", "items": {"type": "string"}, "uniqueItems": True},
+        },
+        "required": ["name", "age"],
+        "additionalProperties": False,
+    }
+    docs = []
+    for i in range(m):
+        d: dict[str, Any] = {"name": f"n{i}", "age": int(rng.integers(-2, 90)), "tags": ["a", "b"]}
+        if rng.random() < 0.1:
+            d["extra"] = 1
+        if rng.random() < 0.1:
+            del d["name"]
+        docs.append(d)
+    import json as _json
+
+    docs_text = [_json.dumps(d) for d in docs]
+
+    cases: list[Case] = [
+        (
+            f"text: corpus BLEU and chrF ({m} sentences)",
+            "sacreBLEU",
+            lambda: [float(es.bleu(refs_, preds)), float(es.chrf(refs_, preds))],
+            None,
+        ),
+        (
+            f"text: ROUGE-1, ROUGE-2, ROUGE-L ({m} sentences)",
+            "rouge-score",
+            lambda: [
+                float(es.rouge_1(refs_, preds)),
+                float(es.rouge_2(refs_, preds)),
+                float(es.rouge_l(refs_, preds)),
+            ],
+            None,
+        ),
+        (
+            f"text: METEOR, exact and stem matches ({m} sentences)",
+            "NLTK",
+            lambda: [float(es.meteor(refs_, preds))],
+            None,
+        ),
+        (
+            f"retrieval: MRR, MAP@20, NDCG@10 ({m} queries)",
+            "ranx",
+            lambda: [
+                float(es.mrr(rel, ret)),
+                float(es.mean_average_precision_at_k(rel, ret, k=20)),
+                float(es.ndcg_at_k(rel, ret, k=10)),
+            ],
+            None,
+        ),
+        (
+            f"agreement: Krippendorff's alpha, interval (4 raters × {m} items)",
+            "krippendorff",
+            lambda: [float(es.krippendorff_alpha(ratings, level="interval"))],
+            None,
+        ),
+        (
+            f"structured: JSON Schema compliance ({m} documents)",
+            "jsonschema",
+            lambda: [float(es.json_schema_compliance(docs_text, schema))],
+            None,
+        ),
+    ]
+    refs: dict[str, Callable[[], Any]] = {}
+    with contextlib.suppress(ImportError):
+        import sacrebleu
+
+        refs[cases[0][0]] = lambda: [
+            sacrebleu.corpus_bleu(preds, [refs_], force=True).score,
+            sacrebleu.corpus_chrf(preds, [refs_]).score,
+        ]
+    with contextlib.suppress(ImportError):
+        from rouge_score import rouge_scorer
+
+        def rouge_ref() -> list[float]:
+            sc = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"])
+            out = [sc.score(r, p) for r, p in zip(refs_, preds)]
+            return [float(np.mean([o[k].fmeasure for o in out])) for k in ("rouge1", "rouge2", "rougeL")]
+
+        refs[cases[1][0]] = rouge_ref
+    with contextlib.suppress(ImportError):
+        from nltk.translate.meteor_score import meteor_score
+
+        class _NoWordNet:
+            def synsets(self, _w: str) -> list[Any]:
+                return []
+
+        refs[cases[2][0]] = lambda: [
+            float(
+                np.mean([meteor_score([r.split()], p.split(), wordnet=_NoWordNet()) for r, p in zip(refs_, preds)])
+            )
+        ]
+    with contextlib.suppress(ImportError):
+        from ranx import Qrels, Run
+        from ranx import evaluate as ranx_evaluate
+
+        def ranx_ref() -> list[float]:
+            qrels = Qrels({f"q{i}": {f"d{d}": 1 for d in r} for i, r in enumerate(rel)})
+            run = Run({f"q{i}": {f"d{d}": float(len(r) - j) for j, d in enumerate(r)} for i, r in enumerate(ret)})
+            res = ranx_evaluate(qrels, run, ["mrr", "map@20", "ndcg@10"])
+            return [float(res["mrr"]), float(res["map@20"]), float(res["ndcg@10"])]
+
+        refs[cases[3][0]] = ranx_ref
+    with contextlib.suppress(ImportError):
+        import krippendorff
+
+        refs[cases[4][0]] = lambda: [
+            float(krippendorff.alpha(reliability_data=ratings, level_of_measurement="interval"))
+        ]
+    with contextlib.suppress(ImportError):
+        import jsonschema
+
+        def jsonschema_ref() -> list[float]:
+            v = jsonschema.Draft202012Validator(schema)
+            return [float(np.mean([v.is_valid(_json.loads(t)) for t in docs_text]))]
+
+        refs[cases[5][0]] = jsonschema_ref
+    return [(name, ref, es_fn, refs.get(name)) for name, ref, es_fn, _ in cases]
+
+
 def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     """v0.2.0: diagnostic accuracy, calibration, decision curves and statistical tests."""
     import evalsuite as es
@@ -371,6 +522,29 @@ def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, own if own is not None else refs.get(name)) for name, ref, es_fn, own in cases]
 
 
+_GROUPS = (
+    ("Classification and regression", ("binary", "10 classes", "regression")),
+    ("Clinical, calibration and statistics", ("clinical", "calibration", "decision", "statistics", "multiple")),
+    ("Segmentation and object detection", ("segmentation", "detection")),
+    ("LLM evaluation", ("text", "retrieval", "agreement", "structured")),
+)
+MATCH_TOLERANCE = 1e-9
+
+
+def benchmark_group(case: str) -> str:
+    """The suite a benchmark case belongs to, by its name."""
+    lowered = case.lower()
+    for title, prefixes in _GROUPS:
+        if lowered.startswith(prefixes):
+            return title
+    return "Other"
+
+
+def _geomean(values: Sequence[float]) -> Optional[float]:
+    vals = [v for v in values if v is not None and v > 0]
+    return float(np.exp(np.mean(np.log(vals)))) if vals else None
+
+
 @dataclass(frozen=True, eq=False)
 class BenchmarkResult:
     """Rows of timings (milliseconds), peak memory (MiB) and agreement, plus the environment they ran in."""
@@ -382,6 +556,60 @@ class BenchmarkResult:
         object.__setattr__(self, "rows", tuple(MappingProxyType(dict(r)) for r in self.rows))
         object.__setattr__(self, "environment", MappingProxyType(dict(self.environment)))
 
+    def sorted_rows(self) -> list[Any]:
+        """Rows in alphabetical order of case, then by size."""
+        return sorted(self.rows, key=lambda r: (r["case"].lower(), r["n"]))
+
+    def overall(self) -> list[dict[str, Any]]:
+        """One summary row per suite and one for everything: cases, how many agree with the reference
+        (max |difference| <= 1e-9), and the geometric-mean, smallest and largest speed-up."""
+
+        def summarise(name: str, rows: Sequence[Any]) -> dict[str, Any]:
+            compared = [r for r in rows if r["speedup"] is not None]
+            speedups = [r["speedup"] for r in compared]
+            return {
+                "group": name,
+                "cases": len({re.sub(r"\s*\((?=[^)]*\d)[^)]*\)$", "", r["case"]) for r in rows}),
+                "rows": len(rows),
+                "compared": len(compared),
+                "matching": sum(
+                    r["max_abs_diff"] is not None and r["max_abs_diff"] <= MATCH_TOLERANCE for r in rows
+                ),
+                "faster": sum(sp > 1 for sp in speedups),
+                "geomean_speedup": _geomean(speedups),
+                "min_speedup": min(speedups) if speedups else None,
+                "max_speedup": max(speedups) if speedups else None,
+            }
+
+        groups = sorted({benchmark_group(r["case"]) for r in self.rows})
+        out = [summarise(g, [r for r in self.rows if benchmark_group(r["case"]) == g]) for g in groups]
+        out.append(summarise("Overall", self.rows))
+        return out
+
+    _OVERALL_TITLES = ("Suite", "Cases", "Rows", "Match reference", "Faster", "Geo-mean speed-up", "Range")
+
+    def _overall_cells(self) -> list[list[str]]:
+        def x(v: Optional[float]) -> str:
+            return "–" if v is None else f"{v:.2f}×"
+
+        return [
+            [
+                o["group"],
+                str(o["cases"]),
+                str(o["rows"]),
+                f"{o['matching']}/{o['compared']}",
+                f"{o['faster']}/{o['compared']}",
+                x(o["geomean_speedup"]),
+                "–" if o["min_speedup"] is None else f"{x(o['min_speedup'])}–{x(o['max_speedup'])}",
+            ]
+            for o in self.overall()
+        ]
+
+    def overall_markdown(self) -> str:
+        """The overall summary as a Markdown table."""
+        lines = ["| " + " | ".join(self._OVERALL_TITLES) + " |", "| --- |" + " ---: |" * 6]
+        return "\n".join(lines + ["| " + " | ".join(c) + " |" for c in self._overall_cells()])
+
     def _cells(self, digits: int) -> list[list[str]]:
         def f(v: Any, d: int = digits) -> str:
             return "–" if v is None else f"{v:.{d}f}"
@@ -398,7 +626,7 @@ class BenchmarkResult:
                 f(r.get("reference_peak_mb"), 2),
                 "–" if r["max_abs_diff"] is None else f"{r['max_abs_diff']:.1e}",
             ]
-            for r in self.rows
+            for r in self.sorted_rows()
         ]
 
     _TITLES = (
@@ -430,7 +658,14 @@ class BenchmarkResult:
             + f" | {env['machine']} | fastest of {env['repeat']} runs"
         )
         note = "Speed-up > 1 means EvalSuite is faster. Max |difference| compares EvalSuite with the reference."
-        return "\n".join([head, "", line(self._TITLES), *(line(c) for c in cells), "", note])
+        ocells = self._overall_cells()
+        owidths = [max(len(t), *(len(c[i]) for c in ocells)) for i, t in enumerate(self._OVERALL_TITLES)]
+
+        def oline(c: Sequence[str]) -> str:
+            return "  ".join(x.ljust(owidths[i]) if i == 0 else x.rjust(owidths[i]) for i, x in enumerate(c))
+
+        overall = ["Overall", oline(self._OVERALL_TITLES), *(oline(c) for c in ocells), ""]
+        return "\n".join([head, "", *overall, line(self._TITLES), *(line(c) for c in cells), "", note])
 
     def __repr__(self) -> str:
         return self.summary()
@@ -438,7 +673,13 @@ class BenchmarkResult:
     def to_dict(self) -> dict[str, Any]:
         return cast(
             "dict[str, Any]",
-            _json_safe({"environment": dict(self.environment), "rows": [dict(r) for r in self.rows]}),
+            _json_safe(
+                {
+                    "environment": dict(self.environment),
+                    "overall": self.overall(),
+                    "rows": [dict(r) for r in self.rows],
+                }
+            ),
         )
 
     def to_json(self, *, indent: Optional[int] = 2) -> str:
@@ -506,15 +747,15 @@ def run_benchmarks(
 
     if repeat < 1:
         raise ValueError("repeat must be at least 1.")
-    if suite not in ("all", "core", "clinical", "vision"):
-        raise ValueError("suite must be 'all', 'core', 'clinical' or 'vision'.")
+    if suite not in ("all", "core", "clinical", "vision", "llm"):
+        raise ValueError("suite must be 'all', 'core', 'clinical', 'vision' or 'llm'.")
     rng = np.random.default_rng(random_state)
     rows: list[dict[str, Any]] = []
     versions: dict[str, Optional[str]] = {"sklearn": None, "statsmodels": None, "pycocotools": None}
     if compare_sklearn:
         for mod in versions:
             with contextlib.suppress(ImportError):
-                __import__(mod)
+                importlib.import_module(mod)  # fixed names only: sklearn, statsmodels, pycocotools
                 from importlib.metadata import version as _dist_version
 
                 versions[mod] = _dist_version("scikit-learn" if mod == "sklearn" else mod)
@@ -522,7 +763,8 @@ def run_benchmarks(
         "core": [_core_cases],
         "clinical": [_clinical_cases],
         "vision": [_vision_cases],
-        "all": [_core_cases, _clinical_cases, _vision_cases],
+        "llm": [_llm_cases],
+        "all": [_core_cases, _clinical_cases, _vision_cases, _llm_cases],
     }[suite]
     for n in sizes:
         for build in builders:

evalsuite/cli/main.py

@@ -297,6 +297,24 @@ def cmd_detection(args: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_text(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    df = read_table(args.file)
+    preds = [str(v) for v in column(df, args.prediction, "--prediction")]
+    ref_cols = [column(df, c, "--reference") for c in args.reference]
+    refs = [[str(col[i]) for col in ref_cols if not _missing(col[i])] for i in range(len(preds))]
+    if any(not r for r in refs):
+        raise CLIError("Every row needs at least one non-empty reference.")
+    report = es.text_report(refs, preds, metrics=metric_list(args.metrics))
+    emit(report, args)
+    return 0
+
+
+def _missing(v: Any) -> bool:
+    return v is None or (isinstance(v, float) and v != v) or (isinstance(v, str) and not v.strip())
+
+
 def cmd_compare(args: argparse.Namespace) -> int:
     import evalsuite as es
 
@@ -506,6 +524,18 @@ def build_parser() -> argparse.ArgumentParser:
     add_output(p, digits=3)
     p.set_defaults(func=cmd_detection)
 
+    p = sub.add_parser(
+        "text", help="text generation: BLEU, chrF, chrF++, TER, ROUGE, METEOR, exact match, token F1"
+    )
+    p.add_argument("file", help="CSV, TSV, Parquet or JSON-lines file with one example per row ('-' for stdin)")
+    p.add_argument("--prediction", required=True, help="column with the model output")
+    p.add_argument(
+        "--reference", required=True, action="append", help="column with a reference (repeat for several)"
+    )
+    p.add_argument("--metrics", help="comma-separated subset, e.g. bleu,chrf,rouge_l")
+    add_output(p)
+    p.set_defaults(func=cmd_text)
+
     p = sub.add_parser(
         "plot", help="save a ROC, PR, calibration, decision-curve, confusion-matrix or residual plot"
     )
@@ -529,7 +559,7 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--repeat", type=int, default=5, help="timed repetitions per case; the fastest is reported")
     p.add_argument(
         "--suite",
-        choices=("all", "core", "clinical", "vision"),
+        choices=("all", "core", "clinical", "vision", "llm"),
         default="all",
         help="core: classification/regression; clinical: clinical, calibration, tests; vision: segmentation, "
         "detection (default all)",

evalsuite/core/validation.py

@@ -45,9 +45,25 @@ def to_numpy(x: ArrayLike, name: str, *, allow_2d: bool = False) -> NDArray[Any]
         raise InputValidationError(f"{name} must be {expected}; received an array with shape {arr.shape}.")
     if arr.shape[0] == 0:
         raise InputValidationError(f"{name} is empty. Provide at least one observation.")
+    if (
+        arr.ndim == 1
+        and arr.dtype.kind in "US"
+        and isinstance(x, (list, tuple))
+        and not all(isinstance(v, (str, bytes)) for v in x)
+    ):
+        # NumPy silently turns [0, "a"] into ["0", "a"]; a mix of numbers and strings is almost always a bug.
+        raise InputValidationError(f"{name} mixes numbers and strings; use one label type throughout.")
     return arr
 
 
+def _label_kind(arr: NDArray[Any]) -> str:
+    if arr.dtype.kind in "US":
+        return "string"
+    if arr.dtype.kind in "biuf":
+        return "number"
+    return "other"
+
+
 def check_finite(arr: NDArray[Any], name: str) -> None:
     if np.issubdtype(arr.dtype, np.number) and not np.all(np.isfinite(arr)):
         n_nan = int(np.isnan(arr).sum())
@@ -126,6 +142,11 @@ def resolve_labels(
         if np.unique(lab).shape[0] != lab.shape[0]:
             raise InputValidationError("labels contains duplicates.")
         return lab
+    if y_pred is not None and {_label_kind(y_true), _label_kind(y_pred)} == {"string", "number"}:
+        raise InputValidationError(
+            f"y_true has {_label_kind(y_true)} labels but y_pred has {_label_kind(y_pred)} labels; "
+            "use the same label type in both (for example map class names to integers first)."
+        )
     present = y_true if y_pred is None else np.concatenate([y_true, y_pred])
     try:
         return unique_labels(present)

evalsuite/plot.py

@@ -503,3 +503,149 @@ def detection_pr(
     )
     ax.legend(loc="lower left", frameon=False, fontsize="small")
     return ax
+
+
+# ---------------------------------------------------------------- v0.4.0: preferences and text
+def _rating_fit(method: str, rows: list[Any]) -> Any:
+    from .text.judge import bradley_terry, elo_ratings
+
+    if method == "bradley_terry":
+        return bradley_terry(rows, scale="elo")
+    if method == "elo":
+        return elo_ratings(rows)
+    raise InputValidationError("method must be 'bradley_terry' or 'elo'.")
+
+
+def ratings(
+    comparisons: Any,
+    *,
+    method: str = "bradley_terry",
+    n_resamples: int = 200,
+    level: float = 0.95,
+    random_state: Optional[int] = 0,
+    ax: Optional[Axes] = None,
+) -> Axes:
+    """Leaderboard of model ratings from pairwise ``(model_a, model_b, outcome)`` comparisons, with percentile
+    bootstrap intervals over comparisons (Chatbot Arena style). ``method``: ``"bradley_terry"`` (Elo scale) or
+    ``"elo"`` (online Elo). Resamples on which the Bradley–Terry fit does not exist are skipped and counted in
+    the x-axis label."""
+    rows = list(comparisons)
+    point = _rating_fit(method, rows)
+    names = list(point.labels or ())
+    est = np.asarray(point.value, dtype=float)
+    rng = np.random.default_rng(random_state)
+    boot, failed = [], 0
+    for _ in range(int(n_resamples)):
+        sample = [rows[i] for i in rng.integers(0, len(rows), len(rows))]
+        try:
+            r = _rating_fit(method, sample)
+        except InputValidationError:
+            failed += 1
+            continue
+        lookup = dict(zip(r.labels or (), np.asarray(r.value, dtype=float)))
+        boot.append([lookup.get(n, np.nan) for n in names])
+    order = np.argsort(est)
+    ax = _axes(ax, (5.5, max(2.6, 0.45 * len(names) + 1.2)))
+    y = np.arange(len(names))
+    if boot:
+        b = np.asarray(boot)
+        a = (1 - level) / 2
+        lo, hi = np.nanquantile(b, a, axis=0), np.nanquantile(b, 1 - a, axis=0)
+        ax.errorbar(
+            est[order],
+            y,
+            xerr=[est[order] - lo[order], hi[order] - est[order]],
+            fmt="o",
+            color="#2a6fb0",
+            ecolor="0.45",
+            capsize=3,
+        )
+    else:
+        ax.plot(est[order], y, "o", color="#2a6fb0")
+    ax.set_yticks(y, [str(names[i]) for i in order])
+    title = "Bradley–Terry rating" if method == "bradley_terry" else "Elo rating"
+    note = f"{int(level * 100)}% bootstrap CI, {len(boot)} resamples" + (f", {failed} skipped" if failed else "")
+    ax.set(xlabel=f"{title} ({note})", title=f"{title}s from {len(rows)} comparisons")
+    ax.grid(axis="x", alpha=0.3)
+    return ax
+
+
+def win_matrix(comparisons: Any, *, ax: Optional[Axes] = None, annotate: bool = True) -> Axes:
+    """Heatmap of pairwise win rates: cell (row, column) is how often the row model beat the column model (ties
+    count half); blank where the two were never compared."""
+    from .text.judge import _comparisons
+
+    names, rows = _comparisons(comparisons)
+    m = len(names)
+    wins, games = np.zeros((m, m)), np.zeros((m, m))
+    for a, b, o in rows:
+        wins[a, b] += o
+        wins[b, a] += 1 - o
+        games[a, b] += 1
+        games[b, a] += 1
+    with np.errstate(invalid="ignore", divide="ignore"):
+        rate = np.where(games > 0, wins / np.where(games > 0, games, 1), np.nan)
+    order = np.argsort(-np.nanmean(np.where(np.eye(m, dtype=bool), np.nan, rate), axis=1))  # strongest first
+    rate = rate[np.ix_(order, order)]
+    names = [names[i] for i in order]
+    ax = _axes(ax, (1.2 + 0.7 * m, 0.9 + 0.6 * m))
+    im = ax.imshow(rate, cmap="RdBu", vmin=0, vmax=1)
+    ax.set_xticks(range(m), [str(n) for n in names], rotation=45, ha="right")
+    ax.set_yticks(range(m), [str(n) for n in names])
+    if annotate:
+        for i in range(m):
+            for j in range(m):
+                if not np.isnan(rate[i, j]):
+                    ax.text(
+                        j,
+                        i,
+                        f"{rate[i, j]:.2f}",
+                        ha="center",
+                        va="center",
+                        fontsize=8,
+                        color="white" if abs(rate[i, j] - 0.5) > 0.3 else "black",
+                    )
+    ax.figure.colorbar(im, ax=ax, label="win rate (row vs column)")
+    ax.set_title("Pairwise win rates")
+    return ax
+
+
+def text_scores(
+    references: Any,
+    predictions: Mapping[str, Any],
+    *,
+    metric: Any = "token_f1",
+    ax: Optional[Axes] = None,
+    **metric_kwargs: Any,
+) -> Axes:
+    """Distribution of a per-example text metric for several systems (box plots with the mean marked), e.g.
+    ``metric="rouge_l"``, ``"token_f1"``, ``"meteor"``, ``"sentence_bleu"`` or ``"bertscore"`` (with
+    embeddings). Pair it with ``es.compare`` for intervals and tests."""
+    from .stats._resolve import resolve_metric
+
+    fn, name = resolve_metric(metric)
+    import inspect
+
+    if "average" not in inspect.signature(fn).parameters:
+        raise InputValidationError(
+            f"{name} is a corpus-level metric without per-example scores; use e.g. sentence_bleu, rouge_l, "
+            "token_f1 or meteor, or es.compare for corpus metrics."
+        )
+    if not isinstance(predictions, Mapping) or not predictions:
+        raise InputValidationError("predictions must be a dict of system name -> predictions.")
+    data, labels = [], []
+    for sys_name, preds in predictions.items():
+        res = fn(references, preds, average=None, **metric_kwargs)
+        vals = np.asarray(res.value, dtype=float)
+        if vals.ndim != 1:
+            raise InputValidationError(f"{name} does not give one score per example.")
+        data.append(vals[~np.isnan(vals)])
+        labels.append(str(sys_name))
+    ax = _axes(ax, (1.6 + 1.1 * len(labels), 4.2))
+    ax.boxplot(
+        data, showmeans=True, meanprops={"marker": "D", "markerfacecolor": "#d9822b", "markeredgecolor": "#d9822b"}
+    )
+    ax.set_xticks(range(1, len(labels) + 1), labels)
+    ax.set(ylabel=f"{res.name} per example", title=f"{res.name} by system (diamond = mean)")
+    ax.grid(axis="y", alpha=0.3)
+    return ax

evalsuite/regression/metrics.py

@@ -202,7 +202,15 @@ def rmse(
 ) -> MetricResult:
     """Root mean squared error (square root taken per output, then averaged)."""
     inp = _inputs(y_true, y_pred, sample_weight)
-    return _finish(np.sqrt(inp.mean(inp.error**2)), inp, multioutput, "rmse", "RMSE")
+    with np.errstate(over="ignore"):
+        per_output = np.sqrt(inp.mean(inp.error**2))
+    if not np.all(np.isfinite(per_output)):
+        # Errors above ~1e154 overflow when squared although the RMSE itself is representable:
+        # rescale by the largest error and square again.
+        scale = np.abs(inp.error).max(axis=0)
+        scale = np.where(np.isfinite(scale) & (scale > 0), scale, 1.0)
+        per_output = scale * np.sqrt(inp.mean((inp.error / scale) ** 2))
+    return _finish(per_output, inp, multioutput, "rmse", "RMSE")
 
 
 def _r2_per_output(inp: _Inputs) -> FloatArray: