errorbars 0.1.2__py3-none-any.whl

errorbars/compare.py

@@ -0,0 +1,178 @@
+"""Paired comparisons between two models on the same questions."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+from errorbars.stats import cluster_robust_se, is_binary, t_for_confidence, t_two_sided_p
+
+__all__ = ["PairedComparison", "paired_compare", "McNemarResult", "mcnemar_exact"]
+
+
+@dataclass(frozen=True)
+class PairedComparison:
+    """Result of a paired comparison between model A and model B."""
+
+    mean_a: float
+    mean_b: float
+    mean_diff: float
+    se_paired: float
+    ci_low: float
+    ci_high: float
+    p_value: float
+    correlation: float
+    se_unpaired: float
+    variance_reduction: float
+    n: int
+    confidence: float
+    se_clustered: float | None = None
+    ci_low_clustered: float | None = None
+    ci_high_clustered: float | None = None
+    p_value_clustered: float | None = None
+    mcnemar: McNemarResult | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {
+            "mean_a": self.mean_a,
+            "mean_b": self.mean_b,
+            "mean_diff": self.mean_diff,
+            "se_paired": self.se_paired,
+            "ci_low": self.ci_low,
+            "ci_high": self.ci_high,
+            "p_value": self.p_value,
+            "correlation": self.correlation,
+            "se_unpaired": self.se_unpaired,
+            "variance_reduction": self.variance_reduction,
+            "n": self.n,
+            "confidence": self.confidence,
+        }
+        if self.se_clustered is not None:
+            d["se_clustered"] = self.se_clustered
+            d["ci_low_clustered"] = self.ci_low_clustered
+            d["ci_high_clustered"] = self.ci_high_clustered
+            d["p_value_clustered"] = self.p_value_clustered
+        if self.mcnemar is not None:
+            d["mcnemar"] = self.mcnemar.as_dict()
+        return d
+
+
+def paired_compare(
+    scores_a: ArrayLike,
+    scores_b: ArrayLike,
+    clusters: ArrayLike | None = None,
+    confidence: float = 0.95,
+) -> PairedComparison:
+    """Paired difference test: mean(A) - mean(B) over the same questions.
+
+    Reports the standard paired t-test SE/CI/p-value (Student t, n - 1
+    degrees of freedom), the correlation between the two models'
+    per-question scores, and how much pairing shrank the SE relative to an
+    unpaired (independent two-sample) SE at the same n. When ``clusters`` is
+    given, also reports a cluster-robust paired SE/CI/p-value (see
+    ``errorbars.stats.cluster_robust_se``) on t with G - 1 degrees of
+    freedom for G clusters, the usual reference for a clustered mean (Cameron
+    & Miller 2015). When both score vectors are binary, also runs McNemar's
+    exact test.
+    """
+    a = np.asarray(scores_a, dtype=float)
+    b = np.asarray(scores_b, dtype=float)
+    if a.shape != b.shape:
+        raise ValueError("scores_a and scores_b must have the same length (paired)")
+    n = a.size
+    if n < 2:
+        raise ValueError("need at least 2 paired observations")
+
+    diff = a - b
+    mean_diff = float(diff.mean())
+    sd_diff = float(diff.std(ddof=1))
+    se_paired = sd_diff / math.sqrt(n)
+    t_crit = t_for_confidence(confidence, n - 1)
+    ci_low, ci_high = mean_diff - t_crit * se_paired, mean_diff + t_crit * se_paired
+    t_stat = mean_diff / se_paired if se_paired > 0 else 0.0
+    p_value = t_two_sided_p(t_stat, n - 1) if se_paired > 0 else 1.0
+
+    var_a, var_b = float(a.var(ddof=1)), float(b.var(ddof=1))
+    corr = float(np.corrcoef(a, b)[0, 1]) if var_a > 0 and var_b > 0 else 0.0
+    var_unpaired = var_a + var_b
+    se_unpaired = math.sqrt(var_unpaired / n)
+    variance_reduction = 1.0 - (se_paired**2 / se_unpaired**2) if se_unpaired > 0 else 0.0
+
+    se_clustered: float | None = None
+    ci_low_c: float | None = None
+    ci_high_c: float | None = None
+    p_value_c: float | None = None
+    if clusters is not None:
+        cluster_arr = np.asarray(clusters)
+        se_clustered = cluster_robust_se(diff, cluster_arr)
+        n_clusters = len(set(cluster_arr.tolist()))
+        # With a single cluster cluster_robust_se falls back to the plain SE, so its reference is n - 1.
+        dof_c = n_clusters - 1 if n_clusters > 1 else n - 1
+        t_crit_c = t_for_confidence(confidence, dof_c)
+        ci_low_c = mean_diff - t_crit_c * se_clustered
+        ci_high_c = mean_diff + t_crit_c * se_clustered
+        t_stat_c = mean_diff / se_clustered if se_clustered > 0 else 0.0
+        p_value_c = float(t_two_sided_p(t_stat_c, dof_c)) if se_clustered > 0 else 1.0
+
+    scores_list_a: list[float] = a.tolist()
+    scores_list_b: list[float] = b.tolist()
+    mcnemar = mcnemar_exact(scores_list_a, scores_list_b) if is_binary(a) and is_binary(b) else None
+
+    return PairedComparison(
+        mean_a=float(a.mean()),
+        mean_b=float(b.mean()),
+        mean_diff=mean_diff,
+        se_paired=se_paired,
+        ci_low=ci_low,
+        ci_high=ci_high,
+        p_value=float(p_value),
+        correlation=corr,
+        se_unpaired=se_unpaired,
+        variance_reduction=float(variance_reduction),
+        n=n,
+        confidence=confidence,
+        se_clustered=se_clustered,
+        ci_low_clustered=ci_low_c,
+        ci_high_clustered=ci_high_c,
+        p_value_clustered=p_value_c,
+        mcnemar=mcnemar,
+    )
+
+
+@dataclass(frozen=True)
+class McNemarResult:
+    """Exact McNemar test on discordant pairs of a 2x2 paired-binary table."""
+
+    n01: int  # A wrong, B right
+    n10: int  # A right, B wrong
+    p_value: float
+
+    def as_dict(self) -> dict[str, Any]:
+        return {"n01": self.n01, "n10": self.n10, "p_value": self.p_value}
+
+
+def mcnemar_exact(scores_a: ArrayLike, scores_b: ArrayLike) -> McNemarResult:
+    """Exact (binomial) two-sided McNemar test for paired binary outcomes.
+
+    Uses only the discordant pairs b = #(A=0,B=1), c = #(A=1,B=0); under the
+    null they are Binomial(b+c, 0.5). Reference: McNemar (1947); the exact
+    version is preferred over the chi-square approximation when b+c is
+    small. Matches ``statsmodels.stats.contingency_tables.mcnemar(...,
+    exact=True)``.
+    """
+    a = np.asarray(scores_a)
+    b = np.asarray(scores_b)
+    n01 = int(np.sum((a == 0) & (b == 1)))
+    n10 = int(np.sum((a == 1) & (b == 0)))
+    total = n01 + n10
+    if total == 0:
+        return McNemarResult(n01, n10, 1.0)
+    k = min(n01, n10)
+    # two-sided exact binomial test, p=0.5: sum both tails via symmetry
+    tail = sum(math.comb(total, i) for i in range(0, k + 1)) / (2**total)
+    p_value = min(1.0, 2 * tail)
+    return McNemarResult(n01, n10, float(p_value))

errorbars/io.py

@@ -0,0 +1,197 @@
+"""Loading per-item eval scores from CSV, JSONL, or a pandas DataFrame.
+
+The canonical shape is "long format": one row per (question, model[, sample])
+with columns ``question_id``, ``model``, ``score``, and optional
+``cluster_id`` / ``sample``. Column names are configurable so you don't have
+to reshape your existing logs.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import math
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+__all__ = ["EvalData", "ColumnMap", "load_csv", "load_jsonl", "load_dataframe", "write_csv"]
+
+
+@dataclass(frozen=True)
+class ColumnMap:
+    """Names of the columns in your source data, if not the defaults."""
+
+    question_id: str = "question_id"
+    model: str = "model"
+    score: str = "score"
+    cluster_id: str | None = "cluster_id"
+    sample: str | None = "sample"
+
+
+@dataclass
+class EvalData:
+    """Normalized long-format eval records, one row per observation."""
+
+    question_id: list[str]
+    model: list[str]
+    score: list[float]
+    cluster_id: list[str] | None = None
+    sample: list[str] | None = None
+    columns: ColumnMap = field(default_factory=ColumnMap)
+
+    def __len__(self) -> int:
+        return len(self.score)
+
+    def models(self) -> list[str]:
+        seen: dict[str, None] = {}
+        for m in self.model:
+            seen.setdefault(m, None)
+        return list(seen)
+
+    def filter_model(self, model: str) -> EvalData:
+        idx = [i for i, m in enumerate(self.model) if m == model]
+        return EvalData(
+            question_id=[self.question_id[i] for i in idx],
+            model=[self.model[i] for i in idx],
+            score=[self.score[i] for i in idx],
+            cluster_id=[self.cluster_id[i] for i in idx] if self.cluster_id else None,
+            sample=[self.sample[i] for i in idx] if self.sample else None,
+            columns=self.columns,
+        )
+
+    def scores_by_question(self) -> dict[str, float]:
+        """Mean score per question_id (collapses repeated samples)."""
+        totals: dict[str, list[float]] = {}
+        for qid, s in zip(self.question_id, self.score, strict=True):
+            totals.setdefault(qid, []).append(s)
+        return {qid: sum(v) / len(v) for qid, v in totals.items()}
+
+    def cluster_by_question(self) -> dict[str, str]:
+        if not self.cluster_id:
+            return {}
+        out: dict[str, str] = {}
+        for qid, c in zip(self.question_id, self.cluster_id, strict=True):
+            out.setdefault(qid, c)
+        return out
+
+
+def _coerce_score(raw: Any, row: int) -> float:
+    if isinstance(raw, bool):
+        return float(raw)
+    try:
+        value = float(raw)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"row {row}: could not parse score value {raw!r} as a number") from exc
+    # A NaN would propagate into every mean, CI and p-value (and sort to the top of a leaderboard);
+    # a missing or failed grade has to be decided on explicitly, not averaged in as "not a number".
+    if not math.isfinite(value):
+        raise ValueError(f"row {row}: score {raw!r} is not a finite number")
+    return value
+
+
+def _from_records(records: Iterable[dict[str, Any]], columns: ColumnMap) -> EvalData:
+    question_id: list[str] = []
+    model: list[str] = []
+    score: list[float] = []
+    cluster_id: list[str] | None = [] if columns.cluster_id else None
+    sample: list[str] | None = [] if columns.sample else None
+
+    n = 0
+    seen: dict[tuple[str, str, str], int] = {}
+    for row in records:
+        n += 1
+        if columns.question_id not in row:
+            raise ValueError(f"missing required column '{columns.question_id}' in row {n}")
+        if columns.model not in row:
+            raise ValueError(f"missing required column '{columns.model}' in row {n}")
+        if columns.score not in row:
+            raise ValueError(f"missing required column '{columns.score}' in row {n}")
+        question_id.append(str(row[columns.question_id]))
+        model.append(str(row[columns.model]))
+        score.append(_coerce_score(row[columns.score], n))
+        if cluster_id is not None and columns.cluster_id is not None:
+            cluster_id.append(str(row.get(columns.cluster_id, question_id[-1])))
+        sample_id = str(row.get(columns.sample, "0")) if columns.sample is not None else "0"
+        if sample is not None:
+            sample.append(sample_id)
+        # The same (model, question, sample) twice would be counted as two questions, inflating n and
+        # shrinking every standard error. Repeated generations need distinct sample ids.
+        key = (model[-1], question_id[-1], sample_id)
+        if key in seen:
+            raise ValueError(
+                f"row {n}: model {key[0]!r} already has a score for question {key[1]!r} (row {seen[key]}); "
+                f"give repeated generations distinct values in a '{columns.sample or 'sample'}' column"
+            )
+        seen[key] = n
+
+    if n == 0:
+        raise ValueError("no rows found in input data")
+
+    return EvalData(question_id, model, score, cluster_id, sample, columns)
+
+
+def load_csv(path: str | Path, columns: ColumnMap | None = None) -> EvalData:
+    """Load per-item scores from a CSV file."""
+    columns = columns or ColumnMap()
+    with open(path, newline="", encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        if reader.fieldnames is None:
+            raise ValueError(f"{path}: empty or headerless CSV")
+        return _from_records(reader, columns)
+
+
+def load_jsonl(path: str | Path, columns: ColumnMap | None = None) -> EvalData:
+    """Load per-item scores from a JSON Lines file (one record per line)."""
+    columns = columns or ColumnMap()
+    records: list[dict[str, Any]] = []
+    with open(path, encoding="utf-8") as f:
+        for lineno, line in enumerate(f, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{lineno}: invalid JSON: {exc}") from exc
+    if not records:
+        raise ValueError(f"{path}: no records found")
+    return _from_records(records, columns)
+
+
+def load_dataframe(df: Any, columns: ColumnMap | None = None) -> EvalData:
+    """Load per-item scores from a pandas DataFrame."""
+    columns = columns or ColumnMap()
+    records = df.to_dict(orient="records")
+    return _from_records(records, columns)
+
+
+def write_csv(data: EvalData, path: str | Path) -> None:
+    """Write an ``EvalData`` back out as a canonical long-format CSV.
+
+    Used by ``errorbars import`` to turn an adapter's output into a file
+    ``summarize``/``compare``/``leaderboard`` can load directly. Only
+    includes ``cluster_id`` / ``sample`` columns when the data actually has
+    them.
+    """
+    fieldnames = ["question_id", "model", "score"]
+    if data.cluster_id:
+        fieldnames.append("cluster_id")
+    if data.sample:
+        fieldnames.append("sample")
+
+    with open(path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for i in range(len(data)):
+            row: dict[str, Any] = {
+                "question_id": data.question_id[i],
+                "model": data.model[i],
+                "score": data.score[i],
+            }
+            if data.cluster_id:
+                row["cluster_id"] = data.cluster_id[i]
+            if data.sample:
+                row["sample"] = data.sample[i]
+            writer.writerow(row)

errorbars/leaderboard.py

@@ -0,0 +1,181 @@
+"""Multi-model leaderboards with paired pairwise tests and Holm correction."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from itertools import combinations
+from typing import Any
+
+from errorbars.compare import PairedComparison, paired_compare
+from errorbars.io import EvalData
+from errorbars.stats import MeanEstimate, is_binary, mean_ci_clt, wilson_ci
+
+__all__ = ["LeaderboardEntry", "PairwiseResult", "Leaderboard", "build_leaderboard", "holm_correction"]
+
+
+def holm_correction(p_values: list[float]) -> list[float]:
+    """Holm-Bonferroni step-down adjusted p-values (Holm, 1979).
+
+    Controls the family-wise error rate without assuming independence,
+    less conservative than plain Bonferroni. Matches
+    ``statsmodels.stats.multitest.multipletests(p, method="holm")``.
+    """
+    m = len(p_values)
+    order = sorted(range(m), key=lambda i: p_values[i])
+    adjusted = [0.0] * m
+    running_max = 0.0
+    for rank, i in enumerate(order):
+        raw = (m - rank) * p_values[i]
+        running_max = max(running_max, raw)
+        adjusted[i] = min(1.0, running_max)
+    return adjusted
+
+
+@dataclass(frozen=True)
+class LeaderboardEntry:
+    model: str
+    mean: float
+    se: float
+    ci_low: float
+    ci_high: float
+    n: int
+    method: str
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "model": self.model,
+            "mean": self.mean,
+            "se": self.se,
+            "ci_low": self.ci_low,
+            "ci_high": self.ci_high,
+            "n": self.n,
+            "method": self.method,
+        }
+
+
+@dataclass(frozen=True)
+class PairwiseResult:
+    model_a: str
+    model_b: str
+    comparison: PairedComparison
+    p_holm: float
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "model_a": self.model_a,
+            "model_b": self.model_b,
+            "mean_diff": self.comparison.mean_diff,
+            "p_value": self.comparison.p_value,
+            "p_holm": self.p_holm,
+        }
+
+
+@dataclass(frozen=True)
+class Leaderboard:
+    entries: list[LeaderboardEntry]  # sorted by mean, descending
+    pairwise: list[PairwiseResult]
+    groups: list[list[str]]  # each group: models not significantly different (Holm alpha)
+    alpha: float
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "entries": [e.as_dict() for e in self.entries],
+            "pairwise": [p.as_dict() for p in self.pairwise],
+            "groups": self.groups,
+            "alpha": self.alpha,
+        }
+
+
+def _maximal_cliques(nodes: list[str], edges: set[frozenset[str]]) -> list[list[str]]:
+    """Bron-Kerbosch without pivoting; fine for leaderboard-sized graphs."""
+
+    def neighbors(v: str) -> set[str]:
+        return {u for u in nodes if u != v and frozenset((u, v)) in edges}
+
+    adj = {v: neighbors(v) for v in nodes}
+    cliques: list[set[str]] = []
+
+    def bron_kerbosch(r: set[str], p: set[str], x: set[str]) -> None:
+        if not p and not x:
+            cliques.append(r)
+            return
+        for v in list(p):
+            bron_kerbosch(r | {v}, p & adj[v], x & adj[v])
+            p = p - {v}
+            x = x | {v}
+
+    bron_kerbosch(set(), set(nodes), set())
+    # sort each clique by the leaderboard's original node order for stable display
+    order = {v: i for i, v in enumerate(nodes)}
+    cliques_sorted = [sorted(c, key=lambda v: order[v]) for c in cliques]
+    cliques_sorted.sort(key=lambda c: (order[c[0]], -len(c)))
+    return cliques_sorted
+
+
+def build_leaderboard(
+    data: EvalData,
+    confidence: float = 0.95,
+    alpha: float = 0.05,
+    use_wilson_below_n: int = 30,
+) -> Leaderboard:
+    """Build a leaderboard: per-model CIs, Holm-corrected pairwise tests, groups.
+
+    ``use_wilson_below_n``: for binary scores with fewer than this many
+    questions, per-model CIs use the Wilson interval instead of the CLT
+    interval (see ``errorbars.stats.wilson_ci``).
+    """
+    models = data.models()
+    if len(models) < 2:
+        raise ValueError("leaderboard needs at least 2 models")
+
+    per_model_scores: dict[str, list[float]] = {m: data.filter_model(m).score for m in models}
+
+    entries: list[LeaderboardEntry] = []
+    for m in models:
+        scores = per_model_scores[m]
+        est = _summarize_mean(scores, confidence, use_wilson_below_n)
+        entries.append(LeaderboardEntry(m, est.mean, est.se, est.ci_low, est.ci_high, est.n, est.method))
+    entries.sort(key=lambda e: e.mean, reverse=True)
+    order = [e.model for e in entries]
+
+    # Align questions for paired tests: use the intersection of question_ids
+    # common to both models, sorted for determinism.
+    qmap = {m: data.filter_model(m).scores_by_question() for m in models}
+    cluster_of_q = data.cluster_by_question()
+
+    pairwise: list[PairwiseResult] = []
+    raw_p: list[float] = []
+    pair_keys: list[tuple[str, str]] = []
+    for a, b in combinations(order, 2):
+        common = sorted(set(qmap[a]) & set(qmap[b]))
+        if len(common) < 2:
+            continue
+        sa = [qmap[a][q] for q in common]
+        sb = [qmap[b][q] for q in common]
+        clusters = [cluster_of_q[q] for q in common] if cluster_of_q else None
+        has_real_clusters = clusters is not None and len(set(clusters)) < len(clusters)
+        comp = paired_compare(sa, sb, clusters=clusters if has_real_clusters else None, confidence=confidence)
+        pair_keys.append((a, b))
+        # Prefer the cluster-robust p-value when clusters are present: it is
+        # the honest one when questions are correlated within a cluster.
+        raw_p.append(comp.p_value_clustered if comp.p_value_clustered is not None else comp.p_value)
+        pairwise.append(PairwiseResult(a, b, comp, p_holm=raw_p[-1]))  # placeholder, fixed below
+
+    adjusted = holm_correction(raw_p) if raw_p else []
+    pairwise = [
+        PairwiseResult(a, b, pr.comparison, p_holm)
+        for (a, b), pr, p_holm in zip(pair_keys, pairwise, adjusted, strict=True)
+    ]
+
+    edges = {frozenset((pr.model_a, pr.model_b)) for pr in pairwise if pr.p_holm >= alpha}
+    groups = _maximal_cliques(order, edges)
+
+    return Leaderboard(entries=entries, pairwise=pairwise, groups=groups, alpha=alpha)
+
+
+def _summarize_mean(scores: list[float], confidence: float, use_wilson_below_n: int) -> MeanEstimate:
+    n = len(scores)
+    if is_binary(scores) and n < use_wilson_below_n:
+        successes = int(round(sum(scores)))
+        return wilson_ci(successes, n, confidence)
+    return mean_ci_clt(scores, confidence)

errorbars/plot.py

@@ -0,0 +1,141 @@
+"""Forest plots for leaderboards.
+
+A dependency-free SVG writer always works; ``forest_plot_matplotlib`` is
+used automatically by the CLI when matplotlib is installed (the ``plot``
+extra) and a raster/PDF output is requested.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from errorbars.leaderboard import Leaderboard
+
+if TYPE_CHECKING:
+    from matplotlib.figure import Figure
+
+__all__ = ["forest_plot_svg", "forest_plot_matplotlib"]
+
+_FONT = "system-ui, -apple-system, 'Segoe UI', sans-serif"
+
+
+def forest_plot_svg(
+    leaderboard: Leaderboard,
+    width: int = 640,
+    row_height: int = 44,
+    title: str | None = None,
+) -> str:
+    """Render a forest plot (mean + CI per model) as a standalone SVG string."""
+    entries = leaderboard.entries
+    n = len(entries)
+    margin_left, margin_right, margin_top, margin_bottom = 160, 40, 50 if title else 20, 30
+    plot_h = n * row_height
+    height = margin_top + plot_h + margin_bottom
+    plot_w = width - margin_left - margin_right
+
+    lo = min(e.ci_low for e in entries)
+    hi = max(e.ci_high for e in entries)
+    pad = (hi - lo) * 0.1 or 0.05
+    x_min, x_max = lo - pad, hi + pad
+
+    def x(v: float) -> float:
+        return margin_left + (v - x_min) / (x_max - x_min) * plot_w
+
+    def y(i: int) -> float:
+        return margin_top + i * row_height + row_height / 2
+
+    parts: list[str] = []
+    parts.append(
+        f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" '
+        f'viewBox="0 0 {width} {height}" font-family="{_FONT}">'
+    )
+    parts.append(f'<rect width="{width}" height="{height}" fill="#ffffff"/>')
+    if title:
+        parts.append(
+            f'<text x="{width / 2}" y="24" text-anchor="middle" font-size="15" '
+            f'font-weight="600" fill="#111827">{_escape(title)}</text>'
+        )
+
+    # gridlines + x axis ticks
+    n_ticks = 5
+    for t in range(n_ticks + 1):
+        v = x_min + (x_max - x_min) * t / n_ticks
+        gx = x(v)
+        parts.append(
+            f'<line x1="{gx:.1f}" y1="{margin_top}" x2="{gx:.1f}" '
+            f'y2="{margin_top + plot_h}" stroke="#e5e7eb" stroke-width="1"/>'
+        )
+        parts.append(
+            f'<text x="{gx:.1f}" y="{margin_top + plot_h + 18}" text-anchor="middle" '
+            f'font-size="10" fill="#6b7280">{v:.3f}</text>'
+        )
+
+    groups = leaderboard.groups
+    letter_of: dict[str, str] = {}
+    for i, group in enumerate(groups):
+        letter = chr(ord("a") + i)
+        for m in group:
+            letter_of[m] = letter_of.get(m, "") + letter
+
+    for i, e in enumerate(entries):
+        cy = y(i)
+        parts.append(
+            f'<text x="{margin_left - 12}" y="{cy + 4:.1f}" text-anchor="end" '
+            f'font-size="12" fill="#111827">{_escape(e.model)}</text>'
+        )
+        x_lo, x_hi, x_mean = x(e.ci_low), x(e.ci_high), x(e.mean)
+        parts.append(
+            f'<line x1="{x_lo:.1f}" y1="{cy:.1f}" x2="{x_hi:.1f}" y2="{cy:.1f}" '
+            f'stroke="#2563eb" stroke-width="2"/>'
+        )
+        for edge_x in (x_lo, x_hi):
+            parts.append(
+                f'<line x1="{edge_x:.1f}" y1="{cy - 5:.1f}" x2="{edge_x:.1f}" '
+                f'y2="{cy + 5:.1f}" stroke="#2563eb" stroke-width="2"/>'
+            )
+        parts.append(f'<circle cx="{x_mean:.1f}" cy="{cy:.1f}" r="4.5" fill="#1d4ed8"/>')
+        label = f"{e.mean:.3f}"
+        if letter_of.get(e.model):
+            label += f"  ({letter_of[e.model]})"
+        parts.append(
+            f'<text x="{x_hi + 10:.1f}" y="{cy + 4:.1f}" font-size="11" '
+            f'fill="#374151">{_escape(label)}</text>'
+        )
+
+    parts.append(
+        f'<line x1="{margin_left}" y1="{margin_top + plot_h}" x2="{margin_left + plot_w}" '
+        f'y2="{margin_top + plot_h}" stroke="#9ca3af" stroke-width="1"/>'
+    )
+    parts.append("</svg>")
+    return "\n".join(parts)
+
+
+def _escape(s: str) -> str:
+    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+
+
+def forest_plot_matplotlib(leaderboard: Leaderboard, title: str | None = None) -> Figure:
+    """Render a forest plot with matplotlib. Requires the ``plot`` extra."""
+    try:
+        import matplotlib.pyplot as plt
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError(
+            "forest_plot_matplotlib requires matplotlib (the 'plot' extra): pip install matplotlib"
+        ) from exc
+
+    entries = leaderboard.entries
+    fig, ax = plt.subplots(figsize=(7, 0.6 * len(entries) + 1.2))
+    ys = list(range(len(entries), 0, -1))
+    means = [e.mean for e in entries]
+    los = [e.mean - e.ci_low for e in entries]
+    his = [e.ci_high - e.mean for e in entries]
+    ax.errorbar(means, ys, xerr=[los, his], fmt="o", color="#1d4ed8", ecolor="#2563eb", capsize=4)
+    ax.set_yticks(ys)
+    ax.set_yticklabels([e.model for e in entries])
+    ax.set_xlabel("mean score")
+    if title:
+        ax.set_title(title)
+    ax.grid(axis="x", color="#e5e7eb", linewidth=0.8)
+    ax.set_axisbelow(True)
+    fig.tight_layout()
+    return fig