evalsuite-python 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

evalsuite/__init__.py

@@ -5,7 +5,7 @@
 >>> print(result.summary())  # doctest: +SKIP
 """
 
-from . import calibration, classification, clinical, plot, regression, stats, vision
+from . import calibration, classification, clinical, plot, regression, stats, text, vision
 from .api import evaluate
 from .calibration import (
     CalibrationReport,
@@ -110,6 +110,8 @@ from .stats import (
     t_test,
     wilcoxon_test,
 )
+from .text import *  # noqa: F403  (every LLM-evaluation function, listed in text.__all__)
+from .text import __all__ as _text_all
 from .version import __version__
 from .vision import (
     DetectionReport,
@@ -134,6 +136,7 @@ from .vision import (
 )
 
 __all__ = [
+    "text",
     "SegmentationReport",
     "segmentation_report",
     "detection_pr_curve",
@@ -258,3 +261,4 @@ __all__ = [
     "specificity",
     "top_k_accuracy",
 ]
+__all__ += list(_text_all)

evalsuite/benchmarks.py

@@ -213,8 +213,8 @@ def _vision_cases(n: int, rng: np.random.Generator) -> list[Case]:
         import contextlib as _ctx
         import io
 
-        from pycocotools.coco import COCO  # type: ignore[import-untyped]
-        from pycocotools.cocoeval import COCOeval  # type: ignore[import-untyped]
+        from pycocotools.coco import COCO
+        from pycocotools.cocoeval import COCOeval
 
         def coco_map() -> list[float]:
             images, anns, dets, aid = [], [], [], 1
@@ -262,6 +262,155 @@ def _vision_cases(n: int, rng: np.random.Generator) -> list[Case]:
     return [(name, ref, es_fn, refs.get(name)) for name, ref, es_fn, _ in cases]
 
 
+_LLM_VOCAB = (  # noqa: SIM905
+    "the a cat dog sat on mat quickly model models data run running ran 2.5 3,000 well-known state-of-the-art "
+    "( ) , . ! ? don't U.S. e-mail Hello WORLD naive cafe results show improves baseline evaluation"
+).split()
+
+
+def _llm_cases(n: int, rng: np.random.Generator) -> list[Case]:
+    """v0.4.0: text generation, retrieval, rater agreement and structured output (n / 100 examples)."""
+    import evalsuite as es
+
+    m = max(10, n // 100)
+    preds, refs_ = [], []
+    for _ in range(m):
+        base = list(rng.choice(_LLM_VOCAB, int(rng.integers(5, 25))))
+        keep = [w for w in base if rng.random() > 0.2] + list(rng.choice(_LLM_VOCAB, int(rng.integers(0, 4))))
+        preds.append(" ".join(keep))
+        refs_.append(" ".join(base))
+    queries = [
+        (set(rng.choice(200, int(rng.integers(1, 6)), replace=False).tolist()), rng.permutation(200)[:20].tolist())
+        for _ in range(m)
+    ]
+    rel, ret = [q[0] for q in queries], [q[1] for q in queries]
+    ratings = rng.integers(1, 6, (4, m)).astype(float)
+    ratings[rng.random(ratings.shape) < 0.1] = np.nan
+    schema = {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string", "minLength": 1},
+            "age": {"type": "integer", "minimum": 0},
+            "tags": {"type": "array", "items": {"type": "string"}, "uniqueItems": True},
+        },
+        "required": ["name", "age"],
+        "additionalProperties": False,
+    }
+    docs = []
+    for i in range(m):
+        d: dict[str, Any] = {"name": f"n{i}", "age": int(rng.integers(-2, 90)), "tags": ["a", "b"]}
+        if rng.random() < 0.1:
+            d["extra"] = 1
+        if rng.random() < 0.1:
+            del d["name"]
+        docs.append(d)
+    import json as _json
+
+    docs_text = [_json.dumps(d) for d in docs]
+
+    cases: list[Case] = [
+        (
+            f"text: corpus BLEU and chrF ({m} sentences)",
+            "sacreBLEU",
+            lambda: [float(es.bleu(refs_, preds)), float(es.chrf(refs_, preds))],
+            None,
+        ),
+        (
+            f"text: ROUGE-1, ROUGE-2, ROUGE-L ({m} sentences)",
+            "rouge-score",
+            lambda: [
+                float(es.rouge_1(refs_, preds)),
+                float(es.rouge_2(refs_, preds)),
+                float(es.rouge_l(refs_, preds)),
+            ],
+            None,
+        ),
+        (
+            f"text: METEOR, exact and stem matches ({m} sentences)",
+            "NLTK",
+            lambda: [float(es.meteor(refs_, preds))],
+            None,
+        ),
+        (
+            f"retrieval: MRR, MAP@20, NDCG@10 ({m} queries)",
+            "ranx",
+            lambda: [
+                float(es.mrr(rel, ret)),
+                float(es.mean_average_precision_at_k(rel, ret, k=20)),
+                float(es.ndcg_at_k(rel, ret, k=10)),
+            ],
+            None,
+        ),
+        (
+            f"agreement: Krippendorff's alpha, interval (4 raters × {m} items)",
+            "krippendorff",
+            lambda: [float(es.krippendorff_alpha(ratings, level="interval"))],
+            None,
+        ),
+        (
+            f"structured: JSON Schema compliance ({m} documents)",
+            "jsonschema",
+            lambda: [float(es.json_schema_compliance(docs_text, schema))],
+            None,
+        ),
+    ]
+    refs: dict[str, Callable[[], Any]] = {}
+    with contextlib.suppress(ImportError):
+        import sacrebleu
+
+        refs[cases[0][0]] = lambda: [
+            sacrebleu.corpus_bleu(preds, [refs_], force=True).score,
+            sacrebleu.corpus_chrf(preds, [refs_]).score,
+        ]
+    with contextlib.suppress(ImportError):
+        from rouge_score import rouge_scorer
+
+        def rouge_ref() -> list[float]:
+            sc = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"])
+            out = [sc.score(r, p) for r, p in zip(refs_, preds)]
+            return [float(np.mean([o[k].fmeasure for o in out])) for k in ("rouge1", "rouge2", "rougeL")]
+
+        refs[cases[1][0]] = rouge_ref
+    with contextlib.suppress(ImportError):
+        from nltk.translate.meteor_score import meteor_score
+
+        class _NoWordNet:
+            def synsets(self, _w: str) -> list[Any]:
+                return []
+
+        refs[cases[2][0]] = lambda: [
+            float(
+                np.mean([meteor_score([r.split()], p.split(), wordnet=_NoWordNet()) for r, p in zip(refs_, preds)])
+            )
+        ]
+    with contextlib.suppress(ImportError):
+        from ranx import Qrels, Run
+        from ranx import evaluate as ranx_evaluate
+
+        def ranx_ref() -> list[float]:
+            qrels = Qrels({f"q{i}": {f"d{d}": 1 for d in r} for i, r in enumerate(rel)})
+            run = Run({f"q{i}": {f"d{d}": float(len(r) - j) for j, d in enumerate(r)} for i, r in enumerate(ret)})
+            res = ranx_evaluate(qrels, run, ["mrr", "map@20", "ndcg@10"])
+            return [float(res["mrr"]), float(res["map@20"]), float(res["ndcg@10"])]
+
+        refs[cases[3][0]] = ranx_ref
+    with contextlib.suppress(ImportError):
+        import krippendorff
+
+        refs[cases[4][0]] = lambda: [
+            float(krippendorff.alpha(reliability_data=ratings, level_of_measurement="interval"))
+        ]
+    with contextlib.suppress(ImportError):
+        import jsonschema
+
+        def jsonschema_ref() -> list[float]:
+            v = jsonschema.Draft202012Validator(schema)
+            return [float(np.mean([v.is_valid(_json.loads(t)) for t in docs_text]))]
+
+        refs[cases[5][0]] = jsonschema_ref
+    return [(name, ref, es_fn, refs.get(name)) for name, ref, es_fn, _ in cases]
+
+
 def _clinical_cases(n: int, rng: np.random.Generator) -> list[Case]:
     """v0.2.0: diagnostic accuracy, calibration, decision curves and statistical tests."""
     import evalsuite as es
@@ -377,6 +526,7 @@ _GROUPS = (
     ("Classification and regression", ("binary", "10 classes", "regression")),
     ("Clinical, calibration and statistics", ("clinical", "calibration", "decision", "statistics", "multiple")),
     ("Segmentation and object detection", ("segmentation", "detection")),
+    ("LLM evaluation", ("text", "retrieval", "agreement", "structured")),
 )
 MATCH_TOLERANCE = 1e-9
 
@@ -419,7 +569,7 @@ class BenchmarkResult:
             speedups = [r["speedup"] for r in compared]
             return {
                 "group": name,
-                "cases": len({re.sub(r"\s*\(\d+ images?\)", "", r["case"]) for r in rows}),
+                "cases": len({re.sub(r"\s*\((?=[^)]*\d)[^)]*\)$", "", r["case"]) for r in rows}),
                 "rows": len(rows),
                 "compared": len(compared),
                 "matching": sum(
@@ -597,8 +747,8 @@ def run_benchmarks(
 
     if repeat < 1:
         raise ValueError("repeat must be at least 1.")
-    if suite not in ("all", "core", "clinical", "vision"):
-        raise ValueError("suite must be 'all', 'core', 'clinical' or 'vision'.")
+    if suite not in ("all", "core", "clinical", "vision", "llm"):
+        raise ValueError("suite must be 'all', 'core', 'clinical', 'vision' or 'llm'.")
     rng = np.random.default_rng(random_state)
     rows: list[dict[str, Any]] = []
     versions: dict[str, Optional[str]] = {"sklearn": None, "statsmodels": None, "pycocotools": None}
@@ -613,7 +763,8 @@ def run_benchmarks(
         "core": [_core_cases],
         "clinical": [_clinical_cases],
         "vision": [_vision_cases],
-        "all": [_core_cases, _clinical_cases, _vision_cases],
+        "llm": [_llm_cases],
+        "all": [_core_cases, _clinical_cases, _vision_cases, _llm_cases],
     }[suite]
     for n in sizes:
         for build in builders:

evalsuite/cli/main.py

@@ -297,6 +297,24 @@ def cmd_detection(args: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_text(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    df = read_table(args.file)
+    preds = [str(v) for v in column(df, args.prediction, "--prediction")]
+    ref_cols = [column(df, c, "--reference") for c in args.reference]
+    refs = [[str(col[i]) for col in ref_cols if not _missing(col[i])] for i in range(len(preds))]
+    if any(not r for r in refs):
+        raise CLIError("Every row needs at least one non-empty reference.")
+    report = es.text_report(refs, preds, metrics=metric_list(args.metrics))
+    emit(report, args)
+    return 0
+
+
+def _missing(v: Any) -> bool:
+    return v is None or (isinstance(v, float) and v != v) or (isinstance(v, str) and not v.strip())
+
+
 def cmd_compare(args: argparse.Namespace) -> int:
     import evalsuite as es
 
@@ -506,6 +524,18 @@ def build_parser() -> argparse.ArgumentParser:
     add_output(p, digits=3)
     p.set_defaults(func=cmd_detection)
 
+    p = sub.add_parser(
+        "text", help="text generation: BLEU, chrF, chrF++, TER, ROUGE, METEOR, exact match, token F1"
+    )
+    p.add_argument("file", help="CSV, TSV, Parquet or JSON-lines file with one example per row ('-' for stdin)")
+    p.add_argument("--prediction", required=True, help="column with the model output")
+    p.add_argument(
+        "--reference", required=True, action="append", help="column with a reference (repeat for several)"
+    )
+    p.add_argument("--metrics", help="comma-separated subset, e.g. bleu,chrf,rouge_l")
+    add_output(p)
+    p.set_defaults(func=cmd_text)
+
     p = sub.add_parser(
         "plot", help="save a ROC, PR, calibration, decision-curve, confusion-matrix or residual plot"
     )
@@ -529,7 +559,7 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument("--repeat", type=int, default=5, help="timed repetitions per case; the fastest is reported")
     p.add_argument(
         "--suite",
-        choices=("all", "core", "clinical", "vision"),
+        choices=("all", "core", "clinical", "vision", "llm"),
         default="all",
         help="core: classification/regression; clinical: clinical, calibration, tests; vision: segmentation, "
         "detection (default all)",

evalsuite/plot.py

@@ -503,3 +503,149 @@ def detection_pr(
     )
     ax.legend(loc="lower left", frameon=False, fontsize="small")
     return ax
+
+
+# ---------------------------------------------------------------- v0.4.0: preferences and text
+def _rating_fit(method: str, rows: list[Any]) -> Any:
+    from .text.judge import bradley_terry, elo_ratings
+
+    if method == "bradley_terry":
+        return bradley_terry(rows, scale="elo")
+    if method == "elo":
+        return elo_ratings(rows)
+    raise InputValidationError("method must be 'bradley_terry' or 'elo'.")
+
+
+def ratings(
+    comparisons: Any,
+    *,
+    method: str = "bradley_terry",
+    n_resamples: int = 200,
+    level: float = 0.95,
+    random_state: Optional[int] = 0,
+    ax: Optional[Axes] = None,
+) -> Axes:
+    """Leaderboard of model ratings from pairwise ``(model_a, model_b, outcome)`` comparisons, with percentile
+    bootstrap intervals over comparisons (Chatbot Arena style). ``method``: ``"bradley_terry"`` (Elo scale) or
+    ``"elo"`` (online Elo). Resamples on which the Bradley–Terry fit does not exist are skipped and counted in
+    the x-axis label."""
+    rows = list(comparisons)
+    point = _rating_fit(method, rows)
+    names = list(point.labels or ())
+    est = np.asarray(point.value, dtype=float)
+    rng = np.random.default_rng(random_state)
+    boot, failed = [], 0
+    for _ in range(int(n_resamples)):
+        sample = [rows[i] for i in rng.integers(0, len(rows), len(rows))]
+        try:
+            r = _rating_fit(method, sample)
+        except InputValidationError:
+            failed += 1
+            continue
+        lookup = dict(zip(r.labels or (), np.asarray(r.value, dtype=float)))
+        boot.append([lookup.get(n, np.nan) for n in names])
+    order = np.argsort(est)
+    ax = _axes(ax, (5.5, max(2.6, 0.45 * len(names) + 1.2)))
+    y = np.arange(len(names))
+    if boot:
+        b = np.asarray(boot)
+        a = (1 - level) / 2
+        lo, hi = np.nanquantile(b, a, axis=0), np.nanquantile(b, 1 - a, axis=0)
+        ax.errorbar(
+            est[order],
+            y,
+            xerr=[est[order] - lo[order], hi[order] - est[order]],
+            fmt="o",
+            color="#2a6fb0",
+            ecolor="0.45",
+            capsize=3,
+        )
+    else:
+        ax.plot(est[order], y, "o", color="#2a6fb0")
+    ax.set_yticks(y, [str(names[i]) for i in order])
+    title = "Bradley–Terry rating" if method == "bradley_terry" else "Elo rating"
+    note = f"{int(level * 100)}% bootstrap CI, {len(boot)} resamples" + (f", {failed} skipped" if failed else "")
+    ax.set(xlabel=f"{title} ({note})", title=f"{title}s from {len(rows)} comparisons")
+    ax.grid(axis="x", alpha=0.3)
+    return ax
+
+
+def win_matrix(comparisons: Any, *, ax: Optional[Axes] = None, annotate: bool = True) -> Axes:
+    """Heatmap of pairwise win rates: cell (row, column) is how often the row model beat the column model (ties
+    count half); blank where the two were never compared."""
+    from .text.judge import _comparisons
+
+    names, rows = _comparisons(comparisons)
+    m = len(names)
+    wins, games = np.zeros((m, m)), np.zeros((m, m))
+    for a, b, o in rows:
+        wins[a, b] += o
+        wins[b, a] += 1 - o
+        games[a, b] += 1
+        games[b, a] += 1
+    with np.errstate(invalid="ignore", divide="ignore"):
+        rate = np.where(games > 0, wins / np.where(games > 0, games, 1), np.nan)
+    order = np.argsort(-np.nanmean(np.where(np.eye(m, dtype=bool), np.nan, rate), axis=1))  # strongest first
+    rate = rate[np.ix_(order, order)]
+    names = [names[i] for i in order]
+    ax = _axes(ax, (1.2 + 0.7 * m, 0.9 + 0.6 * m))
+    im = ax.imshow(rate, cmap="RdBu", vmin=0, vmax=1)
+    ax.set_xticks(range(m), [str(n) for n in names], rotation=45, ha="right")
+    ax.set_yticks(range(m), [str(n) for n in names])
+    if annotate:
+        for i in range(m):
+            for j in range(m):
+                if not np.isnan(rate[i, j]):
+                    ax.text(
+                        j,
+                        i,
+                        f"{rate[i, j]:.2f}",
+                        ha="center",
+                        va="center",
+                        fontsize=8,
+                        color="white" if abs(rate[i, j] - 0.5) > 0.3 else "black",
+                    )
+    ax.figure.colorbar(im, ax=ax, label="win rate (row vs column)")
+    ax.set_title("Pairwise win rates")
+    return ax
+
+
+def text_scores(
+    references: Any,
+    predictions: Mapping[str, Any],
+    *,
+    metric: Any = "token_f1",
+    ax: Optional[Axes] = None,
+    **metric_kwargs: Any,
+) -> Axes:
+    """Distribution of a per-example text metric for several systems (box plots with the mean marked), e.g.
+    ``metric="rouge_l"``, ``"token_f1"``, ``"meteor"``, ``"sentence_bleu"`` or ``"bertscore"`` (with
+    embeddings). Pair it with ``es.compare`` for intervals and tests."""
+    from .stats._resolve import resolve_metric
+
+    fn, name = resolve_metric(metric)
+    import inspect
+
+    if "average" not in inspect.signature(fn).parameters:
+        raise InputValidationError(
+            f"{name} is a corpus-level metric without per-example scores; use e.g. sentence_bleu, rouge_l, "
+            "token_f1 or meteor, or es.compare for corpus metrics."
+        )
+    if not isinstance(predictions, Mapping) or not predictions:
+        raise InputValidationError("predictions must be a dict of system name -> predictions.")
+    data, labels = [], []
+    for sys_name, preds in predictions.items():
+        res = fn(references, preds, average=None, **metric_kwargs)
+        vals = np.asarray(res.value, dtype=float)
+        if vals.ndim != 1:
+            raise InputValidationError(f"{name} does not give one score per example.")
+        data.append(vals[~np.isnan(vals)])
+        labels.append(str(sys_name))
+    ax = _axes(ax, (1.6 + 1.1 * len(labels), 4.2))
+    ax.boxplot(
+        data, showmeans=True, meanprops={"marker": "D", "markerfacecolor": "#d9822b", "markeredgecolor": "#d9822b"}
+    )
+    ax.set_xticks(range(1, len(labels) + 1), labels)
+    ax.set(ylabel=f"{res.name} per example", title=f"{res.name} by system (diamond = mean)")
+    ax.grid(axis="y", alpha=0.3)
+    return ax

evalsuite/stats/_resolve.py

@@ -54,6 +54,19 @@ def as_observations(x: Any, name: str) -> NDArray[Any]:
     return to_numpy(x, name, allow_2d=True)
 
 
+def as_items(x: Any, name: str) -> NDArray[Any]:
+    """A 1-D object array holding one item per observation, whatever each item is."""
+    if x is None or isinstance(x, (str, bytes)):
+        raise InputValidationError(f"{name} must be a list with one item per example.")
+    seq = list(x.tolist() if isinstance(x, np.ndarray) and x.dtype != object and x.ndim == 1 else x)
+    if not seq:
+        raise InputValidationError(f"{name} is empty. Provide at least one observation.")
+    out = np.empty(len(seq), dtype=object)
+    for i, v in enumerate(seq):
+        out[i] = v
+    return out
+
+
 def is_categorical(y: NDArray[Any]) -> bool:
     if y.ndim > 2 or (y.dtype == object and y.size and not isinstance(y.flat[0], (str, bytes))):
         return False
@@ -80,9 +93,16 @@ class MetricCall:
                 "Pass exactly one of y_pred (label/value metrics) or y_prob (probability metrics)."
             )
         self.fn = fn
-        self.y_true = as_observations(y_true, "y_true")
         second = y_pred if y_pred is not None else y_prob
-        self.second = as_observations(second, "y_pred" if y_pred is not None else "y_prob")
+        self.items = bool(getattr(fn, "__evalsuite_items__", False))
+        if self.items:
+            # Text, retrieval and other item-level metrics: each observation is a whole item (a string, a list
+            # of references, a ranked list, an embedding matrix); resample items, never stratify by value.
+            self.y_true = as_items(y_true, "y_true")
+            self.second = as_items(second, "y_pred" if y_pred is not None else "y_prob")
+        else:
+            self.y_true = as_observations(y_true, "y_true")
+            self.second = as_observations(second, "y_pred" if y_pred is not None else "y_prob")
         if self.second.shape[0] != self.y_true.shape[0]:
             raise InputValidationError(
                 f"y_true and {'y_pred' if y_pred is not None else 'y_prob'} must contain the same number of "
@@ -97,7 +117,7 @@ class MetricCall:
         self.weight = None if sample_weight is None else to_numpy(sample_weight, "sample_weight")
         self.kwargs = dict(kwargs or {})
         params = inspect.signature(fn).parameters
-        self.categorical = is_categorical(self.y_true)
+        self.categorical = not self.items and is_categorical(self.y_true)
         # Keep every class in every resample so per-class and averaged metrics stay comparable.
         if self.categorical and "labels" in params and "labels" not in self.kwargs and self.y_true.ndim == 1:
             present = self.y_true if y_pred is None else np.concatenate([self.y_true, self.second])

evalsuite/stats/compare.py

@@ -15,7 +15,7 @@ from ..core.exceptions import InputValidationError
 from ..core.registry import metric_info
 from ..core.result import _json_safe, _latex_escape, _latex_table
 from ..core.types import ArrayLike
-from ._resolve import MetricCall, as_observations, is_categorical, resolve_metric
+from ._resolve import MetricCall, as_items, as_observations, is_categorical, resolve_metric
 from .effect import adjust_pvalues
 from .intervals import _check_level, _check_resamples, bootstrap_distribution, resample_indices
 from .paired import delong_test, mcnemar_test
@@ -295,8 +295,12 @@ def compare(
         raise InputValidationError("compare() needs at least two models (a dict of name -> predictions).")
     if baseline is not None and baseline not in names:
         raise InputValidationError(f"baseline={baseline!r} is not one of the models: {', '.join(names)}.")
-    yt = as_observations(y_true, "y_true")
-    categorical = is_categorical(yt)
+    item_level = metrics is not None and any(
+        getattr(resolve_metric(m)[0], "__evalsuite_items__", False) for m in metrics
+    )
+    # Text, retrieval and other item-level metrics resample whole items and never stratify by value.
+    yt = as_items(y_true, "y_true") if item_level else as_observations(y_true, "y_true")
+    categorical = not item_level and is_categorical(yt)
     if metrics is None:
         if yt.ndim > 2 or (yt.dtype == object and yt.size and not isinstance(yt[0], (str, bytes))):
             is_detection = yt.dtype == object and isinstance(yt[0], Mapping)

evalsuite/text/__init__.py

@@ -0,0 +1,152 @@
+"""Language-model evaluation (v0.4.0): text generation, semantic similarity, factuality and hallucination,
+LLM-as-a-judge and preferences, reasoning benchmarks, retrieval-augmented generation, and instruction following
+and structured output."""
+
+from .factuality import (
+    abstention_accuracy,
+    answer_correctness,
+    answer_relevance,
+    citation_precision,
+    citation_recall,
+    claim_verification_accuracy,
+    faithfulness,
+    groundedness,
+    hallucination_rate,
+    knowledge_consistency,
+)
+from .generation import (
+    bleu,
+    chrf,
+    cider,
+    cross_entropy,
+    distinct_n,
+    meteor,
+    perplexity,
+    rouge_1,
+    rouge_2,
+    rouge_l,
+    rouge_lsum,
+    self_bleu,
+    sentence_bleu,
+    ter,
+)
+from .judge import (
+    bradley_terry,
+    elo_ratings,
+    fleiss_kappa,
+    judge_agreement,
+    krippendorff_alpha,
+    position_consistency,
+    rubric_score,
+    self_preference_bias,
+    verbosity_bias,
+    win_rate,
+)
+from .qa import exact_match, normalize_answer, token_f1
+from .rag import (
+    context_precision,
+    context_recall,
+    context_relevance,
+    failure_attribution,
+    latency_summary,
+    task_success_rate,
+)
+from .reasoning import benchmark_accuracy, extract_answer, majority_vote_accuracy, pass_at_k
+from .report import TextReport, text_report
+from .retrieval import hit_rate_at_k, mean_average_precision_at_k, mrr, ndcg_at_k, precision_at_k, recall_at_k
+from .semantic import bertscore, embedding_similarity, mauve, model_score, moverscore
+from .structured import (
+    api_call_success_rate,
+    constraint_satisfaction_rate,
+    extra_content_rate,
+    extract_json,
+    format_compliance,
+    instruction_compliance_rate,
+    instruction_retention,
+    json_schema_compliance,
+    json_validity,
+    required_field_accuracy,
+    tool_argument_accuracy,
+    tool_call_f1,
+    tool_selection_accuracy,
+    validate_json_schema,
+    xml_validity,
+)
+
+__all__ = [
+    "TextReport",
+    "text_report",
+    "abstention_accuracy",
+    "answer_correctness",
+    "answer_relevance",
+    "api_call_success_rate",
+    "benchmark_accuracy",
+    "bertscore",
+    "bleu",
+    "bradley_terry",
+    "chrf",
+    "cider",
+    "citation_precision",
+    "citation_recall",
+    "claim_verification_accuracy",
+    "constraint_satisfaction_rate",
+    "context_precision",
+    "context_recall",
+    "context_relevance",
+    "cross_entropy",
+    "distinct_n",
+    "elo_ratings",
+    "embedding_similarity",
+    "exact_match",
+    "extra_content_rate",
+    "extract_answer",
+    "extract_json",
+    "failure_attribution",
+    "faithfulness",
+    "fleiss_kappa",
+    "format_compliance",
+    "groundedness",
+    "hallucination_rate",
+    "hit_rate_at_k",
+    "instruction_compliance_rate",
+    "instruction_retention",
+    "json_schema_compliance",
+    "json_validity",
+    "judge_agreement",
+    "knowledge_consistency",
+    "krippendorff_alpha",
+    "latency_summary",
+    "majority_vote_accuracy",
+    "mauve",
+    "mean_average_precision_at_k",
+    "meteor",
+    "model_score",
+    "moverscore",
+    "mrr",
+    "ndcg_at_k",
+    "normalize_answer",
+    "pass_at_k",
+    "perplexity",
+    "position_consistency",
+    "precision_at_k",
+    "recall_at_k",
+    "required_field_accuracy",
+    "rouge_1",
+    "rouge_2",
+    "rouge_l",
+    "rouge_lsum",
+    "rubric_score",
+    "self_bleu",
+    "self_preference_bias",
+    "sentence_bleu",
+    "task_success_rate",
+    "ter",
+    "token_f1",
+    "tool_argument_accuracy",
+    "tool_call_f1",
+    "tool_selection_accuracy",
+    "validate_json_schema",
+    "verbosity_bias",
+    "win_rate",
+    "xml_validity",
+]