evalsuite-python 0.1.0__py3-none-any.whl

evalsuite/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Command-line interface: ``evalsuite --help``."""
+
+from .main import main
+
+__all__ = ["main"]

evalsuite/cli/main.py

@@ -0,0 +1,389 @@
+"""``evalsuite``: evaluate, compare, report, plot and benchmark from the command line.
+
+Examples::
+
+    evalsuite evaluate predictions.csv --y-true label --y-pred pred --y-prob prob
+    evalsuite report predictions.csv --y-true label --y-pred pred --format markdown
+    evalsuite compare predictions.csv --y-true label --pred lr=pred_lr --pred rf=pred_rf \\
+        --prob lr=p_lr --prob rf=p_rf
+    evalsuite plot roc predictions.csv --y-true label --y-prob prob --output roc.png
+    evalsuite metrics --category regression
+    evalsuite info classification.mcc
+    evalsuite benchmark --quick
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from collections.abc import Sequence
+from typing import Any, Optional
+
+from ..core.exceptions import EvalSuiteError
+from ..version import __version__
+
+FORMATS = ("text", "json", "csv", "markdown", "latex", "html")
+_EXT = {
+    ".json": "json",
+    ".csv": "csv",
+    ".md": "markdown",
+    ".tex": "latex",
+    ".html": "html",
+    ".htm": "html",
+    ".txt": "text",
+}
+
+
+class CLIError(Exception):
+    """A user-facing error: printed without a traceback, exit code 2."""
+
+
+# ---- input ---------------------------------------------------------------------------------------
+def read_table(path: str) -> Any:
+    import pandas as pd
+
+    if path == "-":
+        return pd.read_csv(sys.stdin)
+    if not os.path.exists(path):
+        raise CLIError(f"File not found: {path}")
+    ext = os.path.splitext(path)[1].lower()
+    try:
+        if ext in (".tsv", ".tab"):
+            return pd.read_csv(path, sep="\t")
+        if ext == ".parquet":
+            return pd.read_parquet(path)
+        if ext in (".json", ".jsonl"):
+            return pd.read_json(path, lines=ext == ".jsonl")
+        return pd.read_csv(path)
+    except ImportError as exc:
+        raise CLIError(f"Reading {ext} files needs an extra package: {exc}") from exc
+    except Exception as exc:
+        raise CLIError(f"Could not read {path}: {exc}") from exc
+
+
+def column(df: Any, name: str, role: str) -> Any:
+    if name not in df.columns:
+        raise CLIError(
+            f"Column '{name}' ({role}) not found. Columns in the file: {', '.join(map(str, df.columns))}."
+        )
+    return df[name].to_numpy()
+
+
+def columns(df: Any, names: Sequence[str], role: str) -> Any:
+    import numpy as np
+
+    cols = [column(df, n, role) for n in names]
+    return cols[0] if len(cols) == 1 else np.column_stack(cols)
+
+
+def pairs(items: Optional[Sequence[str]], flag: str) -> dict[str, list[str]]:
+    """``name=col`` or ``name=col1,col2,col3`` (multiclass probabilities) -> {name: [cols]}."""
+    out: dict[str, list[str]] = {}
+    for item in items or []:
+        name, sep, cols = item.partition("=")
+        if not sep or not name or not cols:
+            raise CLIError(f"{flag} expects NAME=COLUMN (or NAME=COL1,COL2,... for multiclass); got '{item}'.")
+        out[name] = [c for c in cols.split(",") if c]
+    return out
+
+
+def metric_list(text: Optional[str]) -> Optional[list[str]]:
+    return [m.strip() for m in text.split(",") if m.strip()] if text else None
+
+
+def label_value(text: Optional[str]) -> Any:
+    if text is None:
+        return None
+    try:
+        return int(text)
+    except ValueError:
+        return text
+
+
+# ---- output --------------------------------------------------------------------------------------
+def render(obj: Any, fmt: str, digits: int) -> str:
+    if fmt == "text":
+        return obj.summary(digits=digits) if hasattr(obj, "summary") else str(obj)
+    if fmt == "json":
+        return str(obj.to_json())
+    if fmt == "csv":
+        return str(obj.to_csv())
+    if fmt == "markdown":
+        return str(obj.to_markdown(digits=digits))
+    if fmt == "latex":
+        return str(obj.to_latex(digits=digits))
+    return str(obj.to_html(digits=digits, full=True))
+
+
+def emit(obj: Any, args: argparse.Namespace) -> None:
+    fmt = args.format
+    if fmt is None:
+        fmt = _EXT.get(os.path.splitext(args.output)[1].lower(), "text") if args.output else "text"
+    text = render(obj, fmt, args.digits)
+    if not text.endswith("\n"):
+        text += "\n"
+    if args.output:
+        with open(args.output, "w", encoding="utf-8", newline="") as fh:
+            fh.write(text)
+        print(f"Wrote {fmt} to {args.output}", file=sys.stderr)
+    else:
+        sys.stdout.write(text)
+
+
+def add_output(p: argparse.ArgumentParser, digits: int = 4) -> None:
+    p.add_argument(
+        "--format",
+        "-f",
+        choices=FORMATS,
+        default=None,
+        help="output format (default: from --output extension, else text)",
+    )
+    p.add_argument("--output", "-o", help="write to this file instead of standard output")
+    p.add_argument("--digits", type=int, default=digits, help=f"decimal places (default {digits})")
+
+
+def add_targets(p: argparse.ArgumentParser, pred_required: bool = False) -> None:
+    p.add_argument(
+        "file", help="CSV/TSV/Parquet/JSON file with one row per observation ('-' reads CSV from stdin)"
+    )
+    p.add_argument("--y-true", required=True, metavar="COL", help="column with the true labels or values")
+    p.add_argument(
+        "--y-pred", required=pred_required, metavar="COL", help="column with predicted labels or values"
+    )
+    p.add_argument(
+        "--y-prob",
+        nargs="+",
+        metavar="COL",
+        help="probability column(s): one for P(positive), or one per class in label order",
+    )
+    p.add_argument("--weight", metavar="COL", help="column with sample weights")
+
+
+# ---- commands ------------------------------------------------------------------------------------
+def cmd_evaluate(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    df = read_table(args.file)
+    if not args.y_pred and not args.y_prob:
+        raise CLIError("Give --y-pred, --y-prob, or both.")
+    result = es.evaluate(
+        column(df, args.y_true, "--y-true"),
+        column(df, args.y_pred, "--y-pred") if args.y_pred else None,
+        y_prob=columns(df, args.y_prob, "--y-prob") if args.y_prob else None,
+        task=args.task,
+        metrics=metric_list(args.metrics),
+        average=args.average,
+        pos_label=label_value(args.pos_label),
+        sample_weight=column(df, args.weight, "--weight") if args.weight else None,
+    )
+    emit(result, args)
+    return 0
+
+
+def cmd_report(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    df = read_table(args.file)
+    report = es.classification_report(
+        column(df, args.y_true, "--y-true"),
+        column(df, args.y_pred, "--y-pred"),
+        sample_weight=column(df, args.weight, "--weight") if args.weight else None,
+    )
+    emit(report, args)
+    return 0
+
+
+def cmd_compare(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    df = read_table(args.file)
+    preds = {k: columns(df, v, "--pred") for k, v in pairs(args.pred, "--pred").items()}
+    probs = {k: columns(df, v, "--prob") for k, v in pairs(args.prob, "--prob").items()}
+    if len(set(preds) | set(probs)) < 2:
+        raise CLIError(
+            "compare needs at least two models: --pred NAME=COL (and/or --prob NAME=COL), twice or more."
+        )
+    result = es.compare(
+        column(df, args.y_true, "--y-true"),
+        preds or None,
+        probabilities=probs or None,
+        metrics=metric_list(args.metrics),
+        baseline=args.baseline,
+        level=args.level,
+        alpha=args.alpha,
+        n_resamples=args.resamples,
+        correction=args.correction,
+        random_state=args.seed,
+    )
+    emit(result, args)
+    if args.plot:
+        _save_figure(lambda ax: es.plot.comparison(result, ax=ax), args.plot)
+    return 0
+
+
+def _save_figure(draw: Any, path: str) -> None:
+    from ..plot import _plt
+
+    plt = _plt()
+    fig, ax = plt.subplots(figsize=(5.5, 4.4), layout="constrained")
+    draw(ax)
+    fig.savefig(path, dpi=200)
+    plt.close(fig)
+    print(f"Wrote figure to {path}", file=sys.stderr)
+
+
+def cmd_plot(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    df = read_table(args.file)
+    y = column(df, args.y_true, "--y-true")
+    kind = args.kind
+    if kind in ("roc", "pr", "calibration"):
+        if not args.y_prob:
+            raise CLIError(f"'{kind}' needs --y-prob.")
+        prob = columns(df, args.y_prob, "--y-prob")
+        fn: Any = {"roc": es.plot.roc, "pr": es.plot.pr, "calibration": es.plot.calibration}[kind]
+        _save_figure(lambda ax: fn(y, prob, ax=ax), args.output)
+    else:
+        if not args.y_pred:
+            raise CLIError(f"'{kind}' needs --y-pred.")
+        pred = column(df, args.y_pred, "--y-pred")
+        if kind == "confusion":
+            _save_figure(
+                lambda ax: es.plot.confusion_matrix(y, pred, ax=ax, normalize=args.normalize), args.output
+            )
+        else:
+            _save_figure(
+                lambda ax: es.plot.residuals(
+                    y, pred, ax=ax, kind="predicted" if kind == "predicted" else "residuals"
+                ),
+                args.output,
+            )
+    return 0
+
+
+def cmd_metrics(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    for mid in es.list_metrics(args.category):
+        info = es.metric_info(mid)
+        print(f"{mid:<45} {info.name}")
+    return 0
+
+
+def cmd_info(args: argparse.Namespace) -> int:
+    import evalsuite as es
+
+    try:
+        info = es.metric_info(args.metric)
+    except KeyError as exc:
+        raise CLIError(str(exc).strip("'\"")) from exc
+    better = {True: "higher is better", False: "lower is better", None: "closer to 0 is better"}
+    print(f"{info.name}  ({info.id})\n")
+    print(f"Definition:  {info.definition}")
+    print(f"Formula:     {info.formula}")
+    print(f"Range:       {info.range} ({better[info.higher_is_better]})")
+    print(f"Task:        {info.task}")
+    print(f"Needs:       {', '.join(info.input_requirements)}")
+    print(f"Python:      es.{info.id.split('.')[-1]}(...)")
+    print("References:")
+    for ref in info.references:
+        print(f"  - {ref}")
+    return 0
+
+
+def cmd_benchmark(args: argparse.Namespace) -> int:
+    from ..benchmarks import run_benchmarks
+
+    sizes = (1_000, 10_000) if args.quick else tuple(args.sizes)
+    result = run_benchmarks(
+        sizes=sizes, repeat=args.repeat, compare_sklearn=not args.no_sklearn, random_state=args.seed
+    )
+    emit(result, args)
+    return 0
+
+
+# ---- parser --------------------------------------------------------------------------------------
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="evalsuite",
+        description="EvalSuite: unified, reproducible evaluation for machine learning and research.",
+        epilog="Run 'evalsuite COMMAND --help' for the options of each command.",
+    )
+    parser.add_argument("--version", action="version", version=f"evalsuite-python {__version__}")
+    sub = parser.add_subparsers(dest="command", metavar="COMMAND")
+
+    p = sub.add_parser("evaluate", help="evaluate predictions in a file with a standard or chosen set of metrics")
+    add_targets(p)
+    p.add_argument("--task", choices=("classification", "regression"), help="default: inferred from the data")
+    p.add_argument("--metrics", help="comma-separated metric names, e.g. accuracy,f1,mcc")
+    p.add_argument("--average", default="auto", help="binary, micro, macro, weighted, samples (default: auto)")
+    p.add_argument("--pos-label", help="positive class for binary metrics (default: 1)")
+    add_output(p)
+    p.set_defaults(func=cmd_evaluate)
+
+    p = sub.add_parser(
+        "report", help="per-class classification report (precision, recall, F1, specificity, support)"
+    )
+    add_targets(p, pred_required=True)
+    add_output(p)
+    p.set_defaults(func=cmd_report)
+
+    p = sub.add_parser("compare", help="compare models on the same test set with CIs and paired tests")
+    p.add_argument("file", help="CSV/TSV/Parquet/JSON file with one row per observation")
+    p.add_argument("--y-true", required=True, metavar="COL")
+    p.add_argument("--pred", action="append", metavar="NAME=COL", help="a model's predictions (repeat per model)")
+    p.add_argument(
+        "--prob", action="append", metavar="NAME=COL[,COL...]", help="a model's probabilities (repeat per model)"
+    )
+    p.add_argument("--metrics", help="comma-separated metric names")
+    p.add_argument("--baseline", help="compare every model with this one instead of all pairs")
+    p.add_argument("--level", type=float, default=0.95, help="confidence level (default 0.95)")
+    p.add_argument("--alpha", type=float, default=0.05, help="significance level (default 0.05)")
+    p.add_argument("--resamples", type=int, default=1000, help="bootstrap resamples (default 1000)")
+    p.add_argument("--correction", default="holm", choices=("holm", "bonferroni", "bh", "by"))
+    p.add_argument("--seed", type=int, default=0, help="random seed (default 0, for reproducibility)")
+    p.add_argument("--plot", metavar="PATH", help="also save a forest plot (needs matplotlib)")
+    add_output(p, digits=3)
+    p.set_defaults(func=cmd_compare)
+
+    p = sub.add_parser("plot", help="save a ROC, PR, calibration, confusion-matrix or residual plot")
+    p.add_argument("kind", choices=("roc", "pr", "calibration", "confusion", "residuals", "predicted"))
+    add_targets(p)
+    p.add_argument("--normalize", choices=("true", "pred", "all"), help="confusion matrix normalisation")
+    p.add_argument("--output", "-o", required=True, help="image file (.png, .pdf, .svg)")
+    p.set_defaults(func=cmd_plot)
+
+    p = sub.add_parser("metrics", help="list available metrics")
+    p.add_argument("--category", choices=("classification", "regression"))
+    p.set_defaults(func=cmd_metrics)
+
+    p = sub.add_parser("info", help="show a metric's definition, formula, range and references")
+    p.add_argument("metric", help="metric id, e.g. classification.mcc")
+    p.set_defaults(func=cmd_info)
+
+    p = sub.add_parser("benchmark", help="time and memory benchmarks (against scikit-learn if installed)")
+    p.add_argument("--sizes", type=int, nargs="+", default=[1_000, 100_000, 1_000_000])
+    p.add_argument("--quick", action="store_true", help="small sizes only (1k and 10k)")
+    p.add_argument("--repeat", type=int, default=5, help="timed repetitions per case; the fastest is reported")
+    p.add_argument("--no-sklearn", action="store_true", help="skip the scikit-learn comparison")
+    p.add_argument("--seed", type=int, default=0)
+    add_output(p, digits=3)
+    p.set_defaults(func=cmd_benchmark)
+    return parser
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "command", None):
+        parser.print_help()
+        return 0
+    try:
+        return int(args.func(args))
+    except (CLIError, EvalSuiteError, ValueError) as exc:
+        print(f"evalsuite {args.command}: error: {exc}", file=sys.stderr)
+        return 2
+    except BrokenPipeError:  # e.g. piped into `head`
+        return 0

evalsuite/core/__init__.py

@@ -0,0 +1 @@
+"""Core infrastructure: exceptions, validation, results, registry and evaluation context."""

evalsuite/core/context.py

@@ -0,0 +1,181 @@
+"""EvaluationContext: validate inputs once and cache shared intermediate results.
+
+Every classification metric derives from per-class counts (TP, FP, FN, TN). The context computes the
+confusion matrix once; ``evaluate()`` passes one context to all requested metrics, so ten metrics cost one
+pass over the data.
+"""
+
+from __future__ import annotations
+
+from functools import cached_property
+from typing import Any, Optional
+
+import numpy as np
+from numpy.typing import NDArray
+
+from .exceptions import InputValidationError, UnsupportedTaskError
+from .types import ArrayLike, FloatArray
+from .validation import (
+    TargetType,
+    check_consistent_length,
+    check_finite,
+    resolve_labels,
+    target_type,
+    to_numpy,
+    validate_probabilities,
+    validate_sample_weight,
+)
+
+__all__ = ["ClassificationContext"]
+
+
+class ClassificationContext:
+    """Validated classification inputs plus cached counts.
+
+    For binary and multiclass targets, ``labels`` defines the class order used by every per-class output and
+    by the columns of 2-D ``y_prob``. For multilabel targets, labels are column indices.
+    """
+
+    def __init__(
+        self,
+        y_true: ArrayLike,
+        y_pred: Optional[ArrayLike] = None,
+        *,
+        y_prob: Optional[ArrayLike] = None,
+        labels: Optional[ArrayLike] = None,
+        sample_weight: Optional[ArrayLike] = None,
+    ) -> None:
+        yt = to_numpy(y_true, "y_true", allow_2d=True)
+        yp = None if y_pred is None else to_numpy(y_pred, "y_pred", allow_2d=True)
+        check_finite(yt, "y_true")
+        if yp is not None:
+            check_finite(yp, "y_pred")
+        self.n = check_consistent_length(y_true=yt, y_pred=yp)
+        kind = target_type(yt)
+        if kind == "continuous":
+            raise UnsupportedTaskError(
+                "y_true contains non-integer numbers, which looks like a regression target. "
+                'Use evalsuite.regression metrics or evaluate(..., task="regression").'
+            )
+        if yp is not None and yt.ndim != yp.ndim:
+            raise InputValidationError(
+                f"y_true has {yt.ndim} dimension(s) but y_pred has {yp.ndim}. Multilabel targets need both as "
+                "indicator matrices; single-label targets need both as 1-D label arrays."
+            )
+        if kind == "multilabel" and yp is not None:
+            if yp.shape[1] != yt.shape[1]:
+                raise InputValidationError(f"y_true has {yt.shape[1]} label columns but y_pred has {yp.shape[1]}.")
+            if not np.isin(yp, (0, 1)).all():
+                raise InputValidationError("Multilabel y_pred must be a 0/1 indicator matrix.")
+        self.target_type: TargetType = kind
+        self.y_true = yt
+        self.y_pred = yp
+        self.sample_weight = validate_sample_weight(sample_weight, self.n)
+        if kind == "multilabel":
+            self.labels: NDArray[Any] = np.arange(yt.shape[1])
+        else:
+            self.labels = resolve_labels(yt, yp, labels)
+            if labels is not None:
+                observed = np.unique(yt if yp is None else np.concatenate([yt, yp]))
+                missing = np.setdiff1d(observed, self.labels)
+                if missing.size:
+                    raise InputValidationError(
+                        f"y_true/y_pred contain label(s) {missing.tolist()} that are not in labels. "
+                        "Include every label that occurs, or filter the data first."
+                    )
+            if kind == "binary" and self.labels.shape[0] > 2:
+                self.target_type = "multiclass"
+        self._y_prob_raw = y_prob
+
+    # ---- encodings -------------------------------------------------------------------------------
+    @cached_property
+    def _index(self) -> dict[Any, int]:
+        return {lab.item() if hasattr(lab, "item") else lab: i for i, lab in enumerate(self.labels)}
+
+    def encode(self, y: NDArray[Any]) -> NDArray[np.int64]:
+        sorted_labels = np.all(self.labels[:-1] <= self.labels[1:]) if self.labels.dtype != object else False
+        if sorted_labels:
+            return np.searchsorted(self.labels, y).astype(np.int64)
+        idx = self._index
+        return np.fromiter((idx[v.item() if hasattr(v, "item") else v] for v in y), dtype=np.int64, count=len(y))
+
+    @cached_property
+    def true_idx(self) -> NDArray[np.int64]:
+        return self.encode(self.y_true)
+
+    @cached_property
+    def pred_idx(self) -> NDArray[np.int64]:
+        if self.y_pred is None:
+            raise InputValidationError("This metric needs y_pred (predicted labels).")
+        return self.encode(self.y_pred)
+
+    @cached_property
+    def weights(self) -> FloatArray:
+        return np.ones(self.n) if self.sample_weight is None else self.sample_weight
+
+    # ---- cached counts ---------------------------------------------------------------------------
+    @cached_property
+    def confusion_matrix(self) -> FloatArray:
+        """Rows: true labels; columns: predicted labels (weighted counts). Single-label targets only."""
+        if self.target_type == "multilabel":
+            raise UnsupportedTaskError(
+                "A single confusion matrix is not defined for multilabel targets; use multilabel_counts."
+            )
+        k = self.labels.shape[0]
+        flat = self.true_idx * k + self.pred_idx
+        return np.bincount(flat, weights=self.weights, minlength=k * k).reshape(k, k).astype(np.float64)
+
+    @cached_property
+    def counts(self) -> dict[str, FloatArray]:
+        """Per-class (or per-label) weighted tp, fp, fn, tn and support."""
+        if self.target_type == "multilabel":
+            if self.y_pred is None:
+                raise InputValidationError("This metric needs y_pred (predicted labels).")
+            w = self.weights[:, None]
+            t = self.y_true.astype(bool)
+            p = self.y_pred.astype(bool)
+            tp = ((t & p) * w).sum(0)
+            fp = ((~t & p) * w).sum(0)
+            fn = ((t & ~p) * w).sum(0)
+            tn = ((~t & ~p) * w).sum(0)
+        else:
+            cm = self.confusion_matrix
+            tp = np.diag(cm).copy()
+            fp = cm.sum(0) - tp
+            fn = cm.sum(1) - tp
+            tn = cm.sum() - tp - fp - fn
+        return {"tp": tp, "fp": fp, "fn": fn, "tn": tn, "support": tp + fn}
+
+    @cached_property
+    def total_weight(self) -> float:
+        return float(self.weights.sum())
+
+    # ---- probabilities ---------------------------------------------------------------------------
+    @cached_property
+    def y_prob(self) -> FloatArray:
+        if self._y_prob_raw is None:
+            raise InputValidationError(
+                "This metric needs predicted probabilities (y_prob): P(positive class) for binary tasks, "
+                "or one column per class in label order for multiclass tasks."
+            )
+        k = self.labels.shape[0]
+        p = validate_probabilities(
+            self._y_prob_raw,
+            self.n,
+            n_classes=None if self.target_type == "binary" else k,
+            rows_sum_to_one=self.target_type == "multiclass",
+            unit="label" if self.target_type == "multilabel" else "class",
+        )
+        if self.target_type == "binary" and p.ndim == 2:
+            if p.shape[1] != 2:
+                raise InputValidationError(
+                    f"For binary tasks y_prob must be P(positive) or two columns; received {p.shape[1]} columns."
+                )
+            p = p[:, 1]
+        if self.target_type == "multiclass" and p.ndim == 1:
+            raise InputValidationError(
+                f"Multiclass tasks need y_prob with one column per class ({k} columns, in label order)."
+            )
+        if self.target_type == "multilabel" and (p.ndim != 2 or p.shape[1] != self.y_true.shape[1]):
+            raise InputValidationError("Multilabel y_prob must have one probability column per label.")
+        return p

evalsuite/core/exceptions.py

@@ -0,0 +1,52 @@
+"""Exception hierarchy.
+
+Every message says what failed, why, and how to fix it.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "EvalSuiteError",
+    "InputValidationError",
+    "MetricInputError",
+    "OptionalDependencyError",
+    "StatisticalTestError",
+    "UndefinedMetricWarning",
+    "UnsupportedTaskError",
+]
+
+
+class EvalSuiteError(Exception):
+    """Base class for all EvalSuite errors."""
+
+
+class InputValidationError(EvalSuiteError, ValueError):
+    """Inputs have the wrong shape, length, type or values."""
+
+
+class MetricInputError(EvalSuiteError, ValueError):
+    """Inputs are valid arrays but outside the domain a metric is defined on."""
+
+
+class UnsupportedTaskError(EvalSuiteError, ValueError):
+    """The requested task or averaging mode is not supported for this metric."""
+
+
+class OptionalDependencyError(EvalSuiteError, ImportError):
+    """A feature needs an optional dependency that is not installed."""
+
+    def __init__(self, package: str, extra: str, feature: str) -> None:
+        super().__init__(
+            f"{feature} requires the optional dependency '{package}'. "
+            f'Install it with: pip install "evalsuite-python[{extra}]"'
+        )
+        self.package = package
+        self.extra = extra
+
+
+class StatisticalTestError(EvalSuiteError, ValueError):
+    """A statistical procedure cannot be applied to the given data."""
+
+
+class UndefinedMetricWarning(UserWarning):
+    """A metric is undefined for the input (for example a zero denominator)."""