evaluma 0.1.0__py3-none-any.whl

evaluma/cli.py

@@ -0,0 +1,275 @@
+import os
+import sys
+import warnings
+
+import click
+import yaml
+
+import evaluma
+
+
+def _parse_metric_direction(ctx, param, value):
+    """Parse ``KEY:min`` / ``KEY:max`` tokens into a metric-direction dict.
+
+    Args:
+        ctx: Click context (unused; required by the callback protocol).
+        param: Click parameter (unused).
+        value: Tuple of strings, each formatted as ``"KEY:min"`` or
+            ``"KEY:max"``.
+
+    Returns:
+        dict | None: Mapping from dataset name to ``"min"`` or ``"max"``,
+        or ``None`` when ``value`` is empty.
+
+    Raises:
+        click.BadParameter: If a token is malformed or the direction is not
+            ``"min"`` or ``"max"``.
+    """
+    result = {}
+    for token in value:
+        if ":" not in token:  # pragma: no cover
+            raise click.BadParameter(f"Expected KEY:min or KEY:max, got '{token}'")
+        key, direction = token.split(":", 1)
+        if direction not in ("min", "max"):  # pragma: no cover
+            raise click.BadParameter(
+                f"Direction must be 'min' or 'max', got '{direction}'"
+            )
+        result[key] = direction
+    return result or None
+
+
+def _common_options(f):
+    """Attach shared CLI options to a Click command."""
+    f = click.argument("csv_path")(f)
+    f = click.option("--model", default="model")(f)
+    f = click.option("--dataset", default="dataset")(f)
+    f = click.option("--metric", default="metric")(f)
+    f = click.option("--score", default="score")(f)
+    f = click.option("--config", "config_path", default=None, help="YAML config file")(
+        f
+    )
+    f = click.option(
+        "--metric-direction",
+        multiple=True,
+        callback=_parse_metric_direction,
+        is_eager=False,
+        help="KEY:min or KEY:max (repeatable)",
+    )(f)
+    f = click.option("--output", "output_dir", default=".", show_default=True)(f)
+    return f
+
+
+def _load_bench(
+    csv_path,
+    model,
+    dataset,
+    metric,
+    score,
+    config_path,
+    metric_direction,
+    output_dir,
+    seed=None,
+):
+    """Load a CSV and return a normalized Benchmark, merging CLI args with config.
+
+    Args:
+        csv_path: Path to the input CSV file.
+        model: CLI value for the model column name.
+        dataset: CLI value for the dataset column name.
+        metric: CLI value for the metric column name.
+        score: CLI value for the score column name.
+        config_path: Optional path to a YAML config file.
+        metric_direction: Parsed metric-direction dict (or ``None``).
+        output_dir: Path to the output directory (created if absent).
+        seed: Optional column name for the random seed.
+
+    Returns:
+        Benchmark: Loaded and normalized benchmark.
+    """
+    cfg = {}
+    if config_path:
+        with open(config_path) as fh:
+            cfg = yaml.safe_load(fh) or {}
+
+    col_model = model if model != "model" else cfg.get("model", model)
+    col_dataset = dataset if dataset != "dataset" else cfg.get("dataset", dataset)
+    col_metric = metric if metric != "metric" else cfg.get("metric", metric)
+    col_score = score if score != "score" else cfg.get("score", score)
+    md = metric_direction or cfg.get("metric_direction") or None
+
+    raw_mtb = cfg.get("metric_type_bounds")
+    if raw_mtb is not None:
+        mtb = {k: tuple(v) for k, v in raw_mtb.items()}
+    else:
+        mtb = None
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+        if mtb is not None:
+            bench = evaluma.load_csv(
+                csv_path,
+                model=col_model,
+                dataset=col_dataset,
+                metric=col_metric,
+                score=col_score,
+                seed=seed,
+                metric_direction=md,
+                metric_type_bounds=mtb,
+            )
+        else:
+            bench = evaluma.load_csv(
+                csv_path,
+                model=col_model,
+                dataset=col_dataset,
+                metric=col_metric,
+                score=col_score,
+                seed=seed,
+                metric_direction=md,
+                norm_ref_low=0.0,
+                norm_ref_high=1.0,
+            )
+    return bench
+
+
+def _save(result, stem, output_dir):
+    """Serialize a result to CSV and PNG inside ``output_dir``."""
+    csv_path = os.path.join(output_dir, f"{stem}.csv")
+    png_path = os.path.join(output_dir, f"{stem}.png")
+    result.table.to_csv(csv_path, index=False)
+    fig = result.plot()
+    fig.savefig(png_path, bbox_inches="tight")
+
+
+@click.group()
+def main():
+    """evaluma — ML benchmark evaluation tools."""
+
+
+@main.command()
+@_common_options
+def report(
+    csv_path, model, dataset, metric, score, config_path, metric_direction, output_dir
+):
+    """Run all three analyses and write results to ``--output``."""
+    bench = _load_bench(
+        csv_path,
+        model,
+        dataset,
+        metric,
+        score,
+        config_path,
+        metric_direction,
+        output_dir,
+    )
+    _save(bench.aggregate_ranking(), "aggregate_ranking", output_dir)
+    _save(bench.bayesian_comparison(), "bayesian_comparison", output_dir)
+    _save(bench.performance_profiles(), "performance_profiles", output_dir)
+
+
+@main.command()
+@_common_options
+@click.option("--seed", default=None, help="Column name for the random seed.")
+def rank(
+    csv_path,
+    model,
+    dataset,
+    metric,
+    score,
+    config_path,
+    metric_direction,
+    output_dir,
+    seed,
+):
+    """Compute IQM rankings (requires seed column) and write iqm_ranking.{csv,png}."""
+    bench = _load_bench(
+        csv_path,
+        model,
+        dataset,
+        metric,
+        score,
+        config_path,
+        metric_direction,
+        output_dir,
+        seed=seed,
+    )
+    if bench._raw_runs is None:
+        click.echo(
+            "Error: iqm_ranking() requires multiple seeds — "
+            "use evaluma aggregate for single-run data.",
+            err=True,
+        )
+        sys.exit(1)
+    _save(bench.iqm_ranking(), "iqm_ranking", output_dir)
+
+
+@main.command()
+@_common_options
+@click.option(
+    "--agg",
+    default="trimmed_mean",
+    show_default=True,
+    help="Aggregation mode: trimmed_mean, mean, or median.",
+)
+def aggregate(
+    csv_path,
+    model,
+    dataset,
+    metric,
+    score,
+    config_path,
+    metric_direction,
+    output_dir,
+    agg,
+):
+    """Compute point-estimate aggregate ranking and write aggregate_ranking.csv/png."""
+    bench = _load_bench(
+        csv_path,
+        model,
+        dataset,
+        metric,
+        score,
+        config_path,
+        metric_direction,
+        output_dir,
+    )
+    _save(bench.aggregate_ranking(agg=agg), "aggregate_ranking", output_dir)
+
+
+@main.command()
+@_common_options
+def compare(
+    csv_path, model, dataset, metric, score, config_path, metric_direction, output_dir
+):
+    """Compute Bayesian pairwise comparisons and write results."""
+    bench = _load_bench(
+        csv_path,
+        model,
+        dataset,
+        metric,
+        score,
+        config_path,
+        metric_direction,
+        output_dir,
+    )
+    _save(bench.bayesian_comparison(), "bayesian_comparison", output_dir)
+
+
+@main.command()
+@_common_options
+def profiles(
+    csv_path, model, dataset, metric, score, config_path, metric_direction, output_dir
+):
+    """Compute Dolan-Moré performance profiles and write results."""
+    bench = _load_bench(
+        csv_path,
+        model,
+        dataset,
+        metric,
+        score,
+        config_path,
+        metric_direction,
+        output_dir,
+    )
+    _save(bench.performance_profiles(), "performance_profiles", output_dir)

evaluma/methods/aggregate.py

@@ -0,0 +1,44 @@
+import numpy as np
+import pandas as pd
+from scipy.stats import trim_mean
+
+from evaluma.results import AggregateResult
+
+_AGG_MODES = {"trimmed_mean", "mean", "median"}
+
+
+def compute_aggregate(
+    scores_matrix: pd.DataFrame, agg="trimmed_mean"
+) -> AggregateResult:
+    """Compute a point-estimate descriptive ranking from a normalized score matrix.
+
+    Args:
+        scores_matrix: Normalized model × dataset score matrix.
+        agg: Aggregation mode — one of ``"trimmed_mean"``, ``"mean"``,
+            ``"median"``.
+
+    Returns:
+        AggregateResult: Result with ``.table`` sorted descending by ``score``.
+
+    Raises:
+        ValueError: If ``agg`` is not one of the supported modes.
+    """
+    if agg not in _AGG_MODES:
+        raise ValueError(f"agg must be one of {sorted(_AGG_MODES)!r}, got {agg!r}")
+
+    models = scores_matrix.index.tolist()
+    data = scores_matrix.values  # shape (n_models, n_datasets)
+
+    if agg == "trimmed_mean":
+        scores = np.array([trim_mean(row, proportiontocut=0.25) for row in data])
+    elif agg == "mean":
+        scores = data.mean(axis=1)
+    else:
+        scores = np.median(data, axis=1)
+
+    table = (
+        pd.DataFrame({"model": models, "score": scores})
+        .sort_values("score", ascending=False)
+        .reset_index(drop=True)
+    )
+    return AggregateResult(table)

evaluma/methods/bayesian.py

@@ -0,0 +1,64 @@
+from itertools import combinations
+
+import baycomp
+import pandas as pd
+
+from evaluma.results import BayesianResult
+
+
+def compute_bayesian(
+    scores_matrix: pd.DataFrame,
+    *,
+    rope=0.01,
+    reference=None,
+    pairs=None,
+    random_state=None,
+) -> BayesianResult:
+    """Compute pairwise Bayesian comparisons using a signed-rank test.
+
+    For each pair, :func:`baycomp.two_on_multiple` returns the posterior
+    probability that model A is better, that they are practically
+    equivalent (within ``rope``), and that model B is better.
+
+    Args:
+        scores_matrix: Normalized model × dataset score matrix.
+        rope: Region of practical equivalence half-width. Differences
+            smaller than ``rope`` are treated as ties.
+        reference: If provided, only compare every other model against this
+            reference model.
+        pairs: Explicit list of ``(model_a, model_b)`` pairs to test.
+            Overrides ``reference``.
+        random_state: Seed forwarded to baycomp.
+
+    Returns:
+        BayesianResult: Result with ``.table`` containing columns
+        ``model_a``, ``model_b``, ``p_a_better``, ``p_equiv``,
+        ``p_b_better``.
+    """
+    models = scores_matrix.index.tolist()
+
+    if pairs is not None:
+        pair_list = list(pairs)
+    elif reference is not None:
+        pair_list = [(reference, m) for m in models if m != reference]
+    else:
+        pair_list = list(combinations(models, 2))
+
+    rows = []
+    for model_a, model_b in pair_list:
+        vec_a = scores_matrix.loc[model_a].values.astype(float)
+        vec_b = scores_matrix.loc[model_b].values.astype(float)
+        p_a, p_eq, p_b = baycomp.two_on_multiple(
+            vec_a, vec_b, rope=rope, random_state=random_state
+        )
+        rows.append(
+            {
+                "model_a": model_a,
+                "model_b": model_b,
+                "p_a_better": p_a,
+                "p_equiv": p_eq,
+                "p_b_better": p_b,
+            }
+        )
+
+    return BayesianResult(pd.DataFrame(rows), reference=reference)

evaluma/methods/iqm.py

@@ -0,0 +1,93 @@
+import numpy as np
+import pandas as pd
+from scipy.stats import trim_mean
+
+from evaluma.results import IQMResult
+
+
+def compute_iqm(raw_runs, norm_bounds, n_bootstrap=1000, random_state=None):
+    """Compute Agarwal IQM on the flat run×dataset array with stratified bootstrap CIs.
+
+    Implements the IQM from Agarwal et al. 2021 (rliable): trim the outer 25%
+    of the concatenated per-dataset, per-seed normalized scores and average the
+    remainder. Bootstrap CIs are stratified — seeds are resampled independently
+    within each dataset stratum.
+
+    Args:
+        raw_runs: Long-format DataFrame with columns
+            ``["model", "dataset", "seed", "score"]``.
+        norm_bounds: ``(low, high, metric_direction)`` where ``low`` and
+            ``high`` are per-dataset ``pd.Series`` of normalization bounds
+            and ``metric_direction`` is a ``{dataset: "min"|"max"}`` dict
+            (or ``None``).
+        n_bootstrap: Number of stratified bootstrap replicates for the 95% CI.
+        random_state: Seed for :func:`numpy.random.default_rng`.
+
+    Returns:
+        IQMResult: Result with ``.table`` sorted descending by IQM.
+    """
+    low, high, metric_direction = norm_bounds
+    rng = np.random.default_rng(random_state)
+
+    models = raw_runs["model"].unique().tolist()
+    datasets = raw_runs["dataset"].unique().tolist()
+
+    def _norm_score(score, dataset):
+        lo = float(low[dataset])
+        hi = float(high[dataset])
+        if metric_direction and metric_direction.get(dataset) == "min":
+            return (hi - score) / (hi - lo)
+        return (score - lo) / (hi - lo)
+
+    # Build per-(model, dataset) arrays of normalized scores
+    per_dataset = {m: [] for m in models}
+    for d in datasets:
+        lo = float(low[d])
+        hi = float(high[d])
+        is_min = metric_direction and metric_direction.get(d) == "min"
+        for m in models:
+            mask = (raw_runs["model"] == m) & (raw_runs["dataset"] == d)
+            scores = raw_runs.loc[mask, "score"].values.astype(float)
+            if is_min:
+                norm = (hi - scores) / (hi - lo)
+            else:
+                norm = (scores - lo) / (hi - lo)
+            per_dataset[m].append(norm)
+
+    # Point estimates: flat-array IQM
+    iqms_arr = np.array(
+        [trim_mean(np.concatenate(per_dataset[m]), 0.25) for m in models]
+    )
+
+    if n_bootstrap == 0:
+        table = (
+            pd.DataFrame(
+                {"model": models, "IQM": iqms_arr, "CI_low": np.nan, "CI_high": np.nan}
+            )
+            .sort_values("IQM", ascending=False)
+            .reset_index(drop=True)
+        )
+        return IQMResult(table)
+
+    # Stratified bootstrap: resample seeds within each dataset independently
+    boot_iqms = np.empty((n_bootstrap, len(models)))
+    for b in range(n_bootstrap):
+        for mi, m in enumerate(models):
+            resampled = []
+            for d_scores in per_dataset[m]:
+                n = len(d_scores)
+                idx = rng.integers(0, n, size=n)
+                resampled.append(d_scores[idx])
+            boot_iqms[b, mi] = trim_mean(np.concatenate(resampled), 0.25)
+
+    ci_low = np.percentile(boot_iqms, 2.5, axis=0)
+    ci_high = np.percentile(boot_iqms, 97.5, axis=0)
+
+    table = (
+        pd.DataFrame(
+            {"model": models, "IQM": iqms_arr, "CI_low": ci_low, "CI_high": ci_high}
+        )
+        .sort_values("IQM", ascending=False)
+        .reset_index(drop=True)
+    )
+    return IQMResult(table)

evaluma/methods/profiles.py

@@ -0,0 +1,72 @@
+import numpy as np
+import pandas as pd
+
+from evaluma.results import ProfileResult
+
+
+def compute_profiles(
+    scores_matrix: pd.DataFrame,
+    metric_direction: dict | None = None,
+) -> ProfileResult:
+    """Compute Dolan-Moré performance profiles.
+
+    Ratios are computed directly from raw scores without normalization:
+
+    - **max metrics** (higher is better): r_ij = best_j / score_ij
+    - **min metrics** (lower is better): r_ij = score_ij / best_j
+
+    where best_j is the best score across all models on dataset j. Both
+    definitions yield r_ij ≥ 1, with r_ij = 1 when model i is the best on
+    dataset j.
+
+    The tau grid uses the exact observed ratio values as breakpoints, giving
+    the exact Dolan-Moré step function. The plot renders on a log₁₀(τ) axis,
+    following ML-GYM (Batra et al., 2025) and the AutoML Decathlon (Roberts
+    et al., 2022), which extended Dolan & Moré (2002). Use
+    ``ProfileResult.aup`` for the scalar Area Under the Profile summary.
+
+    Args:
+        scores_matrix: Raw model × dataset score matrix. All values must be
+            strictly positive.
+        metric_direction: Dict mapping dataset (column) names to ``"min"`` or
+            ``"max"``. Datasets absent from the dict default to ``"max"``.
+
+    Returns:
+        ProfileResult with ``.table`` (long-format: ``tau``, ``model``,
+        ``fraction_within_tau``).
+
+    Raises:
+        ValueError: If any value in ``scores_matrix`` is zero or negative.
+    """
+    if (scores_matrix.values <= 0).any():
+        raise ValueError(
+            "Cannot compute performance profiles: score of 0 or below detected. "
+            "All scores must be strictly positive."
+        )
+
+    metric_direction = metric_direction or {}
+    models = scores_matrix.index.tolist()
+    data = scores_matrix.values.astype(float)  # (n_models, n_datasets)
+    datasets = scores_matrix.columns.tolist()
+
+    # Compute per-dataset ratios respecting direction.
+    ratios = np.ones_like(data)
+    for j, dataset in enumerate(datasets):
+        col = data[:, j]
+        if metric_direction.get(dataset) == "min":
+            best = col.min()
+            ratios[:, j] = col / best
+        else:
+            best = col.max()
+            ratios[:, j] = best / col
+
+    # Exact breakpoints: every observed ratio value, plus 1.0 as anchor.
+    tau_grid = np.sort(np.unique(np.concatenate([[1.0], ratios.flatten()])))
+
+    rows = []
+    for tau in tau_grid:
+        for i, model in enumerate(models):
+            frac = (ratios[i] <= tau).mean()
+            rows.append({"tau": tau, "model": model, "fraction_within_tau": frac})
+
+    return ProfileResult(pd.DataFrame(rows))

evaluma/metric_registry.py

@@ -0,0 +1,69 @@
+"""Metric name registry: maps names to optimization direction and natural bounds."""
+
+KNOWN_METRICS = {
+    # name: (direction, (natural_low, natural_high))  — None high: user must specify
+    "accuracy": ("max", (0.0, 1.0)),
+    "acc": ("max", (0.0, 1.0)),
+    "iou": ("max", (0.0, 1.0)),
+    "miou": ("max", (0.0, 1.0)),
+    "f1": ("max", (0.0, 1.0)),
+    "auc": ("max", (0.0, 1.0)),
+    "ap": ("max", (0.0, 1.0)),
+    "map": ("max", (0.0, 1.0)),
+    "precision": ("max", (0.0, 1.0)),
+    "recall": ("max", (0.0, 1.0)),
+    "r2": ("max", (0.0, 1.0)),
+    "rmse": ("min", (0.0, None)),
+    "mae": ("min", (0.0, None)),
+    "mse": ("min", (0.0, None)),
+    "mape": ("min", (0.0, None)),
+}
+
+_KNOWN_LIST = ", ".join(sorted(KNOWN_METRICS))
+
+
+def get_direction(metric_name: str) -> str:
+    """Return ``"min"`` or ``"max"`` for a known metric name.
+
+    Args:
+        metric_name: Metric identifier (case-insensitive).
+
+    Returns:
+        str: ``"min"`` (lower is better) or ``"max"`` (higher is better).
+
+    Raises:
+        ValueError: If ``metric_name`` is not in the registry. The message
+            lists all known names and suggests using ``metric_direction`` for
+            custom metrics.
+    """
+    name = metric_name.lower()
+    if name not in KNOWN_METRICS:
+        raise ValueError(
+            f"Unknown metric '{metric_name}'. Known metrics: {_KNOWN_LIST}. "
+            f"For custom metrics, specify direction via metric_direction and "
+            f"bounds via metric_type_bounds."
+        )
+    return KNOWN_METRICS[name][0]
+
+
+def get_natural_bounds(metric_name: str) -> tuple:
+    """Return ``(natural_low, natural_high)`` for a known metric name.
+
+    ``natural_high`` is ``None`` for unbounded metrics (e.g. ``rmse``, ``mae``),
+    meaning the upper bound must be supplied by the caller.
+
+    Args:
+        metric_name: Metric identifier (case-insensitive).
+
+    Returns:
+        tuple: ``(float, float | None)`` natural lower and upper bounds.
+
+    Raises:
+        ValueError: If ``metric_name`` is not in the registry.
+    """
+    name = metric_name.lower()
+    if name not in KNOWN_METRICS:
+        raise ValueError(
+            f"Unknown metric '{metric_name}'. Known metrics: {_KNOWN_LIST}."
+        )
+    return KNOWN_METRICS[name][1]