evaluma 0.1.0__py3-none-any.whl

evaluma/__init__.py

@@ -0,0 +1,258 @@
+import pandas as pd
+
+from evaluma._version import __version__  # noqa: F401
+from evaluma.benchmark import Benchmark  # noqa: F401
+
+
+def load_df(
+    df,
+    *,
+    model="model",
+    dataset="dataset",
+    metric="metric",
+    score="score",
+    seed=None,
+    metric_type_bounds=None,
+    norm_ref_low=None,
+    norm_ref_high=None,
+    metric_direction=None,
+    drop_incomplete=False,
+):
+    """Load a DataFrame and return a ready-to-use Benchmark object.
+
+    Args:
+        df: A pandas DataFrame in long format (one row per model/dataset pair).
+            To load from a CSV file, use :func:`evaluma.load_csv` instead.
+        model: Column name for the model identifier.
+        dataset: Column name for the dataset identifier.
+        metric: Column name for the metric identifier.
+        score: Column name for the score values.
+        seed: Column name for the random seed. When provided, all seed rows
+            are preserved and a ``seed`` column is included in the loaded
+            DataFrame.
+        metric_type_bounds: Dict mapping metric names to ``(low, high)``
+            bound tuples. ``high`` may be a model name string, resolved
+            per-dataset to that model's score. When provided, ``norm_ref_low``
+            and ``norm_ref_high`` must be ``None``. Metric direction is inferred
+            from the built-in registry; unknown metrics raise ``ValueError``.
+            Bounded metrics not listed here (e.g. accuracy, iou, f1) use their
+            natural ``[0, 1]`` bounds automatically. Unbounded metrics (rmse,
+            mae, mse) must be listed; omitting them raises ``ValueError``.
+        norm_ref_low: Lower normalization reference — scalar, model name,
+            or per-dataset dict. If ``None``, the per-dataset minimum is
+            used and a ``UserWarning`` is emitted. Cannot be combined with
+            ``metric_type_bounds``.
+        norm_ref_high: Upper normalization reference, same format as
+            ``norm_ref_low``. If ``None``, the per-dataset maximum is used.
+            Cannot be combined with ``metric_type_bounds``.
+        metric_direction: Dict mapping dataset names to ``"min"`` or
+            ``"max"``. When used with ``metric_type_bounds``, these entries
+            take precedence over the registry-inferred direction. Without
+            ``metric_type_bounds``, datasets mapped to ``"min"`` are negated
+            before normalization so that higher is always better.
+        drop_incomplete: If ``True``, silently drop models with missing
+            scores instead of raising.
+
+    Returns:
+        Benchmark: Normalized benchmark ready for analysis.
+
+    Raises:
+        TypeError: If ``df`` is not a pandas DataFrame.
+        ValueError: If ``metric_type_bounds`` is provided together with
+            ``norm_ref_low`` or ``norm_ref_high``.
+        ValueError: If the data contains more than one metric per
+            (model, dataset) pair, or if the score matrix is incomplete
+            and ``drop_incomplete`` is ``False``.
+        ValueError: If a metric referenced by a dataset is not in the
+            registry and not covered by ``metric_type_bounds``.
+        ValueError: If a regression metric (rmse, mae, mse) is present but
+            no upper bound is specified in ``metric_type_bounds``.
+    """
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError(
+            f"load_df() expects a pandas DataFrame, got {type(df).__name__}. "
+            "To load from a file, use evaluma.load_csv()."
+        )
+
+    if metric_type_bounds is not None and (
+        norm_ref_low is not None or norm_ref_high is not None
+    ):
+        raise ValueError(
+            "metric_type_bounds cannot be combined with norm_ref_low or norm_ref_high. "
+            "Use metric_type_bounds to specify all bounds, "
+            "or use norm_ref_low/norm_ref_high."
+        )
+
+    df = df.copy()
+    rename = {model: "model", dataset: "dataset", metric: "metric", score: "score"}
+    if seed is not None:
+        rename[seed] = "seed"
+    df = df.rename(columns=rename)
+
+    metrics_per_dataset = df.groupby(["model", "dataset"])["metric"].nunique()
+    if (metrics_per_dataset > 1).any():
+        raise ValueError("more than one metric found per (model, dataset) cell")
+
+    check_df = df.drop_duplicates(subset=["model", "dataset"])
+    pivot = check_df.pivot(index="model", columns="dataset", values="score")
+    missing = pivot.isna()
+    if missing.to_numpy().any():
+        missing_cells = [
+            (m, d)
+            for m in pivot.index
+            for d in pivot.columns
+            if pd.isna(pivot.loc[m, d])
+        ]
+        if drop_incomplete:
+            complete_models = pivot.index[~missing.any(axis=1)]
+            df = df[df["model"].isin(complete_models)].reset_index(drop=True)
+        else:
+            cell_str = ", ".join(f"({m}, {d})" for m, d in missing_cells)
+            raise ValueError(f"Incomplete score matrix: missing cells {cell_str}")
+
+    if seed is not None:
+        df = df[["model", "dataset", "metric", "score", "seed"]]
+    else:
+        df = df[["model", "dataset", "metric", "score"]]
+
+    raw_runs = None
+    if seed is not None:
+        raw_runs = df[["model", "dataset", "seed", "score"]].reset_index(drop=True)
+        raw_matrix = (
+            df.groupby(["model", "dataset"], as_index=False)["score"]
+            .mean()
+            .pivot(index="model", columns="dataset", values="score")
+        )
+    else:
+        raw_matrix = df.pivot(index="model", columns="dataset", values="score")
+    raw_matrix.columns.name = None
+
+    if metric_type_bounds is not None:
+        dataset_metric_map = (
+            df.drop_duplicates("dataset").set_index("dataset")["metric"].to_dict()
+        )
+        norm_ref_low, norm_ref_high, metric_direction = _resolve_metric_type_bounds(
+            metric_type_bounds, dataset_metric_map, raw_matrix, metric_direction
+        )
+
+    return Benchmark(
+        raw_matrix,
+        norm_ref_low=norm_ref_low,
+        norm_ref_high=norm_ref_high,
+        metric_direction=metric_direction,
+        raw_runs=raw_runs,
+    )
+
+
+def load_csv(
+    path,
+    *,
+    model="model",
+    dataset="dataset",
+    metric="metric",
+    score="score",
+    seed=None,
+    metric_type_bounds=None,
+    norm_ref_low=None,
+    norm_ref_high=None,
+    metric_direction=None,
+    drop_incomplete=False,
+):
+    """Load a benchmark CSV file and return a ready-to-use Benchmark object.
+
+    Args:
+        path: Path to the CSV file.
+        model: Column name for the model identifier.
+        dataset: Column name for the dataset identifier.
+        metric: Column name for the metric identifier.
+        score: Column name for the score values.
+        seed: Column name for the random seed.
+        metric_type_bounds: See :func:`evaluma.load_df`.
+        norm_ref_low: See :func:`evaluma.load_df`.
+        norm_ref_high: See :func:`evaluma.load_df`.
+        metric_direction: See :func:`evaluma.load_df`.
+        drop_incomplete: See :func:`evaluma.load_df`.
+
+    Returns:
+        Benchmark: Normalized benchmark ready for analysis.
+    """
+    df = pd.read_csv(path)
+    return load_df(
+        df,
+        model=model,
+        dataset=dataset,
+        metric=metric,
+        score=score,
+        seed=seed,
+        metric_type_bounds=metric_type_bounds,
+        norm_ref_low=norm_ref_low,
+        norm_ref_high=norm_ref_high,
+        metric_direction=metric_direction,
+        drop_incomplete=drop_incomplete,
+    )
+
+
+def _resolve_metric_type_bounds(
+    metric_type_bounds, dataset_metric_map, raw_matrix, metric_direction_override
+):
+    """Resolve per-dataset normalization bounds and directions from the metric registry.
+
+    For each dataset, consults ``metric_type_bounds`` first, then falls back to the
+    built-in registry for metrics with natural bounds. Raises if an unbounded metric
+    (rmse, mae, mse) has no entry in ``metric_type_bounds``.
+
+    Args:
+        metric_type_bounds: Dict mapping metric names → ``(low, high)`` tuples.
+        dataset_metric_map: Dict mapping dataset names → metric name strings.
+        raw_matrix: Model × dataset score DataFrame (used to resolve model-name bounds).
+        metric_direction_override: Optional dict mapping dataset names →
+            ``"min"``/``"max"``; these entries override registry-inferred directions.
+
+    Returns:
+        tuple: ``(norm_ref_low, norm_ref_high, metric_direction)`` where the first
+        two are
+        ``pd.Series`` keyed by dataset and the last is a dict (or ``None`` if empty).
+    """
+    from evaluma.metric_registry import get_direction, get_natural_bounds
+
+    mtb_lower = {k.lower(): v for k, v in metric_type_bounds.items()}
+
+    low_dict = {}
+    high_dict = {}
+    direction_dict = {}
+
+    for dataset, metric_name in dataset_metric_map.items():
+        metric_lower = metric_name.lower()
+
+        if metric_lower in mtb_lower:
+            low, high = mtb_lower[metric_lower]
+            if isinstance(high, str):
+                model_name = high
+                if model_name not in raw_matrix.index:
+                    raise ValueError(
+                        f"Reference model '{model_name}' not found in score matrix. "
+                        f"Available models: {list(raw_matrix.index)}"
+                    )
+                high = raw_matrix.loc[model_name, dataset]
+            low_dict[dataset] = float(low)
+            high_dict[dataset] = float(high)
+            direction_dict[dataset] = get_direction(metric_lower)
+        else:
+            direction = get_direction(metric_lower)  # raises ValueError for unknowns
+            nat_low, nat_high = get_natural_bounds(metric_lower)
+            if nat_high is None:
+                raise ValueError(
+                    f"Metric '{metric_name}' on dataset '{dataset}' has no natural "
+                    f"upper bound: add '{metric_lower}' to metric_type_bounds "
+                    f"in your config."
+                )
+            low_dict[dataset] = nat_low
+            high_dict[dataset] = nat_high
+            direction_dict[dataset] = direction
+
+    direction_dict.update(metric_direction_override or {})
+    return (
+        pd.Series(low_dict),
+        pd.Series(high_dict),
+        direction_dict or None,
+    )

evaluma/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

evaluma/benchmark.py

@@ -0,0 +1,211 @@
+from functools import cached_property
+
+import pandas as pd
+
+from evaluma.normalize import normalize
+
+
+class Benchmark:
+    """Container for a normalized model-vs-dataset score matrix.
+
+    After construction the normalized scores are available as ``scores_``.
+    Use the analysis methods to compute rankings, comparisons, and profiles.
+    """
+
+    def __init__(
+        self,
+        raw_matrix: pd.DataFrame,
+        *,
+        norm_ref_low=None,
+        norm_ref_high=None,
+        metric_direction=None,
+        raw_runs=None,
+    ):
+        """Initialize and normalize the score matrix.
+
+        Args:
+            raw_matrix: Model × dataset score matrix (models as row index,
+                datasets as columns).
+            norm_ref_low: Lower normalization reference — scalar, model
+                name, or per-dataset dict. ``None`` triggers data-dependent
+                bounds and a ``UserWarning``.
+            norm_ref_high: Upper normalization reference, same format as
+                ``norm_ref_low``.
+            metric_direction: Dict mapping dataset names to ``"min"`` or
+                ``"max"``; datasets mapped to ``"min"`` are negated before
+                normalization.
+            raw_runs: Long-format DataFrame with columns
+                ``["model", "dataset", "seed", "score"]`` containing
+                per-seed scores. When provided, ``iqm_ranking()`` uses
+                stratified bootstrap over seeds.
+        """
+        self._raw = raw_matrix
+        self._norm_ref_low = norm_ref_low
+        self._norm_ref_high = norm_ref_high
+        self._metric_direction = metric_direction
+        self._raw_runs = raw_runs
+
+    def _normalize(self, matrix):
+        import warnings
+
+        with warnings.catch_warnings():
+            if self._norm_ref_low is not None or self._norm_ref_high is not None:
+                warnings.simplefilter("ignore", UserWarning)
+            return normalize(
+                matrix,
+                norm_ref_low=self._norm_ref_low,
+                norm_ref_high=self._norm_ref_high,
+                metric_direction=self._metric_direction,
+            )
+
+    @cached_property
+    def scores_(self):
+        """Normalized model × dataset score matrix."""
+        return self._normalize(self._raw)
+
+    def _new(self, raw_matrix, raw_runs=None):
+        return Benchmark(
+            raw_matrix,
+            norm_ref_low=self._norm_ref_low,
+            norm_ref_high=self._norm_ref_high,
+            metric_direction=self._metric_direction,
+            raw_runs=raw_runs,
+        )
+
+    def select_models(self, models):
+        """Subset the benchmark to the given models.
+
+        Args:
+            models: List of model names to retain.
+
+        Returns:
+            Benchmark: New benchmark containing only the selected models.
+        """
+        raw_runs = None
+        if self._raw_runs is not None:
+            raw_runs = self._raw_runs[self._raw_runs["model"].isin(models)].reset_index(
+                drop=True
+            )
+        return self._new(self._raw.loc[models], raw_runs=raw_runs)
+
+    def select_datasets(self, datasets):
+        """Subset the benchmark to the given datasets.
+
+        Args:
+            datasets: List of dataset names to retain.
+
+        Returns:
+            Benchmark: New benchmark containing only the selected datasets.
+        """
+        raw_runs = None
+        if self._raw_runs is not None:
+            raw_runs = self._raw_runs[
+                self._raw_runs["dataset"].isin(datasets)
+            ].reset_index(drop=True)
+        return self._new(self._raw[datasets], raw_runs=raw_runs)
+
+    def drop_incomplete(self):
+        """Remove models that have missing scores for any dataset.
+
+        Returns:
+            Benchmark: New benchmark with incomplete models removed.
+        """
+        complete = self._raw.index[~self._raw.isna().any(axis=1)]
+        raw_runs = None
+        if self._raw_runs is not None:
+            raw_runs = self._raw_runs[
+                self._raw_runs["model"].isin(complete)
+            ].reset_index(drop=True)
+        return self._new(self._raw.loc[complete], raw_runs=raw_runs)
+
+    def iqm_ranking(self, n_bootstrap=1000, random_state=None):
+        """Compute IQM rankings with stratified bootstrap confidence intervals.
+
+        Implements the Agarwal et al. 2021 (rliable) IQM on the flat
+        run×dataset score array. Requires multiple seeds; use
+        ``aggregate_ranking()`` for single-run data.
+
+        Args:
+            n_bootstrap: Number of bootstrap samples for the 95 % CI.
+            random_state: Seed for the random number generator.
+
+        Returns:
+            IQMResult: Result with ``.table`` and ``.plot()``.
+
+        Raises:
+            ValueError: If no seed data is available (``_raw_runs is None``).
+        """
+        if self._raw_runs is None:
+            raise ValueError(
+                "iqm_ranking() requires multiple seeds — "
+                "use aggregate_ranking() for single-run data."
+            )
+        from evaluma.methods.iqm import compute_iqm
+        from evaluma.normalize import _resolve_bound
+
+        low = _resolve_bound(self._raw, self._norm_ref_low, use_min=True)
+        high = _resolve_bound(self._raw, self._norm_ref_high, use_min=False)
+        return compute_iqm(
+            self._raw_runs,
+            norm_bounds=(low, high, self._metric_direction),
+            n_bootstrap=n_bootstrap,
+            random_state=random_state,
+        )
+
+    def aggregate_ranking(self, agg="trimmed_mean"):
+        """Compute a point-estimate descriptive ranking (no CI).
+
+        Works on any benchmark regardless of whether seed data is present.
+
+        Args:
+            agg: Aggregation mode — ``"trimmed_mean"`` (default), ``"mean"``,
+                or ``"median"``.
+
+        Returns:
+            AggregateResult: Result with ``.table`` and ``.plot()``.
+
+        Raises:
+            ValueError: If ``agg`` is not a supported mode.
+        """
+        from evaluma.methods.aggregate import compute_aggregate
+
+        return compute_aggregate(self.scores_, agg=agg)
+
+    def bayesian_comparison(
+        self, rope=0.01, reference=None, pairs=None, random_state=None
+    ):
+        """Compute pairwise Bayesian comparisons via signed-rank test.
+
+        Args:
+            rope: Region of practical equivalence half-width.
+            reference: If given, only compare each other model against this
+                one.
+            pairs: Explicit list of ``(model_a, model_b)`` pairs to test.
+                Overrides ``reference``.
+            random_state: Seed for baycomp's sampler.
+
+        Returns:
+            BayesianResult: Result with ``.table`` and ``.plot()``.
+        """
+        from evaluma.methods.bayesian import compute_bayesian
+
+        return compute_bayesian(
+            self.scores_,
+            rope=rope,
+            reference=reference,
+            pairs=pairs,
+            random_state=random_state,
+        )
+
+    def performance_profiles(self):
+        """Compute Dolan-Moré performance profiles.
+
+        Returns:
+            ProfileResult: Result with ``.table`` and ``.plot()``.
+
+        Raises:
+            ValueError: If any raw score is zero or negative.
+        """
+        from evaluma.methods.profiles import compute_profiles
+
+        return compute_profiles(self._raw, metric_direction=self._metric_direction)