eval-toolkit 1.7.0__tar.gz → 1.8.0__tar.gz

{eval_toolkit-1.7.0 → eval_toolkit-1.8.0}/CHANGELOG.md

@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.8.0] — 2026-06-04 — composite multi-stratum cluster bootstrap (#92)
+
+### Added — `bootstrap.stratified_cluster_bootstrap_ci` (composite multi-stratum cluster bootstrap)
+
+`eval_toolkit.bootstrap` ADDITIVE per [ADR 0003](docs/source/adr/0003-stability-contract-and-gate3-methodology.md) — backward-compatible. Generalises the v1.7.0 single-block `cluster_bootstrap_ci` to the shape leave-one-group-out transfer gaps actually take: a **composite statistic reduced over several independently-resampled cluster strata**.
+
+- **`stratified_cluster_bootstrap_ci(strata, per_stratum_metric, combine, *, resample_labels=(0,1), …)`** — `strata` is a mapping `{key: (y, score, groups)}` of independent resample-units (e.g. `seed`, `(carrier, seed)`, `(attack_type, seed)`); each bootstrap iteration resamples every stratum's `(label, group)` clusters, computes `per_stratum_metric` on each, and reduces the `{key: metric}` map with `combine` to one scalar (a seed-averaged ROC-AUC gap, a mean-over-carriers gap, a top−bottom per-type AUPRC contrast, …). Percentile `BootstrapCI` (`method="stratified_cluster_percentile"`). `cluster_bootstrap_ci` is the single-stratum, identity-reduce special case.
+- **Why:** the v1.7.0 single-block primitive could not express the **seed-averaging** that real LODO estimators do inside the bootstrap (`Gx = val − mean_seed(test_roc)`), so it did not actually fit the consumer portfolio's attack-type / carrier / dialect bootstraps. This is the correct primitive for them.
+- **Parallel + reproducible:** built on `parallel_map` + `spawn_seed_sequences` ⇒ `n_jobs` gives bit-for-bit-identical CIs; `n_jobs=-1` all cores.
+- Exported via `from eval_toolkit import stratified_cluster_bootstrap_ci`; `__all__` + `_EXPORTS` + doctest + n_jobs-reproducibility / seed-averaged / composite-statistic tests; mypy-strict clean.
+
 ## [1.7.0] — 2026-06-04 — label-stratified cluster bootstrap (#90, #91)
 
 ### Added — `bootstrap.cluster_bootstrap_ci` (label-stratified cluster bootstrap)

{eval_toolkit-1.7.0 → eval_toolkit-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-toolkit
-Version: 1.7.0
+Version: 1.8.0
 Summary: Reusable evaluation contracts for binary classification: metrics, bootstrap CIs, calibration, artifacts, and evidence gates.
 Project-URL: Homepage, https://github.com/brandon-behring/eval-toolkit
 Project-URL: Documentation, https://brandon-behring.github.io/eval-toolkit/

{eval_toolkit-1.7.0 → eval_toolkit-1.8.0}/src/eval_toolkit/__init__.py

@@ -140,6 +140,7 @@ _EXPORTS: dict[str, str] = {
     "paired_bootstrap_ece_diff": "eval_toolkit.bootstrap",
     "paired_bootstrap_op_point_diff": "eval_toolkit.bootstrap",
     "paired_mde": "eval_toolkit.bootstrap",
+    "stratified_cluster_bootstrap_ci": "eval_toolkit.bootstrap",
     # --- calibration ---
     "DEFAULT_FN_COST": "eval_toolkit.calibration",
     "DEFAULT_FP_COST": "eval_toolkit.calibration",

{eval_toolkit-1.7.0 → eval_toolkit-1.8.0}/src/eval_toolkit/_version.py

@@ -2,4 +2,4 @@
 
 __all__ = ["__version__"]
 
-__version__ = "1.7.0"
+__version__ = "1.8.0"

{eval_toolkit-1.7.0 → eval_toolkit-1.8.0}/src/eval_toolkit/bootstrap.py

@@ -22,7 +22,7 @@ from __future__ import annotations
 import functools
 import logging
 import warnings
-from collections.abc import Callable
+from collections.abc import Callable, Hashable, Mapping
 from dataclasses import dataclass
 from typing import Final, Literal
 
@@ -63,6 +63,7 @@ __all__ = [
     "paired_bootstrap_ece_diff",
     "paired_bootstrap_op_point_diff",
     "paired_mde",
+    "stratified_cluster_bootstrap_ci",
 ]
 
 DEFAULT_N_RESAMPLES: Final[int] = 1000
@@ -1647,6 +1648,193 @@ def cluster_bootstrap_ci(
     )
 
 
+def _stratified_cluster_step(
+    seed_seq: np.random.SeedSequence,
+    *,
+    strata_data: dict[Hashable, tuple[np.ndarray, np.ndarray]],
+    strata_units: dict[Hashable, dict[int, list[np.ndarray]]],
+    per_stratum_metric: MetricFn,
+    combine: Callable[[Mapping[Hashable, float]], float],
+    resample_labels: tuple[int, ...],
+) -> float | None:
+    """One stratified-cluster draw of ``combine({key: metric})`` (module-level for picklability).
+
+    Each stratum's ``(label, group)`` units are resampled independently (per ``resample_labels``;
+    labels not listed are held fixed) with a single per-resample RNG, the per-stratum metric is
+    computed on the gathered rows, and ``combine`` reduces the ``{key: metric}`` map to one scalar.
+    Returns ``None`` if any stratum metric or ``combine`` raises (a degenerate draw).
+    """
+    rng = np.random.default_rng(seed_seq)
+    by_key: dict[Hashable, float] = {}
+    for key, units in strata_units.items():
+        y_k, s_k = strata_data[key]
+        parts: list[np.ndarray] = []
+        for lab, group_rows in units.items():
+            if lab in resample_labels:
+                chosen = rng.integers(0, len(group_rows), size=len(group_rows))
+                parts.extend(group_rows[c] for c in chosen)
+            else:
+                parts.extend(group_rows)
+        idx = np.concatenate(parts)
+        try:
+            by_key[key] = float(per_stratum_metric(y_k[idx], s_k[idx]))
+        except (ValueError, RuntimeError):
+            return None
+    try:
+        return float(combine(by_key))
+    except (ValueError, RuntimeError, KeyError, ZeroDivisionError):
+        return None
+
+
+def stratified_cluster_bootstrap_ci(
+    strata: Mapping[Hashable, tuple[np.ndarray, np.ndarray, np.ndarray]],
+    per_stratum_metric: MetricFn,
+    combine: Callable[[Mapping[Hashable, float]], float],
+    *,
+    resample_labels: tuple[int, ...] = (0, 1),
+    n_resamples: int = DEFAULT_N_RESAMPLES,
+    confidence: float = DEFAULT_CONFIDENCE,
+    rng: RNGLike | SeedLike | None = DEFAULT_SEED,
+    n_jobs: int = 1,
+) -> BootstrapCI:
+    r"""Cluster bootstrap of a **composite** statistic over several independent strata.
+
+    Generalises :func:`cluster_bootstrap_ci` (one condition, one metric) to a statistic that is a
+    user-supplied **reduction over several independently-resampled cluster strata** — the shape that
+    leave-one-group-out transfer gaps take in practice: a per-seed (and per-group) cluster resample,
+    averaged/combined into one scalar (a seed-averaged ROC-AUC gap, a mean-over-carriers gap, a
+    top-minus-bottom per-type AUPRC contrast, …). Each bootstrap iteration resamples every stratum's
+    ``(label, group)`` clusters (label-stratified, per ``resample_labels``; labels not listed are
+    held fixed), computes ``per_stratum_metric`` on each, and reduces the ``{key: metric}`` map with
+    ``combine``; the percentile CI is over those reduced values.
+
+    :func:`cluster_bootstrap_ci` is the single-stratum, identity-reduce special case
+    (``strata={0: (y, score, groups)}``, ``combine=lambda m: m[0]``).
+
+    Parameters
+    ----------
+    strata : Mapping[key, (y_true, y_score, groups)]
+        Independent resample-units keyed by any hashable (e.g. ``seed`` / ``(carrier, seed)`` /
+        ``(attack_type, seed)``). Each value is three aligned 1-D arrays. Iteration order is the
+        mapping's order (stable ⇒ deterministic).
+    per_stratum_metric : callable ``(y_true, y_score) -> float``
+        Metric computed on each stratum's resampled rows (e.g. ``roc_auc``, ``pr_auc``). Must be
+        **picklable** when ``n_jobs != 1``.
+    combine : callable ``Mapping[key, float] -> float``
+        Reduces the per-stratum metrics to the composite statistic (e.g.
+        ``val − mean_seed(m)``; ``mean_carrier(val[c] − mean_seed(m[c, ·]))``; a top−bottom contrast).
+        Closes over any fixed quantities (val ROC, the type partition) — pass a **picklable**
+        top-level function or ``functools.partial`` when ``n_jobs != 1`` (lambdas are fine at
+        ``n_jobs == 1``).
+    resample_labels : tuple[int, ...], optional
+        Which label strata are cluster-resampled within each stratum (default ``(0, 1)``);
+        ``(1,)`` resamples only positive clusters, holding negatives fixed (the payload-cluster
+        convention). Labels not present in a given stratum are simply skipped there.
+    n_resamples, confidence, rng : standard bootstrap params (``rng`` per SPEC 7).
+    n_jobs : int, optional
+        Parallel workers (default 1). Per-resample seeding via :func:`spawn_seed_sequences` makes the
+        CI **bit-for-bit identical across ``n_jobs``**; ``n_jobs=-1`` uses all cores. See
+        :ref:`methodology/parallelism`.
+
+    Returns
+    -------
+    BootstrapCI
+        ``method="stratified_cluster_percentile"``; ``point_estimate = combine({key:
+        per_stratum_metric(y, score)})`` on the full data; ``[alpha/2, 1 - alpha/2]`` percentile CI.
+
+    Raises
+    ------
+    ValueError
+        On empty ``strata``, a stratum whose arrays mismatch shape / are not 1-D, empty
+        ``resample_labels``, ``confidence`` outside (0, 1), ``n_jobs == 0``, or > 5% degenerate
+        resamples.
+    TypeError
+        If ``n_jobs != 1`` and ``per_stratum_metric`` / ``combine`` are not picklable.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from eval_toolkit.metrics import roc_auc
+    >>> def _block(seed):
+    ...     g = np.repeat(np.arange(20), 4)                 # 20 clusters of 4
+    ...     y = (g % 2).astype(int)
+    ...     s = y + np.random.default_rng(seed).normal(0, 0.3, size=y.size)
+    ...     return y, s, g
+    >>> strata = {0: _block(0), 1: _block(1)}               # two seeds
+    >>> ci = stratified_cluster_bootstrap_ci(
+    ...     strata, roc_auc, lambda m: float(np.mean(list(m.values()))),
+    ...     n_resamples=200, rng=0)
+    >>> ci.method
+    'stratified_cluster_percentile'
+    >>> bool(0.0 <= ci.ci_low <= ci.ci_high <= 1.0)
+    True
+
+    Notes
+    -----
+    For a *gap* with a fixed offset (``Gx = val − stat``, ``val`` fixed), fold the offset into
+    ``combine`` (``combine`` returns ``val − mean(m.values())``) so the CI is on ``Gx`` directly.
+
+    References
+    ----------
+    .. [1] Efron, B. & Tibshirani, R. "An Introduction to the Bootstrap." Chapman & Hall, 1993.
+           (§8 — stratified / clustered resampling.)
+    .. [2] Field, C. A. & Welsh, A. H. "Bootstrapping clustered data." JRSS-B 69(3), 2007.
+    """
+    if not strata:
+        raise ValueError("strata must be non-empty")
+    if not 0.0 < confidence < 1.0:
+        raise ValueError(f"confidence must be in (0, 1), got {confidence}")
+    resample_labels = tuple(int(x) for x in resample_labels)
+    if not resample_labels:
+        raise ValueError("resample_labels must be non-empty (nothing would be resampled)")
+
+    strata_data: dict[Hashable, tuple[np.ndarray, np.ndarray]] = {}
+    strata_units: dict[Hashable, dict[int, list[np.ndarray]]] = {}
+    for key, triple in strata.items():
+        y_a, s_a, g_a = (np.asarray(triple[0]), np.asarray(triple[1]), np.asarray(triple[2]))
+        if not (y_a.shape == s_a.shape == g_a.shape) or y_a.ndim != 1:
+            raise ValueError(
+                f"stratum {key!r}: y/score/groups must be aligned 1-D arrays; got "
+                f"{y_a.shape}, {s_a.shape}, {g_a.shape}"
+            )
+        strata_data[key] = (y_a, s_a)
+        strata_units[key] = _label_cluster_units(y_a, g_a)
+
+    point = float(combine({k: float(per_stratum_metric(*strata_data[k])) for k in strata_data}))
+    seed_seqs = spawn_seed_sequences(rng, n_resamples)
+    step = functools.partial(
+        _stratified_cluster_step,
+        strata_data=strata_data,
+        strata_units=strata_units,
+        per_stratum_metric=per_stratum_metric,
+        combine=combine,
+        resample_labels=resample_labels,
+    )
+    raw = parallel_map(
+        step, seed_seqs, n_jobs=n_jobs, description="stratified_cluster_bootstrap_ci"
+    )
+    failures = sum(1 for r in raw if r is None)
+    vals = [r for r in raw if r is not None]
+    if failures > 0.05 * n_resamples:
+        raise ValueError(
+            f"stratified_cluster_bootstrap_ci: {failures}/{n_resamples} resamples degenerate "
+            "(a per-stratum metric or combine raised); refusing CI on > 5% degenerate"
+        )
+    if not vals:
+        raise ValueError("stratified_cluster_bootstrap_ci: no usable resamples")
+    arr = np.asarray(vals, dtype=np.float64)
+    alpha = 1.0 - confidence
+    ci_low, ci_high = np.quantile(arr, [alpha / 2.0, 1.0 - alpha / 2.0])
+    return BootstrapCI(
+        point_estimate=point,
+        ci_low=float(ci_low),
+        ci_high=float(ci_high),
+        confidence=confidence,
+        n_resamples=int(len(vals)),
+        method="stratified_cluster_percentile",
+    )
+
+
 def cross_validate_metric(
     y_true: np.ndarray,
     y_score: np.ndarray,

{eval_toolkit-1.7.0 → eval_toolkit-1.8.0}/tests/golden/public_api/snapshot.json

@@ -242,6 +242,7 @@
     "skipped_metric",
     "source_role_gate",
     "split_provenance_config",
+    "stratified_cluster_bootstrap_ci",
     "stratified_recall",
     "strict_artifact_gate",
     "sweep",
@@ -1430,7 +1431,7 @@
       "doc_first_line": "str(object='') -> str",
       "kind": "value",
       "type": "str",
-      "value": "'1.7.0'"
+      "value": "'1.8.0'"
     },
     "apply_operating_points": {
       "doc_first_line": "Apply fitted thresholds to a mixed-class or single-class target slice.",
@@ -1947,6 +1948,11 @@
       "kind": "function",
       "signature": "(config: 'Mapping[str, Any]', repo_root: 'Path | str | None' = None, *, path_keys: 'tuple[str, ...]' = ('path', 'dir', 'file', 'splits_dir', 'model_path')) -> 'dict[str, Any]'"
     },
+    "stratified_cluster_bootstrap_ci": {
+      "doc_first_line": "Cluster bootstrap of a **composite** statistic over several independent strata.",
+      "kind": "function",
+      "signature": "(strata: 'Mapping[Hashable, tuple[np.ndarray, np.ndarray, np.ndarray]]', per_stratum_metric: 'MetricFn', combine: 'Callable[[Mapping[Hashable, float]], float]', *, resample_labels: 'tuple[int, ...]' = (0, 1), n_resamples: 'int' = 1000, confidence: 'float' = 0.95, rng: 'RNGLike | SeedLike | None' = 42, n_jobs: 'int' = 1) -> 'BootstrapCI'"
+    },
     "stratified_recall": {
       "doc_first_line": "Recall (TPR) per categorical stratum.",
       "kind": "function",

eval_toolkit-1.8.0/tests/test_stratified_cluster_bootstrap.py

@@ -0,0 +1,153 @@
+"""Tests for :func:`eval_toolkit.bootstrap.stratified_cluster_bootstrap_ci`.
+
+Covers the public contract, the **single-stratum identity reduction == `cluster_bootstrap_ci`**
+equivalence, the **seed-averaged gap** shape (`Gx = val − mean_seed(metric)`), a **composite
+top−bottom** statistic over strata, the v0.34.0 n_jobs-reproducibility contract, and validation.
+"""
+
+from __future__ import annotations
+
+import functools
+
+import numpy as np
+import pytest
+
+from eval_toolkit.bootstrap import (
+    BootstrapCI,
+    cluster_bootstrap_ci,
+    stratified_cluster_bootstrap_ci,
+)
+from eval_toolkit.metrics import roc_auc
+
+
+def _stratum(
+    seed: int, *, n_clusters: int = 30, per: int = 4
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Cluster-pure labels + a separable score with cluster-level noise (one resample-unit)."""
+    rng = np.random.default_rng(seed)
+    g = np.repeat(np.arange(n_clusters), per)
+    y = (g % 2).astype(int)
+    s = y + rng.normal(0, 0.3, size=y.size) + rng.normal(0, 0.3, size=n_clusters)[g]
+    return y, s, g
+
+
+def _mean_combine(m: dict) -> float:  # picklable (module-level)
+    return float(np.mean(list(m.values())))
+
+
+def _gap_combine(m: dict, *, val: float) -> float:  # picklable via functools.partial
+    return float(val - np.mean(list(m.values())))
+
+
+def _top_bottom_combine(m: dict, *, top: tuple, bottom: tuple) -> float:  # picklable via partial
+    return float(np.mean([m[k] for k in bottom]) - np.mean([m[k] for k in top]))
+
+
+@pytest.mark.unit
+def test_returns_bootstrap_ci_with_stratified_method() -> None:
+    """Basic contract: BootstrapCI, stratified_cluster_percentile method, ordered CI."""
+    strata = {0: _stratum(0), 1: _stratum(1)}
+    ci = stratified_cluster_bootstrap_ci(strata, roc_auc, _mean_combine, n_resamples=300, rng=0)
+    assert isinstance(ci, BootstrapCI)
+    assert ci.method == "stratified_cluster_percentile"
+    assert ci.ci_low <= ci.point_estimate <= ci.ci_high
+
+
+@pytest.mark.unit
+def test_single_stratum_identity_equals_cluster_bootstrap_ci() -> None:
+    """One stratum + identity reduce reproduces cluster_bootstrap_ci bit-for-bit (same rng path)."""
+    y, s, g = _stratum(7)
+    strat = stratified_cluster_bootstrap_ci(
+        {0: (y, s, g)}, roc_auc, lambda m: m[0], n_resamples=400, rng=11
+    )
+    single = cluster_bootstrap_ci(y, s, g, roc_auc, n_resamples=400, rng=11)
+    assert (strat.point_estimate, strat.ci_low, strat.ci_high) == (
+        single.point_estimate,
+        single.ci_low,
+        single.ci_high,
+    )
+
+
+@pytest.mark.unit
+def test_seed_averaged_gap_shape() -> None:
+    """The dialect/carrier shape: Gx = val − mean_seed(roc_auc) folded into combine."""
+    strata = {s: _stratum(s) for s in (0, 1, 2)}
+    val = 0.95
+    point_metric = float(np.mean([roc_auc(strata[s][0], strata[s][1]) for s in strata]))
+    ci = stratified_cluster_bootstrap_ci(
+        strata, roc_auc, functools.partial(_gap_combine, val=val), n_resamples=400, rng=0
+    )
+    assert ci.method == "stratified_cluster_percentile"
+    assert np.isclose(ci.point_estimate, val - point_metric)  # combine reduces correctly
+    assert ci.ci_low <= ci.ci_high
+
+
+@pytest.mark.unit
+def test_composite_top_bottom_statistic() -> None:
+    """The §6.5 shape: a top−bottom contrast over strata keyed by group (positives-only resample)."""
+    # 4 strata, separability ordered worst->best by key, positives resampled, negatives fixed.
+    strata = {k: _stratum(k, n_clusters=20) for k in range(4)}
+    ci = stratified_cluster_bootstrap_ci(
+        strata,
+        roc_auc,
+        functools.partial(_top_bottom_combine, top=(0, 1), bottom=(2, 3)),
+        resample_labels=(1,),
+        n_resamples=300,
+        rng=0,
+    )
+    assert ci.method == "stratified_cluster_percentile"
+    assert ci.ci_low <= ci.point_estimate <= ci.ci_high
+
+
+@pytest.mark.unit
+@pytest.mark.slow
+def test_njobs_reproducibility() -> None:
+    """Same seed → bit-for-bit-identical CI across n_jobs (spawn_seed_sequences)."""
+    strata = {0: _stratum(0), 1: _stratum(1), 2: _stratum(2)}
+    r1 = stratified_cluster_bootstrap_ci(
+        strata, roc_auc, _mean_combine, n_resamples=200, rng=42, n_jobs=1
+    )
+    r2 = stratified_cluster_bootstrap_ci(
+        strata, roc_auc, _mean_combine, n_resamples=200, rng=42, n_jobs=2
+    )
+    assert (r1.point_estimate, r1.ci_low, r1.ci_high) == (r2.point_estimate, r2.ci_low, r2.ci_high)
+
+
+@pytest.mark.unit
+@pytest.mark.slow
+def test_njobs_minus_one_runs() -> None:
+    """n_jobs=-1 (all cores) completes without error."""
+    strata = {0: _stratum(0), 1: _stratum(1)}
+    ci = stratified_cluster_bootstrap_ci(
+        strata, roc_auc, _mean_combine, n_resamples=100, rng=42, n_jobs=-1
+    )
+    assert ci.ci_low <= ci.ci_high
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    ("kwargs", "strata_override", "match"),
+    [
+        ({}, {}, "strata must be non-empty"),
+        ({"confidence": 1.5}, None, r"confidence must be in \(0, 1\)"),
+        ({"resample_labels": ()}, None, "non-empty"),
+        ({"n_jobs": 0}, None, "n_jobs"),
+    ],
+)
+def test_validation_errors(kwargs: dict, strata_override, match: str) -> None:
+    """Invalid parameters raise ValueError with a diagnostic message."""
+    strata = {} if strata_override == {} else {0: _stratum(0), 1: _stratum(1)}
+    with pytest.raises(ValueError, match=match):
+        stratified_cluster_bootstrap_ci(
+            strata, roc_auc, _mean_combine, n_resamples=50, rng=0, **kwargs
+        )
+
+
+@pytest.mark.unit
+def test_shape_mismatch_in_a_stratum_raises() -> None:
+    """A stratum with misaligned arrays raises ValueError naming the stratum."""
+    y, s, g = _stratum(0)
+    with pytest.raises(ValueError, match="aligned 1-D"):
+        stratified_cluster_bootstrap_ci(
+            {"bad": (y, s[:-1], g)}, roc_auc, _mean_combine, n_resamples=50, rng=0
+        )