eval-toolkit 1.6.0__tar.gz → 1.7.0__tar.gz

{eval_toolkit-1.6.0 → eval_toolkit-1.7.0}/CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.7.0] — 2026-06-04 — label-stratified cluster bootstrap (#90, #91)
+
+### Added — `bootstrap.cluster_bootstrap_ci` (label-stratified cluster bootstrap)
+
+`eval_toolkit.bootstrap` ADDITIVE per [ADR 0003](docs/source/adr/0003-stability-contract-and-gate3-methodology.md) — backward-compatible. Closes the gap between the row-level (`bootstrap_ci`) and fold-level (`block_bootstrap_on_folds`) resamplers: **the missing middle — resampling clusters of rows.**
+
+- **`cluster_bootstrap_ci(y_true, y_score, groups, statistic, *, resample_labels=(0, 1), …)`** — percentile CI for a single-condition metric that resamples whole **clusters** (`groups`) with replacement, so the CI is honest under intra-cluster correlation (prompts sharing one attack payload; a document contributing both a poisoned and a benign row). The resample unit is `(label, group)`: by default positive- and negative-clusters are resampled **separately** (never a single-class draw); `resample_labels=(1,)` resamples only positive clusters with negatives held fixed (the payload-cluster convention). Returns a `BootstrapCI` with `method="cluster_percentile"`.
+- **Parallel + reproducible:** built on `parallel_map` + `spawn_seed_sequences`, so `n_jobs` gives bit-for-bit-identical CIs across worker counts (the v0.34.0 reproducibility contract). `n_jobs=-1` uses all cores.
+- Motivation: the analytic row-level AUROC-difference CI (`delong_roc_variance`) assumes row independence and under-covers on clustered eval data (LODO transfer gaps with payload/document/page clusters). Dogfooded by the consumer portfolio's attack-type / carrier / dialect leave-one-group-out bootstraps (Rule of Three).
+- Exported via `from eval_toolkit import cluster_bootstrap_ci`; `__all__` + `_EXPORTS` updated; doctest + n_jobs-reproducibility tests; mypy-strict clean.
+
+### Fixed — stale `seed=` kwarg in 2 bootstrap benchmarks (#91)
+
+`tests/benchmarks/test_kernel_benchmarks.py` passed `seed=` to `bootstrap_ci` / `paired_bootstrap_diff`, but those parameters migrated to `rng=` (SPEC 7) — the two bootstrap benchmark tests `TypeError`'d on the nightly-benchmarks workflow (excluded from PR CI via `-m "not benchmark"`, so it went unnoticed). 2-line `seed=`→`rng=` rename.
+
 ## [1.6.0] — 2026-05-29 — Tier-2 `eda` Job-2 + Job-3: shortcut + shift diagnostics (#86, #87)
 
 `eval_toolkit.eda.*` ADDITIVE per [ADR 0003](docs/source/adr/0003-stability-contract-and-gate3-methodology.md) — Tier-2, backward-compatible. Completes the EDA-first analytic layer above the v1.5.0 Job-1 integrity gate: **Job-2** lexical shortcut diagnostics (`lexical_association`, #86) and **Job-3** distribution-shift quantification (`distribution_shift`, #87). Both are dogfooded by the consumer portfolio's pre-modeling OOD-wall prediction (V5 + V9).

{eval_toolkit-1.6.0 → eval_toolkit-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-toolkit
-Version: 1.6.0
+Version: 1.7.0
 Summary: Reusable evaluation contracts for binary classification: metrics, bootstrap CIs, calibration, artifacts, and evidence gates.
 Project-URL: Homepage, https://github.com/brandon-behring/eval-toolkit
 Project-URL: Documentation, https://brandon-behring.github.io/eval-toolkit/

{eval_toolkit-1.6.0 → eval_toolkit-1.7.0}/src/eval_toolkit/__init__.py

@@ -129,6 +129,7 @@ _EXPORTS: dict[str, str] = {
     "block_bootstrap_on_folds": "eval_toolkit.bootstrap",
     "bonferroni_correct": "eval_toolkit.bootstrap",
     "bootstrap_ci": "eval_toolkit.bootstrap",
+    "cluster_bootstrap_ci": "eval_toolkit.bootstrap",
     "correct_p_values": "eval_toolkit.bootstrap",
     "cross_validate_metric": "eval_toolkit.bootstrap",
     "cv_clt_ci": "eval_toolkit.bootstrap",

{eval_toolkit-1.6.0 → eval_toolkit-1.7.0}/src/eval_toolkit/_version.py

@@ -2,4 +2,4 @@
 
 __all__ = ["__version__"]
 
-__version__ = "1.6.0"
+__version__ = "1.7.0"

{eval_toolkit-1.6.0 → eval_toolkit-1.7.0}/src/eval_toolkit/bootstrap.py

@@ -52,6 +52,7 @@ __all__ = [
     "block_bootstrap_on_folds",
     "bonferroni_correct",
     "bootstrap_ci",
+    "cluster_bootstrap_ci",
     "correct_p_values",
     "cross_validate_metric",
     "cv_clt_ci",
@@ -1447,6 +1448,205 @@ def block_bootstrap_on_folds(
     )
 
 
+def _label_cluster_units(y_true: np.ndarray, groups: np.ndarray) -> dict[int, list[np.ndarray]]:
+    """Index rows by ``(label, group)``: per label, a list of per-group row-index arrays.
+
+    A group that appears under both labels contributes a **separate** index array to each label's
+    list — the resample unit is ``(label, group)``, so a mixed-label group (e.g. a document with
+    both a poisoned and a benign variant sharing one id) splits into one positive unit and one
+    negative unit, resampled independently. Helper for :func:`cluster_bootstrap_ci`.
+    """
+    units: dict[int, list[np.ndarray]] = {}
+    for lab in np.unique(y_true):
+        lab_rows = np.flatnonzero(y_true == lab)
+        order = np.argsort(groups[lab_rows], kind="stable")
+        sorted_rows = lab_rows[order]
+        sorted_groups = groups[lab_rows][order]
+        # Split at the boundaries between consecutive distinct group ids (post-sort).
+        cut = np.flatnonzero(sorted_groups[1:] != sorted_groups[:-1]) + 1
+        units[int(lab)] = np.split(sorted_rows, cut)
+    return units
+
+
+def _cluster_bootstrap_step(
+    seed_seq: np.random.SeedSequence,
+    *,
+    y_true: np.ndarray,
+    y_score: np.ndarray,
+    units: dict[int, list[np.ndarray]],
+    statistic: MetricFn,
+    resample_labels: tuple[int, ...],
+) -> float | None:
+    """One cluster-resampled draw of ``statistic`` (module-level for parallel_map picklability).
+
+    For each label, its ``(label, group)`` units are resampled with replacement when the label is
+    in ``resample_labels``, else all its rows are held fixed. Returns the statistic on the gathered
+    rows, or ``None`` if the draw is degenerate (statistic raises — e.g. a single-class draw).
+    """
+    rng = np.random.default_rng(seed_seq)
+    parts: list[np.ndarray] = []
+    for lab, group_rows in units.items():
+        if lab in resample_labels:
+            chosen = rng.integers(0, len(group_rows), size=len(group_rows))
+            parts.extend(group_rows[c] for c in chosen)
+        else:
+            parts.extend(group_rows)
+    idx = np.concatenate(parts)
+    try:
+        return float(statistic(y_true[idx], y_score[idx]))
+    except (ValueError, RuntimeError):
+        return None
+
+
+def cluster_bootstrap_ci(
+    y_true: np.ndarray,
+    y_score: np.ndarray,
+    groups: np.ndarray,
+    statistic: MetricFn,
+    *,
+    resample_labels: tuple[int, ...] = (0, 1),
+    n_resamples: int = DEFAULT_N_RESAMPLES,
+    confidence: float = DEFAULT_CONFIDENCE,
+    rng: RNGLike | SeedLike | None = DEFAULT_SEED,
+    n_jobs: int = 1,
+) -> BootstrapCI:
+    r"""Label-stratified **cluster** (group) bootstrap percentile CI for a single-condition metric.
+
+    Resamples whole ``groups`` (clusters) with replacement rather than individual rows, so the CI
+    is honest under intra-cluster correlation (multiple prompts sharing one attack payload; a
+    document contributing both a poisoned and a benign row). The resample unit is ``(label,
+    group)``: by default (``resample_labels=(0, 1)``) positive-clusters and negative-clusters are
+    resampled **separately**, preserving the per-class cluster split so a draw is never
+    single-class. Pass ``resample_labels=(1,)`` to resample only the positive clusters while holding
+    all negatives fixed (the payload-cluster convention).
+
+    Where :func:`bootstrap_ci` resamples **rows** (i.i.d. assumption) and
+    :func:`block_bootstrap_on_folds` resamples **per-fold scalars**, this resamples **clusters of
+    rows** — the missing middle for grouped eval data. The analytic row-level AUROC-difference CI
+    (:func:`delong_roc_variance`) assumes row independence and so under-covers on clustered data,
+    which is the motivation for this estimator.
+
+    Parameters
+    ----------
+    y_true : np.ndarray, shape (n,)
+        Binary labels in ``{0, 1}``.
+    y_score : np.ndarray, shape (n,)
+        Scores aligned with ``y_true``.
+    groups : np.ndarray, shape (n,)
+        Cluster id per row (any sortable dtype — ints or strings).
+    statistic : callable ``(y_true, y_score) -> float``
+        Metric to bootstrap (e.g. ``roc_auc``). Must be **picklable** when ``n_jobs != 1`` (a named
+        top-level function — lambdas / closures are rejected).
+    resample_labels : tuple[int, ...], optional
+        Which label strata are cluster-resampled (default ``(0, 1)`` — both). Labels not listed are
+        held fixed (all their rows always included). Must be non-empty.
+    n_resamples, confidence, rng : standard bootstrap params (``rng`` per SPEC 7).
+    n_jobs : int, optional
+        Parallel workers (default 1 — sequential). ``n_jobs=-1`` uses all cores; ``n_jobs=0`` is
+        rejected. Per-resample seeding via :func:`spawn_seed_sequences` makes the CI **bit-for-bit
+        identical across ``n_jobs``** for a fixed ``rng``. See :ref:`methodology/parallelism`.
+
+    Returns
+    -------
+    BootstrapCI
+        ``method="cluster_percentile"``; ``point_estimate = statistic(y_true, y_score)`` on the full
+        data; ``[alpha/2, 1 - alpha/2]`` percentile CI over the cluster-resampled distribution.
+
+    Raises
+    ------
+    ValueError
+        On shape mismatch, non-1-D input, ``n < 10``, ``confidence`` outside (0, 1), empty
+        ``resample_labels``, a ``resample_labels`` entry absent from ``y_true``, ``n_jobs == 0``, or
+        > 5% degenerate resamples.
+    TypeError
+        If ``n_jobs != 1`` and ``statistic`` is not picklable.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> from eval_toolkit.metrics import roc_auc
+    >>> rng = np.random.default_rng(0)
+    >>> groups = np.repeat(np.arange(40), 5)        # 40 clusters of 5 rows
+    >>> y = (groups % 2).astype(int)                # cluster-pure labels
+    >>> s = y + rng.normal(0, 0.3, size=y.size)
+    >>> ci = cluster_bootstrap_ci(y, s, groups, roc_auc, n_resamples=200, rng=0)
+    >>> ci.method
+    'cluster_percentile'
+    >>> bool(0.0 <= ci.ci_low <= ci.ci_high <= 1.0)
+    True
+
+    Notes
+    -----
+    For a *gap* statistic with a fixed offset (e.g. ``Gx = val_auc − test_auc`` with ``val_auc``
+    held fixed), bootstrap the variable term and shift the bounds: ``Gx_low = val_auc − ci.ci_high``,
+    ``Gx_high = val_auc − ci.ci_low``. For a one-sided 95% bound, pass ``confidence=0.90`` and read
+    the relevant bound.
+
+    References
+    ----------
+    .. [1] Efron, B. & Tibshirani, R. "An Introduction to the Bootstrap." Chapman & Hall, 1993.
+           (§8 — bootstrapping stratified / clustered data.)
+    .. [2] Field, C. A. & Welsh, A. H. "Bootstrapping clustered data." JRSS-B 69(3), 2007.
+    """
+    y_true_arr = np.asarray(y_true)
+    y_score_arr = np.asarray(y_score)
+    groups_arr = np.asarray(groups)
+    if not (y_true_arr.shape == y_score_arr.shape == groups_arr.shape):
+        raise ValueError(
+            f"shapes mismatch: y_true {y_true_arr.shape}, y_score {y_score_arr.shape}, "
+            f"groups {groups_arr.shape}"
+        )
+    if y_true_arr.ndim != 1:
+        raise ValueError(f"inputs must be 1-D; got shape {y_true_arr.shape}")
+    n = int(y_true_arr.size)
+    if n < 10:
+        raise ValueError(f"n={n} too small for cluster bootstrap; need ≥ 10")
+    if not 0.0 < confidence < 1.0:
+        raise ValueError(f"confidence must be in (0, 1), got {confidence}")
+    resample_labels = tuple(int(x) for x in resample_labels)
+    if not resample_labels:
+        raise ValueError("resample_labels must be non-empty (nothing would be resampled)")
+    present = {int(v) for v in np.unique(y_true_arr).tolist()}
+    missing = set(resample_labels) - present
+    if missing:
+        raise ValueError(
+            f"resample_labels {sorted(missing)} absent from y_true (present: {sorted(present)})"
+        )
+
+    point = float(statistic(y_true_arr, y_score_arr))
+    units = _label_cluster_units(y_true_arr, groups_arr)
+    seed_seqs = spawn_seed_sequences(rng, n_resamples)
+    step = functools.partial(
+        _cluster_bootstrap_step,
+        y_true=y_true_arr,
+        y_score=y_score_arr,
+        units=units,
+        statistic=statistic,
+        resample_labels=resample_labels,
+    )
+    raw = parallel_map(step, seed_seqs, n_jobs=n_jobs, description="cluster_bootstrap_ci")
+    failures = sum(1 for r in raw if r is None)
+    vals = [r for r in raw if r is not None]
+    if failures > 0.05 * n_resamples:
+        raise ValueError(
+            f"cluster_bootstrap_ci: {failures}/{n_resamples} resamples degenerate "
+            "(statistic raised — e.g. single-class draws); refusing to compute CI on > 5% degenerate"
+        )
+    if not vals:
+        raise ValueError("cluster_bootstrap_ci: no usable resamples")
+    arr = np.asarray(vals, dtype=np.float64)
+    alpha = 1.0 - confidence
+    ci_low, ci_high = np.quantile(arr, [alpha / 2.0, 1.0 - alpha / 2.0])
+    return BootstrapCI(
+        point_estimate=point,
+        ci_low=float(ci_low),
+        ci_high=float(ci_high),
+        confidence=confidence,
+        n_resamples=int(len(vals)),
+        method="cluster_percentile",
+    )
+
+
 def cross_validate_metric(
     y_true: np.ndarray,
     y_score: np.ndarray,

{eval_toolkit-1.6.0 → eval_toolkit-1.7.0}/tests/benchmarks/test_kernel_benchmarks.py

@@ -136,7 +136,7 @@ def test_benchmark_bootstrap_ci_pr_auc_n200(
     y, s = yspc_n200
 
     def _run() -> float:
-        return bootstrap_ci(y, s, metric=pr_auc, n_resamples=200, seed=42).point_estimate
+        return bootstrap_ci(y, s, metric=pr_auc, n_resamples=200, rng=42).point_estimate
 
     result = benchmark(_run)
     assert 0.0 <= result <= 1.0
@@ -155,7 +155,7 @@ def test_benchmark_paired_bootstrap_diff_pr_auc_n200(
     y, s_a, s_b = y_two_scorers_n200
 
     def _run() -> float:
-        return paired_bootstrap_diff(y, s_a, s_b, metric=pr_auc, n_resamples=200, seed=42).delta
+        return paired_bootstrap_diff(y, s_a, s_b, metric=pr_auc, n_resamples=200, rng=42).delta
 
     result = benchmark(_run)
     # Delta can be negative (B worse than A) — just verify it's finite + in expected range

{eval_toolkit-1.6.0 → eval_toolkit-1.7.0}/tests/golden/public_api/snapshot.json

@@ -148,6 +148,7 @@
     "bootstrap_metric_from_predictions",
     "brier_decomposition",
     "capture_git_sha",
+    "cluster_bootstrap_ci",
     "compute_file_hash",
     "compute_label_overlap",
     "correct_p_values",
@@ -1429,7 +1430,7 @@
       "doc_first_line": "str(object='') -> str",
       "kind": "value",
       "type": "str",
-      "value": "'1.6.0'"
+      "value": "'1.7.0'"
     },
     "apply_operating_points": {
       "doc_first_line": "Apply fitted thresholds to a mixed-class or single-class target slice.",
@@ -1476,6 +1477,11 @@
       "kind": "function",
       "signature": "(repo_root: 'Path | str | None' = None) -> 'str | None'"
     },
+    "cluster_bootstrap_ci": {
+      "doc_first_line": "Label-stratified **cluster** (group) bootstrap percentile CI for a single-condition metric.",
+      "kind": "function",
+      "signature": "(y_true: 'np.ndarray', y_score: 'np.ndarray', groups: 'np.ndarray', statistic: 'MetricFn', *, resample_labels: 'tuple[int, ...]' = (0, 1), n_resamples: 'int' = 1000, confidence: 'float' = 0.95, rng: 'RNGLike | SeedLike | None' = 42, n_jobs: 'int' = 1) -> 'BootstrapCI'"
+    },
     "compute_file_hash": {
       "doc_first_line": "SHA-256 hex digest of an existing file (sentinel-typed).",
       "kind": "function",

eval_toolkit-1.7.0/tests/test_cluster_bootstrap.py

@@ -0,0 +1,131 @@
+"""Tests for :func:`eval_toolkit.bootstrap.cluster_bootstrap_ci`.
+
+Covers the public contract (percentile CI, method tag, validation), the ``(label, group)``
+resample-unit semantics (mixed-label groups split by label), the ``resample_labels`` knob, the
+v0.34.0 n_jobs-reproducibility contract (same seed → identical CI across worker counts), and the
+methodological reason the function exists: under strong intra-cluster correlation the cluster
+bootstrap CI is **wider** than a naive row bootstrap (which would under-cover).
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from eval_toolkit.bootstrap import (
+    BootstrapCI,
+    _label_cluster_units,
+    bootstrap_ci,
+    cluster_bootstrap_ci,
+)
+from eval_toolkit.metrics import roc_auc
+
+
+def _clustered_inputs(
+    n_clusters: int = 40, rows_per_cluster: int = 5, *, seed: int = 0
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    """Cluster-pure labels (cluster parity) + a separable score with cluster-level noise."""
+    rng = np.random.default_rng(seed)
+    groups = np.repeat(np.arange(n_clusters), rows_per_cluster)
+    y = (groups % 2).astype(int)
+    cluster_offset = rng.normal(0, 0.5, size=n_clusters)[groups]
+    s = y + cluster_offset + rng.normal(0, 0.2, size=y.size)
+    return y, s, groups
+
+
+@pytest.mark.unit
+def test_returns_bootstrap_ci_with_cluster_method() -> None:
+    """Basic contract: BootstrapCI, cluster_percentile method, ordered CI, point in range."""
+    y, s, g = _clustered_inputs()
+    ci = cluster_bootstrap_ci(y, s, g, roc_auc, n_resamples=300, rng=0)
+    assert isinstance(ci, BootstrapCI)
+    assert ci.method == "cluster_percentile"
+    assert ci.n_resamples == 300
+    assert 0.0 <= ci.ci_low <= ci.ci_high <= 1.0
+    assert ci.ci_low <= ci.point_estimate <= ci.ci_high
+
+
+@pytest.mark.unit
+def test_label_cluster_units_splits_mixed_label_group() -> None:
+    """A group id present under both labels splits into one positive unit and one negative unit."""
+    y = np.array([1, 1, 0, 0, 1])
+    groups = np.array(["g0", "g0", "g0", "g1", "g2"])  # g0 is mixed-label
+    units = _label_cluster_units(y, groups)
+    assert set(np.concatenate(units[1]).tolist()) == {0, 1, 4}  # positives: g0 + g2
+    assert set(np.concatenate(units[0]).tolist()) == {2, 3}  # negatives: g0 + g1
+    assert len(units[1]) == 2 and len(units[0]) == 2  # g0 counted on both sides
+
+
+@pytest.mark.unit
+def test_resample_labels_positive_only_holds_negatives_fixed() -> None:
+    """resample_labels=(1,) resamples positive clusters and keeps all negatives fixed."""
+    y, s, g = _clustered_inputs()
+    ci_pos = cluster_bootstrap_ci(y, s, g, roc_auc, resample_labels=(1,), n_resamples=300, rng=0)
+    ci_both = cluster_bootstrap_ci(y, s, g, roc_auc, resample_labels=(0, 1), n_resamples=300, rng=0)
+    # Both valid CIs; holding negatives fixed removes one variance source → no wider than both-sides.
+    assert ci_pos.ci_low <= ci_pos.ci_high
+    assert (ci_pos.ci_high - ci_pos.ci_low) <= (ci_both.ci_high - ci_both.ci_low) + 1e-9
+
+
+@pytest.mark.unit
+@pytest.mark.slow
+def test_njobs_reproducibility() -> None:
+    """Same seed produces a bit-for-bit-identical CI regardless of n_jobs (spawn_seed_sequences)."""
+    y, s, g = _clustered_inputs()
+    r1 = cluster_bootstrap_ci(y, s, g, roc_auc, n_resamples=200, rng=42, n_jobs=1)
+    r2 = cluster_bootstrap_ci(y, s, g, roc_auc, n_resamples=200, rng=42, n_jobs=2)
+    assert (r1.point_estimate, r1.ci_low, r1.ci_high) == (r2.point_estimate, r2.ci_low, r2.ci_high)
+
+
+@pytest.mark.unit
+@pytest.mark.slow
+def test_njobs_minus_one_runs() -> None:
+    """n_jobs=-1 (all cores) completes without error."""
+    y, s, g = _clustered_inputs()
+    ci = cluster_bootstrap_ci(y, s, g, roc_auc, n_resamples=100, rng=42, n_jobs=-1)
+    assert ci.ci_low <= ci.ci_high
+
+
+@pytest.mark.unit
+def test_cluster_ci_wider_than_row_under_intracluster_correlation() -> None:
+    """The reason the function exists: with few, strongly-correlated clusters the cluster bootstrap
+    CI is wider than a naive row bootstrap (which would under-cover by treating rows as i.i.d.)."""
+    # 10 clusters × 30 rows; the cluster-level offset dominates → rows within a cluster are highly
+    # correlated, so the effective sample size is ~10 clusters, not 300 rows.
+    rng = np.random.default_rng(7)
+    n_clusters, per = 10, 30
+    groups = np.repeat(np.arange(n_clusters), per)
+    y = (groups % 2).astype(int)
+    s = y + rng.normal(0, 0.8, size=n_clusters)[groups] + rng.normal(0, 0.05, size=y.size)
+    cluster = cluster_bootstrap_ci(y, s, groups, roc_auc, n_resamples=400, rng=0)
+    row = bootstrap_ci(y, s, roc_auc, n_resamples=400, rng=0, method="percentile")
+    cluster_width = cluster.ci_high - cluster.ci_low
+    row_width = row.ci_high - row.ci_low
+    assert cluster_width > row_width, f"cluster {cluster_width:.3f} !> row {row_width:.3f}"
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    ("kwargs", "match"),
+    [
+        ({"resample_labels": ()}, "non-empty"),
+        ({"resample_labels": (2,)}, "absent from y_true"),
+        ({"confidence": 1.5}, r"confidence must be in \(0, 1\)"),
+        ({"n_jobs": 0}, "n_jobs"),
+    ],
+)
+def test_validation_errors(kwargs: dict[str, object], match: str) -> None:
+    """Invalid parameters raise ValueError with a diagnostic message."""
+    y, s, g = _clustered_inputs()
+    with pytest.raises(ValueError, match=match):
+        cluster_bootstrap_ci(y, s, g, roc_auc, n_resamples=50, rng=0, **kwargs)  # type: ignore[arg-type]
+
+
+@pytest.mark.unit
+def test_shape_and_size_validation() -> None:
+    """Shape mismatch and n < 10 raise ValueError."""
+    y, s, g = _clustered_inputs(n_clusters=4, rows_per_cluster=3)  # n=12
+    with pytest.raises(ValueError, match="shapes mismatch"):
+        cluster_bootstrap_ci(y, s[:-1], g, roc_auc, n_resamples=50, rng=0)
+    with pytest.raises(ValueError, match="too small"):
+        cluster_bootstrap_ci(y[:8], s[:8], g[:8], roc_auc, n_resamples=50, rng=0)