surestop 0.2.0__tar.gz

surestop-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Adam Noonan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

surestop-0.2.0/PKG-INFO

@@ -0,0 +1,173 @@
+Metadata-Version: 2.4
+Name: surestop
+Version: 0.2.0
+Summary: Stop bad runs early, with a conformal bound on how often you stop one that would have ended well
+Author: Adam Noonan
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/ACNoonan/surestop
+Project-URL: Issues, https://github.com/ACNoonan/surestop/issues
+Keywords: hyperparameter-optimization,early-stopping,pruning,conformal,optuna
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Provides-Extra: optuna
+Requires-Dist: optuna>=4; extra == "optuna"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: optuna>=4; extra == "test"
+Dynamic: license-file
+
+# SureStop
+
+**Stop bad training runs early, with a stated bound on how often you stop one that would have
+ended well.** You pick α. The rule kills runs so that the expected fraction of runs that were
+good *and* got killed stays at or below α.
+
+Every pruner in common use (median, percentile, successive halving, Hyperband) is a heuristic,
+and none says how often it kills a run that would have finished well. This one does, under a
+condition you can check: the runs you calibrate on and the runs you judge must come from the same
+random process.
+
+It was measured on 400 real LoRA fine-tuning runs under a pre-registered protocol. The curves are
+in this repository, and a script checks that the package reproduces the recorded numbers exactly.
+
+## Install
+
+```
+pip install surestop            # numpy + scipy
+pip install "surestop[optuna]"  # adds the Optuna pruner
+```
+
+## Use
+
+Plain numpy, on any training loop:
+
+```python
+from surestop import KillRule
+
+# completed_curves: array of shape (runs, evaluations), lower is better
+rule = KillRule(alpha=0.05, good_threshold=2.16, min_peek=17).fit(completed_curves)
+
+rule.should_kill(curve_so_far)     # True -> stop this run now
+rule.kill_step(full_curve)         # where it would have fired, or None
+rule.evaluate(held_out_curves)     # false kills, joint rate, savings
+```
+
+With Optuna, the first `n_calibration` trials run unpruned to calibrate, then the rule freezes:
+
+```python
+import optuna
+from surestop import ConformalPruner
+
+study = optuna.create_study(sampler=optuna.samplers.RandomSampler(),
+                            pruner=ConformalPruner(n_calibration=60, alpha=0.05))
+```
+
+Pass `maximize=True` (or a `direction="maximize"` study) when higher is better.
+
+## The rule
+
+A run is *good* if its final value ends at or below a threshold τ. At every evaluation, a
+predictor forecasts the final value from the curve so far. The default predictor is the latest
+observed value.
+
+To calibrate, take n runs that have already finished. For each good one, record its worst moment:
+the largest gap between forecast and τ at any evaluation. Runs that are not good contribute
+nothing. Set λ to the ⌈(1−α)(n+1)⌉-th smallest of these scores. A new run is killed at the first
+evaluation where its forecast exceeds τ + λ.
+
+This is conformal risk control on the loss "good and killed". Taking the worst moment across all
+evaluations means one threshold covers every look at a run, so there is no correction for
+multiple looks. The guarantee needs the calibration runs and the new runs to be exchangeable.
+Random sampling gives that. Adaptive samplers such as TPE do not.
+
+## What was measured
+
+Experiment CC-03c, pre-registered before any run finished:
+
+- **Runs:** 400 LoRA fine-tunes of Qwen2.5-0.5B on dolly-15k, 750 steps each, validation loss
+  measured 50 times per run.
+- **Configs:** i.i.d. draws from a frozen hyperparameter prior (`docs/hyperparameter-prior.md`).
+- **τ:** pinned from an earlier pool before the first run landed.
+- **Splits:** two, one random and one past-to-future by night, each about half calibration and
+  half test.
+
+The pre-registered rule used a fitted power-law predictor. On both splits it kept the bound and
+saved about half the evaluation compute:
+
+| split | false kills | joint rate (95% CI) | eval compute saved |
+|---|---|---|---|
+| random | 13 of 200 | 6.5% (3.5–10.9%) | 52.2% |
+| past-to-future | 11 of 203 | 5.4% (2.7–9.5%) | 52.8% |
+
+A rule that kills every run immediately was scored by the same gate and failed on both splits.
+
+**The flaw that run found:** every kill fired at the 3rd of 50 evaluations. Early rankings only
+match final rankings from about evaluation 18. The rule stayed within budget because the bound
+held. The fitted power law ranked runs worse than the latest observed value at every early
+evaluation.
+
+**What this package ships instead,** and what the reproduction script checks:
+
+| predictor | hold until | random split | past-to-future split | 2000 resplits |
+|---|---|---|---|---|
+| last value | none | 18 of 200 (9.0%), 78% saved | 7 of 203 (3.4%), 75% saved | joint 5.0%, breach 8.5% |
+| last value | eval 18 | 18 of 200 (9.0%), 57% saved | 7 of 203 (3.4%), 56% saved | joint 5.0%, breach 8.6% |
+
+The random split's 18 of 200 is a breach. It sits at the 95th percentile of that rule's own
+resplit distribution, whose mean is 5.0%, so it reads as a split tail rather than a leak. A
+correct rule at this sample size lands on a breach in about 7% of splits. The held version was
+chosen *after* seeing these 400 runs, so its numbers are a hypothesis. It halves
+how often the single best run is killed at the same budget. Its confirmation needs a fresh pool.
+
+Full account: `docs/writeup.md`. Run the check yourself:
+
+```
+python reproduce/reproduce_cc03c.py
+```
+
+## What the bound does not cover
+
+- **It needs exchangeable runs.** With a random or quasi-random sampler the bound holds. With
+  TPE, Gaussian-process or CMA-ES samplers it does not. The Optuna pruner warns once and carries
+  on, but do not rely on the bound there.
+- **It bounds the joint rate rather than the share of good runs killed.** When good runs are rare, a large
+  share of them can still die: about a third in the measurement above, mostly borderline ones.
+  Lower α if that matters.
+- **An estimated τ gives a measured bound rather than a proven one.** `good_quantile` estimates τ from the
+  calibration runs. Simulations put the realized rate at 0.88 to 0.97 times α. Pass a real
+  `good_threshold` when you have one.
+- **α = 0.05 needs at least 19 calibration runs**, and the certified threshold gets loose with
+  few good runs. Sweeps of a few dozen trials are too small for this.
+- **One setting only so far.** One model, one dataset, one horizon.
+
+## Layout
+
+| path | what |
+|---|---|
+| `src/surestop/` | `KillRule` (numpy/scipy) and `ConformalPruner` (Optuna) |
+| `tests/` | the bound holds on synthetic exchangeable runs, and the same assertion fails on a rule that leaks 2× its budget |
+| `reproduce/` | the 400 curves, the pre-registered splits and targets, the script, and its committed output |
+| `docs/` | the write-up, the prior-art sweep, and the frozen hyperparameter prior the curves were drawn from |
+| `optunahub/` | the package as an OptunaHub registry entry |
+
+## Related work
+
+Calibrating a stop threshold on a predictor with a conformal guarantee is an established pattern. Xie et al.
+(arXiv:2602.13935) use the same device, a threshold on the maximum of a running score calibrated
+on one class only, to stop LLM reasoning traces. Related work does the same for mixed-integer
+solvers and LLM agent episodes. As far as a search found, nobody has applied it to pruning
+training runs, and no major tuning library ships a pruner with a stated error guarantee.
+`docs/prior-art.md` has the sweep.
+
+## Citing
+
+Concept DOI: [10.5281/zenodo.22726440](https://doi.org/10.5281/zenodo.22726440), which always resolves
+to the newest version. `CITATION.cff` has the full entry. The curves are a measurement, released
+under the same MIT licence as the code.

surestop-0.2.0/README.md

@@ -0,0 +1,148 @@
+# SureStop
+
+**Stop bad training runs early, with a stated bound on how often you stop one that would have
+ended well.** You pick α. The rule kills runs so that the expected fraction of runs that were
+good *and* got killed stays at or below α.
+
+Every pruner in common use (median, percentile, successive halving, Hyperband) is a heuristic,
+and none says how often it kills a run that would have finished well. This one does, under a
+condition you can check: the runs you calibrate on and the runs you judge must come from the same
+random process.
+
+It was measured on 400 real LoRA fine-tuning runs under a pre-registered protocol. The curves are
+in this repository, and a script checks that the package reproduces the recorded numbers exactly.
+
+## Install
+
+```
+pip install surestop            # numpy + scipy
+pip install "surestop[optuna]"  # adds the Optuna pruner
+```
+
+## Use
+
+Plain numpy, on any training loop:
+
+```python
+from surestop import KillRule
+
+# completed_curves: array of shape (runs, evaluations), lower is better
+rule = KillRule(alpha=0.05, good_threshold=2.16, min_peek=17).fit(completed_curves)
+
+rule.should_kill(curve_so_far)     # True -> stop this run now
+rule.kill_step(full_curve)         # where it would have fired, or None
+rule.evaluate(held_out_curves)     # false kills, joint rate, savings
+```
+
+With Optuna, the first `n_calibration` trials run unpruned to calibrate, then the rule freezes:
+
+```python
+import optuna
+from surestop import ConformalPruner
+
+study = optuna.create_study(sampler=optuna.samplers.RandomSampler(),
+                            pruner=ConformalPruner(n_calibration=60, alpha=0.05))
+```
+
+Pass `maximize=True` (or a `direction="maximize"` study) when higher is better.
+
+## The rule
+
+A run is *good* if its final value ends at or below a threshold τ. At every evaluation, a
+predictor forecasts the final value from the curve so far. The default predictor is the latest
+observed value.
+
+To calibrate, take n runs that have already finished. For each good one, record its worst moment:
+the largest gap between forecast and τ at any evaluation. Runs that are not good contribute
+nothing. Set λ to the ⌈(1−α)(n+1)⌉-th smallest of these scores. A new run is killed at the first
+evaluation where its forecast exceeds τ + λ.
+
+This is conformal risk control on the loss "good and killed". Taking the worst moment across all
+evaluations means one threshold covers every look at a run, so there is no correction for
+multiple looks. The guarantee needs the calibration runs and the new runs to be exchangeable.
+Random sampling gives that. Adaptive samplers such as TPE do not.
+
+## What was measured
+
+Experiment CC-03c, pre-registered before any run finished:
+
+- **Runs:** 400 LoRA fine-tunes of Qwen2.5-0.5B on dolly-15k, 750 steps each, validation loss
+  measured 50 times per run.
+- **Configs:** i.i.d. draws from a frozen hyperparameter prior (`docs/hyperparameter-prior.md`).
+- **τ:** pinned from an earlier pool before the first run landed.
+- **Splits:** two, one random and one past-to-future by night, each about half calibration and
+  half test.
+
+The pre-registered rule used a fitted power-law predictor. On both splits it kept the bound and
+saved about half the evaluation compute:
+
+| split | false kills | joint rate (95% CI) | eval compute saved |
+|---|---|---|---|
+| random | 13 of 200 | 6.5% (3.5–10.9%) | 52.2% |
+| past-to-future | 11 of 203 | 5.4% (2.7–9.5%) | 52.8% |
+
+A rule that kills every run immediately was scored by the same gate and failed on both splits.
+
+**The flaw that run found:** every kill fired at the 3rd of 50 evaluations. Early rankings only
+match final rankings from about evaluation 18. The rule stayed within budget because the bound
+held. The fitted power law ranked runs worse than the latest observed value at every early
+evaluation.
+
+**What this package ships instead,** and what the reproduction script checks:
+
+| predictor | hold until | random split | past-to-future split | 2000 resplits |
+|---|---|---|---|---|
+| last value | none | 18 of 200 (9.0%), 78% saved | 7 of 203 (3.4%), 75% saved | joint 5.0%, breach 8.5% |
+| last value | eval 18 | 18 of 200 (9.0%), 57% saved | 7 of 203 (3.4%), 56% saved | joint 5.0%, breach 8.6% |
+
+The random split's 18 of 200 is a breach. It sits at the 95th percentile of that rule's own
+resplit distribution, whose mean is 5.0%, so it reads as a split tail rather than a leak. A
+correct rule at this sample size lands on a breach in about 7% of splits. The held version was
+chosen *after* seeing these 400 runs, so its numbers are a hypothesis. It halves
+how often the single best run is killed at the same budget. Its confirmation needs a fresh pool.
+
+Full account: `docs/writeup.md`. Run the check yourself:
+
+```
+python reproduce/reproduce_cc03c.py
+```
+
+## What the bound does not cover
+
+- **It needs exchangeable runs.** With a random or quasi-random sampler the bound holds. With
+  TPE, Gaussian-process or CMA-ES samplers it does not. The Optuna pruner warns once and carries
+  on, but do not rely on the bound there.
+- **It bounds the joint rate rather than the share of good runs killed.** When good runs are rare, a large
+  share of them can still die: about a third in the measurement above, mostly borderline ones.
+  Lower α if that matters.
+- **An estimated τ gives a measured bound rather than a proven one.** `good_quantile` estimates τ from the
+  calibration runs. Simulations put the realized rate at 0.88 to 0.97 times α. Pass a real
+  `good_threshold` when you have one.
+- **α = 0.05 needs at least 19 calibration runs**, and the certified threshold gets loose with
+  few good runs. Sweeps of a few dozen trials are too small for this.
+- **One setting only so far.** One model, one dataset, one horizon.
+
+## Layout
+
+| path | what |
+|---|---|
+| `src/surestop/` | `KillRule` (numpy/scipy) and `ConformalPruner` (Optuna) |
+| `tests/` | the bound holds on synthetic exchangeable runs, and the same assertion fails on a rule that leaks 2× its budget |
+| `reproduce/` | the 400 curves, the pre-registered splits and targets, the script, and its committed output |
+| `docs/` | the write-up, the prior-art sweep, and the frozen hyperparameter prior the curves were drawn from |
+| `optunahub/` | the package as an OptunaHub registry entry |
+
+## Related work
+
+Calibrating a stop threshold on a predictor with a conformal guarantee is an established pattern. Xie et al.
+(arXiv:2602.13935) use the same device, a threshold on the maximum of a running score calibrated
+on one class only, to stop LLM reasoning traces. Related work does the same for mixed-integer
+solvers and LLM agent episodes. As far as a search found, nobody has applied it to pruning
+training runs, and no major tuning library ships a pruner with a stated error guarantee.
+`docs/prior-art.md` has the sweep.
+
+## Citing
+
+Concept DOI: [10.5281/zenodo.22726440](https://doi.org/10.5281/zenodo.22726440), which always resolves
+to the newest version. `CITATION.cff` has the full entry. The curves are a measurement, released
+under the same MIT licence as the code.

surestop-0.2.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "surestop"
+version = "0.2.0"
+description = "Stop bad runs early, with a conformal bound on how often you stop one that would have ended well"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Adam Noonan" }]
+requires-python = ">=3.10"
+dependencies = ["numpy>=1.24", "scipy>=1.10"]
+keywords = ["hyperparameter-optimization", "early-stopping", "pruning", "conformal", "optuna"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Science/Research",
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.optional-dependencies]
+optuna = ["optuna>=4"]
+test = ["pytest", "optuna>=4"]
+
+[project.urls]
+Homepage = "https://github.com/ACNoonan/surestop"
+Issues = "https://github.com/ACNoonan/surestop/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["surestop*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

surestop-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

surestop-0.2.0/src/surestop/__init__.py

@@ -0,0 +1,17 @@
+"""surestop — stop bad runs early, with a bound on how often you stop one that would have ended well."""
+
+from .core import KillRule, NotEnoughCalibration, last_value
+
+__version__ = "0.2.0"
+__all__ = ["KillRule", "NotEnoughCalibration", "last_value", "ConformalPruner"]
+
+
+from typing import Any
+
+
+def __getattr__(name: str) -> Any:
+    # Optuna is optional: import the pruner only when someone asks for it.
+    if name == "ConformalPruner":
+        from .optuna_pruner import ConformalPruner
+        return ConformalPruner
+    raise AttributeError(name)

surestop-0.2.0/src/surestop/core.py

@@ -0,0 +1,201 @@
+"""A kill rule for training runs that bounds how often it kills a run that would have ended well.
+
+The construction is the one CC-03c pre-registered and measured on 400 real LoRA runs
+(`experiments/2026-08-17-CC-03c-validity-gate-that-can-fire/`):
+
+1. Calibrate on n **completed** runs, each a learning curve on a shared step grid.
+2. A run is *good* if its final value is at or below a threshold τ (lower is better).
+3. A predictor forecasts the final value from the curve so far, at every step.
+4. Each good calibration run scores max over eligible steps of (prediction − τ).
+   Runs that are not good score −∞.
+5. λ̂ is the ⌈(1−α)(n+1)⌉-th smallest score. This is conformal risk control on the loss
+   "this run is good AND the rule kills it".
+6. A new run is killed at the first eligible step where prediction − λ̂ > τ.
+
+**Guarantee.** If the calibration runs and the new runs are exchangeable — for example, drawn
+i.i.d. by the same random sampler — then the expected fraction of new runs that are good and
+killed is at most α. One kill decision covers every step at once, with no multiplicity
+correction.
+
+**What it does not promise.**
+- **It bounds the joint rate, not the share of good runs killed.** With a small good fraction,
+  the conditional rate can be large: ~30% on CC-03c.
+- **It needs exchangeability.** An adaptive sampler such as TPE shifts later runs away from the
+  calibration runs, and the guarantee no longer applies.
+- **With τ estimated from the calibration finals** (`good_quantile`), the bound is measured, not
+  proven: 0.88–0.97× α in CC-03c's simulations. Pass `good_threshold` when you have a real target.
+"""
+
+from __future__ import annotations
+
+import math
+import warnings
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+import numpy as np
+from numpy.typing import ArrayLike
+from scipy.stats import rankdata
+
+Predictor = Callable[[np.ndarray], np.ndarray]
+
+
+def last_value(curves: ArrayLike) -> np.ndarray:
+    """Forecast the final value as the latest observed value.
+
+    A predictor maps curves of shape (n, T) to forecasts of shape (n, T) and must be causal:
+    column t may use only columns 0..t. On CC-03c's pool this beat a fitted pow3 curve at every
+    early step.
+    """
+    return np.asarray(curves, dtype=float)
+
+
+class NotEnoughCalibration(ValueError):
+    """Too few calibration runs for any threshold to certify at this α."""
+
+
+def _spearman_cols(M: np.ndarray, y: np.ndarray) -> np.ndarray:
+    rm = rankdata(M, axis=0)
+    rm -= rm.mean(axis=0)
+    ry = rankdata(y)
+    ry -= ry.mean()
+    denom = np.sqrt((rm ** 2).sum(axis=0) * (ry ** 2).sum())
+    with np.errstate(invalid="ignore", divide="ignore"):
+        return np.where(denom > 0, (rm * ry[:, None]).sum(axis=0) / denom, 0.0)
+
+
+@dataclass
+class KillRule:
+    """Certified early-kill rule. Fit on completed curves, then ask `should_kill` on partial ones.
+
+    Parameters
+    ----------
+    alpha : bound on the expected fraction of runs that are good and killed.
+    good_threshold : a run is good if its final value is at or below this. Overrides `good_quantile`.
+    good_quantile : if no threshold is given, τ is this quantile of the calibration finals.
+    min_peek : index of the earliest step at which a kill may fire.
+    hold_until_resolved : if set (e.g. 0.90), also hold kills until the first step at or after
+        `min_peek` where Spearman(prediction, final) on the calibration runs reaches this value.
+        Estimated per calibration set, so it varies between sets; pin `min_peek` instead when a
+        prior pool tells you where the ranking settles.
+    maximize : set True when higher values are better (accuracy, reward).
+    predictor : causal forecaster of the final value. Default: last observed value.
+    """
+
+    alpha: float = 0.05
+    good_threshold: float | None = None
+    good_quantile: float = 0.20
+    min_peek: int = 0
+    hold_until_resolved: float | None = None
+    maximize: bool = False
+    predictor: Predictor = last_value
+
+    tau_: float = field(init=False, default=math.nan)
+    lambda_: float = field(init=False, default=math.nan)
+    first_peek_: int = field(init=False, default=-1)
+    n_steps_: int = field(init=False, default=0)
+    n_calibration_: int = field(init=False, default=0)
+    n_good_: int = field(init=False, default=0)
+    resolution_: np.ndarray | None = field(init=False, default=None)
+
+    # ── fitting ────────────────────────────────────────────────────────────────
+    def _oriented(self, curves: ArrayLike) -> np.ndarray:
+        Y = np.asarray(curves, dtype=float)
+        return -Y if self.maximize else Y
+
+    def fit(self, curves: ArrayLike) -> "KillRule":
+        Y = self._oriented(curves)
+        if Y.ndim != 2 or Y.shape[1] < 2:
+            raise ValueError("curves must be a 2-D array (runs × steps) with at least 2 steps")
+        if not np.all(np.isfinite(Y)):
+            raise ValueError("calibration curves must be finite; drop incomplete runs first")
+        if not 0 < self.alpha < 1:
+            raise ValueError("alpha must lie in (0, 1)")
+        n, T = Y.shape
+        order = math.ceil((1 - self.alpha) * (n + 1))
+        if order > n:
+            raise NotEnoughCalibration(
+                f"alpha={self.alpha} needs at least {math.ceil(1 / self.alpha) - 1} calibration "
+                f"runs for any threshold to certify; got {n}")
+
+        finals = Y[:, -1]
+        if self.good_threshold is not None:
+            tau = -self.good_threshold if self.maximize else float(self.good_threshold)
+        else:
+            tau = float(np.sort(finals)[math.ceil(self.good_quantile * n) - 1])
+        P = np.asarray(self.predictor(Y), dtype=float)
+        if P.shape != Y.shape:
+            raise ValueError(f"predictor returned shape {P.shape}, expected {Y.shape}")
+
+        last_eligible = T - 2  # a kill at the final step saves nothing
+        j0 = self.min_peek
+        if self.hold_until_resolved is not None:
+            rho = _spearman_cols(P[:, : T - 1], finals)
+            self.resolution_ = rho
+            hit = np.flatnonzero(rho[self.min_peek:] >= self.hold_until_resolved)
+            j0 = self.min_peek + int(hit[0]) if hit.size else T - 1
+
+        good = finals <= tau
+        if j0 <= last_eligible:
+            scores = np.where(good, P[:, j0: last_eligible + 1].max(axis=1) - tau, -np.inf)
+        else:
+            scores = np.full(n, -np.inf)
+        lam = float(np.sort(scores)[order - 1])
+        if lam == -np.inf and j0 <= last_eligible:
+            warnings.warn(
+                f"only {int(good.sum())} good calibration runs: at alpha={self.alpha} no finite "
+                "threshold certifies, so the rule kills every run at its first eligible step. "
+                "Raise good_quantile or add calibration runs.", RuntimeWarning, stacklevel=2)
+
+        self.tau_, self.lambda_, self.first_peek_ = tau, lam, j0
+        self.n_steps_, self.n_calibration_, self.n_good_ = T, n, int(good.sum())
+        return self
+
+    # ── deciding ───────────────────────────────────────────────────────────────
+    def _check_fitted(self) -> None:
+        if self.n_steps_ == 0:
+            raise RuntimeError("call fit() first")
+
+    def should_kill(self, partial_curve: ArrayLike) -> bool:
+        """True if a run whose curve so far is `partial_curve` should be killed at its latest step."""
+        self._check_fitted()
+        y = self._oriented(partial_curve)
+        t = y.shape[0] - 1
+        if t < self.first_peek_ or t > self.n_steps_ - 2:
+            return False
+        pred = float(np.asarray(self.predictor(y[None, :]))[0, t])
+        return bool(pred - self.lambda_ > self.tau_)
+
+    def kill_step(self, curve: ArrayLike) -> int | None:
+        """Index of the first step at which the rule would kill this curve, or None."""
+        self._check_fitted()
+        Y = self._oriented(curve)[None, :]
+        P = np.asarray(self.predictor(Y), dtype=float)[0]
+        lo, hi = self.first_peek_, min(self.n_steps_, Y.shape[1]) - 2
+        if lo > hi:
+            return None
+        fire = np.flatnonzero(P[lo: hi + 1] - self.lambda_ > self.tau_)
+        return lo + int(fire[0]) if fire.size else None
+
+    def evaluate(self, curves: ArrayLike) -> dict[str, Any]:
+        """Score the rule on held-out COMPLETED curves: false kills, joint rate, savings."""
+        self._check_fitted()
+        Y = self._oriented(curves)
+        n, T = Y.shape
+        P = np.asarray(self.predictor(Y), dtype=float)
+        lo, hi = self.first_peek_, T - 2
+        if lo <= hi:
+            fire = P[:, lo: hi + 1] - self.lambda_ > self.tau_
+            killed = fire.any(axis=1)
+            step = np.where(killed, lo + np.argmax(fire, axis=1), T - 1)
+        else:
+            killed, step = np.zeros(n, bool), np.full(n, T - 1)
+        good = Y[:, -1] <= self.tau_
+        fk = int((killed & good).sum())
+        return {
+            "n": n, "n_good": int(good.sum()), "n_killed": int(killed.sum()),
+            "false_kills": fk, "joint_false_kill_rate": fk / n,
+            "conditional_false_kill_rate": fk / int(good.sum()) if good.any() else None,
+            "savings": float(1 - (step + 1).sum() / (T * n)),
+            "median_kill_step": float(np.median(step[killed])) if killed.any() else None,
+        }

surestop-0.2.0/src/surestop/optuna_pruner.py

@@ -0,0 +1,99 @@
+"""Optuna integration: a pruner that runs in shadow mode first, then kills with a certified bound.
+
+    pruner = ConformalPruner(n_calibration=60, alpha=0.05)
+    study = optuna.create_study(sampler=optuna.samplers.RandomSampler(), pruner=pruner)
+
+**How it behaves.**
+- **Shadow mode first.** The first `n_calibration` completed trials, by trial number, run
+  unpruned. Their reported curves calibrate the rule once.
+- **Then frozen.** The rule never recalibrates on later trials: those are survivors of the
+  rule's own kills, and fitting on them would bias it.
+- **The final outcome is the last reported intermediate value,** not `trial.value`.
+- **Every calibration trial must report on the same step grid.**
+
+**The guarantee needs the later trials to be exchangeable with the calibration trials.** With
+`RandomSampler`, `QMCSampler` or a shuffled `GridSampler` they are. With `TPESampler`, `GPSampler`
+or `CmaEsSampler` they are not, so the pruner warns once and the bound should not be relied on.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import Any
+
+import numpy as np
+import optuna
+from optuna.pruners import BasePruner
+from optuna.study import StudyDirection
+from optuna.trial import TrialState
+
+from .core import KillRule, Predictor, last_value
+
+ArrayLike = Any
+
+_EXCHANGEABLE = (optuna.samplers.RandomSampler, optuna.samplers.QMCSampler,
+                 optuna.samplers.GridSampler, optuna.samplers.BruteForceSampler)
+
+
+class ConformalPruner(BasePruner):
+    def __init__(self, n_calibration: int = 60, alpha: float = 0.05, *,
+                 good_threshold: float | None = None, good_quantile: float = 0.20,
+                 min_peek: int = 0, hold_until_resolved: float | None = None,
+                 predictor: Predictor = last_value) -> None:
+        self.n_calibration = n_calibration
+        self._kw: dict[str, Any] = dict(alpha=alpha, good_threshold=good_threshold, good_quantile=good_quantile,
+                        min_peek=min_peek, hold_until_resolved=hold_until_resolved,
+                        predictor=predictor)
+        self.rule_: KillRule | None = None
+        self.steps_: list[int] | None = None
+        self.calibration_trials_: list[int] = []
+        self._warned = False
+
+    @classmethod
+    def from_curves(cls, curves: ArrayLike, steps: Any, *, maximize: bool = False,
+                    **kw: Any) -> "ConformalPruner":
+        """A pruner already calibrated offline, e.g. on a previous sweep's completed curves."""
+        p = cls(n_calibration=len(curves), **kw)
+        p.rule_ = KillRule(maximize=maximize, **p._kw).fit(curves)
+        p.steps_ = list(steps)
+        return p
+
+    def _calibrate(self, study: optuna.study.Study) -> bool:
+        done = sorted((t for t in study.get_trials(deepcopy=False, states=(TrialState.COMPLETE,))
+                       if t.intermediate_values), key=lambda t: t.number)[: self.n_calibration]
+        if len(done) < self.n_calibration:
+            return False
+        grid = sorted(done[0].intermediate_values)
+        rows = [[t.intermediate_values[s] for s in grid] for t in done
+                if sorted(t.intermediate_values) == grid]
+        if len(rows) < len(done):
+            warnings.warn(f"{len(done) - len(rows)} calibration trials reported a different step "
+                          "grid and were skipped", RuntimeWarning, stacklevel=3)
+            return False
+        maximize = study.direction == StudyDirection.MAXIMIZE
+        self.rule_ = KillRule(maximize=maximize, **self._kw).fit(np.asarray(rows))
+        self.steps_ = grid
+        self.calibration_trials_ = [t.number for t in done]
+        return True
+
+    def prune(self, study: optuna.study.Study, trial: optuna.trial.FrozenTrial) -> bool:
+        if not self._warned and not isinstance(study.sampler, _EXCHANGEABLE):
+            warnings.warn(f"{type(study.sampler).__name__} adapts to earlier trials, so later trials "
+                          "are not exchangeable with the calibration trials and the false-kill "
+                          "bound does not hold. Use RandomSampler or QMCSampler for a certified "
+                          "bound.", RuntimeWarning, stacklevel=2)
+            self._warned = True
+        step = trial.last_step
+        if step is None:
+            return False
+        if self.rule_ is None and not self._calibrate(study):
+            return False
+        assert self.rule_ is not None and self.steps_ is not None
+        if trial.number in self.calibration_trials_ or step not in self.steps_:
+            return False
+        idx = self.steps_.index(step)
+        values = trial.intermediate_values
+        prefix = [values.get(s) for s in self.steps_[: idx + 1]]
+        if any(v is None for v in prefix):
+            return False
+        return self.rule_.should_kill(np.asarray(prefix, dtype=float))

surestop-0.2.0/src/surestop.egg-info/PKG-INFO

@@ -0,0 +1,173 @@
+Metadata-Version: 2.4
+Name: surestop
+Version: 0.2.0
+Summary: Stop bad runs early, with a conformal bound on how often you stop one that would have ended well
+Author: Adam Noonan
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/ACNoonan/surestop
+Project-URL: Issues, https://github.com/ACNoonan/surestop/issues
+Keywords: hyperparameter-optimization,early-stopping,pruning,conformal,optuna
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Provides-Extra: optuna
+Requires-Dist: optuna>=4; extra == "optuna"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: optuna>=4; extra == "test"
+Dynamic: license-file
+
+# SureStop
+
+**Stop bad training runs early, with a stated bound on how often you stop one that would have
+ended well.** You pick α. The rule kills runs so that the expected fraction of runs that were
+good *and* got killed stays at or below α.
+
+Every pruner in common use (median, percentile, successive halving, Hyperband) is a heuristic,
+and none says how often it kills a run that would have finished well. This one does, under a
+condition you can check: the runs you calibrate on and the runs you judge must come from the same
+random process.
+
+It was measured on 400 real LoRA fine-tuning runs under a pre-registered protocol. The curves are
+in this repository, and a script checks that the package reproduces the recorded numbers exactly.
+
+## Install
+
+```
+pip install surestop            # numpy + scipy
+pip install "surestop[optuna]"  # adds the Optuna pruner
+```
+
+## Use
+
+Plain numpy, on any training loop:
+
+```python
+from surestop import KillRule
+
+# completed_curves: array of shape (runs, evaluations), lower is better
+rule = KillRule(alpha=0.05, good_threshold=2.16, min_peek=17).fit(completed_curves)
+
+rule.should_kill(curve_so_far)     # True -> stop this run now
+rule.kill_step(full_curve)         # where it would have fired, or None
+rule.evaluate(held_out_curves)     # false kills, joint rate, savings
+```
+
+With Optuna, the first `n_calibration` trials run unpruned to calibrate, then the rule freezes:
+
+```python
+import optuna
+from surestop import ConformalPruner
+
+study = optuna.create_study(sampler=optuna.samplers.RandomSampler(),
+                            pruner=ConformalPruner(n_calibration=60, alpha=0.05))
+```
+
+Pass `maximize=True` (or a `direction="maximize"` study) when higher is better.
+
+## The rule
+
+A run is *good* if its final value ends at or below a threshold τ. At every evaluation, a
+predictor forecasts the final value from the curve so far. The default predictor is the latest
+observed value.
+
+To calibrate, take n runs that have already finished. For each good one, record its worst moment:
+the largest gap between forecast and τ at any evaluation. Runs that are not good contribute
+nothing. Set λ to the ⌈(1−α)(n+1)⌉-th smallest of these scores. A new run is killed at the first
+evaluation where its forecast exceeds τ + λ.
+
+This is conformal risk control on the loss "good and killed". Taking the worst moment across all
+evaluations means one threshold covers every look at a run, so there is no correction for
+multiple looks. The guarantee needs the calibration runs and the new runs to be exchangeable.
+Random sampling gives that. Adaptive samplers such as TPE do not.
+
+## What was measured
+
+Experiment CC-03c, pre-registered before any run finished:
+
+- **Runs:** 400 LoRA fine-tunes of Qwen2.5-0.5B on dolly-15k, 750 steps each, validation loss
+  measured 50 times per run.
+- **Configs:** i.i.d. draws from a frozen hyperparameter prior (`docs/hyperparameter-prior.md`).
+- **τ:** pinned from an earlier pool before the first run landed.
+- **Splits:** two, one random and one past-to-future by night, each about half calibration and
+  half test.
+
+The pre-registered rule used a fitted power-law predictor. On both splits it kept the bound and
+saved about half the evaluation compute:
+
+| split | false kills | joint rate (95% CI) | eval compute saved |
+|---|---|---|---|
+| random | 13 of 200 | 6.5% (3.5–10.9%) | 52.2% |
+| past-to-future | 11 of 203 | 5.4% (2.7–9.5%) | 52.8% |
+
+A rule that kills every run immediately was scored by the same gate and failed on both splits.
+
+**The flaw that run found:** every kill fired at the 3rd of 50 evaluations. Early rankings only
+match final rankings from about evaluation 18. The rule stayed within budget because the bound
+held. The fitted power law ranked runs worse than the latest observed value at every early
+evaluation.
+
+**What this package ships instead,** and what the reproduction script checks:
+
+| predictor | hold until | random split | past-to-future split | 2000 resplits |
+|---|---|---|---|---|
+| last value | none | 18 of 200 (9.0%), 78% saved | 7 of 203 (3.4%), 75% saved | joint 5.0%, breach 8.5% |
+| last value | eval 18 | 18 of 200 (9.0%), 57% saved | 7 of 203 (3.4%), 56% saved | joint 5.0%, breach 8.6% |
+
+The random split's 18 of 200 is a breach. It sits at the 95th percentile of that rule's own
+resplit distribution, whose mean is 5.0%, so it reads as a split tail rather than a leak. A
+correct rule at this sample size lands on a breach in about 7% of splits. The held version was
+chosen *after* seeing these 400 runs, so its numbers are a hypothesis. It halves
+how often the single best run is killed at the same budget. Its confirmation needs a fresh pool.
+
+Full account: `docs/writeup.md`. Run the check yourself:
+
+```
+python reproduce/reproduce_cc03c.py
+```
+
+## What the bound does not cover
+
+- **It needs exchangeable runs.** With a random or quasi-random sampler the bound holds. With
+  TPE, Gaussian-process or CMA-ES samplers it does not. The Optuna pruner warns once and carries
+  on, but do not rely on the bound there.
+- **It bounds the joint rate rather than the share of good runs killed.** When good runs are rare, a large
+  share of them can still die: about a third in the measurement above, mostly borderline ones.
+  Lower α if that matters.
+- **An estimated τ gives a measured bound rather than a proven one.** `good_quantile` estimates τ from the
+  calibration runs. Simulations put the realized rate at 0.88 to 0.97 times α. Pass a real
+  `good_threshold` when you have one.
+- **α = 0.05 needs at least 19 calibration runs**, and the certified threshold gets loose with
+  few good runs. Sweeps of a few dozen trials are too small for this.
+- **One setting only so far.** One model, one dataset, one horizon.
+
+## Layout
+
+| path | what |
+|---|---|
+| `src/surestop/` | `KillRule` (numpy/scipy) and `ConformalPruner` (Optuna) |
+| `tests/` | the bound holds on synthetic exchangeable runs, and the same assertion fails on a rule that leaks 2× its budget |
+| `reproduce/` | the 400 curves, the pre-registered splits and targets, the script, and its committed output |
+| `docs/` | the write-up, the prior-art sweep, and the frozen hyperparameter prior the curves were drawn from |
+| `optunahub/` | the package as an OptunaHub registry entry |
+
+## Related work
+
+Calibrating a stop threshold on a predictor with a conformal guarantee is an established pattern. Xie et al.
+(arXiv:2602.13935) use the same device, a threshold on the maximum of a running score calibrated
+on one class only, to stop LLM reasoning traces. Related work does the same for mixed-integer
+solvers and LLM agent episodes. As far as a search found, nobody has applied it to pruning
+training runs, and no major tuning library ships a pruner with a stated error guarantee.
+`docs/prior-art.md` has the sweep.
+
+## Citing
+
+Concept DOI: [10.5281/zenodo.22726440](https://doi.org/10.5281/zenodo.22726440), which always resolves
+to the newest version. `CITATION.cff` has the full entry. The curves are a measurement, released
+under the same MIT licence as the code.

surestop-0.2.0/src/surestop.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+README.md
+pyproject.toml
+src/surestop/__init__.py
+src/surestop/core.py
+src/surestop/optuna_pruner.py
+src/surestop.egg-info/PKG-INFO
+src/surestop.egg-info/SOURCES.txt
+src/surestop.egg-info/dependency_links.txt
+src/surestop.egg-info/requires.txt
+src/surestop.egg-info/top_level.txt
+tests/test_core.py
+tests/test_optuna_pruner.py

surestop-0.2.0/src/surestop.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

surestop-0.2.0/src/surestop.egg-info/requires.txt

@@ -0,0 +1,9 @@
+numpy>=1.24
+scipy>=1.10
+
+[optuna]
+optuna>=4
+
+[test]
+pytest
+optuna>=4

surestop-0.2.0/src/surestop.egg-info/top_level.txt

@@ -0,0 +1 @@
+surestop

surestop-0.2.0/tests/test_core.py

@@ -0,0 +1,88 @@
+"""The bound, a negative control that proves the bound check can fail, and the rule's edges."""
+
+import math
+
+import numpy as np
+import pytest
+
+from surestop import KillRule, NotEnoughCalibration
+
+T = 30
+
+
+def synth(n, rng):
+    """Exchangeable learning curves: a final value, plus a decaying offset and noise that shrink with step."""
+    finals = rng.normal(2.17, 0.013, n)
+    t = np.arange(T)
+    offset = rng.uniform(0.05, 0.3, n)[:, None] * np.exp(-t / 6.0)
+    noise = rng.normal(0, 1, (n, T)) * 0.012 * np.exp(-t / 10.0)
+    Y = finals[:, None] + offset + noise
+    Y[:, -1] = finals
+    return Y
+
+
+def mean_joint_rate(alpha_fit, reps=400, n_cal=200, n_test=200, seed=0, **kw):
+    rng = np.random.default_rng(seed)
+    rates = []
+    for _ in range(reps):
+        cal, test = synth(n_cal, rng), synth(n_test, rng)
+        rule = KillRule(alpha=alpha_fit, good_threshold=2.159, **kw).fit(cal)
+        rates.append(rule.evaluate(test)["joint_false_kill_rate"])
+    rates = np.array(rates)
+    return rates.mean(), rates.std(ddof=1) / math.sqrt(reps)
+
+
+def test_joint_false_kill_rate_is_bounded():
+    mean, se = mean_joint_rate(0.05)
+    assert mean <= 0.05 + 3 * se, (mean, se)
+
+
+def test_negative_control_the_bound_check_can_fail():
+    # A rule that leaks 2x its budget: certify at 0.10, hold the result to 0.05. The assertion
+    # the real test makes must fail here. (At 0.25 the synthetic pool has too few good runs for a
+    # finite threshold, so the rule kills everything — too easy a control to prove anything.)
+    mean, se = mean_joint_rate(0.10, reps=400)
+    assert not (mean <= 0.05 + 3 * se), (mean, se)
+
+
+def test_the_rule_actually_saves_compute():
+    rng = np.random.default_rng(1)
+    rule = KillRule(alpha=0.05, good_threshold=2.159).fit(synth(300, rng))
+    assert rule.evaluate(synth(300, rng))["savings"] > 0.2
+
+
+def test_min_peek_blocks_earlier_kills():
+    rng = np.random.default_rng(2)
+    rule = KillRule(alpha=0.05, good_threshold=2.159, min_peek=12).fit(synth(300, rng))
+    steps = [rule.kill_step(c) for c in synth(300, rng)]
+    assert all(s is None or s >= 12 for s in steps)
+    assert any(s is not None for s in steps)
+
+
+def test_should_kill_agrees_with_kill_step():
+    rng = np.random.default_rng(3)
+    rule = KillRule(alpha=0.05, good_threshold=2.159, min_peek=4).fit(synth(300, rng))
+    for c in synth(50, rng):
+        first = next((t for t in range(T) if rule.should_kill(c[: t + 1])), None)
+        assert first == rule.kill_step(c)
+
+
+def test_hold_until_resolved_moves_the_first_peek_later():
+    rng = np.random.default_rng(4)
+    cal = synth(300, rng)
+    held = KillRule(alpha=0.05, good_threshold=2.159, hold_until_resolved=0.9).fit(cal)
+    assert held.first_peek_ > 0
+    assert held.resolution_[held.first_peek_] >= 0.9
+
+
+def test_too_few_calibration_runs_raises():
+    with pytest.raises(NotEnoughCalibration):
+        KillRule(alpha=0.05).fit(synth(18, np.random.default_rng(5)))
+
+
+def test_maximize_mirrors_minimize():
+    rng = np.random.default_rng(6)
+    cal, test = synth(200, rng), synth(100, rng)
+    lo = KillRule(alpha=0.05, good_threshold=2.159).fit(cal)
+    hi = KillRule(alpha=0.05, good_threshold=-2.159, maximize=True).fit(-cal)
+    assert [lo.kill_step(c) for c in test] == [hi.kill_step(-c) for c in test]

surestop-0.2.0/tests/test_optuna_pruner.py

@@ -0,0 +1,48 @@
+"""The Optuna pruner: shadow mode first, pruning after, a warning for adaptive samplers."""
+
+import numpy as np
+import pytest
+
+optuna = pytest.importorskip("optuna")
+from surestop import ConformalPruner  # noqa: E402
+
+optuna.logging.set_verbosity(optuna.logging.WARNING)
+T = 20
+
+
+def objective(trial):
+    x = trial.suggest_float("x", 0.0, 1.0)
+    rng = np.random.default_rng(trial.number)
+    final = 2.15 + 0.05 * x + rng.normal(0, 0.005)
+    for step in range(T):
+        value = final + 0.2 * np.exp(-step / 4) + rng.normal(0, 0.004) * np.exp(-step / 8)
+        if step == T - 1:
+            value = final
+        trial.report(float(value), step)
+        if trial.should_prune():
+            raise optuna.TrialPruned()
+    return final
+
+
+def test_calibrates_in_shadow_mode_then_prunes():
+    pruner = ConformalPruner(n_calibration=40, alpha=0.05)
+    study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed=0), pruner=pruner)
+    study.optimize(objective, n_trials=160)
+    states = [t.state for t in study.trials]
+    assert all(s == optuna.trial.TrialState.COMPLETE for s in states[:40])
+    assert pruner.rule_ is not None and np.isfinite(pruner.rule_.lambda_)
+    assert sum(s == optuna.trial.TrialState.PRUNED for s in states) > 0
+
+
+def test_warns_for_an_adaptive_sampler():
+    pruner = ConformalPruner(n_calibration=20)
+    study = optuna.create_study(sampler=optuna.samplers.TPESampler(seed=0), pruner=pruner)
+    with pytest.warns(RuntimeWarning, match="not exchangeable"):
+        study.optimize(objective, n_trials=2)
+
+
+def test_from_curves_is_calibrated_before_the_first_trial():
+    rng = np.random.default_rng(0)
+    curves = 2.17 + rng.normal(0, 0.01, (60, 1)) + 0.2 * np.exp(-np.arange(T) / 4)
+    pruner = ConformalPruner.from_curves(curves, steps=range(T))
+    assert pruner.rule_.n_calibration_ == 60 and pruner.steps_ == list(range(T))