robustmargin 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robustmargin/__init__.py +90 -0
- robustmargin/censoring.py +652 -0
- robustmargin/empirical.py +348 -0
- robustmargin/intervals.py +785 -0
- robustmargin/models.py +2119 -0
- robustmargin/py.typed +0 -0
- robustmargin/ratio.py +941 -0
- robustmargin/report.py +650 -0
- robustmargin/schema/scorecard-v1.json +777 -0
- robustmargin/testing.py +465 -0
- robustmargin/threshold.py +631 -0
- robustmargin/types.py +541 -0
- robustmargin/validation.py +722 -0
- robustmargin-0.1.1.dist-info/METADATA +281 -0
- robustmargin-0.1.1.dist-info/RECORD +18 -0
- robustmargin-0.1.1.dist-info/WHEEL +5 -0
- robustmargin-0.1.1.dist-info/licenses/LICENSE +201 -0
- robustmargin-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,652 @@
|
|
|
1
|
+
"""Censoring classification and exact one-sided bounds.
|
|
2
|
+
|
|
3
|
+
Public: :func:`exact_one_sided_bound`, the one piece of numeric machinery every rule in this
|
|
4
|
+
module (and :func:`robustmargin.empirical.per_cell_clopper_pearson`) is built from -- the exact
|
|
5
|
+
arithmetic lives in exactly one place, here. Internal: :func:`classify`, which turns a
|
|
6
|
+
:class:`~robustmargin.Fit` plus an already-resolved target performance into a
|
|
7
|
+
:class:`Classification` describing which of the censoring outcomes applies.
|
|
8
|
+
|
|
9
|
+
Classification order (the censoring conditions are a list of conditions, not a sequence; this is
|
|
10
|
+
the sequence):
|
|
11
|
+
|
|
12
|
+
1. ``fit.status is CONTROL_INCOMPATIBLE`` -- terminal mirror.
|
|
13
|
+
2. ``FAILS_AT_BASELINE`` -- the zero-severity control's exact one-sided bound at ``level``
|
|
14
|
+
already fails the target, and no other tested level passes it (a control failure
|
|
15
|
+
contradicted by a later passing level is a monotonicity contradiction, not a clean baseline
|
|
16
|
+
failure, and belongs to step 4 instead). Short-circuits before ``UNREACHABLE``.
|
|
17
|
+
3. ``UNREACHABLE`` -- the target is not strictly inside ``(l, u)``. Only checked when
|
|
18
|
+
``fit.status is OK``, since reachability needs a fitted ``l``/``u``.
|
|
19
|
+
4. Monotonicity contradiction -- over *all* tested levels at ``level``, the highest confidently
|
|
20
|
+
passing nonzero severity is not below the lowest confidently failing severity, so the exact
|
|
21
|
+
tests contradict a monotone curve. A ``NON_MONOTONE: `` warning names both levels. On a failed
|
|
22
|
+
fit, no bounds are reported and the fit's own status is kept. On a converged fit,
|
|
23
|
+
``RIGHT``/``LEFT`` are skipped and the solve path (step 7) runs, carrying the warning. This is
|
|
24
|
+
a diagnostic of the monotonicity assumption, not an interval claim, so it deliberately looks
|
|
25
|
+
at every level rather than scanning in sequence.
|
|
26
|
+
5. ``RIGHT`` -- the highest tested level's one-sided *lower* bound at ``level`` still clears the
|
|
27
|
+
target.
|
|
28
|
+
6. ``LEFT`` -- the lowest *nonzero* tested level's one-sided *upper* bound at ``level`` fails the
|
|
29
|
+
target, and the control's one-sided lower bound clears it (evaluated on the data, so this
|
|
30
|
+
and ``RIGHT`` are reachable on a converged fit too).
|
|
31
|
+
7. ``fit.status is OK`` and neither ``RIGHT`` nor ``LEFT`` fired -- solve (the caller runs the
|
|
32
|
+
requested interval method); ``Classification.solve`` is ``True``.
|
|
33
|
+
8. Otherwise (``SEPARATION``/``NOT_CONVERGED``) -- the exact-bound fallback: attach the grid
|
|
34
|
+
value, then build a two-sided bracket from two fixed-sequence scans, each side at
|
|
35
|
+
``(1 + level) / 2`` (so joint coverage is at least ``level``). The lower bound is the last
|
|
36
|
+
severity in the run of confidently passing nonzero severities that starts at the lowest
|
|
37
|
+
nonzero severity and ascends; the upper bound is the last severity in the run of confidently
|
|
38
|
+
failing severities, control included, that starts at the highest severity and descends. Each
|
|
39
|
+
scan stops at the first severity that does not qualify. If only one side is determined, it is
|
|
40
|
+
rescanned the same way at ``level`` and reported as a one-sided bound; if neither is, no
|
|
41
|
+
bounds are reported. Under a monotone curve the per-level hypotheses are nested, so a scan can
|
|
42
|
+
only err by wrongly passing (or failing) the first level past the threshold, which keeps each
|
|
43
|
+
side's error at most ``1 - c`` at confidence ``c`` whatever the number of levels
|
|
44
|
+
(fixed-sequence testing; Hsu & Berger, 1999, *Journal of the American Statistical
|
|
45
|
+
Association* 94:468-482). All of this is against the resolved target: on a failed fit under
|
|
46
|
+
``baseline_fraction`` that target comes from the control's observed rate.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
from dataclasses import dataclass, replace
|
|
52
|
+
from typing import Literal
|
|
53
|
+
|
|
54
|
+
from scipy.stats import binomtest
|
|
55
|
+
|
|
56
|
+
from robustmargin.empirical import BaselineRate, GridBreakPoint, grid_break_point
|
|
57
|
+
from robustmargin.models import Fit
|
|
58
|
+
from robustmargin.types import Cell, Censoring, Status, _check_finite_number
|
|
59
|
+
|
|
60
|
+
__all__ = ["Classification", "classify", "exact_one_sided_bound"]
|
|
61
|
+
|
|
62
|
+
_VALID_SIDES = frozenset({"lower", "upper"})
|
|
63
|
+
|
|
64
|
+
# The two statuses that mean "the fit failed but the data still supports an exact
|
|
65
|
+
# statement" -- the exact-bound fallback, and the only two `Threshold` allows `grid` on.
|
|
66
|
+
_NONCONVERGENT_STATUSES = frozenset({Status.SEPARATION, Status.NOT_CONVERGED})
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def exact_one_sided_bound(
|
|
70
|
+
successes: int, trials: int, *, level: float, side: Literal["lower", "upper"]
|
|
71
|
+
) -> float:
|
|
72
|
+
"""The exact (Clopper-Pearson) one-sided confidence bound on a binomial proportion at
|
|
73
|
+
confidence ``level``.
|
|
74
|
+
|
|
75
|
+
``scipy.stats.binomtest(successes, trials).proportion_ci`` is two-sided: at
|
|
76
|
+
``confidence_level = c`` it puts ``(1 - c) / 2`` in each tail, so each endpoint is itself a
|
|
77
|
+
one-sided bound at confidence ``(1 + c) / 2``. To get a one-sided bound at confidence
|
|
78
|
+
``level``, this passes ``confidence_level = 2 * level - 1`` and reads the requested side.
|
|
79
|
+
This is the *only* place in robustmargin this conversion is written down; every other exact
|
|
80
|
+
bound in the package (per-cell rates, the censoring rules below) calls this function rather
|
|
81
|
+
than repeating it.
|
|
82
|
+
|
|
83
|
+
Parameters
|
|
84
|
+
----------
|
|
85
|
+
successes, trials
|
|
86
|
+
The observed count and the number of trials, as taken by ``scipy.stats.binomtest``.
|
|
87
|
+
level
|
|
88
|
+
The one-sided confidence level, strictly between 0 and 1.
|
|
89
|
+
side
|
|
90
|
+
``"lower"`` for a one-sided lower bound (how confident a *lower* limit on the true
|
|
91
|
+
proportion is), ``"upper"`` for a one-sided upper bound.
|
|
92
|
+
|
|
93
|
+
Returns
|
|
94
|
+
-------
|
|
95
|
+
float
|
|
96
|
+
The requested bound, in ``[0, 1]``.
|
|
97
|
+
|
|
98
|
+
Raises
|
|
99
|
+
------
|
|
100
|
+
ValueError
|
|
101
|
+
If ``side`` is not ``"lower"`` or ``"upper"``, or if ``level`` is not a finite number
|
|
102
|
+
strictly between 0 and 1.
|
|
103
|
+
"""
|
|
104
|
+
if side not in _VALID_SIDES:
|
|
105
|
+
raise ValueError(
|
|
106
|
+
f"exact_one_sided_bound: side must be one of {sorted(_VALID_SIDES)}, got {side!r}"
|
|
107
|
+
)
|
|
108
|
+
_check_finite_number(level, label="exact_one_sided_bound level")
|
|
109
|
+
# The `2 * level - 1` conversion below is defined only above 0.5: at or under it scipy is
|
|
110
|
+
# handed a non-positive `confidence_level` and raises naming a number the caller never
|
|
111
|
+
# supplied. A one-sided bound at 0.5 or less is not a meaningful request anyway.
|
|
112
|
+
if not (0.5 < level < 1.0):
|
|
113
|
+
raise ValueError(
|
|
114
|
+
f"exact_one_sided_bound: level must satisfy 0.5 < level < 1, got {level!r}"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
confidence_level = 2.0 * level - 1.0
|
|
118
|
+
result = binomtest(successes, trials).proportion_ci(
|
|
119
|
+
confidence_level=confidence_level, method="exact"
|
|
120
|
+
)
|
|
121
|
+
return float(result.low) if side == "lower" else float(result.high)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(frozen=True, kw_only=True)
|
|
125
|
+
class Classification:
|
|
126
|
+
"""The outcome of :func:`classify`: everything :func:`robustmargin.threshold` needs to build a
|
|
127
|
+
:class:`~robustmargin.Threshold`, other than the solved ``value`` itself (which, when
|
|
128
|
+
``solve`` is ``True``, is the caller's job -- computed from ``fit.params`` directly, then
|
|
129
|
+
refined by the requested interval method).
|
|
130
|
+
|
|
131
|
+
Attributes
|
|
132
|
+
----------
|
|
133
|
+
status
|
|
134
|
+
The :class:`~robustmargin.Status` the resulting threshold should report.
|
|
135
|
+
censoring
|
|
136
|
+
The :class:`~robustmargin.Censoring` the resulting threshold should report. ``NONE`` unless
|
|
137
|
+
``RIGHT``/``LEFT`` fired.
|
|
138
|
+
lo, hi
|
|
139
|
+
Severity-scale exact bounds, when this classification determined one directly (``RIGHT``,
|
|
140
|
+
``LEFT``, or a two-sided ``SEPARATION``/``NOT_CONVERGED`` bracket). ``None`` otherwise.
|
|
141
|
+
baseline
|
|
142
|
+
Set exactly when ``status is Status.FAILS_AT_BASELINE``.
|
|
143
|
+
grid
|
|
144
|
+
Set exactly on the ``SEPARATION``/``NOT_CONVERGED`` exact-bound fallback (step 8).
|
|
145
|
+
warnings
|
|
146
|
+
Free-text notes to append to the resulting threshold's own ``warnings``.
|
|
147
|
+
solve
|
|
148
|
+
``True`` only for the ``fit.status is OK``, non-``RIGHT``/``LEFT`` case: the caller must
|
|
149
|
+
solve for ``value`` and run the requested interval method.
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
status: Status
|
|
153
|
+
censoring: Censoring
|
|
154
|
+
lo: float | None
|
|
155
|
+
hi: float | None
|
|
156
|
+
baseline: BaselineRate | None
|
|
157
|
+
grid: GridBreakPoint | None
|
|
158
|
+
warnings: tuple[str, ...] = ()
|
|
159
|
+
solve: bool = False
|
|
160
|
+
|
|
161
|
+
def __post_init__(self) -> None:
|
|
162
|
+
object.__setattr__(self, "warnings", tuple(self.warnings))
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _control_cell(cells: tuple[Cell, ...]) -> Cell | None:
|
|
166
|
+
for cell in cells:
|
|
167
|
+
if cell.severity == 0.0:
|
|
168
|
+
return cell
|
|
169
|
+
return None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _baseline_rate(control: Cell, *, level: float) -> BaselineRate:
|
|
173
|
+
"""The control cell's rate with its exact interval, for a `FAILS_AT_BASELINE` result.
|
|
174
|
+
|
|
175
|
+
This is a **two-sided** Clopper-Pearson interval at `level`, which is how a reported
|
|
176
|
+
per-cell rate is defined: `binomtest(k, n).proportion_ci(confidence_level=...,
|
|
177
|
+
method='exact')`, labelled `per_cell_clopper_pearson`.
|
|
178
|
+
|
|
179
|
+
It is deliberately *not* the one-sided bound that decided the `FAILS_AT_BASELINE`
|
|
180
|
+
classification. Those are two different jobs: the classification asks "does this cell
|
|
181
|
+
provably fail the target", a one-sided question answered at `level` by
|
|
182
|
+
`exact_one_sided_bound`; the reported interval describes where the control rate lies, a
|
|
183
|
+
two-sided question. Reporting the one-sided pair under the label
|
|
184
|
+
`per_cell_clopper_pearson` would give a roughly `2*level - 1` region wearing a `level`
|
|
185
|
+
name.
|
|
186
|
+
"""
|
|
187
|
+
lo, hi = binomtest(control.successes, control.trials).proportion_ci(
|
|
188
|
+
confidence_level=level, method="exact"
|
|
189
|
+
)
|
|
190
|
+
return BaselineRate(
|
|
191
|
+
severity=control.severity,
|
|
192
|
+
successes=control.successes,
|
|
193
|
+
trials=control.trials,
|
|
194
|
+
rate=control.successes / control.trials,
|
|
195
|
+
lo=float(lo),
|
|
196
|
+
hi=float(hi),
|
|
197
|
+
level=level,
|
|
198
|
+
method="per_cell_clopper_pearson",
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _fails_at_baseline(
|
|
203
|
+
cells: tuple[Cell, ...], *, target: float, level: float
|
|
204
|
+
) -> Classification | None:
|
|
205
|
+
control = _control_cell(cells)
|
|
206
|
+
if control is None:
|
|
207
|
+
return None
|
|
208
|
+
control_upper = exact_one_sided_bound(
|
|
209
|
+
control.successes, control.trials, level=level, side="upper"
|
|
210
|
+
)
|
|
211
|
+
if not control_upper < target:
|
|
212
|
+
return None
|
|
213
|
+
# A control that fails is only a *clean* baseline failure if nothing else
|
|
214
|
+
# tested ever confidently passes the same target -- otherwise this is a monotonicity
|
|
215
|
+
# contradiction (a later level passing while the control fails), which step 4 reports
|
|
216
|
+
# instead, naming both levels.
|
|
217
|
+
other_passes = any(
|
|
218
|
+
exact_one_sided_bound(c.successes, c.trials, level=level, side="lower") >= target
|
|
219
|
+
for c in cells
|
|
220
|
+
if c.severity > 0.0
|
|
221
|
+
)
|
|
222
|
+
if other_passes:
|
|
223
|
+
return None
|
|
224
|
+
return Classification(
|
|
225
|
+
status=Status.FAILS_AT_BASELINE,
|
|
226
|
+
censoring=Censoring.NONE,
|
|
227
|
+
lo=None,
|
|
228
|
+
hi=None,
|
|
229
|
+
baseline=_baseline_rate(control, level=level),
|
|
230
|
+
grid=None,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _unreachable(fit: Fit, *, target: float) -> Classification | None:
|
|
235
|
+
if fit.status is not Status.OK:
|
|
236
|
+
return None
|
|
237
|
+
assert fit.params is not None
|
|
238
|
+
lower = fit.params["lower"].value
|
|
239
|
+
upper = fit.params["upper"].value
|
|
240
|
+
if lower < target < upper:
|
|
241
|
+
return None
|
|
242
|
+
return Classification(
|
|
243
|
+
status=Status.UNREACHABLE,
|
|
244
|
+
censoring=Censoring.NONE,
|
|
245
|
+
lo=None,
|
|
246
|
+
hi=None,
|
|
247
|
+
baseline=None,
|
|
248
|
+
grid=None,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _right(
|
|
253
|
+
fit: Fit, cells: tuple[Cell, ...], *, target: float, level: float
|
|
254
|
+
) -> Classification | None:
|
|
255
|
+
highest = cells[-1]
|
|
256
|
+
if exact_one_sided_bound(highest.successes, highest.trials, level=level, side="lower") < target:
|
|
257
|
+
return None
|
|
258
|
+
return Classification(
|
|
259
|
+
status=fit.status,
|
|
260
|
+
censoring=Censoring.RIGHT,
|
|
261
|
+
lo=highest.severity,
|
|
262
|
+
hi=None,
|
|
263
|
+
baseline=None,
|
|
264
|
+
grid=None,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _left(
|
|
269
|
+
fit: Fit, cells: tuple[Cell, ...], *, target: float, level: float
|
|
270
|
+
) -> Classification | None:
|
|
271
|
+
nonzero = [c for c in cells if c.severity > 0.0]
|
|
272
|
+
if not nonzero:
|
|
273
|
+
return None
|
|
274
|
+
lowest_nonzero = nonzero[0]
|
|
275
|
+
lowest_fails = (
|
|
276
|
+
exact_one_sided_bound(
|
|
277
|
+
lowest_nonzero.successes, lowest_nonzero.trials, level=level, side="upper"
|
|
278
|
+
)
|
|
279
|
+
< target
|
|
280
|
+
)
|
|
281
|
+
if not lowest_fails:
|
|
282
|
+
return None
|
|
283
|
+
control = _control_cell(cells)
|
|
284
|
+
if control is None:
|
|
285
|
+
return None
|
|
286
|
+
control_passes = (
|
|
287
|
+
exact_one_sided_bound(control.successes, control.trials, level=level, side="lower")
|
|
288
|
+
>= target
|
|
289
|
+
)
|
|
290
|
+
if not control_passes:
|
|
291
|
+
return None
|
|
292
|
+
return Classification(
|
|
293
|
+
status=fit.status,
|
|
294
|
+
censoring=Censoring.LEFT,
|
|
295
|
+
lo=None,
|
|
296
|
+
hi=lowest_nonzero.severity,
|
|
297
|
+
baseline=None,
|
|
298
|
+
grid=None,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _grid_for(cells: tuple[Cell, ...], *, target: float) -> GridBreakPoint:
|
|
303
|
+
severity = [c.severity for c in cells]
|
|
304
|
+
performance = [c.successes / c.trials for c in cells]
|
|
305
|
+
return grid_break_point(severity, performance, criterion=target, signed=False)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _contradiction_warning(
|
|
309
|
+
*,
|
|
310
|
+
lo_severity: float,
|
|
311
|
+
lo_bound: float,
|
|
312
|
+
hi_severity: float,
|
|
313
|
+
hi_bound: float,
|
|
314
|
+
target: float,
|
|
315
|
+
level: float,
|
|
316
|
+
curve_exists: bool,
|
|
317
|
+
) -> str:
|
|
318
|
+
"""``curve_exists`` picks the one sentence that differs between the two branches of this
|
|
319
|
+
function's one caller, ``_contradiction_or_none``, so the rest of the wording -- and the
|
|
320
|
+
frozen ``NON_MONOTONE: `` token -- is shared.
|
|
321
|
+
|
|
322
|
+
``True`` (``_contradiction_or_none``'s own ``Status.OK`` branch): the fit converged, so
|
|
323
|
+
``threshold()`` still solves for ``value`` and still runs the requested interval method --
|
|
324
|
+
only the *exact* one-sided bounds (the suppressed ``RIGHT``/``LEFT`` claim) go unreported,
|
|
325
|
+
not the profile/delta ``lo``/``hi`` this ``Threshold`` actually carries.
|
|
326
|
+
|
|
327
|
+
``False`` (``_contradiction_or_none``'s non-``OK`` branch, a failed fit): there is no fitted
|
|
328
|
+
curve at all, and this
|
|
329
|
+
``Threshold``'s ``value``/``lo``/``hi`` are all ``None`` -- "no bounds are reported" is
|
|
330
|
+
simply true here, with nothing to qualify.
|
|
331
|
+
"""
|
|
332
|
+
exact_bounds_clause = (
|
|
333
|
+
"the exact one-sided bounds are not reported here -- any lo/hi this Threshold carries "
|
|
334
|
+
"comes from the fitted curve instead, not from these exact tests"
|
|
335
|
+
if curve_exists
|
|
336
|
+
else "no bounds are reported"
|
|
337
|
+
)
|
|
338
|
+
return (
|
|
339
|
+
# The leading `NON_MONOTONE: ` token is the supported way to detect this condition --
|
|
340
|
+
# distinct from `Fit.monotonicity`'s own structured, data-only adjacent-level check,
|
|
341
|
+
# which compares raw counts against each other rather than exact bounds against the
|
|
342
|
+
# *fitted* curve; the two can disagree, and both are reported when they fire (see
|
|
343
|
+
# `threshold.py`'s `"NON_MONOTONE_DATA: "` token). On a converged fit this warning is the
|
|
344
|
+
# *only* signal of *this* kind of contradiction -- `grid` is legal only for
|
|
345
|
+
# SEPARATION/NOT_CONVERGED, so a consumer filtering on `status is OK and censoring is
|
|
346
|
+
# NONE` would otherwise see a clean bounded interval from data the exact tests just
|
|
347
|
+
# proved inconsistent with any monotone curve. Changing or removing this token is a
|
|
348
|
+
# behaviourally breaking change.
|
|
349
|
+
"NON_MONOTONE: censoring.classify: exact one-sided tests at confidence "
|
|
350
|
+
f"{level} contradict monotonicity: severity {lo_severity} confidently "
|
|
351
|
+
f"passes (lower bound {lo_bound!r} >= target {target!r}) while the lower severity "
|
|
352
|
+
f"{hi_severity} confidently fails (upper bound {hi_bound!r} < target {target!r}); "
|
|
353
|
+
f"{exact_bounds_clause}"
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _scan_bounds(
|
|
358
|
+
cells: tuple[Cell, ...], *, target: float, level: float
|
|
359
|
+
) -> tuple[float | None, float, float | None, float]:
|
|
360
|
+
"""The highest severity that confidently passes and the lowest that confidently fails.
|
|
361
|
+
|
|
362
|
+
The passing scan skips severity 0: "the threshold is above 0" is true of every threshold by
|
|
363
|
+
construction, so a control that passes is not a bound. The failing scan keeps it, because a
|
|
364
|
+
control that *fails* is genuinely informative and is what a monotonicity contradiction is
|
|
365
|
+
usually built from.
|
|
366
|
+
|
|
367
|
+
This looks at every cell and is deliberately not a fixed-sequence run (see
|
|
368
|
+
:func:`_run_bounds`): it is a diagnostic of the monotonicity assumption, not an interval
|
|
369
|
+
claim, and a scan that stops at the first gap would hide the contradictions it exists to find.
|
|
370
|
+
"""
|
|
371
|
+
lo_severity: float | None = None
|
|
372
|
+
lo_bound = 0.0
|
|
373
|
+
hi_severity: float | None = None
|
|
374
|
+
hi_bound = 0.0
|
|
375
|
+
for cell in cells:
|
|
376
|
+
if cell.severity > 0.0:
|
|
377
|
+
bound = exact_one_sided_bound(cell.successes, cell.trials, level=level, side="lower")
|
|
378
|
+
if bound >= target and (lo_severity is None or cell.severity > lo_severity):
|
|
379
|
+
lo_severity, lo_bound = cell.severity, bound
|
|
380
|
+
bound_up = exact_one_sided_bound(cell.successes, cell.trials, level=level, side="upper")
|
|
381
|
+
if bound_up < target and (hi_severity is None or cell.severity < hi_severity):
|
|
382
|
+
hi_severity, hi_bound = cell.severity, bound_up
|
|
383
|
+
return lo_severity, lo_bound, hi_severity, hi_bound
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _run_bounds(
|
|
387
|
+
cells: tuple[Cell, ...], *, target: float, level: float
|
|
388
|
+
) -> tuple[float | None, float | None]:
|
|
389
|
+
"""The two fixed-sequence scans: ``(lo, hi)`` severities, each ``None`` if its run is empty.
|
|
390
|
+
|
|
391
|
+
``lo``: walk the nonzero severities in ascending order while each one's one-sided lower bound
|
|
392
|
+
at ``level`` is at least ``target``; ``lo`` is the last severity in that run. The scan stops at
|
|
393
|
+
the first severity that does not qualify, so a later severity that would qualify is never
|
|
394
|
+
reached. Severity 0 is skipped: a control that passes says "the threshold is above 0", which
|
|
395
|
+
every threshold satisfies by construction, so it is not a bound (and on a log axis ``lo = 0``
|
|
396
|
+
is illegal).
|
|
397
|
+
|
|
398
|
+
``hi``: walk all severities, control included, in descending order while each one's one-sided
|
|
399
|
+
upper bound at ``level`` is below ``target``; ``hi`` is the last severity in that run. The
|
|
400
|
+
control stays in because a control that *fails* is genuinely informative.
|
|
401
|
+
|
|
402
|
+
Why a run and not the extreme: under a monotone decreasing true curve the hypotheses
|
|
403
|
+
``H_i: p(x_i) <= target`` are nested (if ``H_i`` is true so is every ``H_j`` with
|
|
404
|
+
``x_j > x_i``). The ascending scan rejects ``H_i`` only after rejecting every ``H_j`` with a
|
|
405
|
+
lower severity, so a wrong ``lo`` needs the first true hypothesis to be rejected, which has
|
|
406
|
+
probability at most ``1 - level`` however many levels lie near the threshold. The descending
|
|
407
|
+
scan is the mirror image. This is fixed-sequence testing of a-priori ordered hypotheses
|
|
408
|
+
(Hsu & Berger, 1999, *Journal of the American Statistical Association* 94:468-482).
|
|
409
|
+
"""
|
|
410
|
+
lo_severity: float | None = None
|
|
411
|
+
for cell in cells:
|
|
412
|
+
if cell.severity <= 0.0:
|
|
413
|
+
continue
|
|
414
|
+
bound = exact_one_sided_bound(cell.successes, cell.trials, level=level, side="lower")
|
|
415
|
+
if not bound >= target:
|
|
416
|
+
break
|
|
417
|
+
lo_severity = cell.severity
|
|
418
|
+
|
|
419
|
+
hi_severity: float | None = None
|
|
420
|
+
for cell in reversed(cells):
|
|
421
|
+
bound = exact_one_sided_bound(cell.successes, cell.trials, level=level, side="upper")
|
|
422
|
+
if not bound < target:
|
|
423
|
+
break
|
|
424
|
+
hi_severity = cell.severity
|
|
425
|
+
return lo_severity, hi_severity
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _contradiction_or_none(
|
|
429
|
+
fit: Fit, cells: tuple[Cell, ...], *, target: float, level: float
|
|
430
|
+
) -> Classification | None:
|
|
431
|
+
"""The exact-bounds-contradict-monotonicity check, applied *before* the `RIGHT`/`LEFT`
|
|
432
|
+
branches rather than only inside the bracket.
|
|
433
|
+
|
|
434
|
+
Both short-circuits return a directional claim about where the threshold lies -- "sigma* >
|
|
435
|
+
x_max", "sigma* < x_min" -- and every such claim rests on the monotonicity assumption stated
|
|
436
|
+
in `Threshold`'s docstring. When the exact tests themselves contradict that assumption, the
|
|
437
|
+
premise of the claim has failed, so the claim is not a weaker true statement: it is a
|
|
438
|
+
statement whose premise is false. Reporting it with no diagnostic is what this check exists
|
|
439
|
+
to prevent.
|
|
440
|
+
|
|
441
|
+
The confidence is `level`, not the bracket's `(1 + level) / 2`, because these are the
|
|
442
|
+
one-sided bounds the `RIGHT`/`LEFT` rules are themselves evaluated at.
|
|
443
|
+
"""
|
|
444
|
+
lo_severity, lo_bound, hi_severity, hi_bound = _scan_bounds(cells, target=target, level=level)
|
|
445
|
+
if lo_severity is None or hi_severity is None or lo_severity < hi_severity:
|
|
446
|
+
return None
|
|
447
|
+
warning = _contradiction_warning(
|
|
448
|
+
lo_severity=lo_severity,
|
|
449
|
+
lo_bound=lo_bound,
|
|
450
|
+
hi_severity=hi_severity,
|
|
451
|
+
hi_bound=hi_bound,
|
|
452
|
+
target=target,
|
|
453
|
+
level=level,
|
|
454
|
+
curve_exists=fit.status is Status.OK,
|
|
455
|
+
)
|
|
456
|
+
if fit.status is Status.OK:
|
|
457
|
+
# A converged fit still has a model-based estimate, and `Threshold` requires `value`
|
|
458
|
+
# whenever status is OK and censoring is NONE -- the all-`None` encoding of a
|
|
459
|
+
# contradiction (no value, no bounds) is only constructible on a failed fit. So the
|
|
460
|
+
# contradiction suppresses the `RIGHT`/`LEFT` branches (whose directional claim rests on
|
|
461
|
+
# the monotonicity the data just contradicted) and falls through to the solve path,
|
|
462
|
+
# carrying the warning. The caller gets the model's answer plus an explicit statement
|
|
463
|
+
# that the exact tests disagree with the model's monotonicity assumption: a diagnostic
|
|
464
|
+
# that is reported and never acted on.
|
|
465
|
+
return Classification(
|
|
466
|
+
status=Status.OK,
|
|
467
|
+
censoring=Censoring.NONE,
|
|
468
|
+
lo=None,
|
|
469
|
+
hi=None,
|
|
470
|
+
baseline=None,
|
|
471
|
+
grid=None,
|
|
472
|
+
warnings=(warning,),
|
|
473
|
+
solve=True,
|
|
474
|
+
)
|
|
475
|
+
return Classification(
|
|
476
|
+
status=fit.status,
|
|
477
|
+
censoring=Censoring.NONE,
|
|
478
|
+
lo=None,
|
|
479
|
+
hi=None,
|
|
480
|
+
baseline=None,
|
|
481
|
+
grid=_grid_for(cells, target=target),
|
|
482
|
+
warnings=(warning,),
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _exact_bound_fallback(
|
|
487
|
+
fit: Fit, cells: tuple[Cell, ...], *, target: float, level: float
|
|
488
|
+
) -> Classification:
|
|
489
|
+
"""The exact bracket (or one-sided bound) for a failed fit, from two fixed-sequence scans.
|
|
490
|
+
|
|
491
|
+
Each side is the end of a run found by :func:`_run_bounds`: ``lo`` ascends through the
|
|
492
|
+
nonzero severities from the lowest and ``hi`` descends through all severities from the
|
|
493
|
+
highest, each stopping at the first severity that does not qualify. A bracket takes both
|
|
494
|
+
scans at ``(1 + level) / 2``, so a Bonferroni step over the two sides gives joint coverage of
|
|
495
|
+
at least ``level``. Under a monotone curve the per-level hypotheses are nested, so a scan can
|
|
496
|
+
only err by wrongly passing (or failing) the first level past the threshold; that keeps each
|
|
497
|
+
side's error at most ``1 - c`` whatever the number of levels, where taking the extreme
|
|
498
|
+
qualifying severity anywhere would let every level near the threshold add its own chance of
|
|
499
|
+
error.
|
|
500
|
+
|
|
501
|
+
A bracket needs both sides. When only one is determined at the bracket confidence, the result
|
|
502
|
+
is a single one-sided statement, which belongs at ``level``: that side is rescanned with the
|
|
503
|
+
same run rule at ``level`` and the other side is ``None``. When neither is determined, no
|
|
504
|
+
bounds are reported.
|
|
505
|
+
"""
|
|
506
|
+
grid = _grid_for(cells, target=target)
|
|
507
|
+
bracket_level = (1.0 + level) / 2.0
|
|
508
|
+
|
|
509
|
+
# The scans themselves are in `_run_bounds`: the lower one skips severity 0, because a
|
|
510
|
+
# control that passes is not a bound, while the upper one keeps the control.
|
|
511
|
+
lo_severity, hi_severity = _run_bounds(cells, target=target, level=bracket_level)
|
|
512
|
+
|
|
513
|
+
# No contradiction check here. A cell cannot both pass (lower bound >= target) and fail
|
|
514
|
+
# (upper bound < target), so under run scans `lo < hi` always, and `classify` has already run
|
|
515
|
+
# the all-cells contradiction check at `level` before reaching this fallback.
|
|
516
|
+
|
|
517
|
+
warnings: tuple[str, ...] = ()
|
|
518
|
+
if lo_severity is None and hi_severity is None:
|
|
519
|
+
warnings = (
|
|
520
|
+
"censoring.classify: there is no run of confidently passing levels from the lowest "
|
|
521
|
+
"nonzero severity, and no run of confidently failing levels from the highest "
|
|
522
|
+
f"severity, at confidence {bracket_level}, for the target {target!r}; no bounds "
|
|
523
|
+
"are reported",
|
|
524
|
+
)
|
|
525
|
+
elif lo_severity is None or hi_severity is None:
|
|
526
|
+
# Only one side is determined, so this is not a bracket and the `(1 + level) / 2` split
|
|
527
|
+
# does not apply: a single one-sided statement belongs at `level`, like every other
|
|
528
|
+
# one-sided bound in the package.
|
|
529
|
+
#
|
|
530
|
+
# Which side that is must be decided from the *bracket-level* scan above, before the
|
|
531
|
+
# rescan, and the other side must be forced back to `None`. The looser `level` can
|
|
532
|
+
# determine a side that `(1 + level) / 2` did not -- a `0/5` cell has an upper bound of
|
|
533
|
+
# 0.5218 at 0.975 but 0.4507 at 0.95, so against a target of 0.5 it is undetermined at
|
|
534
|
+
# the bracket level and failing at `level`. Taking both sides from the rescan therefore
|
|
535
|
+
# manufactures a two-sided bracket with each side at `level`, whose joint coverage is
|
|
536
|
+
# about `2 * level - 1` while `Threshold.level` still reports `level`. That is exactly
|
|
537
|
+
# the mislabelling this branch exists to prevent.
|
|
538
|
+
determined = "lower" if hi_severity is None else "upper"
|
|
539
|
+
rescanned_lo, rescanned_hi = _run_bounds(cells, target=target, level=level)
|
|
540
|
+
if determined == "lower":
|
|
541
|
+
lo_severity, hi_severity = rescanned_lo, None
|
|
542
|
+
else:
|
|
543
|
+
lo_severity, hi_severity = None, rescanned_hi
|
|
544
|
+
missing_run = (
|
|
545
|
+
"failing levels from the highest"
|
|
546
|
+
if determined == "lower"
|
|
547
|
+
else "passing levels from the lowest nonzero"
|
|
548
|
+
)
|
|
549
|
+
warnings = (
|
|
550
|
+
"censoring.classify: the exact-bound fallback determined only the "
|
|
551
|
+
f"{determined} side at the bracket confidence {bracket_level} -- there is no run of "
|
|
552
|
+
f"confidently {missing_run} severity at that confidence, for the target "
|
|
553
|
+
f"{target!r} -- so the {determined} side is reported alone, rescanned at confidence "
|
|
554
|
+
f"{level}, as a one-sided exact bound, not a bracket. Censoring stays NONE because "
|
|
555
|
+
"RIGHT and LEFT are reserved for their own end-of-range conditions, which this "
|
|
556
|
+
"result did not meet",
|
|
557
|
+
)
|
|
558
|
+
return Classification(
|
|
559
|
+
status=fit.status,
|
|
560
|
+
censoring=Censoring.NONE,
|
|
561
|
+
lo=lo_severity,
|
|
562
|
+
hi=hi_severity,
|
|
563
|
+
baseline=None,
|
|
564
|
+
grid=grid,
|
|
565
|
+
warnings=warnings,
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def _with_grid(
|
|
570
|
+
classification: Classification, fit: Fit, cells: tuple[Cell, ...], *, target: float
|
|
571
|
+
) -> Classification:
|
|
572
|
+
"""Attach the grid break-point when, and only when, the fit failed.
|
|
573
|
+
|
|
574
|
+
``Threshold`` permits ``grid`` only for ``SEPARATION`` and ``NOT_CONVERGED``, so a
|
|
575
|
+
converged fit's one-sided bound is returned unchanged.
|
|
576
|
+
"""
|
|
577
|
+
if fit.status not in _NONCONVERGENT_STATUSES:
|
|
578
|
+
return classification
|
|
579
|
+
return replace(classification, grid=_grid_for(cells, target=target))
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def classify(fit: Fit, *, target: float, level: float) -> Classification:
|
|
583
|
+
"""Classify a fit against a resolved target performance (order per the module docstring).
|
|
584
|
+
|
|
585
|
+
Parameters
|
|
586
|
+
----------
|
|
587
|
+
fit
|
|
588
|
+
The :class:`~robustmargin.Fit` to classify. Its ``cells`` (sorted ascending by
|
|
589
|
+
:meth:`~robustmargin.Observations.cells`) are what every exact test here is run against.
|
|
590
|
+
target
|
|
591
|
+
The already-resolved target performance ``P*``. Resolving it from a
|
|
592
|
+
:class:`~robustmargin.Definition` -- including which ``u`` a failed-fit fallback uses --
|
|
593
|
+
is the caller's job.
|
|
594
|
+
level
|
|
595
|
+
The confidence level, strictly between 0 and 1.
|
|
596
|
+
|
|
597
|
+
Returns
|
|
598
|
+
-------
|
|
599
|
+
Classification
|
|
600
|
+
"""
|
|
601
|
+
if fit.status is Status.CONTROL_INCOMPATIBLE:
|
|
602
|
+
return Classification(
|
|
603
|
+
status=Status.CONTROL_INCOMPATIBLE,
|
|
604
|
+
censoring=Censoring.NONE,
|
|
605
|
+
lo=None,
|
|
606
|
+
hi=None,
|
|
607
|
+
baseline=None,
|
|
608
|
+
grid=None,
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
cells = fit.cells
|
|
612
|
+
|
|
613
|
+
fails_at_baseline = _fails_at_baseline(cells, target=target, level=level)
|
|
614
|
+
if fails_at_baseline is not None:
|
|
615
|
+
return fails_at_baseline
|
|
616
|
+
|
|
617
|
+
unreachable = _unreachable(fit, target=target)
|
|
618
|
+
if unreachable is not None:
|
|
619
|
+
return unreachable
|
|
620
|
+
|
|
621
|
+
# `RIGHT`/`LEFT` are reached on a converged fit as well as a failed one (they are evaluated
|
|
622
|
+
# on the data), so the grid statistic is attached here rather than inside `_right`/`_left`:
|
|
623
|
+
# the grid rule accompanies the exact bounds for separation and non-convergence, and a
|
|
624
|
+
# one-sided bound on a failed fit is that same fallback. On an `OK` fit `Threshold` forbids
|
|
625
|
+
# `grid`, so the rule is stated once, in one place, and cannot leak into the `OK` branch.
|
|
626
|
+
# The monotonicity-contradiction check runs before the directional branches: a
|
|
627
|
+
# `RIGHT`/`LEFT` bound asserts where the threshold lies, and that assertion rests on
|
|
628
|
+
# monotonicity. If the exact tests contradict monotonicity, no such bound may be reported.
|
|
629
|
+
contradiction = _contradiction_or_none(fit, cells, target=target, level=level)
|
|
630
|
+
if contradiction is not None:
|
|
631
|
+
return contradiction
|
|
632
|
+
|
|
633
|
+
right = _right(fit, cells, target=target, level=level)
|
|
634
|
+
if right is not None:
|
|
635
|
+
return _with_grid(right, fit, cells, target=target)
|
|
636
|
+
|
|
637
|
+
left = _left(fit, cells, target=target, level=level)
|
|
638
|
+
if left is not None:
|
|
639
|
+
return _with_grid(left, fit, cells, target=target)
|
|
640
|
+
|
|
641
|
+
if fit.status is Status.OK:
|
|
642
|
+
return Classification(
|
|
643
|
+
status=Status.OK,
|
|
644
|
+
censoring=Censoring.NONE,
|
|
645
|
+
lo=None,
|
|
646
|
+
hi=None,
|
|
647
|
+
baseline=None,
|
|
648
|
+
grid=None,
|
|
649
|
+
solve=True,
|
|
650
|
+
)
|
|
651
|
+
|
|
652
|
+
return _exact_bound_fallback(fit, cells, target=target, level=level)
|