robustmargin 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robustmargin/__init__.py +90 -0
- robustmargin/censoring.py +652 -0
- robustmargin/empirical.py +348 -0
- robustmargin/intervals.py +785 -0
- robustmargin/models.py +2119 -0
- robustmargin/py.typed +0 -0
- robustmargin/ratio.py +941 -0
- robustmargin/report.py +650 -0
- robustmargin/schema/scorecard-v1.json +777 -0
- robustmargin/testing.py +465 -0
- robustmargin/threshold.py +631 -0
- robustmargin/types.py +541 -0
- robustmargin/validation.py +722 -0
- robustmargin-0.1.1.dist-info/METADATA +281 -0
- robustmargin-0.1.1.dist-info/RECORD +18 -0
- robustmargin-0.1.1.dist-info/WHEEL +5 -0
- robustmargin-0.1.1.dist-info/licenses/LICENSE +201 -0
- robustmargin-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,631 @@
|
|
|
1
|
+
"""The threshold result and :func:`threshold`, which solves for one.
|
|
2
|
+
|
|
3
|
+
:func:`threshold` resolves the target performance for a :class:`~robustmargin.Definition`,
|
|
4
|
+
classifies the fit against it (:mod:`robustmargin.censoring`), and -- when the
|
|
5
|
+
classification says to -- solves the closed form and runs the requested interval method
|
|
6
|
+
(:mod:`robustmargin.intervals`). ``interval_method`` is a *request*: every censored or failed
|
|
7
|
+
result records ``"exact_bound"`` regardless of what was asked for, because that is the method
|
|
8
|
+
that actually produced its numbers.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
|
|
16
|
+
from robustmargin.censoring import classify
|
|
17
|
+
from robustmargin.empirical import BaselineRate, GridBreakPoint
|
|
18
|
+
from robustmargin.intervals import delta_interval, profile_interval, solve_value
|
|
19
|
+
from robustmargin.models import Fit
|
|
20
|
+
from robustmargin.types import (
|
|
21
|
+
Axis,
|
|
22
|
+
Cell,
|
|
23
|
+
Censoring,
|
|
24
|
+
Definition,
|
|
25
|
+
JSONValue,
|
|
26
|
+
Status,
|
|
27
|
+
_check_finite_number,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
__all__ = ["Threshold", "threshold"]
|
|
31
|
+
|
|
32
|
+
_VALID_INTERVAL_METHODS = frozenset({"profile", "delta", "exact_bound"})
|
|
33
|
+
_TERMINAL_STATUSES = frozenset(
|
|
34
|
+
{Status.UNREACHABLE, Status.CONTROL_INCOMPATIBLE, Status.FAILS_AT_BASELINE}
|
|
35
|
+
)
|
|
36
|
+
_NONCONVERGENT_STATUSES = frozenset({Status.SEPARATION, Status.NOT_CONVERGED})
|
|
37
|
+
_VALID_DIRECTIONS = frozenset({"decreasing", "increasing"})
|
|
38
|
+
|
|
39
|
+
# A threshold with a value, with status OK or UNREACHABLE, or with an open-sided censoring
|
|
40
|
+
# needs the fit underneath it to have actually converged.
|
|
41
|
+
_REQUIRE_FIT_OK_STATUSES = frozenset({Status.OK, Status.UNREACHABLE})
|
|
42
|
+
_REQUIRE_FIT_OK_CENSORING = frozenset({Censoring.OPEN_UPPER, Censoring.OPEN_LOWER})
|
|
43
|
+
# SEPARATION, NOT_CONVERGED and CONTROL_INCOMPATIBLE are properties of the fit itself; a
|
|
44
|
+
# threshold reporting one of these must be built on a fit reporting the very same status.
|
|
45
|
+
_MIRROR_FIT_STATUS_STATUSES = frozenset(
|
|
46
|
+
{Status.SEPARATION, Status.NOT_CONVERGED, Status.CONTROL_INCOMPATIBLE}
|
|
47
|
+
)
|
|
48
|
+
_VALID_DEPENDENCE = frozenset({"independent"})
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _check_value(value: float | None, *, label: str, scale: str) -> None:
|
|
52
|
+
if value is None:
|
|
53
|
+
return
|
|
54
|
+
_check_finite_number(value, label=f"Threshold.{label}")
|
|
55
|
+
if scale == "log":
|
|
56
|
+
if not value > 0.0:
|
|
57
|
+
raise ValueError(f"Threshold.{label} must be > 0 on a log axis, got {value!r}")
|
|
58
|
+
elif value < 0.0:
|
|
59
|
+
raise ValueError(f"Threshold.{label} must be non-negative, got {value!r}")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True, kw_only=True)
|
|
63
|
+
class Threshold:
|
|
64
|
+
"""Where a fitted curve crosses a target performance, with a confidence interval, or an
|
|
65
|
+
explicit record of why no value is reported.
|
|
66
|
+
|
|
67
|
+
Every censoring rule below assumes the **true** performance curve is monotone in severity
|
|
68
|
+
(the fitted curve, being drawn from a monotone parametric family, is monotone by
|
|
69
|
+
construction regardless). A ``RIGHT`` bound reads "threshold > lo (one-sided, exact,
|
|
70
|
+
monotonicity assumed)", never as a value on its own; the mirrored reading applies to
|
|
71
|
+
``LEFT``. A ``SEPARATION``/``NOT_CONVERGED`` bracket is built from *two* one-sided exact
|
|
72
|
+
bounds, each individually computed at confidence ``(1 + level) / 2``, so their **joint**
|
|
73
|
+
coverage is at least ``level`` -- not merely at least ``2 * level - 1``, the floor that two
|
|
74
|
+
bounds each at ``level`` would guarantee. Each bound comes from a fixed-sequence scan, not
|
|
75
|
+
from the most extreme qualifying level anywhere: the lower bound is the last severity in the
|
|
76
|
+
run of confidently passing nonzero severities ascending from the lowest nonzero severity, the
|
|
77
|
+
upper bound is the last severity in the run of confidently failing severities descending from
|
|
78
|
+
the highest, and each scan stops at the first severity that does not qualify (fixed-sequence
|
|
79
|
+
testing; Hsu & Berger, 1999, *Journal of the American Statistical Association* 94:468-482).
|
|
80
|
+
Under a monotone curve this keeps each side's error at most ``1 - c`` at confidence ``c``
|
|
81
|
+
whatever the number of tested levels, at the cost of reporting no bound on a side whose run
|
|
82
|
+
cannot start. If only one side is determined, it is rescanned the same way at ``level`` and
|
|
83
|
+
reported alone. A one-sided ``RIGHT``/``LEFT`` bound is genuinely one-sided and stays at
|
|
84
|
+
``level`` itself.
|
|
85
|
+
|
|
86
|
+
One limitation applies across these results. Five kinds of exact result can be returned: a
|
|
87
|
+
``RIGHT`` bound, a ``LEFT`` bound, a bracket, a one-sided lower bound and a one-sided upper
|
|
88
|
+
bound. For each kind, the probability that ``threshold()`` returns that kind *and* it
|
|
89
|
+
excludes the true threshold -- the severity where performance crosses the resolved target --
|
|
90
|
+
is at most ``1 - level``. That is not the probability that a result of that kind, once
|
|
91
|
+
seen, is wrong. The kinds are also not controlled jointly: which kind is returned depends on
|
|
92
|
+
the data, so the combined rate of a wrong exact result is bounded only by
|
|
93
|
+
``4 * (1 - level)``. In simulation it reached about 8.4% at ``level = 0.95``, when the true
|
|
94
|
+
performance at tested levels sat just above and just below the target.
|
|
95
|
+
|
|
96
|
+
Binomial curves are decreasing-only in this release: ``direction="increasing"`` raises, since
|
|
97
|
+
increasing-is-worse outcomes are not yet supported.
|
|
98
|
+
|
|
99
|
+
Invariants enforced in ``__post_init__`` (the censoring table, plus the fit-status and
|
|
100
|
+
ordering rules):
|
|
101
|
+
|
|
102
|
+
- ``status`` in ``{UNREACHABLE, CONTROL_INCOMPATIBLE, FAILS_AT_BASELINE}``: ``value``,
|
|
103
|
+
``lo`` and ``hi`` are all ``None``; ``censoring`` is ``NONE``; ``interval_method`` is
|
|
104
|
+
``"exact_bound"``. There is no threshold at all in these cases -- either it cannot be
|
|
105
|
+
reached given the fitted asymptotes, or the question of a crossing does not arise
|
|
106
|
+
because the control already fails.
|
|
107
|
+
- ``status`` in ``{SEPARATION, NOT_CONVERGED}``: ``value`` is ``None``; ``censoring`` is one
|
|
108
|
+
of ``{NONE, RIGHT, LEFT}``; ``interval_method`` is ``"exact_bound"``; ``lo`` and ``hi``
|
|
109
|
+
may be set, as the exact-bound fallback.
|
|
110
|
+
- ``censoring == RIGHT``: ``value`` is ``None``, ``lo`` is set, ``hi`` is ``None``,
|
|
111
|
+
``interval_method == "exact_bound"``.
|
|
112
|
+
- ``censoring == LEFT``: ``value`` is ``None``, ``hi`` is set, ``lo`` is ``None``,
|
|
113
|
+
``interval_method == "exact_bound"``.
|
|
114
|
+
- ``censoring == OPEN_UPPER``: ``status == OK``, ``value`` and ``lo`` are set, ``hi`` is
|
|
115
|
+
``None``, ``interval_method == "profile"``. ``OPEN_LOWER`` mirrors it, with ``hi`` set
|
|
116
|
+
and ``lo`` ``None``.
|
|
117
|
+
- ``censoring == NONE`` with ``status == OK``: ``value``, ``lo`` and ``hi`` are all set,
|
|
118
|
+
with ``lo <= value <= hi``, and ``interval_method`` is ``"profile"`` or ``"delta"``.
|
|
119
|
+
- When both ``lo`` and ``hi`` are set: ``lo < hi`` strictly for a ``SEPARATION`` /
|
|
120
|
+
``NOT_CONVERGED`` bracket, ``lo <= hi`` otherwise.
|
|
121
|
+
- On a log axis, ``value``, ``lo`` and ``hi`` must each be ``> 0`` when present (a linear
|
|
122
|
+
axis keeps ``>= 0``).
|
|
123
|
+
- ``fit.status`` must equal ``OK`` whenever ``value`` is set, ``status`` is ``OK`` or
|
|
124
|
+
``UNREACHABLE``, or ``censoring`` is ``OPEN_UPPER``/``OPEN_LOWER``. ``fit.status`` must
|
|
125
|
+
equal ``status`` itself when ``status`` is ``SEPARATION``, ``NOT_CONVERGED`` or
|
|
126
|
+
``CONTROL_INCOMPATIBLE``. ``FAILS_AT_BASELINE`` places no constraint on ``fit.status``
|
|
127
|
+
(the control's exact test does not depend on the rest of the curve having converged).
|
|
128
|
+
|
|
129
|
+
Attributes
|
|
130
|
+
----------
|
|
131
|
+
axis
|
|
132
|
+
The :class:`~robustmargin.Axis` this threshold was solved on. Must equal ``fit.axis``.
|
|
133
|
+
definition
|
|
134
|
+
Which target performance this threshold solves for.
|
|
135
|
+
direction
|
|
136
|
+
``"decreasing"`` only, in this release (see above). Carried over from ``fit``.
|
|
137
|
+
value
|
|
138
|
+
The estimated crossing severity, or ``None`` per the rules above.
|
|
139
|
+
lo
|
|
140
|
+
The interval's lower side, or a one-sided bound, or ``None``.
|
|
141
|
+
hi
|
|
142
|
+
The interval's upper side, or a one-sided bound, or ``None``.
|
|
143
|
+
level
|
|
144
|
+
The confidence level, strictly between 0 and 1.
|
|
145
|
+
interval_method
|
|
146
|
+
``"profile"``, ``"delta"`` or ``"exact_bound"``.
|
|
147
|
+
censoring
|
|
148
|
+
The :class:`~robustmargin.Censoring` classification.
|
|
149
|
+
status
|
|
150
|
+
The :class:`~robustmargin.Status` of this threshold.
|
|
151
|
+
dependence
|
|
152
|
+
``"independent"`` in this release. Required, with no default: a threshold built from
|
|
153
|
+
clustered data must say so explicitly rather than silently assuming independence.
|
|
154
|
+
fit
|
|
155
|
+
The :class:`~robustmargin.Fit` this threshold was solved from. Per-level trial counts
|
|
156
|
+
live on ``fit.cells``, not on a separate field.
|
|
157
|
+
warnings
|
|
158
|
+
Free-text notes surfaced alongside the threshold. Empty by default. Carries a
|
|
159
|
+
``"NON_MONOTONE_DATA: "``-prefixed entry whenever ``fit.monotonicity.any_flagged``,
|
|
160
|
+
whatever ``status``/``censoring`` turn out to be -- distinct from
|
|
161
|
+
:mod:`robustmargin.censoring`'s own ``"NON_MONOTONE: "`` token, which compares exact
|
|
162
|
+
one-sided bounds against the *fitted* curve rather than testing raw adjacent levels; the two
|
|
163
|
+
may disagree and both are reported when they fire.
|
|
164
|
+
schema_version
|
|
165
|
+
The serialised-result schema version this object belongs to. Defaults to ``"1"``, the
|
|
166
|
+
only version ``from_dict`` reads; any other version is rejected.
|
|
167
|
+
provenance
|
|
168
|
+
An opaque mapping the caller may attach to record where the inputs came from.
|
|
169
|
+
robustmargin stores it and never interprets it. Empty by default.
|
|
170
|
+
baseline
|
|
171
|
+
The zero-severity control's exact rate, set exactly when ``status`` is
|
|
172
|
+
``FAILS_AT_BASELINE``; ``None`` otherwise.
|
|
173
|
+
grid
|
|
174
|
+
The grid break-point value computed as part of the ``SEPARATION``/``NOT_CONVERGED``
|
|
175
|
+
exact-bound fallback, set exactly on that path; ``None`` otherwise.
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
axis: Axis
|
|
179
|
+
definition: Definition
|
|
180
|
+
direction: str
|
|
181
|
+
value: float | None
|
|
182
|
+
lo: float | None
|
|
183
|
+
hi: float | None
|
|
184
|
+
level: float
|
|
185
|
+
interval_method: str
|
|
186
|
+
censoring: Censoring
|
|
187
|
+
status: Status
|
|
188
|
+
dependence: str
|
|
189
|
+
fit: Fit
|
|
190
|
+
warnings: tuple[str, ...] = ()
|
|
191
|
+
schema_version: str = "1"
|
|
192
|
+
provenance: Mapping[str, JSONValue] = field(default_factory=dict)
|
|
193
|
+
baseline: BaselineRate | None = None
|
|
194
|
+
grid: GridBreakPoint | None = None
|
|
195
|
+
|
|
196
|
+
def __post_init__(self) -> None:
|
|
197
|
+
object.__setattr__(self, "warnings", tuple(self.warnings))
|
|
198
|
+
|
|
199
|
+
if self.direction not in _VALID_DIRECTIONS:
|
|
200
|
+
raise ValueError(
|
|
201
|
+
f"Threshold.direction must be one of {sorted(_VALID_DIRECTIONS)}, "
|
|
202
|
+
f"got {self.direction!r}"
|
|
203
|
+
)
|
|
204
|
+
if self.direction != "decreasing":
|
|
205
|
+
raise ValueError(
|
|
206
|
+
"Threshold: direction='increasing' is not supported in this release; an "
|
|
207
|
+
"increasing-is-worse definition is planned for v0.2"
|
|
208
|
+
)
|
|
209
|
+
if self.interval_method not in _VALID_INTERVAL_METHODS:
|
|
210
|
+
raise ValueError(
|
|
211
|
+
f"Threshold.interval_method must be one of {sorted(_VALID_INTERVAL_METHODS)}, "
|
|
212
|
+
f"got {self.interval_method!r}"
|
|
213
|
+
)
|
|
214
|
+
if self.dependence not in _VALID_DEPENDENCE:
|
|
215
|
+
raise ValueError(
|
|
216
|
+
f"Threshold.dependence must be one of {sorted(_VALID_DEPENDENCE)}, "
|
|
217
|
+
f"got {self.dependence!r}"
|
|
218
|
+
)
|
|
219
|
+
_check_finite_number(self.level, label="Threshold.level")
|
|
220
|
+
if not (0.0 < self.level < 1.0):
|
|
221
|
+
raise ValueError(f"Threshold.level must satisfy 0 < level < 1, got {self.level!r}")
|
|
222
|
+
if self.axis != self.fit.axis:
|
|
223
|
+
raise ValueError("Threshold.axis must equal Threshold.fit.axis")
|
|
224
|
+
if self.direction != self.fit.direction:
|
|
225
|
+
raise ValueError("Threshold.direction must equal Threshold.fit.direction")
|
|
226
|
+
|
|
227
|
+
_check_value(self.value, label="value", scale=self.axis.scale)
|
|
228
|
+
_check_value(self.lo, label="lo", scale=self.axis.scale)
|
|
229
|
+
_check_value(self.hi, label="hi", scale=self.axis.scale)
|
|
230
|
+
|
|
231
|
+
requires_fit_ok = (
|
|
232
|
+
self.value is not None
|
|
233
|
+
or self.status in _REQUIRE_FIT_OK_STATUSES
|
|
234
|
+
or self.censoring in _REQUIRE_FIT_OK_CENSORING
|
|
235
|
+
)
|
|
236
|
+
if requires_fit_ok and self.fit.status is not Status.OK:
|
|
237
|
+
raise ValueError(
|
|
238
|
+
"Threshold.fit.status must be OK when value is set, status is OK or "
|
|
239
|
+
"UNREACHABLE, or censoring is OPEN_UPPER/OPEN_LOWER; got "
|
|
240
|
+
f"fit.status={self.fit.status!r}"
|
|
241
|
+
)
|
|
242
|
+
if self.status in _MIRROR_FIT_STATUS_STATUSES and self.fit.status is not self.status:
|
|
243
|
+
raise ValueError(
|
|
244
|
+
f"Threshold.fit.status must equal Threshold.status ({self.status!r}) when "
|
|
245
|
+
"status is SEPARATION, NOT_CONVERGED or CONTROL_INCOMPATIBLE; got "
|
|
246
|
+
f"fit.status={self.fit.status!r}"
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
if self.lo is not None and self.hi is not None:
|
|
250
|
+
if self.status in _NONCONVERGENT_STATUSES:
|
|
251
|
+
if not self.lo < self.hi:
|
|
252
|
+
raise ValueError(
|
|
253
|
+
"Threshold.lo must be strictly less than .hi for a "
|
|
254
|
+
f"SEPARATION/NOT_CONVERGED bracket, got lo={self.lo!r}, hi={self.hi!r}"
|
|
255
|
+
)
|
|
256
|
+
elif not self.lo <= self.hi:
|
|
257
|
+
raise ValueError(
|
|
258
|
+
f"Threshold.lo must not exceed .hi, got lo={self.lo!r}, hi={self.hi!r}"
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
if self.status in _TERMINAL_STATUSES:
|
|
262
|
+
if self.value is not None or self.lo is not None or self.hi is not None:
|
|
263
|
+
raise ValueError(
|
|
264
|
+
f"Threshold.value, .lo and .hi must all be None when status={self.status!r}"
|
|
265
|
+
)
|
|
266
|
+
if self.censoring is not Censoring.NONE:
|
|
267
|
+
raise ValueError(
|
|
268
|
+
f"Threshold.censoring must be NONE when status={self.status!r}, "
|
|
269
|
+
f"got {self.censoring!r}"
|
|
270
|
+
)
|
|
271
|
+
if self.interval_method != "exact_bound":
|
|
272
|
+
raise ValueError(
|
|
273
|
+
"Threshold.interval_method must be 'exact_bound' when "
|
|
274
|
+
f"status={self.status!r}, got {self.interval_method!r}"
|
|
275
|
+
)
|
|
276
|
+
return
|
|
277
|
+
|
|
278
|
+
if self.status in _NONCONVERGENT_STATUSES:
|
|
279
|
+
if self.interval_method != "exact_bound":
|
|
280
|
+
raise ValueError(
|
|
281
|
+
f"Threshold.interval_method must be 'exact_bound' when "
|
|
282
|
+
f"status={self.status!r}, got {self.interval_method!r}"
|
|
283
|
+
)
|
|
284
|
+
if self.value is not None:
|
|
285
|
+
raise ValueError(f"Threshold.value must be None when status={self.status!r}")
|
|
286
|
+
if self.censoring not in (Censoring.NONE, Censoring.RIGHT, Censoring.LEFT):
|
|
287
|
+
raise ValueError(
|
|
288
|
+
f"Threshold.censoring must be one of NONE, RIGHT, LEFT when "
|
|
289
|
+
f"status={self.status!r}, got {self.censoring!r}"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
if self.censoring is Censoring.RIGHT:
|
|
293
|
+
if self.interval_method != "exact_bound":
|
|
294
|
+
raise ValueError(
|
|
295
|
+
"Threshold.interval_method must be 'exact_bound' when censoring is RIGHT, "
|
|
296
|
+
f"got {self.interval_method!r}"
|
|
297
|
+
)
|
|
298
|
+
if self.value is not None:
|
|
299
|
+
raise ValueError("Threshold.value must be None when censoring is RIGHT")
|
|
300
|
+
if self.lo is None:
|
|
301
|
+
raise ValueError("Threshold.lo must be set when censoring is RIGHT")
|
|
302
|
+
if self.hi is not None:
|
|
303
|
+
raise ValueError("Threshold.hi must be None when censoring is RIGHT")
|
|
304
|
+
elif self.censoring is Censoring.LEFT:
|
|
305
|
+
if self.interval_method != "exact_bound":
|
|
306
|
+
raise ValueError(
|
|
307
|
+
"Threshold.interval_method must be 'exact_bound' when censoring is LEFT, "
|
|
308
|
+
f"got {self.interval_method!r}"
|
|
309
|
+
)
|
|
310
|
+
if self.value is not None:
|
|
311
|
+
raise ValueError("Threshold.value must be None when censoring is LEFT")
|
|
312
|
+
if self.hi is None:
|
|
313
|
+
raise ValueError("Threshold.hi must be set when censoring is LEFT")
|
|
314
|
+
if self.lo is not None:
|
|
315
|
+
raise ValueError("Threshold.lo must be None when censoring is LEFT")
|
|
316
|
+
elif self.censoring is Censoring.OPEN_UPPER:
|
|
317
|
+
if self.status is not Status.OK:
|
|
318
|
+
raise ValueError("Threshold.censoring OPEN_UPPER requires status OK")
|
|
319
|
+
if self.interval_method != "profile":
|
|
320
|
+
raise ValueError(
|
|
321
|
+
"Threshold.interval_method must be 'profile' when censoring is OPEN_UPPER, "
|
|
322
|
+
f"got {self.interval_method!r}"
|
|
323
|
+
)
|
|
324
|
+
if self.value is None or self.lo is None:
|
|
325
|
+
raise ValueError("Threshold.value and .lo must be set when censoring is OPEN_UPPER")
|
|
326
|
+
if self.hi is not None:
|
|
327
|
+
raise ValueError("Threshold.hi must be None when censoring is OPEN_UPPER")
|
|
328
|
+
elif self.censoring is Censoring.OPEN_LOWER:
|
|
329
|
+
if self.status is not Status.OK:
|
|
330
|
+
raise ValueError("Threshold.censoring OPEN_LOWER requires status OK")
|
|
331
|
+
if self.interval_method != "profile":
|
|
332
|
+
raise ValueError(
|
|
333
|
+
"Threshold.interval_method must be 'profile' when censoring is OPEN_LOWER, "
|
|
334
|
+
f"got {self.interval_method!r}"
|
|
335
|
+
)
|
|
336
|
+
if self.value is None or self.hi is None:
|
|
337
|
+
raise ValueError("Threshold.value and .hi must be set when censoring is OPEN_LOWER")
|
|
338
|
+
if self.lo is not None:
|
|
339
|
+
raise ValueError("Threshold.lo must be None when censoring is OPEN_LOWER")
|
|
340
|
+
elif self.censoring is Censoring.NONE and self.status is Status.OK:
|
|
341
|
+
# `value` is still required. `lo`/`hi` are not: a converged fit on a design too
|
|
342
|
+
# coarse to constrain the parameter can have a profile that crosses the chi-square
|
|
343
|
+
# target on neither side, and v1 has no `Censoring` member for "open on both sides"
|
|
344
|
+
# (`OPEN_UPPER`/`OPEN_LOWER` each require their closed side to exist). Reporting the
|
|
345
|
+
# estimate with no bounds, and the reason in `warnings`, is the honest encoding;
|
|
346
|
+
# inventing a bound at the search limit would not be. Loosening an invariant is a
|
|
347
|
+
# safe (additive) change, but a card that relies on it needs a reader at least as
|
|
348
|
+
# new as the writer.
|
|
349
|
+
if self.value is None:
|
|
350
|
+
raise ValueError(
|
|
351
|
+
"Threshold.value must be set when censoring is NONE and status is OK"
|
|
352
|
+
)
|
|
353
|
+
if (self.lo is None) != (self.hi is None):
|
|
354
|
+
raise ValueError(
|
|
355
|
+
"Threshold.lo and .hi must either both be set or both be None when "
|
|
356
|
+
f"censoring is NONE and status is OK, got lo={self.lo!r}, hi={self.hi!r}"
|
|
357
|
+
)
|
|
358
|
+
if (
|
|
359
|
+
self.lo is not None
|
|
360
|
+
and self.hi is not None
|
|
361
|
+
and not (self.lo <= self.value <= self.hi)
|
|
362
|
+
):
|
|
363
|
+
raise ValueError(
|
|
364
|
+
f"Threshold requires lo <= value <= hi, got lo={self.lo!r}, "
|
|
365
|
+
f"value={self.value!r}, hi={self.hi!r}"
|
|
366
|
+
)
|
|
367
|
+
if self.interval_method not in ("profile", "delta"):
|
|
368
|
+
raise ValueError(
|
|
369
|
+
"Threshold.interval_method must be 'profile' or 'delta' when censoring is "
|
|
370
|
+
f"NONE and status is OK, got {self.interval_method!r}"
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
# baseline/grid only ever *tighten* -- an existing invariant above is never weakened to
|
|
374
|
+
# admit them. Each is legal only alongside the status it documents; neither is required
|
|
375
|
+
# by that status (a threshold built without them has both `None`).
|
|
376
|
+
if self.baseline is not None and self.status is not Status.FAILS_AT_BASELINE:
|
|
377
|
+
raise ValueError(
|
|
378
|
+
"Threshold.baseline may only be set when status is FAILS_AT_BASELINE, got "
|
|
379
|
+
f"status={self.status!r}"
|
|
380
|
+
)
|
|
381
|
+
if self.grid is not None and self.status not in _NONCONVERGENT_STATUSES:
|
|
382
|
+
raise ValueError(
|
|
383
|
+
"Threshold.grid may only be set when status is SEPARATION or NOT_CONVERGED, got "
|
|
384
|
+
f"status={self.status!r}"
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
_VALID_INTERVAL_METHOD_REQUESTS = frozenset({"profile", "delta"})
|
|
389
|
+
_VALID_DEPENDENCE_INPUTS = frozenset({"independent"})
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _control_cell_or_none(fit: Fit) -> Cell | None:
|
|
393
|
+
for cell in fit.cells:
|
|
394
|
+
if cell.severity == 0.0:
|
|
395
|
+
return cell
|
|
396
|
+
return None
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _resolve_target(definition: Definition, *, fit: Fit) -> tuple[float, str | None]:
|
|
400
|
+
"""The already-resolved target performance ``P*`` plus an optional warning naming where an
|
|
401
|
+
asymptote came from.
|
|
402
|
+
|
|
403
|
+
``absolute(a)`` never needs an asymptote at all: ``P* = a``. ``baseline_fraction(p)`` and
|
|
404
|
+
``relative(p)`` need ``upper`` (and, for ``relative``, ``lower``): the fitted values when
|
|
405
|
+
``fit.status is Status.OK``, or -- since there are no fitted asymptotes on the exact-bound
|
|
406
|
+
fallback path -- the zero-severity control cell's own observed rate for ``upper`` (with
|
|
407
|
+
``lower`` taken as ``0.0``, since nothing in the data speaks to a floor without a fit) and a
|
|
408
|
+
warning naming that substitution.
|
|
409
|
+
"""
|
|
410
|
+
if definition.kind == "absolute":
|
|
411
|
+
return definition.value, None
|
|
412
|
+
|
|
413
|
+
if fit.status is Status.OK:
|
|
414
|
+
assert fit.params is not None
|
|
415
|
+
upper = fit.params["upper"].value
|
|
416
|
+
lower = fit.params["lower"].value
|
|
417
|
+
warning = None
|
|
418
|
+
else:
|
|
419
|
+
if definition.kind == "relative":
|
|
420
|
+
# `relative(p)` needs BOTH asymptotes: `P* = u - p*(u - l)`. When the fit failed,
|
|
421
|
+
# `u` comes from the control cell's observed rate, but there is no source for `l`,
|
|
422
|
+
# because nothing in the data speaks to a lower asymptote.
|
|
423
|
+
# Substituting `l = 0` would silently turn `relative(p)` into
|
|
424
|
+
# `baseline_fraction(1 - p)` and, whenever the true floor is above zero, aim at a
|
|
425
|
+
# target that is easier to clear -- an optimistic threshold produced on a failed
|
|
426
|
+
# fit by an assumption nothing in the data supports, with no warning naming it.
|
|
427
|
+
# Refuse instead, as this function already refuses when there is no control cell to
|
|
428
|
+
# supply `u`.
|
|
429
|
+
raise ValueError(
|
|
430
|
+
f"threshold: definition.kind='relative' needs both asymptotes to resolve a "
|
|
431
|
+
f"target, and fit.status is {fit.status!r} (no fitted asymptotes). There is no "
|
|
432
|
+
"defensible fallback for the lower asymptote -- nothing in the data speaks to "
|
|
433
|
+
"it -- so no target is resolved. Use absolute(a), which needs no asymptote, or "
|
|
434
|
+
"baseline_fraction(p), which needs only the control rate"
|
|
435
|
+
)
|
|
436
|
+
control = _control_cell_or_none(fit)
|
|
437
|
+
if control is None:
|
|
438
|
+
raise ValueError(
|
|
439
|
+
f"threshold: definition.kind={definition.kind!r} needs an upper asymptote to "
|
|
440
|
+
f"resolve a target, fit.status is {fit.status!r} (no fitted asymptotes), and "
|
|
441
|
+
"there is no zero-severity control cell to fall back to"
|
|
442
|
+
)
|
|
443
|
+
upper = control.successes / control.trials
|
|
444
|
+
lower = 0.0
|
|
445
|
+
warning = (
|
|
446
|
+
f"threshold: fit.status is {fit.status!r} (no fitted asymptotes); the target for "
|
|
447
|
+
f"definition.kind={definition.kind!r} was resolved from the zero-severity control "
|
|
448
|
+
f"cell's observed rate ({control.successes}/{control.trials} = {upper!r}), not a "
|
|
449
|
+
"fitted upper asymptote"
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
if definition.kind == "baseline_fraction":
|
|
453
|
+
return definition.value * upper, warning
|
|
454
|
+
return upper - definition.value * (upper - lower), warning # relative
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def threshold(
|
|
458
|
+
fit: Fit,
|
|
459
|
+
*,
|
|
460
|
+
definition: Definition,
|
|
461
|
+
interval_method: str,
|
|
462
|
+
level: float,
|
|
463
|
+
dependence: str | None = None,
|
|
464
|
+
) -> Threshold:
|
|
465
|
+
"""Solve for where ``fit``'s curve crosses ``definition``'s target performance, with a
|
|
466
|
+
confidence interval, or report why no value exists.
|
|
467
|
+
|
|
468
|
+
``interval_method`` is a *request*, not a promise: every censored or failed result records
|
|
469
|
+
``"exact_bound"`` regardless of what was requested, because that is the method that actually
|
|
470
|
+
produced its numbers. The order in which the outcomes below are checked is
|
|
471
|
+
:mod:`robustmargin.censoring`'s module docstring, not this one's.
|
|
472
|
+
|
|
473
|
+
Parameters
|
|
474
|
+
----------
|
|
475
|
+
fit
|
|
476
|
+
The fit to solve a threshold from.
|
|
477
|
+
definition
|
|
478
|
+
Which target performance to solve for.
|
|
479
|
+
interval_method
|
|
480
|
+
``"profile"`` (primary) or ``"delta"`` (cross-check). Required, with no default.
|
|
481
|
+
level
|
|
482
|
+
The confidence level, strictly between 0 and 1.
|
|
483
|
+
dependence
|
|
484
|
+
``None`` (the default) resolves to ``"independent"`` unless ``fit.cluster_ids`` is set,
|
|
485
|
+
in which case omitting it raises (a method assuming independence must refuse clustered
|
|
486
|
+
input unless the caller explicitly opts in). ``"independent"`` may always be passed
|
|
487
|
+
explicitly. ``"paired"`` raises, naming that it is not implemented in this release. The
|
|
488
|
+
resulting ``Threshold.dependence`` is always ``"independent"`` in this release.
|
|
489
|
+
|
|
490
|
+
Returns
|
|
491
|
+
-------
|
|
492
|
+
Threshold
|
|
493
|
+
|
|
494
|
+
Raises
|
|
495
|
+
------
|
|
496
|
+
ValueError
|
|
497
|
+
If ``interval_method`` or ``level`` is invalid; if ``dependence`` is omitted while
|
|
498
|
+
``fit.cluster_ids`` is set, or is any value other than ``None``/``"independent"``/
|
|
499
|
+
``"paired"``; if ``dependence="paired"`` is requested (not yet implemented); or if
|
|
500
|
+
``definition.kind`` is ``"baseline_fraction"``/``"relative"``, ``fit.status`` is not
|
|
501
|
+
``OK``, and no zero-severity control cell exists to resolve a target from.
|
|
502
|
+
"""
|
|
503
|
+
if interval_method not in _VALID_INTERVAL_METHOD_REQUESTS:
|
|
504
|
+
raise ValueError(
|
|
505
|
+
"threshold: interval_method must be one of "
|
|
506
|
+
f"{sorted(_VALID_INTERVAL_METHOD_REQUESTS)}, got {interval_method!r}"
|
|
507
|
+
)
|
|
508
|
+
_check_finite_number(level, label="threshold level")
|
|
509
|
+
if not (0.0 < level < 1.0):
|
|
510
|
+
raise ValueError(f"threshold: level must satisfy 0 < level < 1, got {level!r}")
|
|
511
|
+
|
|
512
|
+
if dependence == "paired":
|
|
513
|
+
raise ValueError(
|
|
514
|
+
"threshold: dependence='paired' is not implemented in this release; pass "
|
|
515
|
+
"dependence='independent' explicitly to acknowledge independence is being assumed "
|
|
516
|
+
"anyway, or wait for a future release that adds it"
|
|
517
|
+
)
|
|
518
|
+
if dependence is not None and dependence not in _VALID_DEPENDENCE_INPUTS:
|
|
519
|
+
raise ValueError(
|
|
520
|
+
"threshold: dependence must be None, 'independent', or (v0.2, not yet implemented) "
|
|
521
|
+
f"'paired', got {dependence!r}"
|
|
522
|
+
)
|
|
523
|
+
if fit.cluster_ids is not None and dependence is None:
|
|
524
|
+
raise ValueError(
|
|
525
|
+
"threshold: fit carries cluster ids (fit.cluster_ids is set); a method that assumes "
|
|
526
|
+
"independence must not run on clustered data unless the caller passes "
|
|
527
|
+
"dependence='independent' explicitly"
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
target, target_warning = _resolve_target(definition, fit=fit)
|
|
531
|
+
collected_warnings: list[str] = [] if target_warning is None else [target_warning]
|
|
532
|
+
|
|
533
|
+
# Applies whatever the resulting censoring or status, because every threshold rule and every
|
|
534
|
+
# fitted model assume the true performance curve is monotone in severity. Never changes
|
|
535
|
+
# value/lo/hi/status/censoring -- only adds a warning.
|
|
536
|
+
if fit.monotonicity is not None and fit.monotonicity.any_flagged:
|
|
537
|
+
flagged_pairs = ", ".join(
|
|
538
|
+
f"({pair.severity_low!r}, {pair.severity_high!r})"
|
|
539
|
+
for pair in fit.monotonicity.pairs
|
|
540
|
+
if pair.flagged
|
|
541
|
+
)
|
|
542
|
+
collected_warnings.append(
|
|
543
|
+
"NON_MONOTONE_DATA: threshold: fit.monotonicity flags an adjacent-level reversal "
|
|
544
|
+
f"at severity pair(s) {flagged_pairs}; every threshold rule and every fitted model "
|
|
545
|
+
"here assume the true performance curve is monotone in severity"
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
classification = classify(fit, target=target, level=level)
|
|
549
|
+
collected_warnings.extend(classification.warnings)
|
|
550
|
+
|
|
551
|
+
if classification.solve:
|
|
552
|
+
theta_hat, value = solve_value(fit, target=target)
|
|
553
|
+
if value <= 0.0:
|
|
554
|
+
# A linear axis can solve to a crossing at or below zero severity. v0.1 severity is
|
|
555
|
+
# non-negative (`Threshold` enforces it), so such a crossing is not reachable on the
|
|
556
|
+
# axis as tested. It is reported as UNREACHABLE with a warning that names the solved
|
|
557
|
+
# value, so it is distinguishable from the asymptote-driven UNREACHABLE. Allowing
|
|
558
|
+
# negative thresholds later is a loosening, and therefore safe.
|
|
559
|
+
collected_warnings.append(
|
|
560
|
+
"threshold: the fitted curve is already below the target at severity 0 -- it "
|
|
561
|
+
f"crosses at {value!r}, at or below the axis origin, so on this axis the "
|
|
562
|
+
"criterion fails before any severity is applied. Reported as UNREACHABLE "
|
|
563
|
+
"because this release's severity axis is non-negative and there is no "
|
|
564
|
+
"model-based equivalent of FAILS_AT_BASELINE for this case"
|
|
565
|
+
)
|
|
566
|
+
return Threshold(
|
|
567
|
+
axis=fit.axis,
|
|
568
|
+
definition=definition,
|
|
569
|
+
direction=fit.direction,
|
|
570
|
+
value=None,
|
|
571
|
+
lo=None,
|
|
572
|
+
hi=None,
|
|
573
|
+
level=level,
|
|
574
|
+
interval_method="exact_bound",
|
|
575
|
+
censoring=Censoring.NONE,
|
|
576
|
+
status=Status.UNREACHABLE,
|
|
577
|
+
dependence="independent",
|
|
578
|
+
fit=fit,
|
|
579
|
+
warnings=tuple(collected_warnings),
|
|
580
|
+
baseline=None,
|
|
581
|
+
grid=None,
|
|
582
|
+
)
|
|
583
|
+
if interval_method == "profile":
|
|
584
|
+
interval_result = profile_interval(
|
|
585
|
+
fit, definition=definition, target=target, level=level, theta_hat=theta_hat
|
|
586
|
+
)
|
|
587
|
+
else:
|
|
588
|
+
interval_result = delta_interval(
|
|
589
|
+
fit,
|
|
590
|
+
definition=definition,
|
|
591
|
+
target=target,
|
|
592
|
+
level=level,
|
|
593
|
+
theta_hat=theta_hat,
|
|
594
|
+
value=value,
|
|
595
|
+
)
|
|
596
|
+
collected_warnings.extend(interval_result.warnings)
|
|
597
|
+
return Threshold(
|
|
598
|
+
axis=fit.axis,
|
|
599
|
+
definition=definition,
|
|
600
|
+
direction=fit.direction,
|
|
601
|
+
value=value,
|
|
602
|
+
lo=interval_result.lo,
|
|
603
|
+
hi=interval_result.hi,
|
|
604
|
+
level=level,
|
|
605
|
+
interval_method=interval_method,
|
|
606
|
+
censoring=interval_result.censoring,
|
|
607
|
+
status=Status.OK,
|
|
608
|
+
dependence="independent",
|
|
609
|
+
fit=fit,
|
|
610
|
+
warnings=tuple(collected_warnings),
|
|
611
|
+
baseline=None,
|
|
612
|
+
grid=None,
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
return Threshold(
|
|
616
|
+
axis=fit.axis,
|
|
617
|
+
definition=definition,
|
|
618
|
+
direction=fit.direction,
|
|
619
|
+
value=None,
|
|
620
|
+
lo=classification.lo,
|
|
621
|
+
hi=classification.hi,
|
|
622
|
+
level=level,
|
|
623
|
+
interval_method="exact_bound",
|
|
624
|
+
censoring=classification.censoring,
|
|
625
|
+
status=classification.status,
|
|
626
|
+
dependence="independent",
|
|
627
|
+
fit=fit,
|
|
628
|
+
warnings=tuple(collected_warnings),
|
|
629
|
+
baseline=classification.baseline,
|
|
630
|
+
grid=classification.grid,
|
|
631
|
+
)
|