robustmargin 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robustmargin/__init__.py +90 -0
- robustmargin/censoring.py +652 -0
- robustmargin/empirical.py +348 -0
- robustmargin/intervals.py +785 -0
- robustmargin/models.py +2119 -0
- robustmargin/py.typed +0 -0
- robustmargin/ratio.py +941 -0
- robustmargin/report.py +650 -0
- robustmargin/schema/scorecard-v1.json +777 -0
- robustmargin/testing.py +465 -0
- robustmargin/threshold.py +631 -0
- robustmargin/types.py +541 -0
- robustmargin/validation.py +722 -0
- robustmargin-0.1.1.dist-info/METADATA +281 -0
- robustmargin-0.1.1.dist-info/RECORD +18 -0
- robustmargin-0.1.1.dist-info/WHEEL +5 -0
- robustmargin-0.1.1.dist-info/licenses/LICENSE +201 -0
- robustmargin-0.1.1.dist-info/top_level.txt +1 -0
robustmargin/models.py
ADDED
|
@@ -0,0 +1,2119 @@
|
|
|
1
|
+
"""The dose-response fit result, its fitting entry point, and prediction.
|
|
2
|
+
|
|
3
|
+
This module holds :class:`Fit`, the two small dataclasses it embeds (:class:`Parameter`,
|
|
4
|
+
:class:`Covariance`), :func:`fit_dose_response`, and :class:`Prediction` (:meth:`Fit.predict`'s
|
|
5
|
+
return type). Two fitting paths exist: statsmodels ``GLM`` when both asymptotes are fixed at
|
|
6
|
+
exactly ``1.0``/``0.0``, and ``statsmodels.base.model.GenericLikelihoodModel`` -- given a
|
|
7
|
+
likelihood function, never a hand-rolled optimizer or IRLS loop -- otherwise. A fit with no
|
|
8
|
+
interior maximum (wrong-sign slope, or an estimated asymptote pinned to a parameter-space
|
|
9
|
+
boundary) is reported as ``Status.NOT_CONVERGED`` with the reason in ``Fit.warnings``, never as
|
|
10
|
+
a number that happens to be where the optimizer stopped.
|
|
11
|
+
|
|
12
|
+
**Known limitation.** A near-step curve (``s -> 0``) with an estimated asymptote, or a fixed
|
|
13
|
+
one other than ``1``/``0``, is not detected: it can return ``Status.OK`` with Wald standard
|
|
14
|
+
errors that carry little information. With such asymptotes the pre-fit overlap check is
|
|
15
|
+
necessary but not sufficient for an interior maximum; it is exact only for ``u = 1, l = 0``.
|
|
16
|
+
|
|
17
|
+
**v1 is binomial-only.** ``model`` accepts only ``"binomial"``; a new model family (``"ll4"``,
|
|
18
|
+
``"isotonic"``) is a v0.2 addition that requires a new ``schema_version``, not a silent
|
|
19
|
+
extension of the current one. :data:`robustmargin.report.SCHEMA_VERSION` is the one version
|
|
20
|
+
:func:`robustmargin.report.from_dict` reads and the single source of that number.
|
|
21
|
+
|
|
22
|
+
:class:`Fit` also carries :attr:`~Fit.diagnostics` (model-only goodness of fit, dispersion and
|
|
23
|
+
link comparison, filled only when ``status`` is ``OK`` -- see :func:`robustmargin.diagnose`) and
|
|
24
|
+
:attr:`~Fit.monotonicity` (a data-only adjacent-level check, filled on *every* return path
|
|
25
|
+
regardless of ``status`` -- see :func:`robustmargin.check_monotonicity` in
|
|
26
|
+
:mod:`robustmargin.validation`). Both are reported and never acted on: neither changes ``link``,
|
|
27
|
+
``params``, or any other field of the ``Fit`` they are attached to.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import math
|
|
33
|
+
import warnings
|
|
34
|
+
from collections.abc import Callable, Mapping
|
|
35
|
+
from dataclasses import dataclass, field, replace
|
|
36
|
+
from typing import Literal
|
|
37
|
+
|
|
38
|
+
import numpy as np
|
|
39
|
+
from numpy.typing import ArrayLike, NDArray
|
|
40
|
+
from scipy.special import expit, gammaln, xlogy
|
|
41
|
+
from scipy.special import logit as _logit
|
|
42
|
+
from scipy.stats import norm
|
|
43
|
+
from statsmodels.base.model import GenericLikelihoodModel
|
|
44
|
+
from statsmodels.genmod import families
|
|
45
|
+
from statsmodels.genmod.generalized_linear_model import GLM
|
|
46
|
+
from statsmodels.tools.sm_exceptions import ConvergenceWarning, PerfectSeparationWarning
|
|
47
|
+
from statsmodels.tools.tools import add_constant
|
|
48
|
+
|
|
49
|
+
from robustmargin.types import Axis, Cell, JSONValue, Observations, Status, _check_finite_number
|
|
50
|
+
from robustmargin.validation import Diagnostics, MonotonicityCheck, check_monotonicity, diagnose
|
|
51
|
+
|
|
52
|
+
# `robustmargin.validation` only imports `Fit` from this module under `TYPE_CHECKING` (see that
|
|
53
|
+
# module's own docstring), so this is not a circular import at runtime: by the time
|
|
54
|
+
# `fit_dose_response` below actually calls `diagnose`/`check_monotonicity`, this module has
|
|
55
|
+
# already finished executing this very import statement.
|
|
56
|
+
|
|
57
|
+
# Narrow statsmodels imports above, never `statsmodels.api`: that module also imports graphics and
|
|
58
|
+
# tsa, and statsmodels 0.14.0's tsa fails to import against current pandas
|
|
59
|
+
# (`deprecate_kwarg() missing 'new_arg_name'`), which breaks the dependency-floor CI job.
|
|
60
|
+
|
|
61
|
+
__all__ = ["Covariance", "Fit", "Parameter", "Prediction", "fit_dose_response"]
|
|
62
|
+
|
|
63
|
+
_VALID_MODELS = frozenset({"binomial"})
|
|
64
|
+
_VALID_LINKS: dict[str, frozenset[str]] = {
|
|
65
|
+
"binomial": frozenset({"probit", "logit", "cloglog"}),
|
|
66
|
+
}
|
|
67
|
+
_BINOMIAL_PARAM_NAMES = frozenset({"mu", "s", "upper", "lower"})
|
|
68
|
+
_VALID_DIRECTIONS = frozenset({"decreasing", "increasing"})
|
|
69
|
+
_INVALID_FIT_STATUSES = frozenset({Status.UNREACHABLE, Status.FAILS_AT_BASELINE})
|
|
70
|
+
|
|
71
|
+
# Tolerance for Covariance's positive-definiteness check: the minimum eigenvalue of the
|
|
72
|
+
# correlation matrix ``D^-1/2 . matrix . D^-1/2`` must exceed this, unscaled, so the test does
|
|
73
|
+
# not depend on the severity units. Strictly positive (not merely semi-definite): a singular
|
|
74
|
+
# matrix, such as ``((0.0, 0.0), (0.0, 1.0))``, is rejected, because a singular
|
|
75
|
+
# covariance means at least one estimated parameter carries no information at all.
|
|
76
|
+
_POSITIVE_DEFINITE_RELATIVE_TOLERANCE = 1e-12
|
|
77
|
+
|
|
78
|
+
# ------------------------------------------------------------------------------------------
|
|
79
|
+
# fit_dose_response: module constants (the numeric boundary test for an estimated asymptote)
|
|
80
|
+
# ------------------------------------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
_VALID_UPPER_LOWER = "estimate"
|
|
83
|
+
_STATSMODELS_LINKS = {
|
|
84
|
+
"probit": families.links.Probit(),
|
|
85
|
+
"logit": families.links.Logit(),
|
|
86
|
+
"cloglog": families.links.CLogLog(),
|
|
87
|
+
}
|
|
88
|
+
# An estimated asymptote's logit-scale parameter (a for lower, b for upper) beyond this
|
|
89
|
+
# magnitude, or the asymptote itself within this distance of a probability boundary (0 or 1),
|
|
90
|
+
# means the likelihood has no interior maximum -- reported as Status.NOT_CONVERGED with a
|
|
91
|
+
# boundary warning, never as a number. Changing either cut-off is a statistical-method change.
|
|
92
|
+
_ASYMPTOTE_LOGIT_BOUND = 15.0
|
|
93
|
+
_ASYMPTOTE_PROBABILITY_EPS = 1e-6
|
|
94
|
+
# Separation is decided by the exact pre-fit overlap check (`_check_separation`), never by
|
|
95
|
+
# statsmodels' PerfectSeparationWarning. That warning tests `allclose(fitted, observed)`, which on
|
|
96
|
+
# grouped binomial data also fires on an exact fit -- every two-level design with overlap -- so it
|
|
97
|
+
# is filtered to "ignore" around both GLM calls below. Every fit reaching those calls has already
|
|
98
|
+
# passed the exact check. The check is exact for fixed u = 1, l = 0 (Albert & Anderson, 1984); with
|
|
99
|
+
# estimated or other fixed asymptotes it is necessary but not sufficient, and a near-step curve
|
|
100
|
+
# there is not yet detected (see the module docstring).
|
|
101
|
+
|
|
102
|
+
# A fitted `s` greater than this factor times the span of the likelihood-contributing transformed
|
|
103
|
+
# levels (the tested range on `log x` or `x`, excluding a log-axis zero-severity control, which
|
|
104
|
+
# carries no covariate value) is flat up to rounding -- the wrong-sign case's degenerate limit,
|
|
105
|
+
# `s -> infinity` -- and gives Status.NOT_CONVERGED rather than a covariance built from a near-zero
|
|
106
|
+
# slope (which would otherwise pass Covariance's own positive-definiteness check while being
|
|
107
|
+
# numerically meaningless, for example ~1e61). Changing this cut-off is a statistical-method change.
|
|
108
|
+
_FLAT_SLOPE_SPAN_FACTOR = 1e8
|
|
109
|
+
|
|
110
|
+
# Likelihood-path convergence: Newton's own `converged` flag only bounds the last step taken, so
|
|
111
|
+
# two more checks apply at the final parameters. The Newton polish's log-likelihood must not
|
|
112
|
+
# have regressed below the BFGS stage's, within this *relative* tolerance (an absolute tolerance
|
|
113
|
+
# does not scale with the log-likelihood's own magnitude, which grows with sample size and is
|
|
114
|
+
# not something a caller controls), applied as
|
|
115
|
+
# `bfgs_llf - _LLF_REGRESSION_TOLERANCE * (1 + |bfgs_llf|)` -- floating-point noise between two
|
|
116
|
+
# different optimizers evaluating the same likelihood at very close points, scaled the same way
|
|
117
|
+
# `_SCORE_TOLERANCE_FACTOR` below is.
|
|
118
|
+
_LLF_REGRESSION_TOLERANCE = 1e-8
|
|
119
|
+
# The score (gradient of the log-likelihood) at the final parameters must be near zero. There is
|
|
120
|
+
# no universal absolute tolerance for a gradient, because its scale depends on the
|
|
121
|
+
# log-likelihood's own scale (more data means both the log-likelihood and its gradient are
|
|
122
|
+
# larger in magnitude even at the true optimum); `1 + |llf|` is a defensible, simple proxy for
|
|
123
|
+
# that scale that never reaches zero, so the tolerance below is always well-defined.
|
|
124
|
+
_SCORE_TOLERANCE_FACTOR = 1e-4
|
|
125
|
+
|
|
126
|
+
# Shared wording for a wrong-sign slope (the GLM path's own beta1 <= 0 check) and for the flat
|
|
127
|
+
# curve it degenerates into (`_FLAT_SLOPE_SPAN_FACTOR` above) -- the same underlying phenomenon
|
|
128
|
+
# (no finite (mu, s) exists once the constrained maximum is pushed to the s -> infinity
|
|
129
|
+
# boundary), so one reason is used for both, on both fitting paths.
|
|
130
|
+
_WRONG_SIGN_OR_FLAT_WARNING = (
|
|
131
|
+
"fit_dose_response: the fitted curve has no interior maximum for direction='decreasing' "
|
|
132
|
+
"(a wrong-sign slope, or a curve that is flat up to rounding) -- no finite (mu, s) exists"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@dataclass(frozen=True, kw_only=True)
|
|
137
|
+
class Parameter:
|
|
138
|
+
"""One fitted or fixed parameter value, on the **natural scale** -- the curve's own units
|
|
139
|
+
(for example ``s`` in log-severity units, ``upper``/``lower`` as probabilities), never the
|
|
140
|
+
optimiser's internal reparametrisation (``log s``, or ``upper``/``lower`` on a logit scale).
|
|
141
|
+
An estimated-asymptote fit delta-transforms its optimiser-scale covariance onto this scale
|
|
142
|
+
before it is ever stored here (see :class:`Fit`).
|
|
143
|
+
|
|
144
|
+
Attributes
|
|
145
|
+
----------
|
|
146
|
+
value
|
|
147
|
+
The parameter's value on its natural scale. Must be finite, and must be a genuine
|
|
148
|
+
``float`` or ``int``, never a ``bool``.
|
|
149
|
+
fixed
|
|
150
|
+
``True`` if this parameter was held fixed rather than estimated (for example
|
|
151
|
+
``upper=1.0`` fixed in a two-parameter probit). Must be a genuine ``bool``.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
value: float
|
|
155
|
+
fixed: bool
|
|
156
|
+
|
|
157
|
+
def __post_init__(self) -> None:
|
|
158
|
+
_check_finite_number(self.value, label="Parameter.value")
|
|
159
|
+
if not isinstance(self.fixed, bool):
|
|
160
|
+
raise ValueError(f"Parameter.fixed must be a real bool, got {self.fixed!r}")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@dataclass(frozen=True, kw_only=True)
|
|
164
|
+
class Covariance:
|
|
165
|
+
"""The variance-covariance matrix of a fit's estimated (non-fixed) parameters, on the same
|
|
166
|
+
natural scale as :class:`Parameter`.
|
|
167
|
+
|
|
168
|
+
Attributes
|
|
169
|
+
----------
|
|
170
|
+
names
|
|
171
|
+
The estimated parameter names, in the order ``matrix`` is indexed by. Unique.
|
|
172
|
+
matrix
|
|
173
|
+
A square, finite, symmetric, **positive-definite** matrix with side ``len(names)``.
|
|
174
|
+
Every entry must be a genuine finite number, never a ``bool``. Symmetry is checked with
|
|
175
|
+
:func:`math.isclose` (``rel_tol=1e-9``, ``abs_tol=1e-12``), not exact equality, to
|
|
176
|
+
tolerate floating-point round-trip noise. Positive-definiteness is checked on the
|
|
177
|
+
**correlation matrix** ``D^-1/2 . matrix . D^-1/2`` (``D = diag(matrix)``): every
|
|
178
|
+
variance must be positive, and the correlation matrix's minimum eigenvalue (via
|
|
179
|
+
:func:`numpy.linalg.eigvalsh`) must exceed ``1e-12``. The rule is therefore invariant to
|
|
180
|
+
rescaling any single parameter, which matters because the natural scale mixes severity
|
|
181
|
+
units (``mu``, ``s``) with probabilities (``upper``, ``lower``). It accepts every matrix
|
|
182
|
+
that a raw-eigenvalue rule (minimum eigenvalue ``> 1e-12 * max(1, max|entry|)``) would
|
|
183
|
+
accept. A singular matrix -- for example ``((0.0, 0.0), (0.0, 1.0))``, or any matrix with a
|
|
184
|
+
zero eigenvalue -- is rejected, not merely a matrix with a negative one.
|
|
185
|
+
"""
|
|
186
|
+
|
|
187
|
+
names: tuple[str, ...]
|
|
188
|
+
matrix: tuple[tuple[float, ...], ...]
|
|
189
|
+
|
|
190
|
+
def __post_init__(self) -> None:
|
|
191
|
+
object.__setattr__(self, "names", tuple(self.names))
|
|
192
|
+
object.__setattr__(self, "matrix", tuple(tuple(row) for row in self.matrix))
|
|
193
|
+
|
|
194
|
+
if len(set(self.names)) != len(self.names):
|
|
195
|
+
raise ValueError(f"Covariance.names must be unique, got {self.names!r}")
|
|
196
|
+
|
|
197
|
+
n = len(self.names)
|
|
198
|
+
if len(self.matrix) != n:
|
|
199
|
+
raise ValueError(
|
|
200
|
+
f"Covariance.matrix must have {n} rows to match names, got {len(self.matrix)}"
|
|
201
|
+
)
|
|
202
|
+
for row in self.matrix:
|
|
203
|
+
if len(row) != n:
|
|
204
|
+
raise ValueError(
|
|
205
|
+
f"Covariance.matrix must be square with side {n}, got a row of length "
|
|
206
|
+
f"{len(row)}"
|
|
207
|
+
)
|
|
208
|
+
for entry in row:
|
|
209
|
+
_check_finite_number(entry, label="Covariance.matrix entries")
|
|
210
|
+
|
|
211
|
+
for i in range(n):
|
|
212
|
+
for j in range(n):
|
|
213
|
+
if not math.isclose(
|
|
214
|
+
self.matrix[i][j], self.matrix[j][i], rel_tol=1e-9, abs_tol=1e-12
|
|
215
|
+
):
|
|
216
|
+
raise ValueError(
|
|
217
|
+
"Covariance.matrix must be symmetric: "
|
|
218
|
+
f"matrix[{i}][{j}]={self.matrix[i][j]!r} != "
|
|
219
|
+
f"matrix[{j}][{i}]={self.matrix[j][i]!r}"
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
if n > 0 and not _is_positive_definite(np.array(self.matrix, dtype=float)):
|
|
223
|
+
raise ValueError(
|
|
224
|
+
"Covariance.matrix must be positive-definite: every variance must be positive "
|
|
225
|
+
"and the correlation matrix's minimum eigenvalue must exceed "
|
|
226
|
+
f"{_POSITIVE_DEFINITE_RELATIVE_TOLERANCE!r}"
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _check_cluster_ids(cluster_ids: tuple[str | int, ...] | None, *, label: str) -> None:
|
|
231
|
+
if cluster_ids is None:
|
|
232
|
+
return
|
|
233
|
+
if len(cluster_ids) == 0:
|
|
234
|
+
raise ValueError(f"{label} must be non-empty when given, got an empty tuple")
|
|
235
|
+
if len(set(cluster_ids)) != len(cluster_ids):
|
|
236
|
+
raise ValueError(f"{label} must be unique, got {cluster_ids!r}")
|
|
237
|
+
for cluster_id in cluster_ids:
|
|
238
|
+
if cluster_id is None or isinstance(cluster_id, bool):
|
|
239
|
+
raise ValueError(
|
|
240
|
+
f"{label} elements must be a str or int, never None or bool, got {cluster_id!r}"
|
|
241
|
+
)
|
|
242
|
+
if not isinstance(cluster_id, (str, int)):
|
|
243
|
+
raise ValueError(f"{label} elements must be a str or int, got {cluster_id!r}")
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
@dataclass(frozen=True, kw_only=True, eq=False)
|
|
247
|
+
class Prediction:
|
|
248
|
+
"""The curve and a confidence band evaluated at a set of severities (:meth:`Fit.predict`).
|
|
249
|
+
Never serialised (``report.py`` does not know this type): it is a throwaway array bundle,
|
|
250
|
+
not a stored result.
|
|
251
|
+
|
|
252
|
+
Two band scales are used, chosen by :meth:`Fit.predict` from which asymptotes were
|
|
253
|
+
estimated -- documented in full on :meth:`Fit.predict` itself, since the choice depends on
|
|
254
|
+
the ``Fit`` being predicted from, not on this type.
|
|
255
|
+
|
|
256
|
+
``eq``/``hash`` are disabled (unlike every other result dataclass in this package): the
|
|
257
|
+
default dataclass equality would compare ``numpy`` arrays element-by-element and then try to
|
|
258
|
+
coerce that into a single ``bool``, which raises. Two ``Prediction`` instances are therefore
|
|
259
|
+
never equal unless they are the same object (Python's default identity-based ``__eq__``,
|
|
260
|
+
from ``object``, applies instead) -- do not write ``fit.predict(x) == fit.predict(x)``
|
|
261
|
+
expecting a value comparison; compare the four arrays individually instead.
|
|
262
|
+
|
|
263
|
+
All four arrays are read-only (``.flags.writeable is False``): this is a frozen dataclass,
|
|
264
|
+
but freezing only stops an attribute from being *reassigned* (``prediction.lo = ...``), not
|
|
265
|
+
the ``numpy`` array it refers to from being mutated in place (``prediction.lo[:] = ...``),
|
|
266
|
+
so ``__post_init__`` marks each array non-writeable itself to close that gap.
|
|
267
|
+
|
|
268
|
+
A scalar ``severity`` argument to :meth:`Fit.predict` (for example ``fit.predict(0.5)``,
|
|
269
|
+
rather than a list or array) gives 0-d arrays here (``severity.shape == ()``), following
|
|
270
|
+
:func:`numpy.asarray`'s own behaviour for a scalar input; it is never promoted to shape
|
|
271
|
+
``(1,)``.
|
|
272
|
+
|
|
273
|
+
Attributes
|
|
274
|
+
----------
|
|
275
|
+
severity
|
|
276
|
+
The severities this prediction was evaluated at, as given (as ``float64``).
|
|
277
|
+
estimate
|
|
278
|
+
The fitted curve's value at each ``severity``, same shape as ``severity``.
|
|
279
|
+
lo
|
|
280
|
+
The band's lower side at each ``severity``. ``lo <= estimate`` everywhere.
|
|
281
|
+
hi
|
|
282
|
+
The band's upper side at each ``severity``. ``estimate <= hi`` everywhere.
|
|
283
|
+
level
|
|
284
|
+
The two-sided confidence level the band was built at, strictly between 0 and 1.
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
severity: NDArray[np.float64]
|
|
288
|
+
estimate: NDArray[np.float64]
|
|
289
|
+
lo: NDArray[np.float64]
|
|
290
|
+
hi: NDArray[np.float64]
|
|
291
|
+
level: float
|
|
292
|
+
|
|
293
|
+
def __post_init__(self) -> None:
|
|
294
|
+
for array in (self.severity, self.estimate, self.lo, self.hi):
|
|
295
|
+
array.flags.writeable = False
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass(frozen=True, kw_only=True)
|
|
299
|
+
class Fit:
|
|
300
|
+
"""A dose-response fit, or an explicit record of why one was not obtained.
|
|
301
|
+
|
|
302
|
+
The fitted curve is ``P(x) = u - (u - l) * F(z)``, where ``z = (log(x) - mu) / s`` on
|
|
303
|
+
a log axis or ``z = (x - mu) / s`` on a linear axis, ``s > 0``, and ``F`` is the link's CDF
|
|
304
|
+
(the standard normal CDF for ``probit``, the logistic CDF for ``logit``, or
|
|
305
|
+
``1 - exp(-exp(z))`` for ``cloglog``, which models the probability of *failure* rising with
|
|
306
|
+
severity and so is not symmetric the way ``probit``/``logit`` are). ``mu`` is on the
|
|
307
|
+
transformed axis (``log x`` on a log axis, ``x`` on a linear axis), not on the original
|
|
308
|
+
severity scale. At ``z = 0`` -- severity ``exp(mu)`` on a log axis, or severity ``mu`` on a
|
|
309
|
+
linear axis -- ``probit`` and ``logit`` both give ``P = (u + l) / 2``: the midpoint of the
|
|
310
|
+
fitted dynamic range, :meth:`~robustmargin.Definition.relative` ``(0.5)``, for **any**
|
|
311
|
+
asymptotes ``u``, ``l``, because the standard normal and logistic CDFs each equal ``0.5``
|
|
312
|
+
at ``z = 0``. That midpoint equals :meth:`~robustmargin.Definition.absolute` ``(0.5)`` and
|
|
313
|
+
:meth:`~robustmargin.Definition.baseline_fraction` ``(0.5)`` only when ``u = 1`` and
|
|
314
|
+
``l = 0``. Under ``cloglog``, which is not symmetric around ``z = 0``,
|
|
315
|
+
``F(0) = 1 - exp(-1)`` (about ``0.632``), so ``mu`` is never the midpoint of the dynamic
|
|
316
|
+
range, for any asymptotes. Each entry in ``params`` records whether it was held fixed or
|
|
317
|
+
estimated, and is on the **natural scale**, never the optimiser's internal
|
|
318
|
+
reparametrisation (``log s``; ``upper``/``lower`` on a logit scale) -- see
|
|
319
|
+
:class:`Parameter`. ``covariance`` is on that same natural scale: when asymptotes are
|
|
320
|
+
estimated, the optimiser's own covariance (which is on its reparametrised scale) is
|
|
321
|
+
delta-transformed onto the natural scale before being stored here.
|
|
322
|
+
|
|
323
|
+
``status == Status.OK`` if and only if both ``params`` and ``log_likelihood`` are present;
|
|
324
|
+
when ``status`` is anything else, ``params``, ``covariance`` and ``log_likelihood`` are all
|
|
325
|
+
``None`` -- no number is ever produced for a failed fit. ``Status.UNREACHABLE`` and
|
|
326
|
+
``Status.FAILS_AT_BASELINE`` describe a *threshold*, not a fit (they depend on a ``Definition``
|
|
327
|
+
a ``Fit`` does not have), and are never a ``Fit.status``.
|
|
328
|
+
|
|
329
|
+
Binomial curves are decreasing-only in this release: ``direction="increasing"`` raises.
|
|
330
|
+
The curve ``P = u - (u - l) * F(z)`` with ``s > 0`` is decreasing by construction, and
|
|
331
|
+
increasing-is-worse outcomes are not yet supported. ``Observations`` itself still accepts
|
|
332
|
+
both directions.
|
|
333
|
+
|
|
334
|
+
``diagnostics`` (goodness of fit, dispersion, link comparison) and ``monotonicity`` (the
|
|
335
|
+
adjacent-level monotonicity check) are two fields that both default to ``None``. The schema's
|
|
336
|
+
``Fit`` object has ``additionalProperties: false``, so a card whose ``Fit`` carries a key the
|
|
337
|
+
schema does not define is rejected, including a key with a ``null`` value. Adding a field to
|
|
338
|
+
``Fit`` is therefore a ``schema_version`` change.
|
|
339
|
+
|
|
340
|
+
Attributes
|
|
341
|
+
----------
|
|
342
|
+
axis
|
|
343
|
+
The :class:`~robustmargin.Axis` the fit was made on.
|
|
344
|
+
outcome
|
|
345
|
+
The outcome name this fit was made against, carried over from the
|
|
346
|
+
:class:`~robustmargin.Observations` it was fit to.
|
|
347
|
+
direction
|
|
348
|
+
``"decreasing"`` only, in this release (see above).
|
|
349
|
+
model
|
|
350
|
+
The model family. Only ``"binomial"`` exists in v0.1; a new family such as ``"ll4"`` or
|
|
351
|
+
``"isotonic"`` is a v0.2 addition requiring a further ``schema_version`` bump beyond the
|
|
352
|
+
current one (:data:`robustmargin.report.SCHEMA_VERSION`).
|
|
353
|
+
link
|
|
354
|
+
The link function: ``"probit"``, ``"logit"`` or ``"cloglog"`` for ``model="binomial"``.
|
|
355
|
+
Required; the schema's ``link`` enum carries no ``null`` alternative, so this is a
|
|
356
|
+
genuine ``str``, never ``None``.
|
|
357
|
+
params
|
|
358
|
+
A mapping with exactly the keys ``{"mu", "s", "upper", "lower"}`` when ``status`` is
|
|
359
|
+
``OK``, else ``None``. ``s.value`` must be positive; ``lower.value`` and
|
|
360
|
+
``upper.value`` must each lie in ``[0, 1]`` with ``lower.value < upper.value``.
|
|
361
|
+
``params["mu"].fixed`` and ``params["s"].fixed`` must both be ``False``: this package
|
|
362
|
+
never produces (and this dataclass never accepts) an ``OK`` fit with a fixed location
|
|
363
|
+
or scale.
|
|
364
|
+
covariance
|
|
365
|
+
The estimated parameters' covariance. Required (not ``None``) when ``status`` is
|
|
366
|
+
``OK``; ``None`` otherwise. If present, ``covariance.names`` is exactly the set of
|
|
367
|
+
non-fixed parameter names.
|
|
368
|
+
log_likelihood
|
|
369
|
+
The fitted log-likelihood, or ``None`` when ``status`` is not ``OK``. Finite when
|
|
370
|
+
present.
|
|
371
|
+
cells
|
|
372
|
+
The per-severity-level pooled counts the fit was made from
|
|
373
|
+
(:meth:`~robustmargin.Observations.cells`). Never empty, and in strictly increasing
|
|
374
|
+
order of severity (one cell per tested level), which is checked on construction and
|
|
375
|
+
never repaired by sorting or pooling.
|
|
376
|
+
status
|
|
377
|
+
The :class:`~robustmargin.Status` of this fit. Never ``UNREACHABLE`` or
|
|
378
|
+
``FAILS_AT_BASELINE`` (those describe a threshold, not a fit).
|
|
379
|
+
cluster_ids
|
|
380
|
+
Optional cluster ids the fitted data carries, one instance per distinct cluster (not
|
|
381
|
+
one per row): unique, non-empty when given, and never ``None`` or ``bool`` elements.
|
|
382
|
+
Set whenever the input :class:`~robustmargin.Observations` carried cluster ids, whatever
|
|
383
|
+
dependence was assumed when this fit was produced -- this field reflects the data's
|
|
384
|
+
structure, not a modelling choice. ``None`` means only that the data had no clusters,
|
|
385
|
+
not that independence was assumed. Used by :class:`~robustmargin.Ratio`'s dependence
|
|
386
|
+
guard.
|
|
387
|
+
warnings
|
|
388
|
+
Free-text notes surfaced alongside the fit (for example, ``CONTROL_INCOMPATIBLE``
|
|
389
|
+
carries a message suggesting ``upper="estimate"``). Empty by default.
|
|
390
|
+
schema_version
|
|
391
|
+
The serialised-result schema version this object belongs to. Defaults to ``"1"``, the
|
|
392
|
+
only version ``from_dict`` reads; any other version is rejected.
|
|
393
|
+
provenance
|
|
394
|
+
An opaque mapping the caller may attach to record where the inputs came from.
|
|
395
|
+
robustmargin stores it and never interprets it. Empty by default.
|
|
396
|
+
diagnostics
|
|
397
|
+
Model-only goodness-of-fit, dispersion and link-comparison diagnostics
|
|
398
|
+
(:func:`robustmargin.diagnose`). Set exactly when ``status`` is
|
|
399
|
+
:data:`~robustmargin.Status.OK` (no model-derived numbers on a failed fit); ``None``
|
|
400
|
+
otherwise.
|
|
401
|
+
monotonicity
|
|
402
|
+
A data-only adjacent-severity-level monotonicity check
|
|
403
|
+
(:func:`robustmargin.check_monotonicity`), computed from the raw
|
|
404
|
+
:class:`~robustmargin.Observations` before a fitting path was even chosen. Unlike
|
|
405
|
+
``diagnostics``, this is set on *every* fresh result regardless of ``status`` -- it needs
|
|
406
|
+
only counts, never a converged fit. A serialised card may omit it (the schema does not
|
|
407
|
+
require it), in which case it reads back as ``None``. The last field, after
|
|
408
|
+
``diagnostics``.
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
axis: Axis
|
|
412
|
+
outcome: str
|
|
413
|
+
direction: str
|
|
414
|
+
model: str
|
|
415
|
+
link: str
|
|
416
|
+
params: Mapping[str, Parameter] | None
|
|
417
|
+
covariance: Covariance | None
|
|
418
|
+
log_likelihood: float | None
|
|
419
|
+
cells: tuple[Cell, ...]
|
|
420
|
+
status: Status
|
|
421
|
+
cluster_ids: tuple[str | int, ...] | None = None
|
|
422
|
+
warnings: tuple[str, ...] = ()
|
|
423
|
+
schema_version: str = "1"
|
|
424
|
+
provenance: Mapping[str, JSONValue] = field(default_factory=dict)
|
|
425
|
+
diagnostics: Diagnostics | None = None
|
|
426
|
+
monotonicity: MonotonicityCheck | None = None
|
|
427
|
+
|
|
428
|
+
def __post_init__(self) -> None:
|
|
429
|
+
object.__setattr__(self, "warnings", tuple(self.warnings))
|
|
430
|
+
object.__setattr__(self, "cells", tuple(self.cells))
|
|
431
|
+
if self.cluster_ids is not None:
|
|
432
|
+
object.__setattr__(self, "cluster_ids", tuple(self.cluster_ids))
|
|
433
|
+
|
|
434
|
+
if not self.outcome:
|
|
435
|
+
raise ValueError("Fit.outcome must be non-empty")
|
|
436
|
+
if self.direction not in _VALID_DIRECTIONS:
|
|
437
|
+
raise ValueError(
|
|
438
|
+
f"Fit.direction must be one of {sorted(_VALID_DIRECTIONS)}, got {self.direction!r}"
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
if self.model not in _VALID_MODELS:
|
|
442
|
+
raise ValueError(
|
|
443
|
+
f"Fit.model must be one of {sorted(_VALID_MODELS)}, got {self.model!r}"
|
|
444
|
+
)
|
|
445
|
+
if self.model == "binomial" and self.direction != "decreasing":
|
|
446
|
+
raise ValueError(
|
|
447
|
+
"Fit: model='binomial' only supports direction='decreasing' in this release; "
|
|
448
|
+
"an increasing-is-worse definition is planned for v0.2"
|
|
449
|
+
)
|
|
450
|
+
allowed_links = _VALID_LINKS[self.model]
|
|
451
|
+
if self.link not in allowed_links:
|
|
452
|
+
raise ValueError(
|
|
453
|
+
f"Fit.link for model={self.model!r} must be one of {sorted(allowed_links)}, "
|
|
454
|
+
f"got {self.link!r}"
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
if self.status in _INVALID_FIT_STATUSES:
|
|
458
|
+
raise ValueError(
|
|
459
|
+
f"Fit.status must not be {self.status!r}: that status describes a threshold, "
|
|
460
|
+
"not a fit"
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
if not self.cells:
|
|
464
|
+
raise ValueError("Fit.cells must not be empty")
|
|
465
|
+
for index in range(1, len(self.cells)):
|
|
466
|
+
previous = self.cells[index - 1].severity
|
|
467
|
+
current = self.cells[index].severity
|
|
468
|
+
if not current > previous:
|
|
469
|
+
raise ValueError(
|
|
470
|
+
"Fit.cells must be in strictly increasing order of severity (one cell per "
|
|
471
|
+
f"tested level), but cells[{index}].severity={current!r} follows "
|
|
472
|
+
f"cells[{index - 1}].severity={previous!r}"
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
_check_cluster_ids(self.cluster_ids, label="Fit.cluster_ids")
|
|
476
|
+
|
|
477
|
+
if self.status is not Status.OK and self.diagnostics is not None:
|
|
478
|
+
raise ValueError(
|
|
479
|
+
"Fit.diagnostics must be None when status is not OK (no model-derived "
|
|
480
|
+
f"numbers on a failed fit), got status={self.status!r} with diagnostics set"
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
is_ok = self.status is Status.OK
|
|
484
|
+
has_params = self.params is not None
|
|
485
|
+
has_log_likelihood = self.log_likelihood is not None
|
|
486
|
+
if is_ok != (has_params and has_log_likelihood):
|
|
487
|
+
raise ValueError(
|
|
488
|
+
"Fit.status is OK if and only if both params and log_likelihood are present: "
|
|
489
|
+
f"status={self.status!r}, params is not None={has_params!r}, "
|
|
490
|
+
f"log_likelihood is not None={has_log_likelihood!r}"
|
|
491
|
+
)
|
|
492
|
+
if not is_ok:
|
|
493
|
+
if self.params is not None or self.covariance is not None:
|
|
494
|
+
raise ValueError(
|
|
495
|
+
"Fit.params and Fit.covariance must both be None when status is not OK"
|
|
496
|
+
)
|
|
497
|
+
return
|
|
498
|
+
|
|
499
|
+
if self.log_likelihood is not None:
|
|
500
|
+
_check_finite_number(self.log_likelihood, label="Fit.log_likelihood")
|
|
501
|
+
|
|
502
|
+
if self.covariance is None:
|
|
503
|
+
raise ValueError("Fit.covariance is required when status is OK")
|
|
504
|
+
|
|
505
|
+
params = self.params
|
|
506
|
+
assert params is not None # established by is_ok above
|
|
507
|
+
if self.model == "binomial":
|
|
508
|
+
if set(params) != _BINOMIAL_PARAM_NAMES:
|
|
509
|
+
raise ValueError(
|
|
510
|
+
f"Fit.params for model='binomial' must have exactly the keys "
|
|
511
|
+
f"{sorted(_BINOMIAL_PARAM_NAMES)}, got {sorted(params)}"
|
|
512
|
+
)
|
|
513
|
+
if params["mu"].fixed or params["s"].fixed:
|
|
514
|
+
raise ValueError(
|
|
515
|
+
"Fit.params['mu'].fixed and Fit.params['s'].fixed must both be False when "
|
|
516
|
+
"status is OK: this package never produces a fit with a fixed location or "
|
|
517
|
+
"scale"
|
|
518
|
+
)
|
|
519
|
+
s = params["s"].value
|
|
520
|
+
lower = params["lower"].value
|
|
521
|
+
upper = params["upper"].value
|
|
522
|
+
if not s > 0.0:
|
|
523
|
+
raise ValueError(f"Fit.params['s'].value must be positive, got {s!r}")
|
|
524
|
+
if not (0.0 <= lower < upper <= 1.0):
|
|
525
|
+
raise ValueError(
|
|
526
|
+
"Fit.params must satisfy 0 <= lower.value < upper.value <= 1, got "
|
|
527
|
+
f"lower={lower!r}, upper={upper!r}"
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
if self.axis.scale == "log" and upper == 1.0 and params["upper"].fixed:
|
|
531
|
+
for cell in self.cells:
|
|
532
|
+
if cell.severity == 0.0 and cell.successes < cell.trials:
|
|
533
|
+
raise ValueError(
|
|
534
|
+
"Fit: the zero-severity control has failures but 'upper' is fixed "
|
|
535
|
+
"at exactly 1.0, so the likelihood is zero for this fit on a log "
|
|
536
|
+
'axis; refit with upper="estimate"'
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
non_fixed = {name for name, param in params.items() if not param.fixed}
|
|
540
|
+
if set(self.covariance.names) != non_fixed:
|
|
541
|
+
raise ValueError(
|
|
542
|
+
"Fit.covariance.names must equal the non-fixed parameter names: "
|
|
543
|
+
f"expected {sorted(non_fixed)}, got {sorted(self.covariance.names)}"
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
def predict(self, severity: ArrayLike, *, level: float = 0.95) -> Prediction:
|
|
547
|
+
"""Evaluate the fitted curve and a delta-method confidence band at each of
|
|
548
|
+
``severity``.
|
|
549
|
+
|
|
550
|
+
Two band scales are used, depending on which asymptotes this fit estimated:
|
|
551
|
+
|
|
552
|
+
- **Both asymptotes fixed:** the delta method is applied to the linear predictor ``z``
|
|
553
|
+
(using ``mu``'s and ``s``'s covariance), then mapped through
|
|
554
|
+
``P = u - (u - l) * F(z)``. Because ``u`` and ``l`` are constants here, the band is
|
|
555
|
+
guaranteed to stay inside ``(l, u)`` -- not just ``(0, 1)`` -- for every severity.
|
|
556
|
+
- **Either asymptote estimated:** the delta method is applied to ``logit(P)``, using
|
|
557
|
+
the gradient of ``logit(P)`` with respect to every *estimated* natural parameter
|
|
558
|
+
(``mu``, ``s``, and whichever of ``lower``/``upper`` are not fixed), then
|
|
559
|
+
back-transformed with the logistic function. The band is guaranteed to stay inside
|
|
560
|
+
``(0, 1)``, but not inside ``(l, u)`` specifically, since ``l``/``u`` themselves now
|
|
561
|
+
carry uncertainty.
|
|
562
|
+
|
|
563
|
+
On a log axis, severity ``0.0`` is always the zero-severity control: its estimate
|
|
564
|
+
is ``upper`` exactly, never computed through ``log(0)``. It is not a special case for
|
|
565
|
+
the band, either: it goes through the same computation as every other severity (so
|
|
566
|
+
``predict(0)`` and ``predict(1e-12)`` agree), which happens to give a zero-width band
|
|
567
|
+
there whenever ``upper`` is fixed (perturbing any other parameter cannot move a
|
|
568
|
+
constant) and a band from ``Var(upper)`` -- correctly propagated through ``logit``, not
|
|
569
|
+
treated as if it were already on the logit scale -- when ``upper`` is estimated.
|
|
570
|
+
|
|
571
|
+
Parameters
|
|
572
|
+
----------
|
|
573
|
+
severity
|
|
574
|
+
Severities to evaluate the curve at. Converted to ``float64``; every entry must be
|
|
575
|
+
finite and non-negative.
|
|
576
|
+
level
|
|
577
|
+
The two-sided confidence level for the band, strictly between 0 and 1 (not a
|
|
578
|
+
``bool``). Defaults to ``0.95``.
|
|
579
|
+
|
|
580
|
+
Returns
|
|
581
|
+
-------
|
|
582
|
+
Prediction
|
|
583
|
+
``severity``, ``estimate``, ``lo`` and ``hi`` are ``numpy`` arrays of the same
|
|
584
|
+
shape as the (converted) ``severity`` input; ``lo <= estimate <= hi`` everywhere.
|
|
585
|
+
|
|
586
|
+
Raises
|
|
587
|
+
------
|
|
588
|
+
ValueError
|
|
589
|
+
If ``self.status`` is not :data:`~robustmargin.Status.OK` (a failed fit has no curve
|
|
590
|
+
to evaluate), if ``level`` is not finite, is a ``bool``, or
|
|
591
|
+
does not satisfy ``0 < level < 1``, or if ``severity`` is empty or contains a
|
|
592
|
+
negative or non-finite entry.
|
|
593
|
+
"""
|
|
594
|
+
if self.status is not Status.OK:
|
|
595
|
+
raise ValueError(
|
|
596
|
+
f"Fit.predict requires status=OK, got {self.status!r}: a failed fit has no "
|
|
597
|
+
"curve to evaluate"
|
|
598
|
+
)
|
|
599
|
+
_check_finite_number(level, label="Fit.predict level")
|
|
600
|
+
if not (0.0 < level < 1.0):
|
|
601
|
+
raise ValueError(f"Fit.predict: level must satisfy 0 < level < 1, got {level!r}")
|
|
602
|
+
|
|
603
|
+
# A copy (`np.array`, not `np.asarray`): Prediction.severity is marked read-only below,
|
|
604
|
+
# and a caller who happened to pass an existing float64 array must never have that
|
|
605
|
+
# array silently frozen out from under them.
|
|
606
|
+
x = np.array(severity, dtype=np.float64)
|
|
607
|
+
if x.size == 0:
|
|
608
|
+
raise ValueError("Fit.predict: severity must not be empty")
|
|
609
|
+
if not bool(np.all(np.isfinite(x))):
|
|
610
|
+
raise ValueError("Fit.predict: severity must be finite (no NaN or inf)")
|
|
611
|
+
if bool(np.any(x < 0.0)):
|
|
612
|
+
raise ValueError("Fit.predict: severity must be non-negative")
|
|
613
|
+
|
|
614
|
+
assert self.params is not None and self.covariance is not None # status is OK
|
|
615
|
+
mu = self.params["mu"].value
|
|
616
|
+
s = self.params["s"].value
|
|
617
|
+
upper_param = self.params["upper"]
|
|
618
|
+
lower_param = self.params["lower"]
|
|
619
|
+
upper = upper_param.value
|
|
620
|
+
lower = lower_param.value
|
|
621
|
+
covariance = self.covariance
|
|
622
|
+
|
|
623
|
+
z_score = float(norm.ppf(0.5 + level / 2.0))
|
|
624
|
+
|
|
625
|
+
# x = 0 on a log axis is not special-cased here: both band functions below route it
|
|
626
|
+
# through the exact same logit-P gradient/delta computation as every other severity
|
|
627
|
+
# so `predict(0)` and `predict(1e-12)` agree, and a *fixed* `upper` still gets a
|
|
628
|
+
# zero-width band there for free (perturbing any other estimated parameter cannot move
|
|
629
|
+
# a constant), with no separate formula to keep in sync.
|
|
630
|
+
if upper_param.fixed and lower_param.fixed:
|
|
631
|
+
estimate, lo, hi = _predict_band_fixed_asymptotes(
|
|
632
|
+
x,
|
|
633
|
+
mu=mu,
|
|
634
|
+
s=s,
|
|
635
|
+
upper=upper,
|
|
636
|
+
lower=lower,
|
|
637
|
+
scale=self.axis.scale,
|
|
638
|
+
link=self.link,
|
|
639
|
+
covariance=covariance,
|
|
640
|
+
z_score=z_score,
|
|
641
|
+
)
|
|
642
|
+
else:
|
|
643
|
+
estimate, lo, hi = _predict_band_estimated_asymptote(
|
|
644
|
+
x,
|
|
645
|
+
params=self.params,
|
|
646
|
+
scale=self.axis.scale,
|
|
647
|
+
link=self.link,
|
|
648
|
+
covariance=covariance,
|
|
649
|
+
z_score=z_score,
|
|
650
|
+
)
|
|
651
|
+
|
|
652
|
+
return Prediction(severity=x, estimate=estimate, lo=lo, hi=hi, level=level)
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
# ================================================================================================
|
|
656
|
+
# Curve evaluation and the full binomial log-likelihood (shared by both fitting paths and by
|
|
657
|
+
# Fit.predict, so a reported log_likelihood and a predicted curve always agree with each other).
|
|
658
|
+
# ================================================================================================
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
def _transform(x: NDArray[np.float64], *, scale: str) -> NDArray[np.float64]:
|
|
662
|
+
"""``log x`` on a log axis, ``x`` unchanged on a linear axis. Never
|
|
663
|
+
called on an exact-zero severity on a log axis -- callers route that case around this
|
|
664
|
+
function entirely, so this never evaluates ``log(0)``.
|
|
665
|
+
"""
|
|
666
|
+
if scale == "log":
|
|
667
|
+
return np.asarray(np.log(x), dtype=np.float64)
|
|
668
|
+
return x
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def _f_cdf(z: NDArray[np.float64], link: str) -> NDArray[np.float64]:
|
|
672
|
+
"""The link's CDF, modelling the probability of *failure* rising with ``z``. ``cloglog``'s
|
|
673
|
+
``1 - exp(-exp(z))`` is written as ``-expm1(-exp(z))`` for numerical stability, not for a
|
|
674
|
+
different value.
|
|
675
|
+
"""
|
|
676
|
+
if link == "probit":
|
|
677
|
+
return np.asarray(norm.cdf(z), dtype=np.float64)
|
|
678
|
+
if link == "logit":
|
|
679
|
+
return np.asarray(expit(z), dtype=np.float64)
|
|
680
|
+
return np.asarray(-np.expm1(-np.exp(z)), dtype=np.float64) # link == "cloglog"
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _f_survival(z: NDArray[np.float64], link: str) -> NDArray[np.float64]:
|
|
684
|
+
"""``1 - _f_cdf(z, link)``, computed directly from each link's own survival identity --
|
|
685
|
+
never as ``1 - _f_cdf(z, link)`` -- so it stays numerically accurate at the tail where that
|
|
686
|
+
subtraction would round to exactly ``0.0``: ``scipy.stats.norm.sf`` for probit;
|
|
687
|
+
``expit(-z) == 1 - expit(z)`` exactly for logit (the logistic CDF's own reflection identity);
|
|
688
|
+
``exp(-exp(z)) == 1 - (-expm1(-exp(z)))`` exactly for cloglog (``_f_cdf``'s own algebraic
|
|
689
|
+
identity, read the other way). ``_f_cdf`` itself is already accurate where *it* is small (large
|
|
690
|
+
negative ``z``); this is its mirror image, accurate where *it* is close to ``1`` (large positive
|
|
691
|
+
``z``).
|
|
692
|
+
"""
|
|
693
|
+
if link == "probit":
|
|
694
|
+
return np.asarray(norm.sf(z), dtype=np.float64)
|
|
695
|
+
if link == "logit":
|
|
696
|
+
return np.asarray(expit(-z), dtype=np.float64)
|
|
697
|
+
return np.asarray(np.exp(-np.exp(z)), dtype=np.float64) # link == "cloglog"
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def _curve_success_probability(
|
|
701
|
+
severity: ArrayLike,
|
|
702
|
+
*,
|
|
703
|
+
mu: float,
|
|
704
|
+
s: float,
|
|
705
|
+
upper: float,
|
|
706
|
+
lower: float,
|
|
707
|
+
scale: str,
|
|
708
|
+
link: str,
|
|
709
|
+
center: float = 0.0,
|
|
710
|
+
spread: float = 1.0,
|
|
711
|
+
) -> NDArray[np.float64]:
|
|
712
|
+
"""``P(x) = u - (u - l) * F(z)``, with the log-axis zero-severity
|
|
713
|
+
control handled directly: ``P(0) = u`` exactly, never through ``log(0)``.
|
|
714
|
+
|
|
715
|
+
``center``/``spread`` let a caller evaluate this on the *centred-and-scaled* transformed
|
|
716
|
+
covariate ``t' = (t - center) / spread`` instead of ``t`` itself (:func:`_centering_and_scale`):
|
|
717
|
+
``z = (t' - mu) / s`` there, so ``mu``/``s`` are then on that same rescaled units, not
|
|
718
|
+
natural ones. Defaulted to the identity (``0.0``, ``1.0``), so every existing caller that
|
|
719
|
+
evaluates the curve at its final, natural-scale ``mu``/``s`` is unaffected.
|
|
720
|
+
"""
|
|
721
|
+
x = np.asarray(severity, dtype=np.float64)
|
|
722
|
+
result = np.empty_like(x)
|
|
723
|
+
zero_control = (scale == "log") & (x == 0.0)
|
|
724
|
+
nonzero = ~zero_control
|
|
725
|
+
if np.any(nonzero):
|
|
726
|
+
t = (_transform(x[nonzero], scale=scale) - center) / spread
|
|
727
|
+
z = (t - mu) / s
|
|
728
|
+
result[nonzero] = upper - (upper - lower) * _f_cdf(z, link)
|
|
729
|
+
if np.any(zero_control):
|
|
730
|
+
result[zero_control] = upper
|
|
731
|
+
return result
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _curve_success_and_failure_probability(
|
|
735
|
+
severity: ArrayLike,
|
|
736
|
+
*,
|
|
737
|
+
mu: float,
|
|
738
|
+
s: float,
|
|
739
|
+
upper: float,
|
|
740
|
+
lower: float,
|
|
741
|
+
scale: str,
|
|
742
|
+
link: str,
|
|
743
|
+
center: float = 0.0,
|
|
744
|
+
spread: float = 1.0,
|
|
745
|
+
) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
|
|
746
|
+
"""``(p, 1 - p)`` for the curve at ``severity``, each computed *directly* rather than one as
|
|
747
|
+
the complement of the other, so both stay numerically accurate at their own informative tail
|
|
748
|
+
-- used by :func:`robustmargin.diagnose`'s goodness-of-fit sums, which need both ``p`` and
|
|
749
|
+
``1 - p`` (and their product) without either rounding to an exact ``0``/``1`` for a cell that is
|
|
750
|
+
not :func:`robustmargin.diagnose`'s own *structural* exclusion.
|
|
751
|
+
|
|
752
|
+
``p = l + (u - l) * (1 - F(z))``, using :func:`_f_survival` -- accurate where ``p`` itself is
|
|
753
|
+
small (high severity, under ``direction="decreasing"``), the tail where :func:`_f_cdf` alone
|
|
754
|
+
would already have rounded to exactly ``1`` and so silently rounded ``p`` to exactly ``l``.
|
|
755
|
+
``1 - p = (1 - u) + (u - l) * F(z)``, using :func:`_f_cdf` directly -- accurate where it is
|
|
756
|
+
small (low severity), the tail :func:`_f_cdf` is already accurate on. Neither value is
|
|
757
|
+
clamped here (that is the caller's job, against whatever floor it needs); the two need not
|
|
758
|
+
sum to exactly ``1.0`` in floating point, by design -- forcing that would reintroduce the
|
|
759
|
+
exact cancellation this function exists to avoid.
|
|
760
|
+
|
|
761
|
+
Shares :func:`_curve_success_probability`'s log-axis zero-severity-control handling (``p(0)
|
|
762
|
+
= u`` exactly) and ``center``/``spread`` reparametrisation -- see that function's
|
|
763
|
+
docstring for both.
|
|
764
|
+
"""
|
|
765
|
+
x = np.asarray(severity, dtype=np.float64)
|
|
766
|
+
p = np.empty_like(x)
|
|
767
|
+
q = np.empty_like(x)
|
|
768
|
+
zero_control = (scale == "log") & (x == 0.0)
|
|
769
|
+
nonzero = ~zero_control
|
|
770
|
+
if np.any(nonzero):
|
|
771
|
+
t = (_transform(x[nonzero], scale=scale) - center) / spread
|
|
772
|
+
z = (t - mu) / s
|
|
773
|
+
p[nonzero] = lower + (upper - lower) * _f_survival(z, link)
|
|
774
|
+
q[nonzero] = (1.0 - upper) + (upper - lower) * _f_cdf(z, link)
|
|
775
|
+
if np.any(zero_control):
|
|
776
|
+
p[zero_control] = upper
|
|
777
|
+
q[zero_control] = 1.0 - upper
|
|
778
|
+
return p, q
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def _cells_to_arrays(
|
|
782
|
+
cells: tuple[Cell, ...],
|
|
783
|
+
) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]:
|
|
784
|
+
severity = np.array([c.severity for c in cells], dtype=np.float64)
|
|
785
|
+
successes = np.array([float(c.successes) for c in cells], dtype=np.float64)
|
|
786
|
+
trials = np.array([float(c.trials) for c in cells], dtype=np.float64)
|
|
787
|
+
return severity, successes, trials
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _transformed_span(severity: NDArray[np.float64], *, scale: str) -> float:
|
|
791
|
+
"""The span (max - min) of the *likelihood-contributing* transformed levels: a log-axis
|
|
792
|
+
zero-severity control is excluded, since it carries no covariate value at all and so
|
|
793
|
+
cannot be part of a "tested range" on the transformed axis. Used by
|
|
794
|
+
``_FLAT_SLOPE_SPAN_FACTOR``'s flat-curve check. Guaranteed non-zero whenever
|
|
795
|
+
``_check_separation`` has already passed, because that check requires at least two distinct
|
|
796
|
+
likelihood-contributing severities.
|
|
797
|
+
"""
|
|
798
|
+
nonzero = severity > 0.0 if scale == "log" else np.ones_like(severity, dtype=bool)
|
|
799
|
+
transformed = _transform(severity[nonzero], scale=scale)
|
|
800
|
+
if transformed.size == 0:
|
|
801
|
+
return 0.0
|
|
802
|
+
return float(np.max(transformed) - np.min(transformed))
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _is_positive_definite(matrix: NDArray[np.float64]) -> bool:
|
|
806
|
+
""":class:`Covariance`'s positive-definiteness rule, shared so the fitters can check a
|
|
807
|
+
natural-scale covariance or information matrix *before* building one: a failure then gives
|
|
808
|
+
``Status.NOT_CONVERGED`` with a reason instead of an uncaught ``ValueError``, or a
|
|
809
|
+
``LinAlgError`` from :func:`numpy.linalg.eigvalsh` on a non-finite matrix. Symmetry is not
|
|
810
|
+
checked here: :func:`numpy.linalg.eigvalsh` reads the lower triangle only, and the fitters
|
|
811
|
+
symmetrise their own matrices before calling this.
|
|
812
|
+
|
|
813
|
+
The test is on the correlation matrix ``D^-1/2 . matrix . D^-1/2``: finite, every diagonal
|
|
814
|
+
entry positive, and minimum eigenvalue ``> 1e-12``. A natural-scale covariance mixes
|
|
815
|
+
severity-unit parameters (``mu``, ``s``, whose variances scale with the square of the
|
|
816
|
+
caller's units) with probabilities (``upper``, ``lower``); a rule on the raw eigenvalues
|
|
817
|
+
would make the fit's status depend on severity units. Since, for a positive semi-definite
|
|
818
|
+
``matrix``, ``min eig(correlation) >= min eig(matrix) / max(diag(matrix))``, every matrix a
|
|
819
|
+
raw-eigenvalue rule (``> 1e-12 * max(1, max|entry|)``) would accept is accepted here too.
|
|
820
|
+
"""
|
|
821
|
+
if not bool(np.all(np.isfinite(matrix))):
|
|
822
|
+
return False
|
|
823
|
+
diagonal = np.diag(matrix)
|
|
824
|
+
if not bool(np.all(diagonal > 0.0)):
|
|
825
|
+
return False
|
|
826
|
+
scale = np.sqrt(diagonal)
|
|
827
|
+
correlation = matrix / np.outer(scale, scale)
|
|
828
|
+
# No symmetrising here: eigvalsh reads the lower triangle only, so a matrix within
|
|
829
|
+
# Covariance's symmetry tolerance is judged on its lower-triangle entries.
|
|
830
|
+
return float(np.min(np.linalg.eigvalsh(correlation))) > _POSITIVE_DEFINITE_RELATIVE_TOLERANCE
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _centering_and_scale(severity: NDArray[np.float64], *, scale: str) -> tuple[float, float]:
|
|
834
|
+
"""The centring constant ``m`` and scale ``d`` used to condition the transformed covariate
|
|
835
|
+
*before* either fitting path ever sees it: fitting ``t' = (t - m) / d`` instead of ``t``
|
|
836
|
+
directly is an exact reparametrisation of the identical curve
|
|
837
|
+
(``mu = m + d * mu'``, ``s = d * s'``; the asymptotes are untouched by it), so it is
|
|
838
|
+
consistent with never hand-rolling a fitter -- it changes the optimiser's working units, not
|
|
839
|
+
the model. Without it, severity units that are very large or very small on the transformed axis
|
|
840
|
+
(for example a linear axis measured in micro-units, or in millions of units) make the
|
|
841
|
+
optimiser's own internal scale blow up or collapse, even though the natural-scale fit itself is
|
|
842
|
+
perfectly well identified.
|
|
843
|
+
|
|
844
|
+
``m`` is the mean and ``d`` the standard deviation of the *likelihood-contributing*
|
|
845
|
+
transformed levels -- the same subset :func:`_transformed_span` and
|
|
846
|
+
:func:`_check_separation` use: a log-axis zero-severity control is excluded, since it
|
|
847
|
+
carries no covariate value at all. Falls back to the span, then to ``1.0``, in the
|
|
848
|
+
degenerate case of a zero standard deviation (should not arise once
|
|
849
|
+
:func:`_check_separation` has passed, since that requires at least two distinct
|
|
850
|
+
likelihood-contributing severities).
|
|
851
|
+
"""
|
|
852
|
+
nonzero = severity > 0.0 if scale == "log" else np.ones_like(severity, dtype=bool)
|
|
853
|
+
transformed = _transform(severity[nonzero], scale=scale)
|
|
854
|
+
if transformed.size == 0:
|
|
855
|
+
return 0.0, 1.0
|
|
856
|
+
m = float(np.mean(transformed))
|
|
857
|
+
d = float(np.std(transformed))
|
|
858
|
+
if not d > 0.0:
|
|
859
|
+
span = float(np.max(transformed) - np.min(transformed))
|
|
860
|
+
d = span if span > 0.0 else 1.0
|
|
861
|
+
return m, d
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def _full_binomial_log_likelihood(
|
|
865
|
+
severity: NDArray[np.float64],
|
|
866
|
+
successes: NDArray[np.float64],
|
|
867
|
+
trials: NDArray[np.float64],
|
|
868
|
+
*,
|
|
869
|
+
mu: float,
|
|
870
|
+
s: float,
|
|
871
|
+
upper: float,
|
|
872
|
+
lower: float,
|
|
873
|
+
scale: str,
|
|
874
|
+
link: str,
|
|
875
|
+
center: float = 0.0,
|
|
876
|
+
spread: float = 1.0,
|
|
877
|
+
) -> float:
|
|
878
|
+
"""The full binomial log-likelihood, including the ``lchoose`` combinatorial term, so it
|
|
879
|
+
matches R ``drc``'s ``logLik()`` on both fitting paths (see
|
|
880
|
+
https://github.com/robustmargin/robustmargin/blob/main/tools/drc_reference/README.md).
|
|
881
|
+
:func:`scipy.special.xlogy` gives a 0-successes or 0-failures cell a term of exactly ``0.0``
|
|
882
|
+
without ever evaluating ``log(0)``.
|
|
883
|
+
|
|
884
|
+
``center``/``spread`` are forwarded to :func:`_curve_success_probability` -- see there.
|
|
885
|
+
Defaulted to the identity, so this still computes the reported ``Fit.log_likelihood`` on
|
|
886
|
+
natural-scale ``mu``/``s`` against raw severity when a caller omits them (both fitting
|
|
887
|
+
paths' final report does); only the estimated-asymptote optimiser's own iterations, which
|
|
888
|
+
work in centred-and-scaled units, pass them explicitly.
|
|
889
|
+
|
|
890
|
+
Summed with the plain Python builtin ``sum`` over a fixed-order list, not ``numpy.sum``:
|
|
891
|
+
``numpy.sum``'s pairwise reduction can regroup its additions differently when the array's
|
|
892
|
+
length changes by one, which would make an exact-zero-contribution cell (an all-success
|
|
893
|
+
zero-severity control under a fixed ``upper=1.0``) change the *other* cells' floating-point
|
|
894
|
+
rounding even though its own contribution is exactly ``0.0``. A left-to-right Python sum
|
|
895
|
+
does not have that failure mode: inserting an exact ``0.0`` term anywhere in a strictly
|
|
896
|
+
sequential accumulation is a bit-exact no-op. This is what keeps the aggregation property
|
|
897
|
+
test's "with control" and "without control" fits bit-identical, not merely close.
|
|
898
|
+
"""
|
|
899
|
+
p = _curve_success_probability(
|
|
900
|
+
severity,
|
|
901
|
+
mu=mu,
|
|
902
|
+
s=s,
|
|
903
|
+
upper=upper,
|
|
904
|
+
lower=lower,
|
|
905
|
+
scale=scale,
|
|
906
|
+
link=link,
|
|
907
|
+
center=center,
|
|
908
|
+
spread=spread,
|
|
909
|
+
)
|
|
910
|
+
failures = trials - successes
|
|
911
|
+
lchoose = gammaln(trials + 1.0) - gammaln(successes + 1.0) - gammaln(failures + 1.0)
|
|
912
|
+
terms = lchoose + xlogy(successes, p) + xlogy(failures, 1.0 - p)
|
|
913
|
+
return float(sum(terms.tolist()))
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
def _covariance_entry(covariance: Covariance, name_a: str, name_b: str) -> float:
|
|
917
|
+
i = covariance.names.index(name_a)
|
|
918
|
+
j = covariance.names.index(name_b)
|
|
919
|
+
return float(covariance.matrix[i][j])
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
# ================================================================================================
|
|
923
|
+
# Fit.predict's two band scales (documented in full on Fit.predict).
|
|
924
|
+
# ================================================================================================
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
def _predict_band_fixed_asymptotes(
|
|
928
|
+
x: NDArray[np.float64],
|
|
929
|
+
*,
|
|
930
|
+
mu: float,
|
|
931
|
+
s: float,
|
|
932
|
+
upper: float,
|
|
933
|
+
lower: float,
|
|
934
|
+
scale: str,
|
|
935
|
+
link: str,
|
|
936
|
+
covariance: Covariance,
|
|
937
|
+
z_score: float,
|
|
938
|
+
) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]:
|
|
939
|
+
"""Delta method on the linear predictor ``z``, mapped through ``P = u - (u - l) * F(z)``.
|
|
940
|
+
Only valid when both asymptotes are fixed constants, which is what keeps the resulting band
|
|
941
|
+
inside ``(l, u)`` for every severity: ``F`` maps into ``[0, 1]`` regardless of how uncertain
|
|
942
|
+
``z`` is, and ``u``, ``l`` themselves carry no uncertainty here.
|
|
943
|
+
|
|
944
|
+
A log-axis zero-severity control is handled directly (``estimate = lo = hi = upper``, a
|
|
945
|
+
zero-width band, since ``u`` is a constant here): ``z`` is undefined at ``x = 0`` on a log
|
|
946
|
+
axis, so those entries are routed around the ``z``-based computation entirely rather
|
|
947
|
+
than computed through ``log(0)``.
|
|
948
|
+
"""
|
|
949
|
+
estimate = np.empty_like(x)
|
|
950
|
+
lo = np.empty_like(x)
|
|
951
|
+
hi = np.empty_like(x)
|
|
952
|
+
|
|
953
|
+
zero_control = (scale == "log") & (x == 0.0)
|
|
954
|
+
nonzero = ~zero_control
|
|
955
|
+
if np.any(zero_control):
|
|
956
|
+
estimate[zero_control] = upper
|
|
957
|
+
lo[zero_control] = upper
|
|
958
|
+
hi[zero_control] = upper
|
|
959
|
+
|
|
960
|
+
if np.any(nonzero):
|
|
961
|
+
x_nonzero = x[nonzero]
|
|
962
|
+
z = (_transform(x_nonzero, scale=scale) - mu) / s
|
|
963
|
+
var_mu = _covariance_entry(covariance, "mu", "mu")
|
|
964
|
+
var_s = _covariance_entry(covariance, "s", "s")
|
|
965
|
+
cov_mu_s = _covariance_entry(covariance, "mu", "s")
|
|
966
|
+
var_z = (var_mu + (z**2) * var_s - 2.0 * z * cov_mu_s) / (s**2)
|
|
967
|
+
se_z = np.sqrt(np.clip(var_z, 0.0, None))
|
|
968
|
+
|
|
969
|
+
estimate[nonzero] = upper - (upper - lower) * _f_cdf(z, link)
|
|
970
|
+
# F is non-decreasing, so the smaller z-side gives the larger P (P falls as z rises).
|
|
971
|
+
hi[nonzero] = upper - (upper - lower) * _f_cdf(z - z_score * se_z, link)
|
|
972
|
+
lo[nonzero] = upper - (upper - lower) * _f_cdf(z + z_score * se_z, link)
|
|
973
|
+
return estimate, lo, hi
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _predict_band_estimated_asymptote(
|
|
977
|
+
x: NDArray[np.float64],
|
|
978
|
+
*,
|
|
979
|
+
params: Mapping[str, Parameter],
|
|
980
|
+
scale: str,
|
|
981
|
+
link: str,
|
|
982
|
+
covariance: Covariance,
|
|
983
|
+
z_score: float,
|
|
984
|
+
) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]:
|
|
985
|
+
"""Delta method on ``logit(P)``, using the gradient of ``logit(P(x))`` with respect to
|
|
986
|
+
every *estimated* natural parameter (``covariance.names``, always ``mu``, ``s`` and
|
|
987
|
+
whichever of ``lower``/``upper`` are not fixed). Guarantees the band stays inside
|
|
988
|
+
``(0, 1)`` regardless of how uncertain an estimated asymptote is, because ``expit`` is used
|
|
989
|
+
to back-transform rather than a linear step on ``P`` itself.
|
|
990
|
+
"""
|
|
991
|
+
names = covariance.names
|
|
992
|
+
natural_vector = np.array([params[name].value for name in names], dtype=np.float64)
|
|
993
|
+
fixed_lower = params["lower"].value if params["lower"].fixed else None
|
|
994
|
+
fixed_upper = params["upper"].value if params["upper"].fixed else None
|
|
995
|
+
matrix = np.array([[float(v) for v in row] for row in covariance.matrix], dtype=np.float64)
|
|
996
|
+
|
|
997
|
+
def _probability_at(natural: NDArray[np.float64], severity_value: float) -> float:
|
|
998
|
+
mapping = dict(zip(names, natural, strict=True))
|
|
999
|
+
mu = mapping["mu"]
|
|
1000
|
+
s = mapping["s"]
|
|
1001
|
+
lower = mapping["lower"] if "lower" in mapping else fixed_lower
|
|
1002
|
+
upper = mapping["upper"] if "upper" in mapping else fixed_upper
|
|
1003
|
+
assert lower is not None and upper is not None
|
|
1004
|
+
return float(
|
|
1005
|
+
_curve_success_probability(
|
|
1006
|
+
np.array([severity_value]),
|
|
1007
|
+
mu=mu,
|
|
1008
|
+
s=s,
|
|
1009
|
+
upper=upper,
|
|
1010
|
+
lower=lower,
|
|
1011
|
+
scale=scale,
|
|
1012
|
+
link=link,
|
|
1013
|
+
)[0]
|
|
1014
|
+
)
|
|
1015
|
+
|
|
1016
|
+
# A flat, at-least-1-D working view: a 0-d `x` (Fit.predict called with a bare scalar) has
|
|
1017
|
+
# no positional index (`x[0]` raises `IndexError` on a 0-d array), so integer-indexed
|
|
1018
|
+
# iteration needs a 1-D view regardless of `x`'s own shape. The result is reshaped back to
|
|
1019
|
+
# `x.shape` at the end, so a 0-d input still gives 0-d output arrays (Prediction's own
|
|
1020
|
+
# documented behaviour), not a shape promoted to `(1,)`.
|
|
1021
|
+
flat_x = np.atleast_1d(x)
|
|
1022
|
+
estimate = np.empty_like(flat_x)
|
|
1023
|
+
lo = np.empty_like(flat_x)
|
|
1024
|
+
hi = np.empty_like(flat_x)
|
|
1025
|
+
for i in range(flat_x.size):
|
|
1026
|
+
severity_value = float(flat_x[i])
|
|
1027
|
+
p0 = _probability_at(natural_vector, severity_value)
|
|
1028
|
+
|
|
1029
|
+
def _logit_p(
|
|
1030
|
+
natural: NDArray[np.float64], sv: float = severity_value
|
|
1031
|
+
) -> NDArray[np.float64]:
|
|
1032
|
+
return np.array([float(_logit(_probability_at(natural, sv)))])
|
|
1033
|
+
|
|
1034
|
+
gradient = _central_difference_jacobian(_logit_p, natural_vector)[0]
|
|
1035
|
+
var_logit_p = float(gradient @ matrix @ gradient)
|
|
1036
|
+
se_logit_p = math.sqrt(max(var_logit_p, 0.0))
|
|
1037
|
+
logit_p0 = float(_logit(p0))
|
|
1038
|
+
estimate[i] = p0
|
|
1039
|
+
lo[i] = expit(logit_p0 - z_score * se_logit_p)
|
|
1040
|
+
hi[i] = expit(logit_p0 + z_score * se_logit_p)
|
|
1041
|
+
return estimate.reshape(x.shape), lo.reshape(x.shape), hi.reshape(x.shape)
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def _central_difference_jacobian(
|
|
1045
|
+
func: Callable[[NDArray[np.float64]], NDArray[np.float64]],
|
|
1046
|
+
x: NDArray[np.float64],
|
|
1047
|
+
*,
|
|
1048
|
+
relative_step: float = 1e-6,
|
|
1049
|
+
) -> NDArray[np.float64]:
|
|
1050
|
+
"""The Jacobian of ``func`` at ``x`` by central finite differences: the delta method's
|
|
1051
|
+
numerical-differentiation step (used both to transform a fit's optimizer-scale covariance
|
|
1052
|
+
onto the natural scale, and inside :func:`Fit.predict`'s estimated-asymptote band), not a
|
|
1053
|
+
fitting procedure. The package never hand-rolls an optimizer or IRLS loop; a numerical
|
|
1054
|
+
derivative of an already-fitted quantity is not one.
|
|
1055
|
+
"""
|
|
1056
|
+
x = np.asarray(x, dtype=np.float64)
|
|
1057
|
+
f0 = np.asarray(func(x), dtype=np.float64)
|
|
1058
|
+
jacobian = np.zeros((f0.size, x.size), dtype=np.float64)
|
|
1059
|
+
for j in range(x.size):
|
|
1060
|
+
step = relative_step * max(1.0, abs(float(x[j])))
|
|
1061
|
+
x_plus = x.copy()
|
|
1062
|
+
x_plus[j] += step
|
|
1063
|
+
x_minus = x.copy()
|
|
1064
|
+
x_minus[j] -= step
|
|
1065
|
+
jacobian[:, j] = (
|
|
1066
|
+
np.asarray(func(x_plus), dtype=np.float64) - np.asarray(func(x_minus), dtype=np.float64)
|
|
1067
|
+
) / (2.0 * step)
|
|
1068
|
+
return jacobian
|
|
1069
|
+
|
|
1070
|
+
|
|
1071
|
+
# ================================================================================================
|
|
1072
|
+
# fit_dose_response: the estimated-asymptote (GenericLikelihoodModel) path
|
|
1073
|
+
# ================================================================================================
|
|
1074
|
+
|
|
1075
|
+
|
|
1076
|
+
def _unpack_optimizer_params(
|
|
1077
|
+
params: NDArray[np.float64],
|
|
1078
|
+
*,
|
|
1079
|
+
estimate_lower: bool,
|
|
1080
|
+
estimate_upper: bool,
|
|
1081
|
+
lower_value: float,
|
|
1082
|
+
upper_value: float,
|
|
1083
|
+
) -> tuple[float, float, float, float]:
|
|
1084
|
+
"""Map the optimizer's internal ``(mu, log_s, [a], [b])`` vector
|
|
1085
|
+
to the natural scale ``(mu, s, lower, upper)``. Whichever of ``a``/``b`` corresponds to a
|
|
1086
|
+
*fixed* asymptote is simply absent from ``params`` ("fixed asymptotes are left out of the
|
|
1087
|
+
parameter vector"). ``l < u`` holds by construction in every combination:
|
|
1088
|
+
|
|
1089
|
+
- Both estimated: ``l = expit(a)`` and ``u = l + (1 - l) * expit(b)``.
|
|
1090
|
+
- ``upper`` estimated, ``lower`` fixed: ``u = l + (1 - l) * expit(b)`` with the fixed ``l``.
|
|
1091
|
+
- ``lower`` estimated, ``upper`` fixed: ``l = u * expit(a)``
|
|
1092
|
+
with the fixed ``u`` -- *not* the unconstrained ``l = expit(a)`` used when ``upper`` is
|
|
1093
|
+
also estimated, which can exceed a fixed ``u`` and converge onto an increasing curve
|
|
1094
|
+
(``Fit`` would then reject ``lower.value < upper.value`` at construction). Anchoring ``l``
|
|
1095
|
+
to a fraction of the fixed ``u`` keeps ``l`` inside ``(0, u)`` for every finite ``a``.
|
|
1096
|
+
"""
|
|
1097
|
+
mu = float(params[0])
|
|
1098
|
+
s = float(np.exp(params[1]))
|
|
1099
|
+
idx = 2
|
|
1100
|
+
if estimate_lower:
|
|
1101
|
+
a_value = float(params[idx])
|
|
1102
|
+
idx += 1
|
|
1103
|
+
lower = float(upper_value * expit(a_value)) if not estimate_upper else float(expit(a_value))
|
|
1104
|
+
else:
|
|
1105
|
+
lower = lower_value
|
|
1106
|
+
if estimate_upper:
|
|
1107
|
+
upper = float(lower + (1.0 - lower) * expit(params[idx]))
|
|
1108
|
+
idx += 1
|
|
1109
|
+
else:
|
|
1110
|
+
upper = upper_value
|
|
1111
|
+
return mu, s, lower, upper
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
@dataclass(frozen=True)
|
|
1115
|
+
class _GenericLogLikelihood:
|
|
1116
|
+
"""The estimated-asymptote likelihood, passed to
|
|
1117
|
+
``statsmodels.base.model.GenericLikelihoodModel`` as its ``loglike=`` callable
|
|
1118
|
+
(specifying a likelihood for a statsmodels optimizer, never a subclass that reimplements the
|
|
1119
|
+
optimizer itself). Held as a plain callable object, not a ``GenericLikelihoodModel`` subclass,
|
|
1120
|
+
so nothing here subclasses a type statsmodels ships with no ``py.typed`` marker.
|
|
1121
|
+
|
|
1122
|
+
``center``/``spread`` are the :func:`_centering_and_scale` constants: the optimizer's own
|
|
1123
|
+
``mu``/``log_s`` therefore live on the *centred-and-scaled* transformed covariate, not on
|
|
1124
|
+
natural units, which is what keeps the optimizer's working scale independent of the caller's
|
|
1125
|
+
severity units (a fit must not depend on them). ``a``/``b`` (asymptote logits) are untouched by
|
|
1126
|
+
this -- only ``mu``/``s`` are rescaled.
|
|
1127
|
+
"""
|
|
1128
|
+
|
|
1129
|
+
severity: NDArray[np.float64]
|
|
1130
|
+
successes: NDArray[np.float64]
|
|
1131
|
+
trials: NDArray[np.float64]
|
|
1132
|
+
scale: str
|
|
1133
|
+
link: str
|
|
1134
|
+
estimate_lower: bool
|
|
1135
|
+
estimate_upper: bool
|
|
1136
|
+
lower_value: float
|
|
1137
|
+
upper_value: float
|
|
1138
|
+
center: float
|
|
1139
|
+
spread: float
|
|
1140
|
+
|
|
1141
|
+
def __call__(self, params: NDArray[np.float64]) -> float:
|
|
1142
|
+
mu, s, lower, upper = _unpack_optimizer_params(
|
|
1143
|
+
params,
|
|
1144
|
+
estimate_lower=self.estimate_lower,
|
|
1145
|
+
estimate_upper=self.estimate_upper,
|
|
1146
|
+
lower_value=self.lower_value,
|
|
1147
|
+
upper_value=self.upper_value,
|
|
1148
|
+
)
|
|
1149
|
+
return _full_binomial_log_likelihood(
|
|
1150
|
+
self.severity,
|
|
1151
|
+
self.successes,
|
|
1152
|
+
self.trials,
|
|
1153
|
+
mu=mu,
|
|
1154
|
+
s=s,
|
|
1155
|
+
upper=upper,
|
|
1156
|
+
lower=lower,
|
|
1157
|
+
scale=self.scale,
|
|
1158
|
+
link=self.link,
|
|
1159
|
+
center=self.center,
|
|
1160
|
+
spread=self.spread,
|
|
1161
|
+
)
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
def _pilot_start(
|
|
1165
|
+
severity: NDArray[np.float64],
|
|
1166
|
+
successes: NDArray[np.float64],
|
|
1167
|
+
trials: NDArray[np.float64],
|
|
1168
|
+
*,
|
|
1169
|
+
scale: str,
|
|
1170
|
+
link: str,
|
|
1171
|
+
center: float,
|
|
1172
|
+
spread: float,
|
|
1173
|
+
) -> tuple[float, float, float, float, bool]:
|
|
1174
|
+
"""A starting point for the estimated-asymptote optimizer -- "only a starting point",
|
|
1175
|
+
never itself reported as a fit. ``upper``/``lower`` start at the
|
|
1176
|
+
control rate (or the maximum rate) and the minimum rate (probabilities, unaffected by
|
|
1177
|
+
``center``/``spread``); ``mu``/``s`` start from a GLM pilot fit of the *rescaled* rates
|
|
1178
|
+
against the same link, on the same centred-and-scaled covariate
|
|
1179
|
+
``t' = (t - center) / spread`` (:func:`_centering_and_scale`) the real optimizer fits on --
|
|
1180
|
+
both far better conditioned than fitting the raw 0/1-bounded rates on raw severity units
|
|
1181
|
+
when the true asymptotes are not 0/1 and/or the severity units are far from order 1.
|
|
1182
|
+
``mu0``/``s0`` are therefore returned already on that same rescaled ("primed") scale, ready
|
|
1183
|
+
to seed the optimizer's own ``mu'``/``log_s'`` directly with no further conversion.
|
|
1184
|
+
|
|
1185
|
+
The pilot GLM itself is never skipped or wrapped in a try/except: a
|
|
1186
|
+
`PerfectSeparationWarning` from it is filtered to "ignore", not treated as a failure, so the
|
|
1187
|
+
only two things that actually change what is returned are (a) fewer than two distinct
|
|
1188
|
+
nonzero severities, when there is nothing to regress against at all, and (b) a wrong-sign
|
|
1189
|
+
pilot slope (``beta1 <= 0``) -- either keeps the neutral default (``mu0' = 0``, ``s0' = 1``,
|
|
1190
|
+
both sensible defaults *because* the rescaled covariate is centred at 0 with spread 1):
|
|
1191
|
+
robustness matters more than precision for a value the real optimizer only starts from.
|
|
1192
|
+
Case (b) is also reported back via ``pilot_wrong_sign``, so a caller whose real fit later
|
|
1193
|
+
fails can attach the wrong-sign reason too.
|
|
1194
|
+
|
|
1195
|
+
Returns
|
|
1196
|
+
-------
|
|
1197
|
+
tuple[float, float, float, float, bool]
|
|
1198
|
+
``(mu0, s0, upper_start, lower_start, pilot_wrong_sign)``, with ``mu0``/``s0`` on the
|
|
1199
|
+
centred-and-scaled covariate. ``pilot_wrong_sign`` is ``True`` only when the pilot GLM
|
|
1200
|
+
actually ran (at least two distinct nonzero severities) and its fitted slope was
|
|
1201
|
+
``<= 0``.
|
|
1202
|
+
"""
|
|
1203
|
+
rate = successes / trials
|
|
1204
|
+
control_mask = severity == 0.0
|
|
1205
|
+
if scale == "log" and bool(np.any(control_mask)):
|
|
1206
|
+
upper_start = float(rate[control_mask][0])
|
|
1207
|
+
else:
|
|
1208
|
+
upper_start = float(np.max(rate))
|
|
1209
|
+
lower_start = float(np.min(rate))
|
|
1210
|
+
if not upper_start > lower_start:
|
|
1211
|
+
upper_start, lower_start = 1.0, 0.0
|
|
1212
|
+
|
|
1213
|
+
nonzero = severity > 0.0 if scale == "log" else np.ones_like(severity, dtype=bool)
|
|
1214
|
+
transformed_scaled = (_transform(severity[nonzero], scale=scale) - center) / spread
|
|
1215
|
+
mu0 = float(np.median(transformed_scaled)) if transformed_scaled.size else 0.0
|
|
1216
|
+
s0 = 1.0
|
|
1217
|
+
pilot_wrong_sign = False
|
|
1218
|
+
|
|
1219
|
+
if transformed_scaled.size and len(np.unique(severity[nonzero])) >= 2:
|
|
1220
|
+
eps = 1e-3
|
|
1221
|
+
rescaled_failure_rate = np.clip(
|
|
1222
|
+
(upper_start - rate[nonzero]) / (upper_start - lower_start), eps, 1.0 - eps
|
|
1223
|
+
)
|
|
1224
|
+
exog = add_constant(transformed_scaled)
|
|
1225
|
+
with warnings.catch_warnings():
|
|
1226
|
+
warnings.simplefilter("ignore", category=PerfectSeparationWarning)
|
|
1227
|
+
pilot = GLM(
|
|
1228
|
+
rescaled_failure_rate,
|
|
1229
|
+
exog,
|
|
1230
|
+
family=families.Binomial(link=_STATSMODELS_LINKS[link]),
|
|
1231
|
+
var_weights=trials[nonzero],
|
|
1232
|
+
).fit()
|
|
1233
|
+
beta0 = float(pilot.params[0])
|
|
1234
|
+
beta1 = float(pilot.params[1])
|
|
1235
|
+
# Start values only: a wrong-sign pilot keeps the neutral (median, 1.0) start.
|
|
1236
|
+
if beta1 > 0.0:
|
|
1237
|
+
mu0 = -beta0 / beta1
|
|
1238
|
+
s0 = 1.0 / beta1
|
|
1239
|
+
else:
|
|
1240
|
+
pilot_wrong_sign = True
|
|
1241
|
+
return mu0, s0, upper_start, lower_start, pilot_wrong_sign
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
def _boundary_warning(
|
|
1245
|
+
final_params: NDArray[np.float64],
|
|
1246
|
+
*,
|
|
1247
|
+
estimate_lower: bool,
|
|
1248
|
+
estimate_upper: bool,
|
|
1249
|
+
lower: float,
|
|
1250
|
+
upper: float,
|
|
1251
|
+
) -> str | None:
|
|
1252
|
+
"""An estimated asymptote pinned to a parameter-space boundary,
|
|
1253
|
+
checked both on the optimizer's own logit-scale parameter (``a``/``b``, ``|.| > 15``) and
|
|
1254
|
+
on the natural-scale probability itself (within ``1e-6`` of 0 or 1). Either test alone can
|
|
1255
|
+
miss a boundary fit the other catches (a huge logit-scale value that still rounds to a
|
|
1256
|
+
natural-scale probability inside the epsilon, or vice versa via the numerics of a
|
|
1257
|
+
particular optimizer step), so both are checked.
|
|
1258
|
+
"""
|
|
1259
|
+
idx = 2
|
|
1260
|
+
if estimate_lower:
|
|
1261
|
+
a_value = float(final_params[idx])
|
|
1262
|
+
idx += 1
|
|
1263
|
+
if abs(a_value) > _ASYMPTOTE_LOGIT_BOUND:
|
|
1264
|
+
return (
|
|
1265
|
+
"fit_dose_response: the estimated lower asymptote reached the parameter "
|
|
1266
|
+
f"boundary (logit-scale a={a_value!r}, |a| > {_ASYMPTOTE_LOGIT_BOUND}) -- no "
|
|
1267
|
+
"interior maximum exists"
|
|
1268
|
+
)
|
|
1269
|
+
if estimate_upper:
|
|
1270
|
+
b_value = float(final_params[idx])
|
|
1271
|
+
if abs(b_value) > _ASYMPTOTE_LOGIT_BOUND:
|
|
1272
|
+
return (
|
|
1273
|
+
"fit_dose_response: the estimated upper asymptote reached the parameter "
|
|
1274
|
+
f"boundary (logit-scale b={b_value!r}, |b| > {_ASYMPTOTE_LOGIT_BOUND}) -- no "
|
|
1275
|
+
"interior maximum exists"
|
|
1276
|
+
)
|
|
1277
|
+
if estimate_lower and estimate_upper and lower < _ASYMPTOTE_PROBABILITY_EPS:
|
|
1278
|
+
return (
|
|
1279
|
+
"fit_dose_response: the estimated lower asymptote reached the probability "
|
|
1280
|
+
f"boundary (lower={lower!r} < {_ASYMPTOTE_PROBABILITY_EPS}) -- no interior maximum "
|
|
1281
|
+
"exists"
|
|
1282
|
+
)
|
|
1283
|
+
if estimate_lower and not estimate_upper and (upper - lower) < _ASYMPTOTE_PROBABILITY_EPS:
|
|
1284
|
+
# With a fixed upper, l = u * expit(a) ranges over (0, u). l -> 0 (a -> -inf) is *not*
|
|
1285
|
+
# exempt from a boundary check here -- the |a| > 15 test above already applies to it, the
|
|
1286
|
+
# same as it would for any other estimated asymptote's logit parameter, so it is not
|
|
1287
|
+
# "ordinary" merely because there is no separate probability-floor test for it. This
|
|
1288
|
+
# eps-based test instead covers the *other* end, l -> u (a -> +inf, expit(a) -> 1): the
|
|
1289
|
+
# degenerate property there is the gap (u - l) collapsing (the curve going flat, u == l, no
|
|
1290
|
+
# dynamic range at all), which a fixed magnitude bound on `a` alone does not directly test,
|
|
1291
|
+
# because how large `a` needs to be for u - l to fall below eps depends on the fixed u
|
|
1292
|
+
# itself.
|
|
1293
|
+
return (
|
|
1294
|
+
"fit_dose_response: the estimated lower asymptote reached the probability "
|
|
1295
|
+
f"boundary near the fixed upper asymptote (upper - lower = {upper - lower!r} < "
|
|
1296
|
+
f"{_ASYMPTOTE_PROBABILITY_EPS}) -- no interior maximum exists"
|
|
1297
|
+
)
|
|
1298
|
+
if estimate_upper and upper > 1.0 - _ASYMPTOTE_PROBABILITY_EPS:
|
|
1299
|
+
return (
|
|
1300
|
+
"fit_dose_response: the estimated upper asymptote reached the probability "
|
|
1301
|
+
f"boundary (upper={upper!r} > 1 - {_ASYMPTOTE_PROBABILITY_EPS}) -- no interior "
|
|
1302
|
+
"maximum exists"
|
|
1303
|
+
)
|
|
1304
|
+
return None
|
|
1305
|
+
|
|
1306
|
+
|
|
1307
|
+
def _failure_warnings(reason: str, *, pilot_wrong_sign: bool) -> tuple[str, ...]:
|
|
1308
|
+
"""The ``warnings`` tuple for a ``NOT_CONVERGED`` estimated-asymptote fit: ``reason`` alone,
|
|
1309
|
+
or ``reason`` plus the shared wrong-sign/flat reason when a pilot GLM slope on the (rescaled)
|
|
1310
|
+
data had the wrong sign for ``direction='decreasing'`` -- unless ``reason`` already *is* that
|
|
1311
|
+
shared reason, in which case it is not duplicated.
|
|
1312
|
+
"""
|
|
1313
|
+
if pilot_wrong_sign and reason != _WRONG_SIGN_OR_FLAT_WARNING:
|
|
1314
|
+
return (reason, _WRONG_SIGN_OR_FLAT_WARNING)
|
|
1315
|
+
return (reason,)
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def _fit_generic(
|
|
1319
|
+
obs: Observations,
|
|
1320
|
+
*,
|
|
1321
|
+
cells: tuple[Cell, ...],
|
|
1322
|
+
cluster_ids: tuple[str | int, ...] | None,
|
|
1323
|
+
link: str,
|
|
1324
|
+
upper_is_estimate: bool,
|
|
1325
|
+
upper_value: float,
|
|
1326
|
+
lower_is_estimate: bool,
|
|
1327
|
+
lower_value: float,
|
|
1328
|
+
) -> Fit:
|
|
1329
|
+
"""The estimated-or-non-trivial-fixed-asymptote path: ``statsmodels``
|
|
1330
|
+
``GenericLikelihoodModel`` given :class:`_GenericLogLikelihood`, fit with ``method="bfgs"``
|
|
1331
|
+
then a ``method="newton"`` polish -- both statsmodels optimizers,
|
|
1332
|
+
never a hand-rolled Newton or IRLS loop.
|
|
1333
|
+
"""
|
|
1334
|
+
axis = obs.axis
|
|
1335
|
+
|
|
1336
|
+
# Drop a structurally zero-contribution zero-severity control row before building the
|
|
1337
|
+
# optimizer's own data: with `upper` fixed at exactly 1.0 on a log axis,
|
|
1338
|
+
# `_curve_success_probability`'s own zero-severity-control rule pins this cell's fitted
|
|
1339
|
+
# probability to exactly `upper` for every possible mu/s/lower, so its log-likelihood
|
|
1340
|
+
# contribution is an exact constant with respect to every parameter this path estimates -- by
|
|
1341
|
+
# the time `_fit_generic` is reached with upper fixed at 1.0, `_check_control_incompatible` has
|
|
1342
|
+
# already ruled out a control with any failures here, so that constant is exactly 0. Leaving
|
|
1343
|
+
# that row in the data had two distinct effects on the fit, not one, and this exclusion removes
|
|
1344
|
+
# both at once: (1) `_pilot_start`'s own `upper_start` reads the *control's* rate when one is
|
|
1345
|
+
# present (here, exactly 1.0) rather than the maximum rate among the other levels (here, about
|
|
1346
|
+
# 0.96), which moved the pilot `mu0`/`s0` the real optimizer started from; (2)
|
|
1347
|
+
# `GenericLikelihoodModel.fit` (statsmodels' own optimizer, not one of ours) minimises
|
|
1348
|
+
# `-loglike(params) / nobs`, where `nobs` is the row count it was built with, so a
|
|
1349
|
+
# zero-contribution row changes `nobs` and, with it, the scale at which `gtol`/`tol` judge
|
|
1350
|
+
# convergence, even though the likelihood function itself is unchanged at every parameter value.
|
|
1351
|
+
# Excluding it here mirrors `_fit_glm`'s own `fit_cells` exclusion below, and keeps both the
|
|
1352
|
+
# pilot start and `nobs` identical to a fit over the same data with no control row at all. An
|
|
1353
|
+
# *estimated* upper is never exempt: the zero_control branch then reports `upper` itself, a free
|
|
1354
|
+
# parameter, so the control genuinely identifies it and must stay in the optimizer's data.
|
|
1355
|
+
fit_cells = (
|
|
1356
|
+
tuple(c for c in cells if c.severity != 0.0)
|
|
1357
|
+
if axis.scale == "log" and not upper_is_estimate and upper_value == 1.0
|
|
1358
|
+
else cells
|
|
1359
|
+
)
|
|
1360
|
+
severity, successes, trials = _cells_to_arrays(fit_cells)
|
|
1361
|
+
|
|
1362
|
+
# Centre and scale the transformed covariate before either the pilot or the real optimiser
|
|
1363
|
+
# ever sees it (a fit must not depend on the caller's choice of severity
|
|
1364
|
+
# units). This is an exact reparametrisation, not a different model -- see
|
|
1365
|
+
# _centering_and_scale and _GenericLogLikelihood.
|
|
1366
|
+
center, spread = _centering_and_scale(severity, scale=axis.scale)
|
|
1367
|
+
|
|
1368
|
+
mu0, s0, u0, l0, pilot_wrong_sign = _pilot_start(
|
|
1369
|
+
severity, successes, trials, scale=axis.scale, link=link, center=center, spread=spread
|
|
1370
|
+
)
|
|
1371
|
+
|
|
1372
|
+
start = [mu0, math.log(s0)]
|
|
1373
|
+
eps = 1e-3
|
|
1374
|
+
lower_for_start = l0 if lower_is_estimate else lower_value
|
|
1375
|
+
upper_for_start = u0 if upper_is_estimate else upper_value
|
|
1376
|
+
if lower_is_estimate:
|
|
1377
|
+
if upper_is_estimate:
|
|
1378
|
+
a0 = float(_logit(np.clip(lower_for_start, eps, 1.0 - eps)))
|
|
1379
|
+
else:
|
|
1380
|
+
# l = u * expit(a) when upper is fixed, so a's start value inverts *that* formula with
|
|
1381
|
+
# the fixed u, not l = expit(a) directly.
|
|
1382
|
+
a0 = float(_logit(np.clip(lower_for_start / upper_for_start, eps, 1.0 - eps)))
|
|
1383
|
+
start.append(a0)
|
|
1384
|
+
if upper_is_estimate:
|
|
1385
|
+
denominator = max(1.0 - lower_for_start, 1e-6)
|
|
1386
|
+
fraction = float(np.clip((upper_for_start - lower_for_start) / denominator, eps, 1.0 - eps))
|
|
1387
|
+
start.append(float(_logit(fraction)))
|
|
1388
|
+
start_params = np.array(start, dtype=np.float64)
|
|
1389
|
+
|
|
1390
|
+
loglike_fn = _GenericLogLikelihood(
|
|
1391
|
+
severity=severity,
|
|
1392
|
+
successes=successes,
|
|
1393
|
+
trials=trials,
|
|
1394
|
+
scale=axis.scale,
|
|
1395
|
+
link=link,
|
|
1396
|
+
estimate_lower=lower_is_estimate,
|
|
1397
|
+
estimate_upper=upper_is_estimate,
|
|
1398
|
+
lower_value=lower_value,
|
|
1399
|
+
upper_value=upper_value,
|
|
1400
|
+
center=center,
|
|
1401
|
+
spread=spread,
|
|
1402
|
+
)
|
|
1403
|
+
exog = np.ones((severity.size, start_params.size))
|
|
1404
|
+
model = GenericLikelihoodModel(successes, exog, loglike=loglike_fn)
|
|
1405
|
+
|
|
1406
|
+
# The BFGS stage is only ever a starting point for the Newton polish: it routinely reports its
|
|
1407
|
+
# own `converged=False` / ConvergenceWarning even when it has landed close enough for Newton to
|
|
1408
|
+
# finish the job, so only the *final* (Newton) result's own convergence signal decides the
|
|
1409
|
+
# status below -- checking a warning merged across both stages would treat that expected
|
|
1410
|
+
# intermediate state as a failure.
|
|
1411
|
+
with warnings.catch_warnings():
|
|
1412
|
+
warnings.simplefilter("ignore")
|
|
1413
|
+
bfgs_result = model.fit(
|
|
1414
|
+
start_params=start_params, method="bfgs", disp=False, maxiter=2000, gtol=1e-10
|
|
1415
|
+
)
|
|
1416
|
+
try:
|
|
1417
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
1418
|
+
warnings.simplefilter("always")
|
|
1419
|
+
result = model.fit(
|
|
1420
|
+
start_params=np.asarray(bfgs_result.params, dtype=np.float64),
|
|
1421
|
+
method="newton",
|
|
1422
|
+
disp=False,
|
|
1423
|
+
maxiter=200,
|
|
1424
|
+
tol=1e-10,
|
|
1425
|
+
)
|
|
1426
|
+
except np.linalg.LinAlgError:
|
|
1427
|
+
# A singular Hessian during the Newton polish is usually the symptom of an estimated
|
|
1428
|
+
# asymptote drifting to its boundary; name that cause when the BFGS
|
|
1429
|
+
# stage already shows it, so the reason does not depend on which platform's linear
|
|
1430
|
+
# algebra happens to fail first.
|
|
1431
|
+
bfgs_params = np.asarray(bfgs_result.params, dtype=np.float64)
|
|
1432
|
+
_, _, bfgs_lower, bfgs_upper = _unpack_optimizer_params(
|
|
1433
|
+
bfgs_params,
|
|
1434
|
+
estimate_lower=lower_is_estimate,
|
|
1435
|
+
estimate_upper=upper_is_estimate,
|
|
1436
|
+
lower_value=lower_value,
|
|
1437
|
+
upper_value=upper_value,
|
|
1438
|
+
)
|
|
1439
|
+
singular_reason = _boundary_warning(
|
|
1440
|
+
bfgs_params,
|
|
1441
|
+
estimate_lower=lower_is_estimate,
|
|
1442
|
+
estimate_upper=upper_is_estimate,
|
|
1443
|
+
lower=bfgs_lower,
|
|
1444
|
+
upper=bfgs_upper,
|
|
1445
|
+
) or (
|
|
1446
|
+
"fit_dose_response: the optimizer's Hessian was singular while fitting -- "
|
|
1447
|
+
"no interior maximum was found"
|
|
1448
|
+
)
|
|
1449
|
+
return _status_only_fit(
|
|
1450
|
+
obs,
|
|
1451
|
+
cells=cells,
|
|
1452
|
+
cluster_ids=cluster_ids,
|
|
1453
|
+
link=link,
|
|
1454
|
+
status=Status.NOT_CONVERGED,
|
|
1455
|
+
fit_warnings=_failure_warnings(singular_reason, pilot_wrong_sign=pilot_wrong_sign),
|
|
1456
|
+
)
|
|
1457
|
+
convergence_warned = any(issubclass(w.category, ConvergenceWarning) for w in caught)
|
|
1458
|
+
converged_flag = bool(result.mle_retvals.get("converged", False))
|
|
1459
|
+
final_params = np.asarray(result.params, dtype=np.float64)
|
|
1460
|
+
|
|
1461
|
+
mu_scaled, s_scaled, lower, upper = _unpack_optimizer_params(
|
|
1462
|
+
final_params,
|
|
1463
|
+
estimate_lower=lower_is_estimate,
|
|
1464
|
+
estimate_upper=upper_is_estimate,
|
|
1465
|
+
lower_value=lower_value,
|
|
1466
|
+
upper_value=upper_value,
|
|
1467
|
+
)
|
|
1468
|
+
# mu/s from the optimizer are on the centred-and-scaled covariate; map back to natural
|
|
1469
|
+
# units (the exact inverse of _centering_and_scale's reparametrisation). lower/upper are
|
|
1470
|
+
# already natural -- asymptotes are untouched by the covariate rescaling.
|
|
1471
|
+
mu = center + spread * mu_scaled
|
|
1472
|
+
s = spread * s_scaled
|
|
1473
|
+
|
|
1474
|
+
# A flat curve up to rounding, checked before anything that would build a covariance from a
|
|
1475
|
+
# near-degenerate slope.
|
|
1476
|
+
span = _transformed_span(severity, scale=axis.scale)
|
|
1477
|
+
if s > _FLAT_SLOPE_SPAN_FACTOR * span:
|
|
1478
|
+
return _status_only_fit(
|
|
1479
|
+
obs,
|
|
1480
|
+
cells=cells,
|
|
1481
|
+
cluster_ids=cluster_ids,
|
|
1482
|
+
link=link,
|
|
1483
|
+
status=Status.NOT_CONVERGED,
|
|
1484
|
+
fit_warnings=_failure_warnings(
|
|
1485
|
+
_WRONG_SIGN_OR_FLAT_WARNING, pilot_wrong_sign=pilot_wrong_sign
|
|
1486
|
+
),
|
|
1487
|
+
)
|
|
1488
|
+
|
|
1489
|
+
boundary_message = _boundary_warning(
|
|
1490
|
+
final_params,
|
|
1491
|
+
estimate_lower=lower_is_estimate,
|
|
1492
|
+
estimate_upper=upper_is_estimate,
|
|
1493
|
+
lower=lower,
|
|
1494
|
+
upper=upper,
|
|
1495
|
+
)
|
|
1496
|
+
if boundary_message is not None:
|
|
1497
|
+
return _status_only_fit(
|
|
1498
|
+
obs,
|
|
1499
|
+
cells=cells,
|
|
1500
|
+
cluster_ids=cluster_ids,
|
|
1501
|
+
link=link,
|
|
1502
|
+
status=Status.NOT_CONVERGED,
|
|
1503
|
+
fit_warnings=_failure_warnings(boundary_message, pilot_wrong_sign=pilot_wrong_sign),
|
|
1504
|
+
)
|
|
1505
|
+
|
|
1506
|
+
# Newton's own `converged` flag only bounds the last step, so the final log-likelihood must
|
|
1507
|
+
# not have regressed below the BFGS stage's (relative tolerance -- an absolute tolerance here
|
|
1508
|
+
# does not scale with the log-likelihood's own magnitude, which depends on the sample size,
|
|
1509
|
+
# not on anything a caller controls), and the score at the final parameters must be near
|
|
1510
|
+
# zero.
|
|
1511
|
+
bfgs_llf = float(bfgs_result.llf)
|
|
1512
|
+
llf_regressed = float(result.llf) < bfgs_llf - _LLF_REGRESSION_TOLERANCE * (1.0 + abs(bfgs_llf))
|
|
1513
|
+
score = np.asarray(model.score(final_params), dtype=np.float64)
|
|
1514
|
+
score_tolerance = _SCORE_TOLERANCE_FACTOR * (1.0 + abs(float(result.llf)))
|
|
1515
|
+
score_not_near_zero = bool(np.any(np.abs(score) > score_tolerance))
|
|
1516
|
+
|
|
1517
|
+
not_converged_reason: str | None = None
|
|
1518
|
+
if not converged_flag or convergence_warned:
|
|
1519
|
+
not_converged_reason = "fit_dose_response: the optimizer did not converge"
|
|
1520
|
+
elif llf_regressed:
|
|
1521
|
+
not_converged_reason = (
|
|
1522
|
+
"fit_dose_response: the Newton polish's log-likelihood is lower than the BFGS "
|
|
1523
|
+
"stage's -- the optimizer did not converge"
|
|
1524
|
+
)
|
|
1525
|
+
elif score_not_near_zero:
|
|
1526
|
+
not_converged_reason = (
|
|
1527
|
+
"fit_dose_response: the score at the optimum is not near zero -- the optimizer "
|
|
1528
|
+
"did not converge"
|
|
1529
|
+
)
|
|
1530
|
+
if not_converged_reason is not None:
|
|
1531
|
+
return _status_only_fit(
|
|
1532
|
+
obs,
|
|
1533
|
+
cells=cells,
|
|
1534
|
+
cluster_ids=cluster_ids,
|
|
1535
|
+
link=link,
|
|
1536
|
+
status=Status.NOT_CONVERGED,
|
|
1537
|
+
fit_warnings=_failure_warnings(not_converged_reason, pilot_wrong_sign=pilot_wrong_sign),
|
|
1538
|
+
)
|
|
1539
|
+
|
|
1540
|
+
# A non-finite Hessian entry is checked before eigvalsh ever sees it
|
|
1541
|
+
# (numpy.linalg.eigvalsh raises LinAlgError on a non-finite array), and the
|
|
1542
|
+
# positive-definite check itself is the same one Covariance applies, done here first so a
|
|
1543
|
+
# broken Hessian never reaches that constructor.
|
|
1544
|
+
hessian = np.asarray(model.hessian(final_params), dtype=np.float64)
|
|
1545
|
+
neg_hessian = -(hessian + hessian.T) / 2.0
|
|
1546
|
+
if not _is_positive_definite(neg_hessian):
|
|
1547
|
+
return _status_only_fit(
|
|
1548
|
+
obs,
|
|
1549
|
+
cells=cells,
|
|
1550
|
+
cluster_ids=cluster_ids,
|
|
1551
|
+
link=link,
|
|
1552
|
+
status=Status.NOT_CONVERGED,
|
|
1553
|
+
fit_warnings=_failure_warnings(
|
|
1554
|
+
"fit_dose_response: the Hessian at the optimum is not finite and "
|
|
1555
|
+
"positive-definite -- no interior maximum was found",
|
|
1556
|
+
pilot_wrong_sign=pilot_wrong_sign,
|
|
1557
|
+
),
|
|
1558
|
+
)
|
|
1559
|
+
|
|
1560
|
+
names: list[str] = ["mu", "s"]
|
|
1561
|
+
if lower_is_estimate:
|
|
1562
|
+
names.append("lower")
|
|
1563
|
+
if upper_is_estimate:
|
|
1564
|
+
names.append("upper")
|
|
1565
|
+
|
|
1566
|
+
def _natural_vector(p: NDArray[np.float64]) -> NDArray[np.float64]:
|
|
1567
|
+
mu_scaled_, s_scaled_, lower_, upper_ = _unpack_optimizer_params(
|
|
1568
|
+
p,
|
|
1569
|
+
estimate_lower=lower_is_estimate,
|
|
1570
|
+
estimate_upper=upper_is_estimate,
|
|
1571
|
+
lower_value=lower_value,
|
|
1572
|
+
upper_value=upper_value,
|
|
1573
|
+
)
|
|
1574
|
+
# Fold the covariate rescaling into the same numeric Jacobian (mu/s only; asymptotes
|
|
1575
|
+
# are already natural).
|
|
1576
|
+
values = [center + spread * mu_scaled_, spread * s_scaled_]
|
|
1577
|
+
if lower_is_estimate:
|
|
1578
|
+
values.append(lower_)
|
|
1579
|
+
if upper_is_estimate:
|
|
1580
|
+
values.append(upper_)
|
|
1581
|
+
return np.array(values, dtype=np.float64)
|
|
1582
|
+
|
|
1583
|
+
jacobian = _central_difference_jacobian(_natural_vector, final_params)
|
|
1584
|
+
optimizer_cov = np.asarray(result.cov_params(), dtype=np.float64)
|
|
1585
|
+
natural_cov = jacobian @ optimizer_cov @ jacobian.T
|
|
1586
|
+
natural_cov = (natural_cov + natural_cov.T) / 2.0
|
|
1587
|
+
|
|
1588
|
+
# Covariance's own positive-definiteness rule, on the correlation scale so the status does
|
|
1589
|
+
# not depend on severity units.
|
|
1590
|
+
if not _is_positive_definite(natural_cov):
|
|
1591
|
+
return _status_only_fit(
|
|
1592
|
+
obs,
|
|
1593
|
+
cells=cells,
|
|
1594
|
+
cluster_ids=cluster_ids,
|
|
1595
|
+
link=link,
|
|
1596
|
+
status=Status.NOT_CONVERGED,
|
|
1597
|
+
fit_warnings=_failure_warnings(
|
|
1598
|
+
"fit_dose_response: the natural-scale covariance is not finite and "
|
|
1599
|
+
"positive-definite -- no interior maximum was found",
|
|
1600
|
+
pilot_wrong_sign=pilot_wrong_sign,
|
|
1601
|
+
),
|
|
1602
|
+
)
|
|
1603
|
+
|
|
1604
|
+
# The reported log-likelihood is always the full binomial log-likelihood over every cell
|
|
1605
|
+
# (`Fit.cells`, including a control `fit_cells` dropped above) -- `fit_cells`/`severity` etc.
|
|
1606
|
+
# above are the optimizer's own reduced view, not what gets reported.
|
|
1607
|
+
all_severity, all_successes, all_trials = _cells_to_arrays(cells)
|
|
1608
|
+
log_likelihood = _full_binomial_log_likelihood(
|
|
1609
|
+
all_severity,
|
|
1610
|
+
all_successes,
|
|
1611
|
+
all_trials,
|
|
1612
|
+
mu=mu,
|
|
1613
|
+
s=s,
|
|
1614
|
+
upper=upper,
|
|
1615
|
+
lower=lower,
|
|
1616
|
+
scale=axis.scale,
|
|
1617
|
+
link=link,
|
|
1618
|
+
)
|
|
1619
|
+
|
|
1620
|
+
params = {
|
|
1621
|
+
"mu": Parameter(value=mu, fixed=False),
|
|
1622
|
+
"s": Parameter(value=s, fixed=False),
|
|
1623
|
+
"lower": Parameter(value=lower, fixed=not lower_is_estimate),
|
|
1624
|
+
"upper": Parameter(value=upper, fixed=not upper_is_estimate),
|
|
1625
|
+
}
|
|
1626
|
+
covariance = Covariance(
|
|
1627
|
+
names=tuple(names), matrix=tuple(tuple(row) for row in natural_cov.tolist())
|
|
1628
|
+
)
|
|
1629
|
+
|
|
1630
|
+
return Fit(
|
|
1631
|
+
axis=axis,
|
|
1632
|
+
outcome=obs.outcome,
|
|
1633
|
+
direction=obs.direction,
|
|
1634
|
+
model="binomial",
|
|
1635
|
+
link=link,
|
|
1636
|
+
params=params,
|
|
1637
|
+
covariance=covariance,
|
|
1638
|
+
log_likelihood=log_likelihood,
|
|
1639
|
+
cells=cells,
|
|
1640
|
+
status=Status.OK,
|
|
1641
|
+
cluster_ids=cluster_ids,
|
|
1642
|
+
warnings=(),
|
|
1643
|
+
provenance={},
|
|
1644
|
+
)
|
|
1645
|
+
|
|
1646
|
+
|
|
1647
|
+
# ================================================================================================
|
|
1648
|
+
# fit_dose_response: the fixed u=1, l=0 path (statsmodels GLM)
|
|
1649
|
+
# ================================================================================================
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def _fit_glm(
|
|
1653
|
+
obs: Observations,
|
|
1654
|
+
*,
|
|
1655
|
+
cells: tuple[Cell, ...],
|
|
1656
|
+
cluster_ids: tuple[str | int, ...] | None,
|
|
1657
|
+
link: str,
|
|
1658
|
+
) -> Fit:
|
|
1659
|
+
axis = obs.axis
|
|
1660
|
+
fit_cells = tuple(c for c in cells if not (axis.scale == "log" and c.severity == 0.0))
|
|
1661
|
+
severity, successes, trials = _cells_to_arrays(fit_cells)
|
|
1662
|
+
failures = trials - successes
|
|
1663
|
+
|
|
1664
|
+
# Centre and scale the transformed covariate before the GLM ever sees it (a fit must not
|
|
1665
|
+
# depend on the caller's choice of severity units). An exact reparametrisation of the
|
|
1666
|
+
# identical model -- see _centering_and_scale.
|
|
1667
|
+
center, spread = _centering_and_scale(severity, scale=axis.scale)
|
|
1668
|
+
exog = add_constant((_transform(severity, scale=axis.scale) - center) / spread)
|
|
1669
|
+
endog = failures / trials
|
|
1670
|
+
|
|
1671
|
+
model = GLM(
|
|
1672
|
+
endog, exog, family=families.Binomial(link=_STATSMODELS_LINKS[link]), var_weights=trials
|
|
1673
|
+
)
|
|
1674
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
1675
|
+
warnings.simplefilter("always")
|
|
1676
|
+
warnings.filterwarnings("ignore", category=PerfectSeparationWarning)
|
|
1677
|
+
glm_result = model.fit()
|
|
1678
|
+
convergence_warned = any(issubclass(w.category, ConvergenceWarning) for w in caught)
|
|
1679
|
+
|
|
1680
|
+
if convergence_warned or not bool(glm_result.converged):
|
|
1681
|
+
return _status_only_fit(
|
|
1682
|
+
obs,
|
|
1683
|
+
cells=cells,
|
|
1684
|
+
cluster_ids=cluster_ids,
|
|
1685
|
+
link=link,
|
|
1686
|
+
status=Status.NOT_CONVERGED,
|
|
1687
|
+
fit_warnings=("fit_dose_response: the GLM optimizer did not converge",),
|
|
1688
|
+
)
|
|
1689
|
+
|
|
1690
|
+
beta0 = float(glm_result.params[0])
|
|
1691
|
+
beta1 = float(glm_result.params[1])
|
|
1692
|
+
if beta1 <= 0.0:
|
|
1693
|
+
return _status_only_fit(
|
|
1694
|
+
obs,
|
|
1695
|
+
cells=cells,
|
|
1696
|
+
cluster_ids=cluster_ids,
|
|
1697
|
+
link=link,
|
|
1698
|
+
status=Status.NOT_CONVERGED,
|
|
1699
|
+
fit_warnings=(_WRONG_SIGN_OR_FLAT_WARNING,),
|
|
1700
|
+
)
|
|
1701
|
+
|
|
1702
|
+
# beta0/beta1 are on the centred-and-scaled covariate; map back to natural units (the exact
|
|
1703
|
+
# inverse of _centering_and_scale's reparametrisation).
|
|
1704
|
+
mu_scaled = -beta0 / beta1
|
|
1705
|
+
s_scaled = 1.0 / beta1
|
|
1706
|
+
mu = center + spread * mu_scaled
|
|
1707
|
+
s = spread * s_scaled
|
|
1708
|
+
|
|
1709
|
+
# A slope that is technically positive but effectively flat (for example beta1 ~ 1e-15) makes s
|
|
1710
|
+
# explode, which would otherwise pass Covariance's own positive-definite check while being
|
|
1711
|
+
# numerically meaningless (s on the order of 1e61).
|
|
1712
|
+
span = _transformed_span(severity, scale=axis.scale)
|
|
1713
|
+
if s > _FLAT_SLOPE_SPAN_FACTOR * span:
|
|
1714
|
+
return _status_only_fit(
|
|
1715
|
+
obs,
|
|
1716
|
+
cells=cells,
|
|
1717
|
+
cluster_ids=cluster_ids,
|
|
1718
|
+
link=link,
|
|
1719
|
+
status=Status.NOT_CONVERGED,
|
|
1720
|
+
fit_warnings=(_WRONG_SIGN_OR_FLAT_WARNING,),
|
|
1721
|
+
)
|
|
1722
|
+
|
|
1723
|
+
# Both fitting paths report covariance from the *observed* information
|
|
1724
|
+
# (the inverse of the negative Hessian of the log-likelihood at the MLE), not statsmodels'
|
|
1725
|
+
# own `cov_params()` (the *expected*/Fisher information), which for a non-canonical link
|
|
1726
|
+
# such as probit or cloglog differs from what drc and GenericLikelihoodModel report.
|
|
1727
|
+
#
|
|
1728
|
+
# There is no positive-definiteness gate on this intermediate,
|
|
1729
|
+
# centred-and-scaled-covariate-scale ("beta-scale") information matrix -- checking it would
|
|
1730
|
+
# be a bug: even a perfectly fine fit's beta-scale matrix can have a large or small
|
|
1731
|
+
# absolute magnitude for reasons that have nothing to do with whether the fit is degenerate
|
|
1732
|
+
# (for example a design with very few levels), and a positive-definiteness rule was never
|
|
1733
|
+
# meant to judge an intermediate quantity. Only a narrow inversion safety net remains here;
|
|
1734
|
+
# the real positive-definiteness decision is made once, below, on the natural-scale
|
|
1735
|
+
# covariance -- the only matrix a caller actually sees.
|
|
1736
|
+
observed_hessian = np.asarray(
|
|
1737
|
+
model.hessian(np.asarray(glm_result.params, dtype=np.float64), observed=True),
|
|
1738
|
+
dtype=np.float64,
|
|
1739
|
+
)
|
|
1740
|
+
neg_observed_information = -(observed_hessian + observed_hessian.T) / 2.0
|
|
1741
|
+
try:
|
|
1742
|
+
sigma_beta = np.linalg.inv(neg_observed_information)
|
|
1743
|
+
except np.linalg.LinAlgError:
|
|
1744
|
+
return _status_only_fit(
|
|
1745
|
+
obs,
|
|
1746
|
+
cells=cells,
|
|
1747
|
+
cluster_ids=cluster_ids,
|
|
1748
|
+
link=link,
|
|
1749
|
+
status=Status.NOT_CONVERGED,
|
|
1750
|
+
fit_warnings=(
|
|
1751
|
+
"fit_dose_response: the GLM's observed information is singular -- no interior "
|
|
1752
|
+
"maximum was found",
|
|
1753
|
+
),
|
|
1754
|
+
)
|
|
1755
|
+
if not bool(np.all(np.isfinite(sigma_beta))):
|
|
1756
|
+
return _status_only_fit(
|
|
1757
|
+
obs,
|
|
1758
|
+
cells=cells,
|
|
1759
|
+
cluster_ids=cluster_ids,
|
|
1760
|
+
link=link,
|
|
1761
|
+
status=Status.NOT_CONVERGED,
|
|
1762
|
+
fit_warnings=(
|
|
1763
|
+
"fit_dose_response: the GLM's observed information inverts to a non-finite "
|
|
1764
|
+
"covariance -- no interior maximum was found",
|
|
1765
|
+
),
|
|
1766
|
+
)
|
|
1767
|
+
jacobian = spread * np.array(
|
|
1768
|
+
[[-1.0 / beta1, beta0 / beta1**2], [0.0, -1.0 / beta1**2]], dtype=np.float64
|
|
1769
|
+
)
|
|
1770
|
+
natural_cov = jacobian @ sigma_beta @ jacobian.T
|
|
1771
|
+
natural_cov = (natural_cov + natural_cov.T) / 2.0
|
|
1772
|
+
|
|
1773
|
+
# Covariance's own positive-definiteness rule, on the correlation scale so the status does
|
|
1774
|
+
# not depend on severity units.
|
|
1775
|
+
if not _is_positive_definite(natural_cov):
|
|
1776
|
+
return _status_only_fit(
|
|
1777
|
+
obs,
|
|
1778
|
+
cells=cells,
|
|
1779
|
+
cluster_ids=cluster_ids,
|
|
1780
|
+
link=link,
|
|
1781
|
+
status=Status.NOT_CONVERGED,
|
|
1782
|
+
fit_warnings=(
|
|
1783
|
+
"fit_dose_response: the natural-scale covariance is not finite and "
|
|
1784
|
+
"positive-definite -- no interior maximum was found",
|
|
1785
|
+
),
|
|
1786
|
+
)
|
|
1787
|
+
|
|
1788
|
+
all_severity, all_successes, all_trials = _cells_to_arrays(cells)
|
|
1789
|
+
log_likelihood = _full_binomial_log_likelihood(
|
|
1790
|
+
all_severity,
|
|
1791
|
+
all_successes,
|
|
1792
|
+
all_trials,
|
|
1793
|
+
mu=mu,
|
|
1794
|
+
s=s,
|
|
1795
|
+
upper=1.0,
|
|
1796
|
+
lower=0.0,
|
|
1797
|
+
scale=axis.scale,
|
|
1798
|
+
link=link,
|
|
1799
|
+
)
|
|
1800
|
+
|
|
1801
|
+
params = {
|
|
1802
|
+
"mu": Parameter(value=mu, fixed=False),
|
|
1803
|
+
"s": Parameter(value=s, fixed=False),
|
|
1804
|
+
"upper": Parameter(value=1.0, fixed=True),
|
|
1805
|
+
"lower": Parameter(value=0.0, fixed=True),
|
|
1806
|
+
}
|
|
1807
|
+
covariance = Covariance(
|
|
1808
|
+
names=("mu", "s"), matrix=tuple(tuple(row) for row in natural_cov.tolist())
|
|
1809
|
+
)
|
|
1810
|
+
|
|
1811
|
+
return Fit(
|
|
1812
|
+
axis=axis,
|
|
1813
|
+
outcome=obs.outcome,
|
|
1814
|
+
direction=obs.direction,
|
|
1815
|
+
model="binomial",
|
|
1816
|
+
link=link,
|
|
1817
|
+
params=params,
|
|
1818
|
+
covariance=covariance,
|
|
1819
|
+
log_likelihood=log_likelihood,
|
|
1820
|
+
cells=cells,
|
|
1821
|
+
status=Status.OK,
|
|
1822
|
+
cluster_ids=cluster_ids,
|
|
1823
|
+
warnings=(),
|
|
1824
|
+
provenance={},
|
|
1825
|
+
)
|
|
1826
|
+
|
|
1827
|
+
|
|
1828
|
+
# ================================================================================================
|
|
1829
|
+
# fit_dose_response: shared checks and the public entry point
|
|
1830
|
+
# ================================================================================================
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
def _resolve_asymptote(value: float | str, *, label: str) -> tuple[bool, float]:
|
|
1834
|
+
"""Validate ``upper``/``lower`` and split it into ``(is_estimate, fixed_value)``.
|
|
1835
|
+
``fixed_value`` is meaningless (``math.nan``) when ``is_estimate`` is ``True``.
|
|
1836
|
+
"""
|
|
1837
|
+
if isinstance(value, str):
|
|
1838
|
+
if value != _VALID_UPPER_LOWER:
|
|
1839
|
+
raise ValueError(
|
|
1840
|
+
f"fit_dose_response: {label} must be a float in [0, 1] or the string "
|
|
1841
|
+
f"'estimate', got {value!r}"
|
|
1842
|
+
)
|
|
1843
|
+
return True, math.nan
|
|
1844
|
+
_check_finite_number(value, label=f"fit_dose_response.{label}")
|
|
1845
|
+
if not (0.0 <= value <= 1.0):
|
|
1846
|
+
raise ValueError(f"fit_dose_response: {label} must lie in [0, 1], got {value!r}")
|
|
1847
|
+
return False, float(value)
|
|
1848
|
+
|
|
1849
|
+
|
|
1850
|
+
def _unique_cluster_ids(
|
|
1851
|
+
cluster: tuple[str | int, ...] | None,
|
|
1852
|
+
) -> tuple[str | int, ...] | None:
|
|
1853
|
+
"""The distinct cluster ids ``Observations`` carries, one instance per cluster (not one per
|
|
1854
|
+
row -- :class:`Fit`'s own invariant), in first-appearance order.
|
|
1855
|
+
"""
|
|
1856
|
+
if cluster is None:
|
|
1857
|
+
return None
|
|
1858
|
+
return tuple(dict.fromkeys(cluster))
|
|
1859
|
+
|
|
1860
|
+
|
|
1861
|
+
def _control_cell(cells: tuple[Cell, ...]) -> Cell | None:
|
|
1862
|
+
for cell in cells:
|
|
1863
|
+
if cell.severity == 0.0:
|
|
1864
|
+
return cell
|
|
1865
|
+
return None
|
|
1866
|
+
|
|
1867
|
+
|
|
1868
|
+
def _check_control_incompatible(
|
|
1869
|
+
cells: tuple[Cell, ...], *, axis: Axis, upper_is_estimate: bool, upper_value: float
|
|
1870
|
+
) -> str | None:
|
|
1871
|
+
"""A fixed ``upper=1.0`` with any failure at the zero-severity control
|
|
1872
|
+
drives the likelihood to zero on a log axis. Only meaningful on a log axis (a linear axis
|
|
1873
|
+
has no zero-severity control concept) and only when ``upper`` is fixed at exactly ``1.0``
|
|
1874
|
+
(a fixed ``upper`` at any other value does not create a zero-likelihood problem).
|
|
1875
|
+
"""
|
|
1876
|
+
if axis.scale != "log" or upper_is_estimate or upper_value != 1.0:
|
|
1877
|
+
return None
|
|
1878
|
+
control = _control_cell(cells)
|
|
1879
|
+
if control is None or control.successes == control.trials:
|
|
1880
|
+
return None
|
|
1881
|
+
return (
|
|
1882
|
+
"fit_dose_response: the zero-severity control has failures but upper=1.0 is fixed, "
|
|
1883
|
+
'which drives the likelihood to zero on a log axis; refit with upper="estimate" '
|
|
1884
|
+
"instead"
|
|
1885
|
+
)
|
|
1886
|
+
|
|
1887
|
+
|
|
1888
|
+
def _check_separation(cells: tuple[Cell, ...], *, axis: Axis) -> str | None:
|
|
1889
|
+
"""The pre-fit separation check: checked on
|
|
1890
|
+
nonzero levels on a log axis (the zero-severity control is excluded, since it carries no
|
|
1891
|
+
covariate information), or on every level on a linear axis (there is no special
|
|
1892
|
+
zero-severity control there). Triggers when no failure exists, no success exists, or the
|
|
1893
|
+
highest severity with a success is at or below the lowest severity with a failure (complete
|
|
1894
|
+
or quasi-complete separation, the latter including an exact tie at one level).
|
|
1895
|
+
"""
|
|
1896
|
+
checked = tuple(c for c in cells if not (axis.scale == "log" and c.severity == 0.0))
|
|
1897
|
+
if not checked:
|
|
1898
|
+
return None
|
|
1899
|
+
has_success = any(c.successes > 0 for c in checked)
|
|
1900
|
+
has_failure = any(c.successes < c.trials for c in checked)
|
|
1901
|
+
if not has_success:
|
|
1902
|
+
return "fit_dose_response: no success was observed at any tested severity (separation)"
|
|
1903
|
+
if not has_failure:
|
|
1904
|
+
return "fit_dose_response: no failure was observed at any tested severity (separation)"
|
|
1905
|
+
max_success_severity = max(c.severity for c in checked if c.successes > 0)
|
|
1906
|
+
min_failure_severity = min(c.severity for c in checked if c.successes < c.trials)
|
|
1907
|
+
if max_success_severity <= min_failure_severity:
|
|
1908
|
+
return (
|
|
1909
|
+
"fit_dose_response: complete or quasi-complete separation -- the highest severity "
|
|
1910
|
+
"with a success is at or below the lowest severity with a failure"
|
|
1911
|
+
)
|
|
1912
|
+
return None
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _status_only_fit(
|
|
1916
|
+
obs: Observations,
|
|
1917
|
+
*,
|
|
1918
|
+
cells: tuple[Cell, ...],
|
|
1919
|
+
cluster_ids: tuple[str | int, ...] | None,
|
|
1920
|
+
link: str,
|
|
1921
|
+
status: Status,
|
|
1922
|
+
fit_warnings: tuple[str, ...],
|
|
1923
|
+
) -> Fit:
|
|
1924
|
+
return Fit(
|
|
1925
|
+
axis=obs.axis,
|
|
1926
|
+
outcome=obs.outcome,
|
|
1927
|
+
direction=obs.direction,
|
|
1928
|
+
model="binomial",
|
|
1929
|
+
link=link,
|
|
1930
|
+
params=None,
|
|
1931
|
+
covariance=None,
|
|
1932
|
+
log_likelihood=None,
|
|
1933
|
+
cells=cells,
|
|
1934
|
+
status=status,
|
|
1935
|
+
cluster_ids=cluster_ids,
|
|
1936
|
+
warnings=fit_warnings,
|
|
1937
|
+
provenance={},
|
|
1938
|
+
)
|
|
1939
|
+
|
|
1940
|
+
|
|
1941
|
+
def _finalize(fit: Fit, *, monotonicity: MonotonicityCheck) -> Fit:
|
|
1942
|
+
"""Attach ``monotonicity`` (every return path, whatever the status) and ``diagnostics``
|
|
1943
|
+
(only when ``fit.status is Status.OK``) to a freshly built ``Fit``. The single place
|
|
1944
|
+
:func:`fit_dose_response` routes every one of its return statements through, so the
|
|
1945
|
+
diagnostics/monotonicity rule is stated once.
|
|
1946
|
+
"""
|
|
1947
|
+
diagnostics = diagnose(fit) if fit.status is Status.OK else None
|
|
1948
|
+
return replace(fit, diagnostics=diagnostics, monotonicity=monotonicity)
|
|
1949
|
+
|
|
1950
|
+
|
|
1951
|
+
def fit_dose_response(
|
|
1952
|
+
obs: Observations,
|
|
1953
|
+
*,
|
|
1954
|
+
model: Literal["binomial"],
|
|
1955
|
+
link: Literal["probit", "logit", "cloglog"],
|
|
1956
|
+
upper: float | Literal["estimate"],
|
|
1957
|
+
lower: float | Literal["estimate"],
|
|
1958
|
+
dependence: str | None = None,
|
|
1959
|
+
) -> Fit:
|
|
1960
|
+
"""Fit a binomial dose-response curve to ``obs``.
|
|
1961
|
+
|
|
1962
|
+
Two fitting paths: statsmodels ``GLM`` when both ``upper`` and ``lower`` are fixed at exactly
|
|
1963
|
+
``1.0``/``0.0``; ``statsmodels`` ``GenericLikelihoodModel`` -- given a likelihood, never a
|
|
1964
|
+
hand-rolled optimizer or IRLS loop -- for every other combination of fixed and estimated
|
|
1965
|
+
asymptotes. Both paths are checked for separation before fitting and for a non-interior maximum
|
|
1966
|
+
after fitting; either gives an explicit :class:`~robustmargin.Status` with no numbers, never a
|
|
1967
|
+
value that merely happens to be where the optimizer stopped. One case is not yet detected: a
|
|
1968
|
+
near-step curve (``s -> 0``) with an estimated asymptote, or a fixed one other than
|
|
1969
|
+
``1``/``0``, can return ``Status.OK`` with Wald standard errors that carry little
|
|
1970
|
+
information. With such asymptotes the overlap check is necessary but not sufficient.
|
|
1971
|
+
|
|
1972
|
+
Before any of that, :func:`robustmargin.check_monotonicity` is run on ``obs`` directly, and
|
|
1973
|
+
its result is attached to the returned ``Fit`` as ``monotonicity`` on *every* return path
|
|
1974
|
+
below, whatever the resulting ``status`` -- the check needs only raw counts, so it is not
|
|
1975
|
+
hidden behind a fit that failed. ``diagnostics`` is filled (via :func:`robustmargin.diagnose`)
|
|
1976
|
+
only when the fit reaches :data:`~robustmargin.Status.OK`.
|
|
1977
|
+
|
|
1978
|
+
Parameters
|
|
1979
|
+
----------
|
|
1980
|
+
obs
|
|
1981
|
+
The severity/outcome data to fit. Aggregated and per-observation ``Observations`` that
|
|
1982
|
+
pool to the same cells give numerically identical fits: everything below is a
|
|
1983
|
+
deterministic function of ``obs.cells()``, not of row order or representation.
|
|
1984
|
+
model
|
|
1985
|
+
Only ``"binomial"`` exists in this release.
|
|
1986
|
+
link
|
|
1987
|
+
``"probit"``, ``"logit"`` or ``"cloglog"``. Fixed by the caller before the data is
|
|
1988
|
+
seen: this function never compares links and picks one for you.
|
|
1989
|
+
upper, lower
|
|
1990
|
+
Each either a fixed probability in ``[0, 1]``, or the literal string ``"estimate"``.
|
|
1991
|
+
Required, with no default (a wrong default would silently change what curve is fit).
|
|
1992
|
+
dependence
|
|
1993
|
+
``None`` (the default) resolves to ``"independent"`` unless ``obs.cluster`` is set, in
|
|
1994
|
+
which case omitting it raises -- forwarded to :func:`robustmargin.check_monotonicity`
|
|
1995
|
+
as-is, so there is only one implementation of this guard. ``"independent"`` may always
|
|
1996
|
+
be passed explicitly.
|
|
1997
|
+
|
|
1998
|
+
Returns
|
|
1999
|
+
-------
|
|
2000
|
+
Fit
|
|
2001
|
+
``status`` is :data:`~robustmargin.Status.OK` with all four natural-scale parameters and
|
|
2002
|
+
a covariance over the estimated ones, or one of :data:`~robustmargin.Status.SEPARATION`,
|
|
2003
|
+
:data:`~robustmargin.Status.NOT_CONVERGED` or
|
|
2004
|
+
:data:`~robustmargin.Status.CONTROL_INCOMPATIBLE`
|
|
2005
|
+
with no numbers and a reason in ``warnings``. ``cluster_ids`` mirrors ``obs.cluster``
|
|
2006
|
+
whatever the outcome. The dependence guard is enforced here too, at fit time (via the
|
|
2007
|
+
``check_monotonicity`` call above) -- :class:`~robustmargin.Ratio` and
|
|
2008
|
+
:func:`robustmargin.threshold` each still enforce it again for their own clustered-fit
|
|
2009
|
+
inputs. ``monotonicity`` is never ``None`` on a freshly built ``Fit``; ``diagnostics`` is
|
|
2010
|
+
``None`` unless ``status`` is ``OK``.
|
|
2011
|
+
|
|
2012
|
+
Raises
|
|
2013
|
+
------
|
|
2014
|
+
ValueError
|
|
2015
|
+
If ``model``/``link`` is not one of the values above, if ``obs.direction`` is not
|
|
2016
|
+
``"decreasing"`` (an increasing-is-worse curve is not yet supported; ``Fit``
|
|
2017
|
+
itself rejects it), if ``upper``/``lower`` is not a valid probability or ``"estimate"``,
|
|
2018
|
+
if both are fixed with ``upper <= lower``, or if ``obs.cluster`` is set and
|
|
2019
|
+
``dependence`` is not ``"independent"`` (raised by :func:`robustmargin.check_monotonicity`).
|
|
2020
|
+
"""
|
|
2021
|
+
if model != "binomial":
|
|
2022
|
+
raise ValueError(f"fit_dose_response: model must be 'binomial', got {model!r}")
|
|
2023
|
+
if link not in _STATSMODELS_LINKS:
|
|
2024
|
+
raise ValueError(
|
|
2025
|
+
f"fit_dose_response: link must be one of {sorted(_STATSMODELS_LINKS)}, got {link!r}"
|
|
2026
|
+
)
|
|
2027
|
+
if obs.direction != "decreasing":
|
|
2028
|
+
raise ValueError(
|
|
2029
|
+
"fit_dose_response: direction='increasing' is not supported in this release "
|
|
2030
|
+
"(model='binomial' only supports 'decreasing'; an increasing-is-worse definition "
|
|
2031
|
+
"is planned for v0.2)"
|
|
2032
|
+
)
|
|
2033
|
+
|
|
2034
|
+
# Computed from the raw Observations before a fitting path is even chosen, and attached to
|
|
2035
|
+
# every return path below. This call is the dependence guard too -- there is no separate
|
|
2036
|
+
# check here, so there is only one implementation.
|
|
2037
|
+
monotonicity = check_monotonicity(obs, dependence=dependence)
|
|
2038
|
+
|
|
2039
|
+
upper_is_estimate, upper_value = _resolve_asymptote(upper, label="upper")
|
|
2040
|
+
lower_is_estimate, lower_value = _resolve_asymptote(lower, label="lower")
|
|
2041
|
+
if not upper_is_estimate and not lower_is_estimate and not upper_value > lower_value:
|
|
2042
|
+
raise ValueError(
|
|
2043
|
+
f"fit_dose_response: upper ({upper_value!r}) must exceed lower ({lower_value!r})"
|
|
2044
|
+
)
|
|
2045
|
+
|
|
2046
|
+
cells = obs.cells()
|
|
2047
|
+
cluster_ids = _unique_cluster_ids(obs.cluster)
|
|
2048
|
+
|
|
2049
|
+
control_incompatible_warning = _check_control_incompatible(
|
|
2050
|
+
cells, axis=obs.axis, upper_is_estimate=upper_is_estimate, upper_value=upper_value
|
|
2051
|
+
)
|
|
2052
|
+
if control_incompatible_warning is not None:
|
|
2053
|
+
return _finalize(
|
|
2054
|
+
_status_only_fit(
|
|
2055
|
+
obs,
|
|
2056
|
+
cells=cells,
|
|
2057
|
+
cluster_ids=cluster_ids,
|
|
2058
|
+
link=link,
|
|
2059
|
+
status=Status.CONTROL_INCOMPATIBLE,
|
|
2060
|
+
fit_warnings=(control_incompatible_warning,),
|
|
2061
|
+
),
|
|
2062
|
+
monotonicity=monotonicity,
|
|
2063
|
+
)
|
|
2064
|
+
|
|
2065
|
+
# A log-axis observation whose only severity level is the zero-severity control (the "no
|
|
2066
|
+
# covariate value at all" case, with nothing *else* in the data either) has no
|
|
2067
|
+
# likelihood-contributing level at all. Left unchecked, this
|
|
2068
|
+
# reaches `_centering_and_scale`/`_transformed_span` (both fitting paths) with an empty
|
|
2069
|
+
# nonzero-severity array and fails with numpy's own unhelpful "zero-size array to reduction
|
|
2070
|
+
# operation maximum which has no identity" rather than a message this package wrote itself.
|
|
2071
|
+
# Checked once, here, before either fitting path is chosen -- after
|
|
2072
|
+
# `_check_control_incompatible` so a control with failures still reports
|
|
2073
|
+
# CONTROL_INCOMPATIBLE, and before `_check_separation` so a control plus exactly one other
|
|
2074
|
+
# level still reaches that check and reports SEPARATION as before.
|
|
2075
|
+
if obs.axis.scale == "log" and not any(c.severity != 0.0 for c in cells):
|
|
2076
|
+
raise ValueError(
|
|
2077
|
+
"fit_dose_response: this observation has no severity level besides the "
|
|
2078
|
+
"zero-severity control; a dose-response curve needs at least one severity level "
|
|
2079
|
+
"besides the control"
|
|
2080
|
+
)
|
|
2081
|
+
|
|
2082
|
+
separation_warning = _check_separation(cells, axis=obs.axis)
|
|
2083
|
+
if separation_warning is not None:
|
|
2084
|
+
return _finalize(
|
|
2085
|
+
_status_only_fit(
|
|
2086
|
+
obs,
|
|
2087
|
+
cells=cells,
|
|
2088
|
+
cluster_ids=cluster_ids,
|
|
2089
|
+
link=link,
|
|
2090
|
+
status=Status.SEPARATION,
|
|
2091
|
+
fit_warnings=(separation_warning,),
|
|
2092
|
+
),
|
|
2093
|
+
monotonicity=monotonicity,
|
|
2094
|
+
)
|
|
2095
|
+
|
|
2096
|
+
if (
|
|
2097
|
+
not upper_is_estimate
|
|
2098
|
+
and not lower_is_estimate
|
|
2099
|
+
and upper_value == 1.0
|
|
2100
|
+
and lower_value == 0.0
|
|
2101
|
+
):
|
|
2102
|
+
return _finalize(
|
|
2103
|
+
_fit_glm(obs, cells=cells, cluster_ids=cluster_ids, link=link),
|
|
2104
|
+
monotonicity=monotonicity,
|
|
2105
|
+
)
|
|
2106
|
+
|
|
2107
|
+
return _finalize(
|
|
2108
|
+
_fit_generic(
|
|
2109
|
+
obs,
|
|
2110
|
+
cells=cells,
|
|
2111
|
+
cluster_ids=cluster_ids,
|
|
2112
|
+
link=link,
|
|
2113
|
+
upper_is_estimate=upper_is_estimate,
|
|
2114
|
+
upper_value=upper_value,
|
|
2115
|
+
lower_is_estimate=lower_is_estimate,
|
|
2116
|
+
lower_value=lower_value,
|
|
2117
|
+
),
|
|
2118
|
+
monotonicity=monotonicity,
|
|
2119
|
+
)
|