robustmargin 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
robustmargin/models.py ADDED
@@ -0,0 +1,2119 @@
1
+ """The dose-response fit result, its fitting entry point, and prediction.
2
+
3
+ This module holds :class:`Fit`, the two small dataclasses it embeds (:class:`Parameter`,
4
+ :class:`Covariance`), :func:`fit_dose_response`, and :class:`Prediction` (:meth:`Fit.predict`'s
5
+ return type). Two fitting paths exist: statsmodels ``GLM`` when both asymptotes are fixed at
6
+ exactly ``1.0``/``0.0``, and ``statsmodels.base.model.GenericLikelihoodModel`` -- given a
7
+ likelihood function, never a hand-rolled optimizer or IRLS loop -- otherwise. A fit with no
8
+ interior maximum (wrong-sign slope, or an estimated asymptote pinned to a parameter-space
9
+ boundary) is reported as ``Status.NOT_CONVERGED`` with the reason in ``Fit.warnings``, never as
10
+ a number that happens to be where the optimizer stopped.
11
+
12
+ **Known limitation.** A near-step curve (``s -> 0``) with an estimated asymptote, or a fixed
13
+ one other than ``1``/``0``, is not detected: it can return ``Status.OK`` with Wald standard
14
+ errors that carry little information. With such asymptotes the pre-fit overlap check is
15
+ necessary but not sufficient for an interior maximum; it is exact only for ``u = 1, l = 0``.
16
+
17
+ **v1 is binomial-only.** ``model`` accepts only ``"binomial"``; a new model family (``"ll4"``,
18
+ ``"isotonic"``) is a v0.2 addition that requires a new ``schema_version``, not a silent
19
+ extension of the current one. :data:`robustmargin.report.SCHEMA_VERSION` is the one version
20
+ :func:`robustmargin.report.from_dict` reads and the single source of that number.
21
+
22
+ :class:`Fit` also carries :attr:`~Fit.diagnostics` (model-only goodness of fit, dispersion and
23
+ link comparison, filled only when ``status`` is ``OK`` -- see :func:`robustmargin.diagnose`) and
24
+ :attr:`~Fit.monotonicity` (a data-only adjacent-level check, filled on *every* return path
25
+ regardless of ``status`` -- see :func:`robustmargin.check_monotonicity` in
26
+ :mod:`robustmargin.validation`). Both are reported and never acted on: neither changes ``link``,
27
+ ``params``, or any other field of the ``Fit`` they are attached to.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import math
33
+ import warnings
34
+ from collections.abc import Callable, Mapping
35
+ from dataclasses import dataclass, field, replace
36
+ from typing import Literal
37
+
38
+ import numpy as np
39
+ from numpy.typing import ArrayLike, NDArray
40
+ from scipy.special import expit, gammaln, xlogy
41
+ from scipy.special import logit as _logit
42
+ from scipy.stats import norm
43
+ from statsmodels.base.model import GenericLikelihoodModel
44
+ from statsmodels.genmod import families
45
+ from statsmodels.genmod.generalized_linear_model import GLM
46
+ from statsmodels.tools.sm_exceptions import ConvergenceWarning, PerfectSeparationWarning
47
+ from statsmodels.tools.tools import add_constant
48
+
49
+ from robustmargin.types import Axis, Cell, JSONValue, Observations, Status, _check_finite_number
50
+ from robustmargin.validation import Diagnostics, MonotonicityCheck, check_monotonicity, diagnose
51
+
52
+ # `robustmargin.validation` only imports `Fit` from this module under `TYPE_CHECKING` (see that
53
+ # module's own docstring), so this is not a circular import at runtime: by the time
54
+ # `fit_dose_response` below actually calls `diagnose`/`check_monotonicity`, this module has
55
+ # already finished executing this very import statement.
56
+
57
+ # Narrow statsmodels imports above, never `statsmodels.api`: that module also imports graphics and
58
+ # tsa, and statsmodels 0.14.0's tsa fails to import against current pandas
59
+ # (`deprecate_kwarg() missing 'new_arg_name'`), which breaks the dependency-floor CI job.
60
+
61
+ __all__ = ["Covariance", "Fit", "Parameter", "Prediction", "fit_dose_response"]
62
+
63
+ _VALID_MODELS = frozenset({"binomial"})
64
+ _VALID_LINKS: dict[str, frozenset[str]] = {
65
+ "binomial": frozenset({"probit", "logit", "cloglog"}),
66
+ }
67
+ _BINOMIAL_PARAM_NAMES = frozenset({"mu", "s", "upper", "lower"})
68
+ _VALID_DIRECTIONS = frozenset({"decreasing", "increasing"})
69
+ _INVALID_FIT_STATUSES = frozenset({Status.UNREACHABLE, Status.FAILS_AT_BASELINE})
70
+
71
+ # Tolerance for Covariance's positive-definiteness check: the minimum eigenvalue of the
72
+ # correlation matrix ``D^-1/2 . matrix . D^-1/2`` must exceed this, unscaled, so the test does
73
+ # not depend on the severity units. Strictly positive (not merely semi-definite): a singular
74
+ # matrix, such as ``((0.0, 0.0), (0.0, 1.0))``, is rejected, because a singular
75
+ # covariance means at least one estimated parameter carries no information at all.
76
+ _POSITIVE_DEFINITE_RELATIVE_TOLERANCE = 1e-12
77
+
78
+ # ------------------------------------------------------------------------------------------
79
+ # fit_dose_response: module constants (the numeric boundary test for an estimated asymptote)
80
+ # ------------------------------------------------------------------------------------------
81
+
82
+ _VALID_UPPER_LOWER = "estimate"
83
+ _STATSMODELS_LINKS = {
84
+ "probit": families.links.Probit(),
85
+ "logit": families.links.Logit(),
86
+ "cloglog": families.links.CLogLog(),
87
+ }
88
+ # An estimated asymptote's logit-scale parameter (a for lower, b for upper) beyond this
89
+ # magnitude, or the asymptote itself within this distance of a probability boundary (0 or 1),
90
+ # means the likelihood has no interior maximum -- reported as Status.NOT_CONVERGED with a
91
+ # boundary warning, never as a number. Changing either cut-off is a statistical-method change.
92
+ _ASYMPTOTE_LOGIT_BOUND = 15.0
93
+ _ASYMPTOTE_PROBABILITY_EPS = 1e-6
94
+ # Separation is decided by the exact pre-fit overlap check (`_check_separation`), never by
95
+ # statsmodels' PerfectSeparationWarning. That warning tests `allclose(fitted, observed)`, which on
96
+ # grouped binomial data also fires on an exact fit -- every two-level design with overlap -- so it
97
+ # is filtered to "ignore" around both GLM calls below. Every fit reaching those calls has already
98
+ # passed the exact check. The check is exact for fixed u = 1, l = 0 (Albert & Anderson, 1984); with
99
+ # estimated or other fixed asymptotes it is necessary but not sufficient, and a near-step curve
100
+ # there is not yet detected (see the module docstring).
101
+
102
+ # A fitted `s` greater than this factor times the span of the likelihood-contributing transformed
103
+ # levels (the tested range on `log x` or `x`, excluding a log-axis zero-severity control, which
104
+ # carries no covariate value) is flat up to rounding -- the wrong-sign case's degenerate limit,
105
+ # `s -> infinity` -- and gives Status.NOT_CONVERGED rather than a covariance built from a near-zero
106
+ # slope (which would otherwise pass Covariance's own positive-definiteness check while being
107
+ # numerically meaningless, for example ~1e61). Changing this cut-off is a statistical-method change.
108
+ _FLAT_SLOPE_SPAN_FACTOR = 1e8
109
+
110
+ # Likelihood-path convergence: Newton's own `converged` flag only bounds the last step taken, so
111
+ # two more checks apply at the final parameters. The Newton polish's log-likelihood must not
112
+ # have regressed below the BFGS stage's, within this *relative* tolerance (an absolute tolerance
113
+ # does not scale with the log-likelihood's own magnitude, which grows with sample size and is
114
+ # not something a caller controls), applied as
115
+ # `bfgs_llf - _LLF_REGRESSION_TOLERANCE * (1 + |bfgs_llf|)` -- floating-point noise between two
116
+ # different optimizers evaluating the same likelihood at very close points, scaled the same way
117
+ # `_SCORE_TOLERANCE_FACTOR` below is.
118
+ _LLF_REGRESSION_TOLERANCE = 1e-8
119
+ # The score (gradient of the log-likelihood) at the final parameters must be near zero. There is
120
+ # no universal absolute tolerance for a gradient, because its scale depends on the
121
+ # log-likelihood's own scale (more data means both the log-likelihood and its gradient are
122
+ # larger in magnitude even at the true optimum); `1 + |llf|` is a defensible, simple proxy for
123
+ # that scale that never reaches zero, so the tolerance below is always well-defined.
124
+ _SCORE_TOLERANCE_FACTOR = 1e-4
125
+
126
+ # Shared wording for a wrong-sign slope (the GLM path's own beta1 <= 0 check) and for the flat
127
+ # curve it degenerates into (`_FLAT_SLOPE_SPAN_FACTOR` above) -- the same underlying phenomenon
128
+ # (no finite (mu, s) exists once the constrained maximum is pushed to the s -> infinity
129
+ # boundary), so one reason is used for both, on both fitting paths.
130
+ _WRONG_SIGN_OR_FLAT_WARNING = (
131
+ "fit_dose_response: the fitted curve has no interior maximum for direction='decreasing' "
132
+ "(a wrong-sign slope, or a curve that is flat up to rounding) -- no finite (mu, s) exists"
133
+ )
134
+
135
+
136
+ @dataclass(frozen=True, kw_only=True)
137
+ class Parameter:
138
+ """One fitted or fixed parameter value, on the **natural scale** -- the curve's own units
139
+ (for example ``s`` in log-severity units, ``upper``/``lower`` as probabilities), never the
140
+ optimiser's internal reparametrisation (``log s``, or ``upper``/``lower`` on a logit scale).
141
+ An estimated-asymptote fit delta-transforms its optimiser-scale covariance onto this scale
142
+ before it is ever stored here (see :class:`Fit`).
143
+
144
+ Attributes
145
+ ----------
146
+ value
147
+ The parameter's value on its natural scale. Must be finite, and must be a genuine
148
+ ``float`` or ``int``, never a ``bool``.
149
+ fixed
150
+ ``True`` if this parameter was held fixed rather than estimated (for example
151
+ ``upper=1.0`` fixed in a two-parameter probit). Must be a genuine ``bool``.
152
+ """
153
+
154
+ value: float
155
+ fixed: bool
156
+
157
+ def __post_init__(self) -> None:
158
+ _check_finite_number(self.value, label="Parameter.value")
159
+ if not isinstance(self.fixed, bool):
160
+ raise ValueError(f"Parameter.fixed must be a real bool, got {self.fixed!r}")
161
+
162
+
163
+ @dataclass(frozen=True, kw_only=True)
164
+ class Covariance:
165
+ """The variance-covariance matrix of a fit's estimated (non-fixed) parameters, on the same
166
+ natural scale as :class:`Parameter`.
167
+
168
+ Attributes
169
+ ----------
170
+ names
171
+ The estimated parameter names, in the order ``matrix`` is indexed by. Unique.
172
+ matrix
173
+ A square, finite, symmetric, **positive-definite** matrix with side ``len(names)``.
174
+ Every entry must be a genuine finite number, never a ``bool``. Symmetry is checked with
175
+ :func:`math.isclose` (``rel_tol=1e-9``, ``abs_tol=1e-12``), not exact equality, to
176
+ tolerate floating-point round-trip noise. Positive-definiteness is checked on the
177
+ **correlation matrix** ``D^-1/2 . matrix . D^-1/2`` (``D = diag(matrix)``): every
178
+ variance must be positive, and the correlation matrix's minimum eigenvalue (via
179
+ :func:`numpy.linalg.eigvalsh`) must exceed ``1e-12``. The rule is therefore invariant to
180
+ rescaling any single parameter, which matters because the natural scale mixes severity
181
+ units (``mu``, ``s``) with probabilities (``upper``, ``lower``). It accepts every matrix
182
+ that a raw-eigenvalue rule (minimum eigenvalue ``> 1e-12 * max(1, max|entry|)``) would
183
+ accept. A singular matrix -- for example ``((0.0, 0.0), (0.0, 1.0))``, or any matrix with a
184
+ zero eigenvalue -- is rejected, not merely a matrix with a negative one.
185
+ """
186
+
187
+ names: tuple[str, ...]
188
+ matrix: tuple[tuple[float, ...], ...]
189
+
190
+ def __post_init__(self) -> None:
191
+ object.__setattr__(self, "names", tuple(self.names))
192
+ object.__setattr__(self, "matrix", tuple(tuple(row) for row in self.matrix))
193
+
194
+ if len(set(self.names)) != len(self.names):
195
+ raise ValueError(f"Covariance.names must be unique, got {self.names!r}")
196
+
197
+ n = len(self.names)
198
+ if len(self.matrix) != n:
199
+ raise ValueError(
200
+ f"Covariance.matrix must have {n} rows to match names, got {len(self.matrix)}"
201
+ )
202
+ for row in self.matrix:
203
+ if len(row) != n:
204
+ raise ValueError(
205
+ f"Covariance.matrix must be square with side {n}, got a row of length "
206
+ f"{len(row)}"
207
+ )
208
+ for entry in row:
209
+ _check_finite_number(entry, label="Covariance.matrix entries")
210
+
211
+ for i in range(n):
212
+ for j in range(n):
213
+ if not math.isclose(
214
+ self.matrix[i][j], self.matrix[j][i], rel_tol=1e-9, abs_tol=1e-12
215
+ ):
216
+ raise ValueError(
217
+ "Covariance.matrix must be symmetric: "
218
+ f"matrix[{i}][{j}]={self.matrix[i][j]!r} != "
219
+ f"matrix[{j}][{i}]={self.matrix[j][i]!r}"
220
+ )
221
+
222
+ if n > 0 and not _is_positive_definite(np.array(self.matrix, dtype=float)):
223
+ raise ValueError(
224
+ "Covariance.matrix must be positive-definite: every variance must be positive "
225
+ "and the correlation matrix's minimum eigenvalue must exceed "
226
+ f"{_POSITIVE_DEFINITE_RELATIVE_TOLERANCE!r}"
227
+ )
228
+
229
+
230
+ def _check_cluster_ids(cluster_ids: tuple[str | int, ...] | None, *, label: str) -> None:
231
+ if cluster_ids is None:
232
+ return
233
+ if len(cluster_ids) == 0:
234
+ raise ValueError(f"{label} must be non-empty when given, got an empty tuple")
235
+ if len(set(cluster_ids)) != len(cluster_ids):
236
+ raise ValueError(f"{label} must be unique, got {cluster_ids!r}")
237
+ for cluster_id in cluster_ids:
238
+ if cluster_id is None or isinstance(cluster_id, bool):
239
+ raise ValueError(
240
+ f"{label} elements must be a str or int, never None or bool, got {cluster_id!r}"
241
+ )
242
+ if not isinstance(cluster_id, (str, int)):
243
+ raise ValueError(f"{label} elements must be a str or int, got {cluster_id!r}")
244
+
245
+
246
+ @dataclass(frozen=True, kw_only=True, eq=False)
247
+ class Prediction:
248
+ """The curve and a confidence band evaluated at a set of severities (:meth:`Fit.predict`).
249
+ Never serialised (``report.py`` does not know this type): it is a throwaway array bundle,
250
+ not a stored result.
251
+
252
+ Two band scales are used, chosen by :meth:`Fit.predict` from which asymptotes were
253
+ estimated -- documented in full on :meth:`Fit.predict` itself, since the choice depends on
254
+ the ``Fit`` being predicted from, not on this type.
255
+
256
+ ``eq``/``hash`` are disabled (unlike every other result dataclass in this package): the
257
+ default dataclass equality would compare ``numpy`` arrays element-by-element and then try to
258
+ coerce that into a single ``bool``, which raises. Two ``Prediction`` instances are therefore
259
+ never equal unless they are the same object (Python's default identity-based ``__eq__``,
260
+ from ``object``, applies instead) -- do not write ``fit.predict(x) == fit.predict(x)``
261
+ expecting a value comparison; compare the four arrays individually instead.
262
+
263
+ All four arrays are read-only (``.flags.writeable is False``): this is a frozen dataclass,
264
+ but freezing only stops an attribute from being *reassigned* (``prediction.lo = ...``), not
265
+ the ``numpy`` array it refers to from being mutated in place (``prediction.lo[:] = ...``),
266
+ so ``__post_init__`` marks each array non-writeable itself to close that gap.
267
+
268
+ A scalar ``severity`` argument to :meth:`Fit.predict` (for example ``fit.predict(0.5)``,
269
+ rather than a list or array) gives 0-d arrays here (``severity.shape == ()``), following
270
+ :func:`numpy.asarray`'s own behaviour for a scalar input; it is never promoted to shape
271
+ ``(1,)``.
272
+
273
+ Attributes
274
+ ----------
275
+ severity
276
+ The severities this prediction was evaluated at, as given (as ``float64``).
277
+ estimate
278
+ The fitted curve's value at each ``severity``, same shape as ``severity``.
279
+ lo
280
+ The band's lower side at each ``severity``. ``lo <= estimate`` everywhere.
281
+ hi
282
+ The band's upper side at each ``severity``. ``estimate <= hi`` everywhere.
283
+ level
284
+ The two-sided confidence level the band was built at, strictly between 0 and 1.
285
+ """
286
+
287
+ severity: NDArray[np.float64]
288
+ estimate: NDArray[np.float64]
289
+ lo: NDArray[np.float64]
290
+ hi: NDArray[np.float64]
291
+ level: float
292
+
293
+ def __post_init__(self) -> None:
294
+ for array in (self.severity, self.estimate, self.lo, self.hi):
295
+ array.flags.writeable = False
296
+
297
+
298
+ @dataclass(frozen=True, kw_only=True)
299
+ class Fit:
300
+ """A dose-response fit, or an explicit record of why one was not obtained.
301
+
302
+ The fitted curve is ``P(x) = u - (u - l) * F(z)``, where ``z = (log(x) - mu) / s`` on
303
+ a log axis or ``z = (x - mu) / s`` on a linear axis, ``s > 0``, and ``F`` is the link's CDF
304
+ (the standard normal CDF for ``probit``, the logistic CDF for ``logit``, or
305
+ ``1 - exp(-exp(z))`` for ``cloglog``, which models the probability of *failure* rising with
306
+ severity and so is not symmetric the way ``probit``/``logit`` are). ``mu`` is on the
307
+ transformed axis (``log x`` on a log axis, ``x`` on a linear axis), not on the original
308
+ severity scale. At ``z = 0`` -- severity ``exp(mu)`` on a log axis, or severity ``mu`` on a
309
+ linear axis -- ``probit`` and ``logit`` both give ``P = (u + l) / 2``: the midpoint of the
310
+ fitted dynamic range, :meth:`~robustmargin.Definition.relative` ``(0.5)``, for **any**
311
+ asymptotes ``u``, ``l``, because the standard normal and logistic CDFs each equal ``0.5``
312
+ at ``z = 0``. That midpoint equals :meth:`~robustmargin.Definition.absolute` ``(0.5)`` and
313
+ :meth:`~robustmargin.Definition.baseline_fraction` ``(0.5)`` only when ``u = 1`` and
314
+ ``l = 0``. Under ``cloglog``, which is not symmetric around ``z = 0``,
315
+ ``F(0) = 1 - exp(-1)`` (about ``0.632``), so ``mu`` is never the midpoint of the dynamic
316
+ range, for any asymptotes. Each entry in ``params`` records whether it was held fixed or
317
+ estimated, and is on the **natural scale**, never the optimiser's internal
318
+ reparametrisation (``log s``; ``upper``/``lower`` on a logit scale) -- see
319
+ :class:`Parameter`. ``covariance`` is on that same natural scale: when asymptotes are
320
+ estimated, the optimiser's own covariance (which is on its reparametrised scale) is
321
+ delta-transformed onto the natural scale before being stored here.
322
+
323
+ ``status == Status.OK`` if and only if both ``params`` and ``log_likelihood`` are present;
324
+ when ``status`` is anything else, ``params``, ``covariance`` and ``log_likelihood`` are all
325
+ ``None`` -- no number is ever produced for a failed fit. ``Status.UNREACHABLE`` and
326
+ ``Status.FAILS_AT_BASELINE`` describe a *threshold*, not a fit (they depend on a ``Definition``
327
+ a ``Fit`` does not have), and are never a ``Fit.status``.
328
+
329
+ Binomial curves are decreasing-only in this release: ``direction="increasing"`` raises.
330
+ The curve ``P = u - (u - l) * F(z)`` with ``s > 0`` is decreasing by construction, and
331
+ increasing-is-worse outcomes are not yet supported. ``Observations`` itself still accepts
332
+ both directions.
333
+
334
+ ``diagnostics`` (goodness of fit, dispersion, link comparison) and ``monotonicity`` (the
335
+ adjacent-level monotonicity check) are two fields that both default to ``None``. The schema's
336
+ ``Fit`` object has ``additionalProperties: false``, so a card whose ``Fit`` carries a key the
337
+ schema does not define is rejected, including a key with a ``null`` value. Adding a field to
338
+ ``Fit`` is therefore a ``schema_version`` change.
339
+
340
+ Attributes
341
+ ----------
342
+ axis
343
+ The :class:`~robustmargin.Axis` the fit was made on.
344
+ outcome
345
+ The outcome name this fit was made against, carried over from the
346
+ :class:`~robustmargin.Observations` it was fit to.
347
+ direction
348
+ ``"decreasing"`` only, in this release (see above).
349
+ model
350
+ The model family. Only ``"binomial"`` exists in v0.1; a new family such as ``"ll4"`` or
351
+ ``"isotonic"`` is a v0.2 addition requiring a further ``schema_version`` bump beyond the
352
+ current one (:data:`robustmargin.report.SCHEMA_VERSION`).
353
+ link
354
+ The link function: ``"probit"``, ``"logit"`` or ``"cloglog"`` for ``model="binomial"``.
355
+ Required; the schema's ``link`` enum carries no ``null`` alternative, so this is a
356
+ genuine ``str``, never ``None``.
357
+ params
358
+ A mapping with exactly the keys ``{"mu", "s", "upper", "lower"}`` when ``status`` is
359
+ ``OK``, else ``None``. ``s.value`` must be positive; ``lower.value`` and
360
+ ``upper.value`` must each lie in ``[0, 1]`` with ``lower.value < upper.value``.
361
+ ``params["mu"].fixed`` and ``params["s"].fixed`` must both be ``False``: this package
362
+ never produces (and this dataclass never accepts) an ``OK`` fit with a fixed location
363
+ or scale.
364
+ covariance
365
+ The estimated parameters' covariance. Required (not ``None``) when ``status`` is
366
+ ``OK``; ``None`` otherwise. If present, ``covariance.names`` is exactly the set of
367
+ non-fixed parameter names.
368
+ log_likelihood
369
+ The fitted log-likelihood, or ``None`` when ``status`` is not ``OK``. Finite when
370
+ present.
371
+ cells
372
+ The per-severity-level pooled counts the fit was made from
373
+ (:meth:`~robustmargin.Observations.cells`). Never empty, and in strictly increasing
374
+ order of severity (one cell per tested level), which is checked on construction and
375
+ never repaired by sorting or pooling.
376
+ status
377
+ The :class:`~robustmargin.Status` of this fit. Never ``UNREACHABLE`` or
378
+ ``FAILS_AT_BASELINE`` (those describe a threshold, not a fit).
379
+ cluster_ids
380
+ Optional cluster ids the fitted data carries, one instance per distinct cluster (not
381
+ one per row): unique, non-empty when given, and never ``None`` or ``bool`` elements.
382
+ Set whenever the input :class:`~robustmargin.Observations` carried cluster ids, whatever
383
+ dependence was assumed when this fit was produced -- this field reflects the data's
384
+ structure, not a modelling choice. ``None`` means only that the data had no clusters,
385
+ not that independence was assumed. Used by :class:`~robustmargin.Ratio`'s dependence
386
+ guard.
387
+ warnings
388
+ Free-text notes surfaced alongside the fit (for example, ``CONTROL_INCOMPATIBLE``
389
+ carries a message suggesting ``upper="estimate"``). Empty by default.
390
+ schema_version
391
+ The serialised-result schema version this object belongs to. Defaults to ``"1"``, the
392
+ only version ``from_dict`` reads; any other version is rejected.
393
+ provenance
394
+ An opaque mapping the caller may attach to record where the inputs came from.
395
+ robustmargin stores it and never interprets it. Empty by default.
396
+ diagnostics
397
+ Model-only goodness-of-fit, dispersion and link-comparison diagnostics
398
+ (:func:`robustmargin.diagnose`). Set exactly when ``status`` is
399
+ :data:`~robustmargin.Status.OK` (no model-derived numbers on a failed fit); ``None``
400
+ otherwise.
401
+ monotonicity
402
+ A data-only adjacent-severity-level monotonicity check
403
+ (:func:`robustmargin.check_monotonicity`), computed from the raw
404
+ :class:`~robustmargin.Observations` before a fitting path was even chosen. Unlike
405
+ ``diagnostics``, this is set on *every* fresh result regardless of ``status`` -- it needs
406
+ only counts, never a converged fit. A serialised card may omit it (the schema does not
407
+ require it), in which case it reads back as ``None``. The last field, after
408
+ ``diagnostics``.
409
+ """
410
+
411
+ axis: Axis
412
+ outcome: str
413
+ direction: str
414
+ model: str
415
+ link: str
416
+ params: Mapping[str, Parameter] | None
417
+ covariance: Covariance | None
418
+ log_likelihood: float | None
419
+ cells: tuple[Cell, ...]
420
+ status: Status
421
+ cluster_ids: tuple[str | int, ...] | None = None
422
+ warnings: tuple[str, ...] = ()
423
+ schema_version: str = "1"
424
+ provenance: Mapping[str, JSONValue] = field(default_factory=dict)
425
+ diagnostics: Diagnostics | None = None
426
+ monotonicity: MonotonicityCheck | None = None
427
+
428
+ def __post_init__(self) -> None:
429
+ object.__setattr__(self, "warnings", tuple(self.warnings))
430
+ object.__setattr__(self, "cells", tuple(self.cells))
431
+ if self.cluster_ids is not None:
432
+ object.__setattr__(self, "cluster_ids", tuple(self.cluster_ids))
433
+
434
+ if not self.outcome:
435
+ raise ValueError("Fit.outcome must be non-empty")
436
+ if self.direction not in _VALID_DIRECTIONS:
437
+ raise ValueError(
438
+ f"Fit.direction must be one of {sorted(_VALID_DIRECTIONS)}, got {self.direction!r}"
439
+ )
440
+
441
+ if self.model not in _VALID_MODELS:
442
+ raise ValueError(
443
+ f"Fit.model must be one of {sorted(_VALID_MODELS)}, got {self.model!r}"
444
+ )
445
+ if self.model == "binomial" and self.direction != "decreasing":
446
+ raise ValueError(
447
+ "Fit: model='binomial' only supports direction='decreasing' in this release; "
448
+ "an increasing-is-worse definition is planned for v0.2"
449
+ )
450
+ allowed_links = _VALID_LINKS[self.model]
451
+ if self.link not in allowed_links:
452
+ raise ValueError(
453
+ f"Fit.link for model={self.model!r} must be one of {sorted(allowed_links)}, "
454
+ f"got {self.link!r}"
455
+ )
456
+
457
+ if self.status in _INVALID_FIT_STATUSES:
458
+ raise ValueError(
459
+ f"Fit.status must not be {self.status!r}: that status describes a threshold, "
460
+ "not a fit"
461
+ )
462
+
463
+ if not self.cells:
464
+ raise ValueError("Fit.cells must not be empty")
465
+ for index in range(1, len(self.cells)):
466
+ previous = self.cells[index - 1].severity
467
+ current = self.cells[index].severity
468
+ if not current > previous:
469
+ raise ValueError(
470
+ "Fit.cells must be in strictly increasing order of severity (one cell per "
471
+ f"tested level), but cells[{index}].severity={current!r} follows "
472
+ f"cells[{index - 1}].severity={previous!r}"
473
+ )
474
+
475
+ _check_cluster_ids(self.cluster_ids, label="Fit.cluster_ids")
476
+
477
+ if self.status is not Status.OK and self.diagnostics is not None:
478
+ raise ValueError(
479
+ "Fit.diagnostics must be None when status is not OK (no model-derived "
480
+ f"numbers on a failed fit), got status={self.status!r} with diagnostics set"
481
+ )
482
+
483
+ is_ok = self.status is Status.OK
484
+ has_params = self.params is not None
485
+ has_log_likelihood = self.log_likelihood is not None
486
+ if is_ok != (has_params and has_log_likelihood):
487
+ raise ValueError(
488
+ "Fit.status is OK if and only if both params and log_likelihood are present: "
489
+ f"status={self.status!r}, params is not None={has_params!r}, "
490
+ f"log_likelihood is not None={has_log_likelihood!r}"
491
+ )
492
+ if not is_ok:
493
+ if self.params is not None or self.covariance is not None:
494
+ raise ValueError(
495
+ "Fit.params and Fit.covariance must both be None when status is not OK"
496
+ )
497
+ return
498
+
499
+ if self.log_likelihood is not None:
500
+ _check_finite_number(self.log_likelihood, label="Fit.log_likelihood")
501
+
502
+ if self.covariance is None:
503
+ raise ValueError("Fit.covariance is required when status is OK")
504
+
505
+ params = self.params
506
+ assert params is not None # established by is_ok above
507
+ if self.model == "binomial":
508
+ if set(params) != _BINOMIAL_PARAM_NAMES:
509
+ raise ValueError(
510
+ f"Fit.params for model='binomial' must have exactly the keys "
511
+ f"{sorted(_BINOMIAL_PARAM_NAMES)}, got {sorted(params)}"
512
+ )
513
+ if params["mu"].fixed or params["s"].fixed:
514
+ raise ValueError(
515
+ "Fit.params['mu'].fixed and Fit.params['s'].fixed must both be False when "
516
+ "status is OK: this package never produces a fit with a fixed location or "
517
+ "scale"
518
+ )
519
+ s = params["s"].value
520
+ lower = params["lower"].value
521
+ upper = params["upper"].value
522
+ if not s > 0.0:
523
+ raise ValueError(f"Fit.params['s'].value must be positive, got {s!r}")
524
+ if not (0.0 <= lower < upper <= 1.0):
525
+ raise ValueError(
526
+ "Fit.params must satisfy 0 <= lower.value < upper.value <= 1, got "
527
+ f"lower={lower!r}, upper={upper!r}"
528
+ )
529
+
530
+ if self.axis.scale == "log" and upper == 1.0 and params["upper"].fixed:
531
+ for cell in self.cells:
532
+ if cell.severity == 0.0 and cell.successes < cell.trials:
533
+ raise ValueError(
534
+ "Fit: the zero-severity control has failures but 'upper' is fixed "
535
+ "at exactly 1.0, so the likelihood is zero for this fit on a log "
536
+ 'axis; refit with upper="estimate"'
537
+ )
538
+
539
+ non_fixed = {name for name, param in params.items() if not param.fixed}
540
+ if set(self.covariance.names) != non_fixed:
541
+ raise ValueError(
542
+ "Fit.covariance.names must equal the non-fixed parameter names: "
543
+ f"expected {sorted(non_fixed)}, got {sorted(self.covariance.names)}"
544
+ )
545
+
546
+ def predict(self, severity: ArrayLike, *, level: float = 0.95) -> Prediction:
547
+ """Evaluate the fitted curve and a delta-method confidence band at each of
548
+ ``severity``.
549
+
550
+ Two band scales are used, depending on which asymptotes this fit estimated:
551
+
552
+ - **Both asymptotes fixed:** the delta method is applied to the linear predictor ``z``
553
+ (using ``mu``'s and ``s``'s covariance), then mapped through
554
+ ``P = u - (u - l) * F(z)``. Because ``u`` and ``l`` are constants here, the band is
555
+ guaranteed to stay inside ``(l, u)`` -- not just ``(0, 1)`` -- for every severity.
556
+ - **Either asymptote estimated:** the delta method is applied to ``logit(P)``, using
557
+ the gradient of ``logit(P)`` with respect to every *estimated* natural parameter
558
+ (``mu``, ``s``, and whichever of ``lower``/``upper`` are not fixed), then
559
+ back-transformed with the logistic function. The band is guaranteed to stay inside
560
+ ``(0, 1)``, but not inside ``(l, u)`` specifically, since ``l``/``u`` themselves now
561
+ carry uncertainty.
562
+
563
+ On a log axis, severity ``0.0`` is always the zero-severity control: its estimate
564
+ is ``upper`` exactly, never computed through ``log(0)``. It is not a special case for
565
+ the band, either: it goes through the same computation as every other severity (so
566
+ ``predict(0)`` and ``predict(1e-12)`` agree), which happens to give a zero-width band
567
+ there whenever ``upper`` is fixed (perturbing any other parameter cannot move a
568
+ constant) and a band from ``Var(upper)`` -- correctly propagated through ``logit``, not
569
+ treated as if it were already on the logit scale -- when ``upper`` is estimated.
570
+
571
+ Parameters
572
+ ----------
573
+ severity
574
+ Severities to evaluate the curve at. Converted to ``float64``; every entry must be
575
+ finite and non-negative.
576
+ level
577
+ The two-sided confidence level for the band, strictly between 0 and 1 (not a
578
+ ``bool``). Defaults to ``0.95``.
579
+
580
+ Returns
581
+ -------
582
+ Prediction
583
+ ``severity``, ``estimate``, ``lo`` and ``hi`` are ``numpy`` arrays of the same
584
+ shape as the (converted) ``severity`` input; ``lo <= estimate <= hi`` everywhere.
585
+
586
+ Raises
587
+ ------
588
+ ValueError
589
+ If ``self.status`` is not :data:`~robustmargin.Status.OK` (a failed fit has no curve
590
+ to evaluate), if ``level`` is not finite, is a ``bool``, or
591
+ does not satisfy ``0 < level < 1``, or if ``severity`` is empty or contains a
592
+ negative or non-finite entry.
593
+ """
594
+ if self.status is not Status.OK:
595
+ raise ValueError(
596
+ f"Fit.predict requires status=OK, got {self.status!r}: a failed fit has no "
597
+ "curve to evaluate"
598
+ )
599
+ _check_finite_number(level, label="Fit.predict level")
600
+ if not (0.0 < level < 1.0):
601
+ raise ValueError(f"Fit.predict: level must satisfy 0 < level < 1, got {level!r}")
602
+
603
+ # A copy (`np.array`, not `np.asarray`): Prediction.severity is marked read-only below,
604
+ # and a caller who happened to pass an existing float64 array must never have that
605
+ # array silently frozen out from under them.
606
+ x = np.array(severity, dtype=np.float64)
607
+ if x.size == 0:
608
+ raise ValueError("Fit.predict: severity must not be empty")
609
+ if not bool(np.all(np.isfinite(x))):
610
+ raise ValueError("Fit.predict: severity must be finite (no NaN or inf)")
611
+ if bool(np.any(x < 0.0)):
612
+ raise ValueError("Fit.predict: severity must be non-negative")
613
+
614
+ assert self.params is not None and self.covariance is not None # status is OK
615
+ mu = self.params["mu"].value
616
+ s = self.params["s"].value
617
+ upper_param = self.params["upper"]
618
+ lower_param = self.params["lower"]
619
+ upper = upper_param.value
620
+ lower = lower_param.value
621
+ covariance = self.covariance
622
+
623
+ z_score = float(norm.ppf(0.5 + level / 2.0))
624
+
625
+ # x = 0 on a log axis is not special-cased here: both band functions below route it
626
+ # through the exact same logit-P gradient/delta computation as every other severity
627
+ # so `predict(0)` and `predict(1e-12)` agree, and a *fixed* `upper` still gets a
628
+ # zero-width band there for free (perturbing any other estimated parameter cannot move
629
+ # a constant), with no separate formula to keep in sync.
630
+ if upper_param.fixed and lower_param.fixed:
631
+ estimate, lo, hi = _predict_band_fixed_asymptotes(
632
+ x,
633
+ mu=mu,
634
+ s=s,
635
+ upper=upper,
636
+ lower=lower,
637
+ scale=self.axis.scale,
638
+ link=self.link,
639
+ covariance=covariance,
640
+ z_score=z_score,
641
+ )
642
+ else:
643
+ estimate, lo, hi = _predict_band_estimated_asymptote(
644
+ x,
645
+ params=self.params,
646
+ scale=self.axis.scale,
647
+ link=self.link,
648
+ covariance=covariance,
649
+ z_score=z_score,
650
+ )
651
+
652
+ return Prediction(severity=x, estimate=estimate, lo=lo, hi=hi, level=level)
653
+
654
+
655
+ # ================================================================================================
656
+ # Curve evaluation and the full binomial log-likelihood (shared by both fitting paths and by
657
+ # Fit.predict, so a reported log_likelihood and a predicted curve always agree with each other).
658
+ # ================================================================================================
659
+
660
+
661
+ def _transform(x: NDArray[np.float64], *, scale: str) -> NDArray[np.float64]:
662
+ """``log x`` on a log axis, ``x`` unchanged on a linear axis. Never
663
+ called on an exact-zero severity on a log axis -- callers route that case around this
664
+ function entirely, so this never evaluates ``log(0)``.
665
+ """
666
+ if scale == "log":
667
+ return np.asarray(np.log(x), dtype=np.float64)
668
+ return x
669
+
670
+
671
+ def _f_cdf(z: NDArray[np.float64], link: str) -> NDArray[np.float64]:
672
+ """The link's CDF, modelling the probability of *failure* rising with ``z``. ``cloglog``'s
673
+ ``1 - exp(-exp(z))`` is written as ``-expm1(-exp(z))`` for numerical stability, not for a
674
+ different value.
675
+ """
676
+ if link == "probit":
677
+ return np.asarray(norm.cdf(z), dtype=np.float64)
678
+ if link == "logit":
679
+ return np.asarray(expit(z), dtype=np.float64)
680
+ return np.asarray(-np.expm1(-np.exp(z)), dtype=np.float64) # link == "cloglog"
681
+
682
+
683
+ def _f_survival(z: NDArray[np.float64], link: str) -> NDArray[np.float64]:
684
+ """``1 - _f_cdf(z, link)``, computed directly from each link's own survival identity --
685
+ never as ``1 - _f_cdf(z, link)`` -- so it stays numerically accurate at the tail where that
686
+ subtraction would round to exactly ``0.0``: ``scipy.stats.norm.sf`` for probit;
687
+ ``expit(-z) == 1 - expit(z)`` exactly for logit (the logistic CDF's own reflection identity);
688
+ ``exp(-exp(z)) == 1 - (-expm1(-exp(z)))`` exactly for cloglog (``_f_cdf``'s own algebraic
689
+ identity, read the other way). ``_f_cdf`` itself is already accurate where *it* is small (large
690
+ negative ``z``); this is its mirror image, accurate where *it* is close to ``1`` (large positive
691
+ ``z``).
692
+ """
693
+ if link == "probit":
694
+ return np.asarray(norm.sf(z), dtype=np.float64)
695
+ if link == "logit":
696
+ return np.asarray(expit(-z), dtype=np.float64)
697
+ return np.asarray(np.exp(-np.exp(z)), dtype=np.float64) # link == "cloglog"
698
+
699
+
700
+ def _curve_success_probability(
701
+ severity: ArrayLike,
702
+ *,
703
+ mu: float,
704
+ s: float,
705
+ upper: float,
706
+ lower: float,
707
+ scale: str,
708
+ link: str,
709
+ center: float = 0.0,
710
+ spread: float = 1.0,
711
+ ) -> NDArray[np.float64]:
712
+ """``P(x) = u - (u - l) * F(z)``, with the log-axis zero-severity
713
+ control handled directly: ``P(0) = u`` exactly, never through ``log(0)``.
714
+
715
+ ``center``/``spread`` let a caller evaluate this on the *centred-and-scaled* transformed
716
+ covariate ``t' = (t - center) / spread`` instead of ``t`` itself (:func:`_centering_and_scale`):
717
+ ``z = (t' - mu) / s`` there, so ``mu``/``s`` are then on that same rescaled units, not
718
+ natural ones. Defaulted to the identity (``0.0``, ``1.0``), so every existing caller that
719
+ evaluates the curve at its final, natural-scale ``mu``/``s`` is unaffected.
720
+ """
721
+ x = np.asarray(severity, dtype=np.float64)
722
+ result = np.empty_like(x)
723
+ zero_control = (scale == "log") & (x == 0.0)
724
+ nonzero = ~zero_control
725
+ if np.any(nonzero):
726
+ t = (_transform(x[nonzero], scale=scale) - center) / spread
727
+ z = (t - mu) / s
728
+ result[nonzero] = upper - (upper - lower) * _f_cdf(z, link)
729
+ if np.any(zero_control):
730
+ result[zero_control] = upper
731
+ return result
732
+
733
+
734
+ def _curve_success_and_failure_probability(
735
+ severity: ArrayLike,
736
+ *,
737
+ mu: float,
738
+ s: float,
739
+ upper: float,
740
+ lower: float,
741
+ scale: str,
742
+ link: str,
743
+ center: float = 0.0,
744
+ spread: float = 1.0,
745
+ ) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
746
+ """``(p, 1 - p)`` for the curve at ``severity``, each computed *directly* rather than one as
747
+ the complement of the other, so both stay numerically accurate at their own informative tail
748
+ -- used by :func:`robustmargin.diagnose`'s goodness-of-fit sums, which need both ``p`` and
749
+ ``1 - p`` (and their product) without either rounding to an exact ``0``/``1`` for a cell that is
750
+ not :func:`robustmargin.diagnose`'s own *structural* exclusion.
751
+
752
+ ``p = l + (u - l) * (1 - F(z))``, using :func:`_f_survival` -- accurate where ``p`` itself is
753
+ small (high severity, under ``direction="decreasing"``), the tail where :func:`_f_cdf` alone
754
+ would already have rounded to exactly ``1`` and so silently rounded ``p`` to exactly ``l``.
755
+ ``1 - p = (1 - u) + (u - l) * F(z)``, using :func:`_f_cdf` directly -- accurate where it is
756
+ small (low severity), the tail :func:`_f_cdf` is already accurate on. Neither value is
757
+ clamped here (that is the caller's job, against whatever floor it needs); the two need not
758
+ sum to exactly ``1.0`` in floating point, by design -- forcing that would reintroduce the
759
+ exact cancellation this function exists to avoid.
760
+
761
+ Shares :func:`_curve_success_probability`'s log-axis zero-severity-control handling (``p(0)
762
+ = u`` exactly) and ``center``/``spread`` reparametrisation -- see that function's
763
+ docstring for both.
764
+ """
765
+ x = np.asarray(severity, dtype=np.float64)
766
+ p = np.empty_like(x)
767
+ q = np.empty_like(x)
768
+ zero_control = (scale == "log") & (x == 0.0)
769
+ nonzero = ~zero_control
770
+ if np.any(nonzero):
771
+ t = (_transform(x[nonzero], scale=scale) - center) / spread
772
+ z = (t - mu) / s
773
+ p[nonzero] = lower + (upper - lower) * _f_survival(z, link)
774
+ q[nonzero] = (1.0 - upper) + (upper - lower) * _f_cdf(z, link)
775
+ if np.any(zero_control):
776
+ p[zero_control] = upper
777
+ q[zero_control] = 1.0 - upper
778
+ return p, q
779
+
780
+
781
+ def _cells_to_arrays(
782
+ cells: tuple[Cell, ...],
783
+ ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]:
784
+ severity = np.array([c.severity for c in cells], dtype=np.float64)
785
+ successes = np.array([float(c.successes) for c in cells], dtype=np.float64)
786
+ trials = np.array([float(c.trials) for c in cells], dtype=np.float64)
787
+ return severity, successes, trials
788
+
789
+
790
+ def _transformed_span(severity: NDArray[np.float64], *, scale: str) -> float:
791
+ """The span (max - min) of the *likelihood-contributing* transformed levels: a log-axis
792
+ zero-severity control is excluded, since it carries no covariate value at all and so
793
+ cannot be part of a "tested range" on the transformed axis. Used by
794
+ ``_FLAT_SLOPE_SPAN_FACTOR``'s flat-curve check. Guaranteed non-zero whenever
795
+ ``_check_separation`` has already passed, because that check requires at least two distinct
796
+ likelihood-contributing severities.
797
+ """
798
+ nonzero = severity > 0.0 if scale == "log" else np.ones_like(severity, dtype=bool)
799
+ transformed = _transform(severity[nonzero], scale=scale)
800
+ if transformed.size == 0:
801
+ return 0.0
802
+ return float(np.max(transformed) - np.min(transformed))
803
+
804
+
805
+ def _is_positive_definite(matrix: NDArray[np.float64]) -> bool:
806
+ """:class:`Covariance`'s positive-definiteness rule, shared so the fitters can check a
807
+ natural-scale covariance or information matrix *before* building one: a failure then gives
808
+ ``Status.NOT_CONVERGED`` with a reason instead of an uncaught ``ValueError``, or a
809
+ ``LinAlgError`` from :func:`numpy.linalg.eigvalsh` on a non-finite matrix. Symmetry is not
810
+ checked here: :func:`numpy.linalg.eigvalsh` reads the lower triangle only, and the fitters
811
+ symmetrise their own matrices before calling this.
812
+
813
+ The test is on the correlation matrix ``D^-1/2 . matrix . D^-1/2``: finite, every diagonal
814
+ entry positive, and minimum eigenvalue ``> 1e-12``. A natural-scale covariance mixes
815
+ severity-unit parameters (``mu``, ``s``, whose variances scale with the square of the
816
+ caller's units) with probabilities (``upper``, ``lower``); a rule on the raw eigenvalues
817
+ would make the fit's status depend on severity units. Since, for a positive semi-definite
818
+ ``matrix``, ``min eig(correlation) >= min eig(matrix) / max(diag(matrix))``, every matrix a
819
+ raw-eigenvalue rule (``> 1e-12 * max(1, max|entry|)``) would accept is accepted here too.
820
+ """
821
+ if not bool(np.all(np.isfinite(matrix))):
822
+ return False
823
+ diagonal = np.diag(matrix)
824
+ if not bool(np.all(diagonal > 0.0)):
825
+ return False
826
+ scale = np.sqrt(diagonal)
827
+ correlation = matrix / np.outer(scale, scale)
828
+ # No symmetrising here: eigvalsh reads the lower triangle only, so a matrix within
829
+ # Covariance's symmetry tolerance is judged on its lower-triangle entries.
830
+ return float(np.min(np.linalg.eigvalsh(correlation))) > _POSITIVE_DEFINITE_RELATIVE_TOLERANCE
831
+
832
+
833
+ def _centering_and_scale(severity: NDArray[np.float64], *, scale: str) -> tuple[float, float]:
834
+ """The centring constant ``m`` and scale ``d`` used to condition the transformed covariate
835
+ *before* either fitting path ever sees it: fitting ``t' = (t - m) / d`` instead of ``t``
836
+ directly is an exact reparametrisation of the identical curve
837
+ (``mu = m + d * mu'``, ``s = d * s'``; the asymptotes are untouched by it), so it is
838
+ consistent with never hand-rolling a fitter -- it changes the optimiser's working units, not
839
+ the model. Without it, severity units that are very large or very small on the transformed axis
840
+ (for example a linear axis measured in micro-units, or in millions of units) make the
841
+ optimiser's own internal scale blow up or collapse, even though the natural-scale fit itself is
842
+ perfectly well identified.
843
+
844
+ ``m`` is the mean and ``d`` the standard deviation of the *likelihood-contributing*
845
+ transformed levels -- the same subset :func:`_transformed_span` and
846
+ :func:`_check_separation` use: a log-axis zero-severity control is excluded, since it
847
+ carries no covariate value at all. Falls back to the span, then to ``1.0``, in the
848
+ degenerate case of a zero standard deviation (should not arise once
849
+ :func:`_check_separation` has passed, since that requires at least two distinct
850
+ likelihood-contributing severities).
851
+ """
852
+ nonzero = severity > 0.0 if scale == "log" else np.ones_like(severity, dtype=bool)
853
+ transformed = _transform(severity[nonzero], scale=scale)
854
+ if transformed.size == 0:
855
+ return 0.0, 1.0
856
+ m = float(np.mean(transformed))
857
+ d = float(np.std(transformed))
858
+ if not d > 0.0:
859
+ span = float(np.max(transformed) - np.min(transformed))
860
+ d = span if span > 0.0 else 1.0
861
+ return m, d
862
+
863
+
864
+ def _full_binomial_log_likelihood(
865
+ severity: NDArray[np.float64],
866
+ successes: NDArray[np.float64],
867
+ trials: NDArray[np.float64],
868
+ *,
869
+ mu: float,
870
+ s: float,
871
+ upper: float,
872
+ lower: float,
873
+ scale: str,
874
+ link: str,
875
+ center: float = 0.0,
876
+ spread: float = 1.0,
877
+ ) -> float:
878
+ """The full binomial log-likelihood, including the ``lchoose`` combinatorial term, so it
879
+ matches R ``drc``'s ``logLik()`` on both fitting paths (see
880
+ https://github.com/robustmargin/robustmargin/blob/main/tools/drc_reference/README.md).
881
+ :func:`scipy.special.xlogy` gives a 0-successes or 0-failures cell a term of exactly ``0.0``
882
+ without ever evaluating ``log(0)``.
883
+
884
+ ``center``/``spread`` are forwarded to :func:`_curve_success_probability` -- see there.
885
+ Defaulted to the identity, so this still computes the reported ``Fit.log_likelihood`` on
886
+ natural-scale ``mu``/``s`` against raw severity when a caller omits them (both fitting
887
+ paths' final report does); only the estimated-asymptote optimiser's own iterations, which
888
+ work in centred-and-scaled units, pass them explicitly.
889
+
890
+ Summed with the plain Python builtin ``sum`` over a fixed-order list, not ``numpy.sum``:
891
+ ``numpy.sum``'s pairwise reduction can regroup its additions differently when the array's
892
+ length changes by one, which would make an exact-zero-contribution cell (an all-success
893
+ zero-severity control under a fixed ``upper=1.0``) change the *other* cells' floating-point
894
+ rounding even though its own contribution is exactly ``0.0``. A left-to-right Python sum
895
+ does not have that failure mode: inserting an exact ``0.0`` term anywhere in a strictly
896
+ sequential accumulation is a bit-exact no-op. This is what keeps the aggregation property
897
+ test's "with control" and "without control" fits bit-identical, not merely close.
898
+ """
899
+ p = _curve_success_probability(
900
+ severity,
901
+ mu=mu,
902
+ s=s,
903
+ upper=upper,
904
+ lower=lower,
905
+ scale=scale,
906
+ link=link,
907
+ center=center,
908
+ spread=spread,
909
+ )
910
+ failures = trials - successes
911
+ lchoose = gammaln(trials + 1.0) - gammaln(successes + 1.0) - gammaln(failures + 1.0)
912
+ terms = lchoose + xlogy(successes, p) + xlogy(failures, 1.0 - p)
913
+ return float(sum(terms.tolist()))
914
+
915
+
916
+ def _covariance_entry(covariance: Covariance, name_a: str, name_b: str) -> float:
917
+ i = covariance.names.index(name_a)
918
+ j = covariance.names.index(name_b)
919
+ return float(covariance.matrix[i][j])
920
+
921
+
922
+ # ================================================================================================
923
+ # Fit.predict's two band scales (documented in full on Fit.predict).
924
+ # ================================================================================================
925
+
926
+
927
+ def _predict_band_fixed_asymptotes(
928
+ x: NDArray[np.float64],
929
+ *,
930
+ mu: float,
931
+ s: float,
932
+ upper: float,
933
+ lower: float,
934
+ scale: str,
935
+ link: str,
936
+ covariance: Covariance,
937
+ z_score: float,
938
+ ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]:
939
+ """Delta method on the linear predictor ``z``, mapped through ``P = u - (u - l) * F(z)``.
940
+ Only valid when both asymptotes are fixed constants, which is what keeps the resulting band
941
+ inside ``(l, u)`` for every severity: ``F`` maps into ``[0, 1]`` regardless of how uncertain
942
+ ``z`` is, and ``u``, ``l`` themselves carry no uncertainty here.
943
+
944
+ A log-axis zero-severity control is handled directly (``estimate = lo = hi = upper``, a
945
+ zero-width band, since ``u`` is a constant here): ``z`` is undefined at ``x = 0`` on a log
946
+ axis, so those entries are routed around the ``z``-based computation entirely rather
947
+ than computed through ``log(0)``.
948
+ """
949
+ estimate = np.empty_like(x)
950
+ lo = np.empty_like(x)
951
+ hi = np.empty_like(x)
952
+
953
+ zero_control = (scale == "log") & (x == 0.0)
954
+ nonzero = ~zero_control
955
+ if np.any(zero_control):
956
+ estimate[zero_control] = upper
957
+ lo[zero_control] = upper
958
+ hi[zero_control] = upper
959
+
960
+ if np.any(nonzero):
961
+ x_nonzero = x[nonzero]
962
+ z = (_transform(x_nonzero, scale=scale) - mu) / s
963
+ var_mu = _covariance_entry(covariance, "mu", "mu")
964
+ var_s = _covariance_entry(covariance, "s", "s")
965
+ cov_mu_s = _covariance_entry(covariance, "mu", "s")
966
+ var_z = (var_mu + (z**2) * var_s - 2.0 * z * cov_mu_s) / (s**2)
967
+ se_z = np.sqrt(np.clip(var_z, 0.0, None))
968
+
969
+ estimate[nonzero] = upper - (upper - lower) * _f_cdf(z, link)
970
+ # F is non-decreasing, so the smaller z-side gives the larger P (P falls as z rises).
971
+ hi[nonzero] = upper - (upper - lower) * _f_cdf(z - z_score * se_z, link)
972
+ lo[nonzero] = upper - (upper - lower) * _f_cdf(z + z_score * se_z, link)
973
+ return estimate, lo, hi
974
+
975
+
976
+ def _predict_band_estimated_asymptote(
977
+ x: NDArray[np.float64],
978
+ *,
979
+ params: Mapping[str, Parameter],
980
+ scale: str,
981
+ link: str,
982
+ covariance: Covariance,
983
+ z_score: float,
984
+ ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.float64]]:
985
+ """Delta method on ``logit(P)``, using the gradient of ``logit(P(x))`` with respect to
986
+ every *estimated* natural parameter (``covariance.names``, always ``mu``, ``s`` and
987
+ whichever of ``lower``/``upper`` are not fixed). Guarantees the band stays inside
988
+ ``(0, 1)`` regardless of how uncertain an estimated asymptote is, because ``expit`` is used
989
+ to back-transform rather than a linear step on ``P`` itself.
990
+ """
991
+ names = covariance.names
992
+ natural_vector = np.array([params[name].value for name in names], dtype=np.float64)
993
+ fixed_lower = params["lower"].value if params["lower"].fixed else None
994
+ fixed_upper = params["upper"].value if params["upper"].fixed else None
995
+ matrix = np.array([[float(v) for v in row] for row in covariance.matrix], dtype=np.float64)
996
+
997
+ def _probability_at(natural: NDArray[np.float64], severity_value: float) -> float:
998
+ mapping = dict(zip(names, natural, strict=True))
999
+ mu = mapping["mu"]
1000
+ s = mapping["s"]
1001
+ lower = mapping["lower"] if "lower" in mapping else fixed_lower
1002
+ upper = mapping["upper"] if "upper" in mapping else fixed_upper
1003
+ assert lower is not None and upper is not None
1004
+ return float(
1005
+ _curve_success_probability(
1006
+ np.array([severity_value]),
1007
+ mu=mu,
1008
+ s=s,
1009
+ upper=upper,
1010
+ lower=lower,
1011
+ scale=scale,
1012
+ link=link,
1013
+ )[0]
1014
+ )
1015
+
1016
+ # A flat, at-least-1-D working view: a 0-d `x` (Fit.predict called with a bare scalar) has
1017
+ # no positional index (`x[0]` raises `IndexError` on a 0-d array), so integer-indexed
1018
+ # iteration needs a 1-D view regardless of `x`'s own shape. The result is reshaped back to
1019
+ # `x.shape` at the end, so a 0-d input still gives 0-d output arrays (Prediction's own
1020
+ # documented behaviour), not a shape promoted to `(1,)`.
1021
+ flat_x = np.atleast_1d(x)
1022
+ estimate = np.empty_like(flat_x)
1023
+ lo = np.empty_like(flat_x)
1024
+ hi = np.empty_like(flat_x)
1025
+ for i in range(flat_x.size):
1026
+ severity_value = float(flat_x[i])
1027
+ p0 = _probability_at(natural_vector, severity_value)
1028
+
1029
+ def _logit_p(
1030
+ natural: NDArray[np.float64], sv: float = severity_value
1031
+ ) -> NDArray[np.float64]:
1032
+ return np.array([float(_logit(_probability_at(natural, sv)))])
1033
+
1034
+ gradient = _central_difference_jacobian(_logit_p, natural_vector)[0]
1035
+ var_logit_p = float(gradient @ matrix @ gradient)
1036
+ se_logit_p = math.sqrt(max(var_logit_p, 0.0))
1037
+ logit_p0 = float(_logit(p0))
1038
+ estimate[i] = p0
1039
+ lo[i] = expit(logit_p0 - z_score * se_logit_p)
1040
+ hi[i] = expit(logit_p0 + z_score * se_logit_p)
1041
+ return estimate.reshape(x.shape), lo.reshape(x.shape), hi.reshape(x.shape)
1042
+
1043
+
1044
+ def _central_difference_jacobian(
1045
+ func: Callable[[NDArray[np.float64]], NDArray[np.float64]],
1046
+ x: NDArray[np.float64],
1047
+ *,
1048
+ relative_step: float = 1e-6,
1049
+ ) -> NDArray[np.float64]:
1050
+ """The Jacobian of ``func`` at ``x`` by central finite differences: the delta method's
1051
+ numerical-differentiation step (used both to transform a fit's optimizer-scale covariance
1052
+ onto the natural scale, and inside :func:`Fit.predict`'s estimated-asymptote band), not a
1053
+ fitting procedure. The package never hand-rolls an optimizer or IRLS loop; a numerical
1054
+ derivative of an already-fitted quantity is not one.
1055
+ """
1056
+ x = np.asarray(x, dtype=np.float64)
1057
+ f0 = np.asarray(func(x), dtype=np.float64)
1058
+ jacobian = np.zeros((f0.size, x.size), dtype=np.float64)
1059
+ for j in range(x.size):
1060
+ step = relative_step * max(1.0, abs(float(x[j])))
1061
+ x_plus = x.copy()
1062
+ x_plus[j] += step
1063
+ x_minus = x.copy()
1064
+ x_minus[j] -= step
1065
+ jacobian[:, j] = (
1066
+ np.asarray(func(x_plus), dtype=np.float64) - np.asarray(func(x_minus), dtype=np.float64)
1067
+ ) / (2.0 * step)
1068
+ return jacobian
1069
+
1070
+
1071
+ # ================================================================================================
1072
+ # fit_dose_response: the estimated-asymptote (GenericLikelihoodModel) path
1073
+ # ================================================================================================
1074
+
1075
+
1076
+ def _unpack_optimizer_params(
1077
+ params: NDArray[np.float64],
1078
+ *,
1079
+ estimate_lower: bool,
1080
+ estimate_upper: bool,
1081
+ lower_value: float,
1082
+ upper_value: float,
1083
+ ) -> tuple[float, float, float, float]:
1084
+ """Map the optimizer's internal ``(mu, log_s, [a], [b])`` vector
1085
+ to the natural scale ``(mu, s, lower, upper)``. Whichever of ``a``/``b`` corresponds to a
1086
+ *fixed* asymptote is simply absent from ``params`` ("fixed asymptotes are left out of the
1087
+ parameter vector"). ``l < u`` holds by construction in every combination:
1088
+
1089
+ - Both estimated: ``l = expit(a)`` and ``u = l + (1 - l) * expit(b)``.
1090
+ - ``upper`` estimated, ``lower`` fixed: ``u = l + (1 - l) * expit(b)`` with the fixed ``l``.
1091
+ - ``lower`` estimated, ``upper`` fixed: ``l = u * expit(a)``
1092
+ with the fixed ``u`` -- *not* the unconstrained ``l = expit(a)`` used when ``upper`` is
1093
+ also estimated, which can exceed a fixed ``u`` and converge onto an increasing curve
1094
+ (``Fit`` would then reject ``lower.value < upper.value`` at construction). Anchoring ``l``
1095
+ to a fraction of the fixed ``u`` keeps ``l`` inside ``(0, u)`` for every finite ``a``.
1096
+ """
1097
+ mu = float(params[0])
1098
+ s = float(np.exp(params[1]))
1099
+ idx = 2
1100
+ if estimate_lower:
1101
+ a_value = float(params[idx])
1102
+ idx += 1
1103
+ lower = float(upper_value * expit(a_value)) if not estimate_upper else float(expit(a_value))
1104
+ else:
1105
+ lower = lower_value
1106
+ if estimate_upper:
1107
+ upper = float(lower + (1.0 - lower) * expit(params[idx]))
1108
+ idx += 1
1109
+ else:
1110
+ upper = upper_value
1111
+ return mu, s, lower, upper
1112
+
1113
+
1114
+ @dataclass(frozen=True)
1115
+ class _GenericLogLikelihood:
1116
+ """The estimated-asymptote likelihood, passed to
1117
+ ``statsmodels.base.model.GenericLikelihoodModel`` as its ``loglike=`` callable
1118
+ (specifying a likelihood for a statsmodels optimizer, never a subclass that reimplements the
1119
+ optimizer itself). Held as a plain callable object, not a ``GenericLikelihoodModel`` subclass,
1120
+ so nothing here subclasses a type statsmodels ships with no ``py.typed`` marker.
1121
+
1122
+ ``center``/``spread`` are the :func:`_centering_and_scale` constants: the optimizer's own
1123
+ ``mu``/``log_s`` therefore live on the *centred-and-scaled* transformed covariate, not on
1124
+ natural units, which is what keeps the optimizer's working scale independent of the caller's
1125
+ severity units (a fit must not depend on them). ``a``/``b`` (asymptote logits) are untouched by
1126
+ this -- only ``mu``/``s`` are rescaled.
1127
+ """
1128
+
1129
+ severity: NDArray[np.float64]
1130
+ successes: NDArray[np.float64]
1131
+ trials: NDArray[np.float64]
1132
+ scale: str
1133
+ link: str
1134
+ estimate_lower: bool
1135
+ estimate_upper: bool
1136
+ lower_value: float
1137
+ upper_value: float
1138
+ center: float
1139
+ spread: float
1140
+
1141
+ def __call__(self, params: NDArray[np.float64]) -> float:
1142
+ mu, s, lower, upper = _unpack_optimizer_params(
1143
+ params,
1144
+ estimate_lower=self.estimate_lower,
1145
+ estimate_upper=self.estimate_upper,
1146
+ lower_value=self.lower_value,
1147
+ upper_value=self.upper_value,
1148
+ )
1149
+ return _full_binomial_log_likelihood(
1150
+ self.severity,
1151
+ self.successes,
1152
+ self.trials,
1153
+ mu=mu,
1154
+ s=s,
1155
+ upper=upper,
1156
+ lower=lower,
1157
+ scale=self.scale,
1158
+ link=self.link,
1159
+ center=self.center,
1160
+ spread=self.spread,
1161
+ )
1162
+
1163
+
1164
+ def _pilot_start(
1165
+ severity: NDArray[np.float64],
1166
+ successes: NDArray[np.float64],
1167
+ trials: NDArray[np.float64],
1168
+ *,
1169
+ scale: str,
1170
+ link: str,
1171
+ center: float,
1172
+ spread: float,
1173
+ ) -> tuple[float, float, float, float, bool]:
1174
+ """A starting point for the estimated-asymptote optimizer -- "only a starting point",
1175
+ never itself reported as a fit. ``upper``/``lower`` start at the
1176
+ control rate (or the maximum rate) and the minimum rate (probabilities, unaffected by
1177
+ ``center``/``spread``); ``mu``/``s`` start from a GLM pilot fit of the *rescaled* rates
1178
+ against the same link, on the same centred-and-scaled covariate
1179
+ ``t' = (t - center) / spread`` (:func:`_centering_and_scale`) the real optimizer fits on --
1180
+ both far better conditioned than fitting the raw 0/1-bounded rates on raw severity units
1181
+ when the true asymptotes are not 0/1 and/or the severity units are far from order 1.
1182
+ ``mu0``/``s0`` are therefore returned already on that same rescaled ("primed") scale, ready
1183
+ to seed the optimizer's own ``mu'``/``log_s'`` directly with no further conversion.
1184
+
1185
+ The pilot GLM itself is never skipped or wrapped in a try/except: a
1186
+ `PerfectSeparationWarning` from it is filtered to "ignore", not treated as a failure, so the
1187
+ only two things that actually change what is returned are (a) fewer than two distinct
1188
+ nonzero severities, when there is nothing to regress against at all, and (b) a wrong-sign
1189
+ pilot slope (``beta1 <= 0``) -- either keeps the neutral default (``mu0' = 0``, ``s0' = 1``,
1190
+ both sensible defaults *because* the rescaled covariate is centred at 0 with spread 1):
1191
+ robustness matters more than precision for a value the real optimizer only starts from.
1192
+ Case (b) is also reported back via ``pilot_wrong_sign``, so a caller whose real fit later
1193
+ fails can attach the wrong-sign reason too.
1194
+
1195
+ Returns
1196
+ -------
1197
+ tuple[float, float, float, float, bool]
1198
+ ``(mu0, s0, upper_start, lower_start, pilot_wrong_sign)``, with ``mu0``/``s0`` on the
1199
+ centred-and-scaled covariate. ``pilot_wrong_sign`` is ``True`` only when the pilot GLM
1200
+ actually ran (at least two distinct nonzero severities) and its fitted slope was
1201
+ ``<= 0``.
1202
+ """
1203
+ rate = successes / trials
1204
+ control_mask = severity == 0.0
1205
+ if scale == "log" and bool(np.any(control_mask)):
1206
+ upper_start = float(rate[control_mask][0])
1207
+ else:
1208
+ upper_start = float(np.max(rate))
1209
+ lower_start = float(np.min(rate))
1210
+ if not upper_start > lower_start:
1211
+ upper_start, lower_start = 1.0, 0.0
1212
+
1213
+ nonzero = severity > 0.0 if scale == "log" else np.ones_like(severity, dtype=bool)
1214
+ transformed_scaled = (_transform(severity[nonzero], scale=scale) - center) / spread
1215
+ mu0 = float(np.median(transformed_scaled)) if transformed_scaled.size else 0.0
1216
+ s0 = 1.0
1217
+ pilot_wrong_sign = False
1218
+
1219
+ if transformed_scaled.size and len(np.unique(severity[nonzero])) >= 2:
1220
+ eps = 1e-3
1221
+ rescaled_failure_rate = np.clip(
1222
+ (upper_start - rate[nonzero]) / (upper_start - lower_start), eps, 1.0 - eps
1223
+ )
1224
+ exog = add_constant(transformed_scaled)
1225
+ with warnings.catch_warnings():
1226
+ warnings.simplefilter("ignore", category=PerfectSeparationWarning)
1227
+ pilot = GLM(
1228
+ rescaled_failure_rate,
1229
+ exog,
1230
+ family=families.Binomial(link=_STATSMODELS_LINKS[link]),
1231
+ var_weights=trials[nonzero],
1232
+ ).fit()
1233
+ beta0 = float(pilot.params[0])
1234
+ beta1 = float(pilot.params[1])
1235
+ # Start values only: a wrong-sign pilot keeps the neutral (median, 1.0) start.
1236
+ if beta1 > 0.0:
1237
+ mu0 = -beta0 / beta1
1238
+ s0 = 1.0 / beta1
1239
+ else:
1240
+ pilot_wrong_sign = True
1241
+ return mu0, s0, upper_start, lower_start, pilot_wrong_sign
1242
+
1243
+
1244
+ def _boundary_warning(
1245
+ final_params: NDArray[np.float64],
1246
+ *,
1247
+ estimate_lower: bool,
1248
+ estimate_upper: bool,
1249
+ lower: float,
1250
+ upper: float,
1251
+ ) -> str | None:
1252
+ """An estimated asymptote pinned to a parameter-space boundary,
1253
+ checked both on the optimizer's own logit-scale parameter (``a``/``b``, ``|.| > 15``) and
1254
+ on the natural-scale probability itself (within ``1e-6`` of 0 or 1). Either test alone can
1255
+ miss a boundary fit the other catches (a huge logit-scale value that still rounds to a
1256
+ natural-scale probability inside the epsilon, or vice versa via the numerics of a
1257
+ particular optimizer step), so both are checked.
1258
+ """
1259
+ idx = 2
1260
+ if estimate_lower:
1261
+ a_value = float(final_params[idx])
1262
+ idx += 1
1263
+ if abs(a_value) > _ASYMPTOTE_LOGIT_BOUND:
1264
+ return (
1265
+ "fit_dose_response: the estimated lower asymptote reached the parameter "
1266
+ f"boundary (logit-scale a={a_value!r}, |a| > {_ASYMPTOTE_LOGIT_BOUND}) -- no "
1267
+ "interior maximum exists"
1268
+ )
1269
+ if estimate_upper:
1270
+ b_value = float(final_params[idx])
1271
+ if abs(b_value) > _ASYMPTOTE_LOGIT_BOUND:
1272
+ return (
1273
+ "fit_dose_response: the estimated upper asymptote reached the parameter "
1274
+ f"boundary (logit-scale b={b_value!r}, |b| > {_ASYMPTOTE_LOGIT_BOUND}) -- no "
1275
+ "interior maximum exists"
1276
+ )
1277
+ if estimate_lower and estimate_upper and lower < _ASYMPTOTE_PROBABILITY_EPS:
1278
+ return (
1279
+ "fit_dose_response: the estimated lower asymptote reached the probability "
1280
+ f"boundary (lower={lower!r} < {_ASYMPTOTE_PROBABILITY_EPS}) -- no interior maximum "
1281
+ "exists"
1282
+ )
1283
+ if estimate_lower and not estimate_upper and (upper - lower) < _ASYMPTOTE_PROBABILITY_EPS:
1284
+ # With a fixed upper, l = u * expit(a) ranges over (0, u). l -> 0 (a -> -inf) is *not*
1285
+ # exempt from a boundary check here -- the |a| > 15 test above already applies to it, the
1286
+ # same as it would for any other estimated asymptote's logit parameter, so it is not
1287
+ # "ordinary" merely because there is no separate probability-floor test for it. This
1288
+ # eps-based test instead covers the *other* end, l -> u (a -> +inf, expit(a) -> 1): the
1289
+ # degenerate property there is the gap (u - l) collapsing (the curve going flat, u == l, no
1290
+ # dynamic range at all), which a fixed magnitude bound on `a` alone does not directly test,
1291
+ # because how large `a` needs to be for u - l to fall below eps depends on the fixed u
1292
+ # itself.
1293
+ return (
1294
+ "fit_dose_response: the estimated lower asymptote reached the probability "
1295
+ f"boundary near the fixed upper asymptote (upper - lower = {upper - lower!r} < "
1296
+ f"{_ASYMPTOTE_PROBABILITY_EPS}) -- no interior maximum exists"
1297
+ )
1298
+ if estimate_upper and upper > 1.0 - _ASYMPTOTE_PROBABILITY_EPS:
1299
+ return (
1300
+ "fit_dose_response: the estimated upper asymptote reached the probability "
1301
+ f"boundary (upper={upper!r} > 1 - {_ASYMPTOTE_PROBABILITY_EPS}) -- no interior "
1302
+ "maximum exists"
1303
+ )
1304
+ return None
1305
+
1306
+
1307
+ def _failure_warnings(reason: str, *, pilot_wrong_sign: bool) -> tuple[str, ...]:
1308
+ """The ``warnings`` tuple for a ``NOT_CONVERGED`` estimated-asymptote fit: ``reason`` alone,
1309
+ or ``reason`` plus the shared wrong-sign/flat reason when a pilot GLM slope on the (rescaled)
1310
+ data had the wrong sign for ``direction='decreasing'`` -- unless ``reason`` already *is* that
1311
+ shared reason, in which case it is not duplicated.
1312
+ """
1313
+ if pilot_wrong_sign and reason != _WRONG_SIGN_OR_FLAT_WARNING:
1314
+ return (reason, _WRONG_SIGN_OR_FLAT_WARNING)
1315
+ return (reason,)
1316
+
1317
+
1318
+ def _fit_generic(
1319
+ obs: Observations,
1320
+ *,
1321
+ cells: tuple[Cell, ...],
1322
+ cluster_ids: tuple[str | int, ...] | None,
1323
+ link: str,
1324
+ upper_is_estimate: bool,
1325
+ upper_value: float,
1326
+ lower_is_estimate: bool,
1327
+ lower_value: float,
1328
+ ) -> Fit:
1329
+ """The estimated-or-non-trivial-fixed-asymptote path: ``statsmodels``
1330
+ ``GenericLikelihoodModel`` given :class:`_GenericLogLikelihood`, fit with ``method="bfgs"``
1331
+ then a ``method="newton"`` polish -- both statsmodels optimizers,
1332
+ never a hand-rolled Newton or IRLS loop.
1333
+ """
1334
+ axis = obs.axis
1335
+
1336
+ # Drop a structurally zero-contribution zero-severity control row before building the
1337
+ # optimizer's own data: with `upper` fixed at exactly 1.0 on a log axis,
1338
+ # `_curve_success_probability`'s own zero-severity-control rule pins this cell's fitted
1339
+ # probability to exactly `upper` for every possible mu/s/lower, so its log-likelihood
1340
+ # contribution is an exact constant with respect to every parameter this path estimates -- by
1341
+ # the time `_fit_generic` is reached with upper fixed at 1.0, `_check_control_incompatible` has
1342
+ # already ruled out a control with any failures here, so that constant is exactly 0. Leaving
1343
+ # that row in the data had two distinct effects on the fit, not one, and this exclusion removes
1344
+ # both at once: (1) `_pilot_start`'s own `upper_start` reads the *control's* rate when one is
1345
+ # present (here, exactly 1.0) rather than the maximum rate among the other levels (here, about
1346
+ # 0.96), which moved the pilot `mu0`/`s0` the real optimizer started from; (2)
1347
+ # `GenericLikelihoodModel.fit` (statsmodels' own optimizer, not one of ours) minimises
1348
+ # `-loglike(params) / nobs`, where `nobs` is the row count it was built with, so a
1349
+ # zero-contribution row changes `nobs` and, with it, the scale at which `gtol`/`tol` judge
1350
+ # convergence, even though the likelihood function itself is unchanged at every parameter value.
1351
+ # Excluding it here mirrors `_fit_glm`'s own `fit_cells` exclusion below, and keeps both the
1352
+ # pilot start and `nobs` identical to a fit over the same data with no control row at all. An
1353
+ # *estimated* upper is never exempt: the zero_control branch then reports `upper` itself, a free
1354
+ # parameter, so the control genuinely identifies it and must stay in the optimizer's data.
1355
+ fit_cells = (
1356
+ tuple(c for c in cells if c.severity != 0.0)
1357
+ if axis.scale == "log" and not upper_is_estimate and upper_value == 1.0
1358
+ else cells
1359
+ )
1360
+ severity, successes, trials = _cells_to_arrays(fit_cells)
1361
+
1362
+ # Centre and scale the transformed covariate before either the pilot or the real optimiser
1363
+ # ever sees it (a fit must not depend on the caller's choice of severity
1364
+ # units). This is an exact reparametrisation, not a different model -- see
1365
+ # _centering_and_scale and _GenericLogLikelihood.
1366
+ center, spread = _centering_and_scale(severity, scale=axis.scale)
1367
+
1368
+ mu0, s0, u0, l0, pilot_wrong_sign = _pilot_start(
1369
+ severity, successes, trials, scale=axis.scale, link=link, center=center, spread=spread
1370
+ )
1371
+
1372
+ start = [mu0, math.log(s0)]
1373
+ eps = 1e-3
1374
+ lower_for_start = l0 if lower_is_estimate else lower_value
1375
+ upper_for_start = u0 if upper_is_estimate else upper_value
1376
+ if lower_is_estimate:
1377
+ if upper_is_estimate:
1378
+ a0 = float(_logit(np.clip(lower_for_start, eps, 1.0 - eps)))
1379
+ else:
1380
+ # l = u * expit(a) when upper is fixed, so a's start value inverts *that* formula with
1381
+ # the fixed u, not l = expit(a) directly.
1382
+ a0 = float(_logit(np.clip(lower_for_start / upper_for_start, eps, 1.0 - eps)))
1383
+ start.append(a0)
1384
+ if upper_is_estimate:
1385
+ denominator = max(1.0 - lower_for_start, 1e-6)
1386
+ fraction = float(np.clip((upper_for_start - lower_for_start) / denominator, eps, 1.0 - eps))
1387
+ start.append(float(_logit(fraction)))
1388
+ start_params = np.array(start, dtype=np.float64)
1389
+
1390
+ loglike_fn = _GenericLogLikelihood(
1391
+ severity=severity,
1392
+ successes=successes,
1393
+ trials=trials,
1394
+ scale=axis.scale,
1395
+ link=link,
1396
+ estimate_lower=lower_is_estimate,
1397
+ estimate_upper=upper_is_estimate,
1398
+ lower_value=lower_value,
1399
+ upper_value=upper_value,
1400
+ center=center,
1401
+ spread=spread,
1402
+ )
1403
+ exog = np.ones((severity.size, start_params.size))
1404
+ model = GenericLikelihoodModel(successes, exog, loglike=loglike_fn)
1405
+
1406
+ # The BFGS stage is only ever a starting point for the Newton polish: it routinely reports its
1407
+ # own `converged=False` / ConvergenceWarning even when it has landed close enough for Newton to
1408
+ # finish the job, so only the *final* (Newton) result's own convergence signal decides the
1409
+ # status below -- checking a warning merged across both stages would treat that expected
1410
+ # intermediate state as a failure.
1411
+ with warnings.catch_warnings():
1412
+ warnings.simplefilter("ignore")
1413
+ bfgs_result = model.fit(
1414
+ start_params=start_params, method="bfgs", disp=False, maxiter=2000, gtol=1e-10
1415
+ )
1416
+ try:
1417
+ with warnings.catch_warnings(record=True) as caught:
1418
+ warnings.simplefilter("always")
1419
+ result = model.fit(
1420
+ start_params=np.asarray(bfgs_result.params, dtype=np.float64),
1421
+ method="newton",
1422
+ disp=False,
1423
+ maxiter=200,
1424
+ tol=1e-10,
1425
+ )
1426
+ except np.linalg.LinAlgError:
1427
+ # A singular Hessian during the Newton polish is usually the symptom of an estimated
1428
+ # asymptote drifting to its boundary; name that cause when the BFGS
1429
+ # stage already shows it, so the reason does not depend on which platform's linear
1430
+ # algebra happens to fail first.
1431
+ bfgs_params = np.asarray(bfgs_result.params, dtype=np.float64)
1432
+ _, _, bfgs_lower, bfgs_upper = _unpack_optimizer_params(
1433
+ bfgs_params,
1434
+ estimate_lower=lower_is_estimate,
1435
+ estimate_upper=upper_is_estimate,
1436
+ lower_value=lower_value,
1437
+ upper_value=upper_value,
1438
+ )
1439
+ singular_reason = _boundary_warning(
1440
+ bfgs_params,
1441
+ estimate_lower=lower_is_estimate,
1442
+ estimate_upper=upper_is_estimate,
1443
+ lower=bfgs_lower,
1444
+ upper=bfgs_upper,
1445
+ ) or (
1446
+ "fit_dose_response: the optimizer's Hessian was singular while fitting -- "
1447
+ "no interior maximum was found"
1448
+ )
1449
+ return _status_only_fit(
1450
+ obs,
1451
+ cells=cells,
1452
+ cluster_ids=cluster_ids,
1453
+ link=link,
1454
+ status=Status.NOT_CONVERGED,
1455
+ fit_warnings=_failure_warnings(singular_reason, pilot_wrong_sign=pilot_wrong_sign),
1456
+ )
1457
+ convergence_warned = any(issubclass(w.category, ConvergenceWarning) for w in caught)
1458
+ converged_flag = bool(result.mle_retvals.get("converged", False))
1459
+ final_params = np.asarray(result.params, dtype=np.float64)
1460
+
1461
+ mu_scaled, s_scaled, lower, upper = _unpack_optimizer_params(
1462
+ final_params,
1463
+ estimate_lower=lower_is_estimate,
1464
+ estimate_upper=upper_is_estimate,
1465
+ lower_value=lower_value,
1466
+ upper_value=upper_value,
1467
+ )
1468
+ # mu/s from the optimizer are on the centred-and-scaled covariate; map back to natural
1469
+ # units (the exact inverse of _centering_and_scale's reparametrisation). lower/upper are
1470
+ # already natural -- asymptotes are untouched by the covariate rescaling.
1471
+ mu = center + spread * mu_scaled
1472
+ s = spread * s_scaled
1473
+
1474
+ # A flat curve up to rounding, checked before anything that would build a covariance from a
1475
+ # near-degenerate slope.
1476
+ span = _transformed_span(severity, scale=axis.scale)
1477
+ if s > _FLAT_SLOPE_SPAN_FACTOR * span:
1478
+ return _status_only_fit(
1479
+ obs,
1480
+ cells=cells,
1481
+ cluster_ids=cluster_ids,
1482
+ link=link,
1483
+ status=Status.NOT_CONVERGED,
1484
+ fit_warnings=_failure_warnings(
1485
+ _WRONG_SIGN_OR_FLAT_WARNING, pilot_wrong_sign=pilot_wrong_sign
1486
+ ),
1487
+ )
1488
+
1489
+ boundary_message = _boundary_warning(
1490
+ final_params,
1491
+ estimate_lower=lower_is_estimate,
1492
+ estimate_upper=upper_is_estimate,
1493
+ lower=lower,
1494
+ upper=upper,
1495
+ )
1496
+ if boundary_message is not None:
1497
+ return _status_only_fit(
1498
+ obs,
1499
+ cells=cells,
1500
+ cluster_ids=cluster_ids,
1501
+ link=link,
1502
+ status=Status.NOT_CONVERGED,
1503
+ fit_warnings=_failure_warnings(boundary_message, pilot_wrong_sign=pilot_wrong_sign),
1504
+ )
1505
+
1506
+ # Newton's own `converged` flag only bounds the last step, so the final log-likelihood must
1507
+ # not have regressed below the BFGS stage's (relative tolerance -- an absolute tolerance here
1508
+ # does not scale with the log-likelihood's own magnitude, which depends on the sample size,
1509
+ # not on anything a caller controls), and the score at the final parameters must be near
1510
+ # zero.
1511
+ bfgs_llf = float(bfgs_result.llf)
1512
+ llf_regressed = float(result.llf) < bfgs_llf - _LLF_REGRESSION_TOLERANCE * (1.0 + abs(bfgs_llf))
1513
+ score = np.asarray(model.score(final_params), dtype=np.float64)
1514
+ score_tolerance = _SCORE_TOLERANCE_FACTOR * (1.0 + abs(float(result.llf)))
1515
+ score_not_near_zero = bool(np.any(np.abs(score) > score_tolerance))
1516
+
1517
+ not_converged_reason: str | None = None
1518
+ if not converged_flag or convergence_warned:
1519
+ not_converged_reason = "fit_dose_response: the optimizer did not converge"
1520
+ elif llf_regressed:
1521
+ not_converged_reason = (
1522
+ "fit_dose_response: the Newton polish's log-likelihood is lower than the BFGS "
1523
+ "stage's -- the optimizer did not converge"
1524
+ )
1525
+ elif score_not_near_zero:
1526
+ not_converged_reason = (
1527
+ "fit_dose_response: the score at the optimum is not near zero -- the optimizer "
1528
+ "did not converge"
1529
+ )
1530
+ if not_converged_reason is not None:
1531
+ return _status_only_fit(
1532
+ obs,
1533
+ cells=cells,
1534
+ cluster_ids=cluster_ids,
1535
+ link=link,
1536
+ status=Status.NOT_CONVERGED,
1537
+ fit_warnings=_failure_warnings(not_converged_reason, pilot_wrong_sign=pilot_wrong_sign),
1538
+ )
1539
+
1540
+ # A non-finite Hessian entry is checked before eigvalsh ever sees it
1541
+ # (numpy.linalg.eigvalsh raises LinAlgError on a non-finite array), and the
1542
+ # positive-definite check itself is the same one Covariance applies, done here first so a
1543
+ # broken Hessian never reaches that constructor.
1544
+ hessian = np.asarray(model.hessian(final_params), dtype=np.float64)
1545
+ neg_hessian = -(hessian + hessian.T) / 2.0
1546
+ if not _is_positive_definite(neg_hessian):
1547
+ return _status_only_fit(
1548
+ obs,
1549
+ cells=cells,
1550
+ cluster_ids=cluster_ids,
1551
+ link=link,
1552
+ status=Status.NOT_CONVERGED,
1553
+ fit_warnings=_failure_warnings(
1554
+ "fit_dose_response: the Hessian at the optimum is not finite and "
1555
+ "positive-definite -- no interior maximum was found",
1556
+ pilot_wrong_sign=pilot_wrong_sign,
1557
+ ),
1558
+ )
1559
+
1560
+ names: list[str] = ["mu", "s"]
1561
+ if lower_is_estimate:
1562
+ names.append("lower")
1563
+ if upper_is_estimate:
1564
+ names.append("upper")
1565
+
1566
+ def _natural_vector(p: NDArray[np.float64]) -> NDArray[np.float64]:
1567
+ mu_scaled_, s_scaled_, lower_, upper_ = _unpack_optimizer_params(
1568
+ p,
1569
+ estimate_lower=lower_is_estimate,
1570
+ estimate_upper=upper_is_estimate,
1571
+ lower_value=lower_value,
1572
+ upper_value=upper_value,
1573
+ )
1574
+ # Fold the covariate rescaling into the same numeric Jacobian (mu/s only; asymptotes
1575
+ # are already natural).
1576
+ values = [center + spread * mu_scaled_, spread * s_scaled_]
1577
+ if lower_is_estimate:
1578
+ values.append(lower_)
1579
+ if upper_is_estimate:
1580
+ values.append(upper_)
1581
+ return np.array(values, dtype=np.float64)
1582
+
1583
+ jacobian = _central_difference_jacobian(_natural_vector, final_params)
1584
+ optimizer_cov = np.asarray(result.cov_params(), dtype=np.float64)
1585
+ natural_cov = jacobian @ optimizer_cov @ jacobian.T
1586
+ natural_cov = (natural_cov + natural_cov.T) / 2.0
1587
+
1588
+ # Covariance's own positive-definiteness rule, on the correlation scale so the status does
1589
+ # not depend on severity units.
1590
+ if not _is_positive_definite(natural_cov):
1591
+ return _status_only_fit(
1592
+ obs,
1593
+ cells=cells,
1594
+ cluster_ids=cluster_ids,
1595
+ link=link,
1596
+ status=Status.NOT_CONVERGED,
1597
+ fit_warnings=_failure_warnings(
1598
+ "fit_dose_response: the natural-scale covariance is not finite and "
1599
+ "positive-definite -- no interior maximum was found",
1600
+ pilot_wrong_sign=pilot_wrong_sign,
1601
+ ),
1602
+ )
1603
+
1604
+ # The reported log-likelihood is always the full binomial log-likelihood over every cell
1605
+ # (`Fit.cells`, including a control `fit_cells` dropped above) -- `fit_cells`/`severity` etc.
1606
+ # above are the optimizer's own reduced view, not what gets reported.
1607
+ all_severity, all_successes, all_trials = _cells_to_arrays(cells)
1608
+ log_likelihood = _full_binomial_log_likelihood(
1609
+ all_severity,
1610
+ all_successes,
1611
+ all_trials,
1612
+ mu=mu,
1613
+ s=s,
1614
+ upper=upper,
1615
+ lower=lower,
1616
+ scale=axis.scale,
1617
+ link=link,
1618
+ )
1619
+
1620
+ params = {
1621
+ "mu": Parameter(value=mu, fixed=False),
1622
+ "s": Parameter(value=s, fixed=False),
1623
+ "lower": Parameter(value=lower, fixed=not lower_is_estimate),
1624
+ "upper": Parameter(value=upper, fixed=not upper_is_estimate),
1625
+ }
1626
+ covariance = Covariance(
1627
+ names=tuple(names), matrix=tuple(tuple(row) for row in natural_cov.tolist())
1628
+ )
1629
+
1630
+ return Fit(
1631
+ axis=axis,
1632
+ outcome=obs.outcome,
1633
+ direction=obs.direction,
1634
+ model="binomial",
1635
+ link=link,
1636
+ params=params,
1637
+ covariance=covariance,
1638
+ log_likelihood=log_likelihood,
1639
+ cells=cells,
1640
+ status=Status.OK,
1641
+ cluster_ids=cluster_ids,
1642
+ warnings=(),
1643
+ provenance={},
1644
+ )
1645
+
1646
+
1647
+ # ================================================================================================
1648
+ # fit_dose_response: the fixed u=1, l=0 path (statsmodels GLM)
1649
+ # ================================================================================================
1650
+
1651
+
1652
+ def _fit_glm(
1653
+ obs: Observations,
1654
+ *,
1655
+ cells: tuple[Cell, ...],
1656
+ cluster_ids: tuple[str | int, ...] | None,
1657
+ link: str,
1658
+ ) -> Fit:
1659
+ axis = obs.axis
1660
+ fit_cells = tuple(c for c in cells if not (axis.scale == "log" and c.severity == 0.0))
1661
+ severity, successes, trials = _cells_to_arrays(fit_cells)
1662
+ failures = trials - successes
1663
+
1664
+ # Centre and scale the transformed covariate before the GLM ever sees it (a fit must not
1665
+ # depend on the caller's choice of severity units). An exact reparametrisation of the
1666
+ # identical model -- see _centering_and_scale.
1667
+ center, spread = _centering_and_scale(severity, scale=axis.scale)
1668
+ exog = add_constant((_transform(severity, scale=axis.scale) - center) / spread)
1669
+ endog = failures / trials
1670
+
1671
+ model = GLM(
1672
+ endog, exog, family=families.Binomial(link=_STATSMODELS_LINKS[link]), var_weights=trials
1673
+ )
1674
+ with warnings.catch_warnings(record=True) as caught:
1675
+ warnings.simplefilter("always")
1676
+ warnings.filterwarnings("ignore", category=PerfectSeparationWarning)
1677
+ glm_result = model.fit()
1678
+ convergence_warned = any(issubclass(w.category, ConvergenceWarning) for w in caught)
1679
+
1680
+ if convergence_warned or not bool(glm_result.converged):
1681
+ return _status_only_fit(
1682
+ obs,
1683
+ cells=cells,
1684
+ cluster_ids=cluster_ids,
1685
+ link=link,
1686
+ status=Status.NOT_CONVERGED,
1687
+ fit_warnings=("fit_dose_response: the GLM optimizer did not converge",),
1688
+ )
1689
+
1690
+ beta0 = float(glm_result.params[0])
1691
+ beta1 = float(glm_result.params[1])
1692
+ if beta1 <= 0.0:
1693
+ return _status_only_fit(
1694
+ obs,
1695
+ cells=cells,
1696
+ cluster_ids=cluster_ids,
1697
+ link=link,
1698
+ status=Status.NOT_CONVERGED,
1699
+ fit_warnings=(_WRONG_SIGN_OR_FLAT_WARNING,),
1700
+ )
1701
+
1702
+ # beta0/beta1 are on the centred-and-scaled covariate; map back to natural units (the exact
1703
+ # inverse of _centering_and_scale's reparametrisation).
1704
+ mu_scaled = -beta0 / beta1
1705
+ s_scaled = 1.0 / beta1
1706
+ mu = center + spread * mu_scaled
1707
+ s = spread * s_scaled
1708
+
1709
+ # A slope that is technically positive but effectively flat (for example beta1 ~ 1e-15) makes s
1710
+ # explode, which would otherwise pass Covariance's own positive-definite check while being
1711
+ # numerically meaningless (s on the order of 1e61).
1712
+ span = _transformed_span(severity, scale=axis.scale)
1713
+ if s > _FLAT_SLOPE_SPAN_FACTOR * span:
1714
+ return _status_only_fit(
1715
+ obs,
1716
+ cells=cells,
1717
+ cluster_ids=cluster_ids,
1718
+ link=link,
1719
+ status=Status.NOT_CONVERGED,
1720
+ fit_warnings=(_WRONG_SIGN_OR_FLAT_WARNING,),
1721
+ )
1722
+
1723
+ # Both fitting paths report covariance from the *observed* information
1724
+ # (the inverse of the negative Hessian of the log-likelihood at the MLE), not statsmodels'
1725
+ # own `cov_params()` (the *expected*/Fisher information), which for a non-canonical link
1726
+ # such as probit or cloglog differs from what drc and GenericLikelihoodModel report.
1727
+ #
1728
+ # There is no positive-definiteness gate on this intermediate,
1729
+ # centred-and-scaled-covariate-scale ("beta-scale") information matrix -- checking it would
1730
+ # be a bug: even a perfectly fine fit's beta-scale matrix can have a large or small
1731
+ # absolute magnitude for reasons that have nothing to do with whether the fit is degenerate
1732
+ # (for example a design with very few levels), and a positive-definiteness rule was never
1733
+ # meant to judge an intermediate quantity. Only a narrow inversion safety net remains here;
1734
+ # the real positive-definiteness decision is made once, below, on the natural-scale
1735
+ # covariance -- the only matrix a caller actually sees.
1736
+ observed_hessian = np.asarray(
1737
+ model.hessian(np.asarray(glm_result.params, dtype=np.float64), observed=True),
1738
+ dtype=np.float64,
1739
+ )
1740
+ neg_observed_information = -(observed_hessian + observed_hessian.T) / 2.0
1741
+ try:
1742
+ sigma_beta = np.linalg.inv(neg_observed_information)
1743
+ except np.linalg.LinAlgError:
1744
+ return _status_only_fit(
1745
+ obs,
1746
+ cells=cells,
1747
+ cluster_ids=cluster_ids,
1748
+ link=link,
1749
+ status=Status.NOT_CONVERGED,
1750
+ fit_warnings=(
1751
+ "fit_dose_response: the GLM's observed information is singular -- no interior "
1752
+ "maximum was found",
1753
+ ),
1754
+ )
1755
+ if not bool(np.all(np.isfinite(sigma_beta))):
1756
+ return _status_only_fit(
1757
+ obs,
1758
+ cells=cells,
1759
+ cluster_ids=cluster_ids,
1760
+ link=link,
1761
+ status=Status.NOT_CONVERGED,
1762
+ fit_warnings=(
1763
+ "fit_dose_response: the GLM's observed information inverts to a non-finite "
1764
+ "covariance -- no interior maximum was found",
1765
+ ),
1766
+ )
1767
+ jacobian = spread * np.array(
1768
+ [[-1.0 / beta1, beta0 / beta1**2], [0.0, -1.0 / beta1**2]], dtype=np.float64
1769
+ )
1770
+ natural_cov = jacobian @ sigma_beta @ jacobian.T
1771
+ natural_cov = (natural_cov + natural_cov.T) / 2.0
1772
+
1773
+ # Covariance's own positive-definiteness rule, on the correlation scale so the status does
1774
+ # not depend on severity units.
1775
+ if not _is_positive_definite(natural_cov):
1776
+ return _status_only_fit(
1777
+ obs,
1778
+ cells=cells,
1779
+ cluster_ids=cluster_ids,
1780
+ link=link,
1781
+ status=Status.NOT_CONVERGED,
1782
+ fit_warnings=(
1783
+ "fit_dose_response: the natural-scale covariance is not finite and "
1784
+ "positive-definite -- no interior maximum was found",
1785
+ ),
1786
+ )
1787
+
1788
+ all_severity, all_successes, all_trials = _cells_to_arrays(cells)
1789
+ log_likelihood = _full_binomial_log_likelihood(
1790
+ all_severity,
1791
+ all_successes,
1792
+ all_trials,
1793
+ mu=mu,
1794
+ s=s,
1795
+ upper=1.0,
1796
+ lower=0.0,
1797
+ scale=axis.scale,
1798
+ link=link,
1799
+ )
1800
+
1801
+ params = {
1802
+ "mu": Parameter(value=mu, fixed=False),
1803
+ "s": Parameter(value=s, fixed=False),
1804
+ "upper": Parameter(value=1.0, fixed=True),
1805
+ "lower": Parameter(value=0.0, fixed=True),
1806
+ }
1807
+ covariance = Covariance(
1808
+ names=("mu", "s"), matrix=tuple(tuple(row) for row in natural_cov.tolist())
1809
+ )
1810
+
1811
+ return Fit(
1812
+ axis=axis,
1813
+ outcome=obs.outcome,
1814
+ direction=obs.direction,
1815
+ model="binomial",
1816
+ link=link,
1817
+ params=params,
1818
+ covariance=covariance,
1819
+ log_likelihood=log_likelihood,
1820
+ cells=cells,
1821
+ status=Status.OK,
1822
+ cluster_ids=cluster_ids,
1823
+ warnings=(),
1824
+ provenance={},
1825
+ )
1826
+
1827
+
1828
+ # ================================================================================================
1829
+ # fit_dose_response: shared checks and the public entry point
1830
+ # ================================================================================================
1831
+
1832
+
1833
+ def _resolve_asymptote(value: float | str, *, label: str) -> tuple[bool, float]:
1834
+ """Validate ``upper``/``lower`` and split it into ``(is_estimate, fixed_value)``.
1835
+ ``fixed_value`` is meaningless (``math.nan``) when ``is_estimate`` is ``True``.
1836
+ """
1837
+ if isinstance(value, str):
1838
+ if value != _VALID_UPPER_LOWER:
1839
+ raise ValueError(
1840
+ f"fit_dose_response: {label} must be a float in [0, 1] or the string "
1841
+ f"'estimate', got {value!r}"
1842
+ )
1843
+ return True, math.nan
1844
+ _check_finite_number(value, label=f"fit_dose_response.{label}")
1845
+ if not (0.0 <= value <= 1.0):
1846
+ raise ValueError(f"fit_dose_response: {label} must lie in [0, 1], got {value!r}")
1847
+ return False, float(value)
1848
+
1849
+
1850
+ def _unique_cluster_ids(
1851
+ cluster: tuple[str | int, ...] | None,
1852
+ ) -> tuple[str | int, ...] | None:
1853
+ """The distinct cluster ids ``Observations`` carries, one instance per cluster (not one per
1854
+ row -- :class:`Fit`'s own invariant), in first-appearance order.
1855
+ """
1856
+ if cluster is None:
1857
+ return None
1858
+ return tuple(dict.fromkeys(cluster))
1859
+
1860
+
1861
+ def _control_cell(cells: tuple[Cell, ...]) -> Cell | None:
1862
+ for cell in cells:
1863
+ if cell.severity == 0.0:
1864
+ return cell
1865
+ return None
1866
+
1867
+
1868
+ def _check_control_incompatible(
1869
+ cells: tuple[Cell, ...], *, axis: Axis, upper_is_estimate: bool, upper_value: float
1870
+ ) -> str | None:
1871
+ """A fixed ``upper=1.0`` with any failure at the zero-severity control
1872
+ drives the likelihood to zero on a log axis. Only meaningful on a log axis (a linear axis
1873
+ has no zero-severity control concept) and only when ``upper`` is fixed at exactly ``1.0``
1874
+ (a fixed ``upper`` at any other value does not create a zero-likelihood problem).
1875
+ """
1876
+ if axis.scale != "log" or upper_is_estimate or upper_value != 1.0:
1877
+ return None
1878
+ control = _control_cell(cells)
1879
+ if control is None or control.successes == control.trials:
1880
+ return None
1881
+ return (
1882
+ "fit_dose_response: the zero-severity control has failures but upper=1.0 is fixed, "
1883
+ 'which drives the likelihood to zero on a log axis; refit with upper="estimate" '
1884
+ "instead"
1885
+ )
1886
+
1887
+
1888
+ def _check_separation(cells: tuple[Cell, ...], *, axis: Axis) -> str | None:
1889
+ """The pre-fit separation check: checked on
1890
+ nonzero levels on a log axis (the zero-severity control is excluded, since it carries no
1891
+ covariate information), or on every level on a linear axis (there is no special
1892
+ zero-severity control there). Triggers when no failure exists, no success exists, or the
1893
+ highest severity with a success is at or below the lowest severity with a failure (complete
1894
+ or quasi-complete separation, the latter including an exact tie at one level).
1895
+ """
1896
+ checked = tuple(c for c in cells if not (axis.scale == "log" and c.severity == 0.0))
1897
+ if not checked:
1898
+ return None
1899
+ has_success = any(c.successes > 0 for c in checked)
1900
+ has_failure = any(c.successes < c.trials for c in checked)
1901
+ if not has_success:
1902
+ return "fit_dose_response: no success was observed at any tested severity (separation)"
1903
+ if not has_failure:
1904
+ return "fit_dose_response: no failure was observed at any tested severity (separation)"
1905
+ max_success_severity = max(c.severity for c in checked if c.successes > 0)
1906
+ min_failure_severity = min(c.severity for c in checked if c.successes < c.trials)
1907
+ if max_success_severity <= min_failure_severity:
1908
+ return (
1909
+ "fit_dose_response: complete or quasi-complete separation -- the highest severity "
1910
+ "with a success is at or below the lowest severity with a failure"
1911
+ )
1912
+ return None
1913
+
1914
+
1915
+ def _status_only_fit(
1916
+ obs: Observations,
1917
+ *,
1918
+ cells: tuple[Cell, ...],
1919
+ cluster_ids: tuple[str | int, ...] | None,
1920
+ link: str,
1921
+ status: Status,
1922
+ fit_warnings: tuple[str, ...],
1923
+ ) -> Fit:
1924
+ return Fit(
1925
+ axis=obs.axis,
1926
+ outcome=obs.outcome,
1927
+ direction=obs.direction,
1928
+ model="binomial",
1929
+ link=link,
1930
+ params=None,
1931
+ covariance=None,
1932
+ log_likelihood=None,
1933
+ cells=cells,
1934
+ status=status,
1935
+ cluster_ids=cluster_ids,
1936
+ warnings=fit_warnings,
1937
+ provenance={},
1938
+ )
1939
+
1940
+
1941
+ def _finalize(fit: Fit, *, monotonicity: MonotonicityCheck) -> Fit:
1942
+ """Attach ``monotonicity`` (every return path, whatever the status) and ``diagnostics``
1943
+ (only when ``fit.status is Status.OK``) to a freshly built ``Fit``. The single place
1944
+ :func:`fit_dose_response` routes every one of its return statements through, so the
1945
+ diagnostics/monotonicity rule is stated once.
1946
+ """
1947
+ diagnostics = diagnose(fit) if fit.status is Status.OK else None
1948
+ return replace(fit, diagnostics=diagnostics, monotonicity=monotonicity)
1949
+
1950
+
1951
+ def fit_dose_response(
1952
+ obs: Observations,
1953
+ *,
1954
+ model: Literal["binomial"],
1955
+ link: Literal["probit", "logit", "cloglog"],
1956
+ upper: float | Literal["estimate"],
1957
+ lower: float | Literal["estimate"],
1958
+ dependence: str | None = None,
1959
+ ) -> Fit:
1960
+ """Fit a binomial dose-response curve to ``obs``.
1961
+
1962
+ Two fitting paths: statsmodels ``GLM`` when both ``upper`` and ``lower`` are fixed at exactly
1963
+ ``1.0``/``0.0``; ``statsmodels`` ``GenericLikelihoodModel`` -- given a likelihood, never a
1964
+ hand-rolled optimizer or IRLS loop -- for every other combination of fixed and estimated
1965
+ asymptotes. Both paths are checked for separation before fitting and for a non-interior maximum
1966
+ after fitting; either gives an explicit :class:`~robustmargin.Status` with no numbers, never a
1967
+ value that merely happens to be where the optimizer stopped. One case is not yet detected: a
1968
+ near-step curve (``s -> 0``) with an estimated asymptote, or a fixed one other than
1969
+ ``1``/``0``, can return ``Status.OK`` with Wald standard errors that carry little
1970
+ information. With such asymptotes the overlap check is necessary but not sufficient.
1971
+
1972
+ Before any of that, :func:`robustmargin.check_monotonicity` is run on ``obs`` directly, and
1973
+ its result is attached to the returned ``Fit`` as ``monotonicity`` on *every* return path
1974
+ below, whatever the resulting ``status`` -- the check needs only raw counts, so it is not
1975
+ hidden behind a fit that failed. ``diagnostics`` is filled (via :func:`robustmargin.diagnose`)
1976
+ only when the fit reaches :data:`~robustmargin.Status.OK`.
1977
+
1978
+ Parameters
1979
+ ----------
1980
+ obs
1981
+ The severity/outcome data to fit. Aggregated and per-observation ``Observations`` that
1982
+ pool to the same cells give numerically identical fits: everything below is a
1983
+ deterministic function of ``obs.cells()``, not of row order or representation.
1984
+ model
1985
+ Only ``"binomial"`` exists in this release.
1986
+ link
1987
+ ``"probit"``, ``"logit"`` or ``"cloglog"``. Fixed by the caller before the data is
1988
+ seen: this function never compares links and picks one for you.
1989
+ upper, lower
1990
+ Each either a fixed probability in ``[0, 1]``, or the literal string ``"estimate"``.
1991
+ Required, with no default (a wrong default would silently change what curve is fit).
1992
+ dependence
1993
+ ``None`` (the default) resolves to ``"independent"`` unless ``obs.cluster`` is set, in
1994
+ which case omitting it raises -- forwarded to :func:`robustmargin.check_monotonicity`
1995
+ as-is, so there is only one implementation of this guard. ``"independent"`` may always
1996
+ be passed explicitly.
1997
+
1998
+ Returns
1999
+ -------
2000
+ Fit
2001
+ ``status`` is :data:`~robustmargin.Status.OK` with all four natural-scale parameters and
2002
+ a covariance over the estimated ones, or one of :data:`~robustmargin.Status.SEPARATION`,
2003
+ :data:`~robustmargin.Status.NOT_CONVERGED` or
2004
+ :data:`~robustmargin.Status.CONTROL_INCOMPATIBLE`
2005
+ with no numbers and a reason in ``warnings``. ``cluster_ids`` mirrors ``obs.cluster``
2006
+ whatever the outcome. The dependence guard is enforced here too, at fit time (via the
2007
+ ``check_monotonicity`` call above) -- :class:`~robustmargin.Ratio` and
2008
+ :func:`robustmargin.threshold` each still enforce it again for their own clustered-fit
2009
+ inputs. ``monotonicity`` is never ``None`` on a freshly built ``Fit``; ``diagnostics`` is
2010
+ ``None`` unless ``status`` is ``OK``.
2011
+
2012
+ Raises
2013
+ ------
2014
+ ValueError
2015
+ If ``model``/``link`` is not one of the values above, if ``obs.direction`` is not
2016
+ ``"decreasing"`` (an increasing-is-worse curve is not yet supported; ``Fit``
2017
+ itself rejects it), if ``upper``/``lower`` is not a valid probability or ``"estimate"``,
2018
+ if both are fixed with ``upper <= lower``, or if ``obs.cluster`` is set and
2019
+ ``dependence`` is not ``"independent"`` (raised by :func:`robustmargin.check_monotonicity`).
2020
+ """
2021
+ if model != "binomial":
2022
+ raise ValueError(f"fit_dose_response: model must be 'binomial', got {model!r}")
2023
+ if link not in _STATSMODELS_LINKS:
2024
+ raise ValueError(
2025
+ f"fit_dose_response: link must be one of {sorted(_STATSMODELS_LINKS)}, got {link!r}"
2026
+ )
2027
+ if obs.direction != "decreasing":
2028
+ raise ValueError(
2029
+ "fit_dose_response: direction='increasing' is not supported in this release "
2030
+ "(model='binomial' only supports 'decreasing'; an increasing-is-worse definition "
2031
+ "is planned for v0.2)"
2032
+ )
2033
+
2034
+ # Computed from the raw Observations before a fitting path is even chosen, and attached to
2035
+ # every return path below. This call is the dependence guard too -- there is no separate
2036
+ # check here, so there is only one implementation.
2037
+ monotonicity = check_monotonicity(obs, dependence=dependence)
2038
+
2039
+ upper_is_estimate, upper_value = _resolve_asymptote(upper, label="upper")
2040
+ lower_is_estimate, lower_value = _resolve_asymptote(lower, label="lower")
2041
+ if not upper_is_estimate and not lower_is_estimate and not upper_value > lower_value:
2042
+ raise ValueError(
2043
+ f"fit_dose_response: upper ({upper_value!r}) must exceed lower ({lower_value!r})"
2044
+ )
2045
+
2046
+ cells = obs.cells()
2047
+ cluster_ids = _unique_cluster_ids(obs.cluster)
2048
+
2049
+ control_incompatible_warning = _check_control_incompatible(
2050
+ cells, axis=obs.axis, upper_is_estimate=upper_is_estimate, upper_value=upper_value
2051
+ )
2052
+ if control_incompatible_warning is not None:
2053
+ return _finalize(
2054
+ _status_only_fit(
2055
+ obs,
2056
+ cells=cells,
2057
+ cluster_ids=cluster_ids,
2058
+ link=link,
2059
+ status=Status.CONTROL_INCOMPATIBLE,
2060
+ fit_warnings=(control_incompatible_warning,),
2061
+ ),
2062
+ monotonicity=monotonicity,
2063
+ )
2064
+
2065
+ # A log-axis observation whose only severity level is the zero-severity control (the "no
2066
+ # covariate value at all" case, with nothing *else* in the data either) has no
2067
+ # likelihood-contributing level at all. Left unchecked, this
2068
+ # reaches `_centering_and_scale`/`_transformed_span` (both fitting paths) with an empty
2069
+ # nonzero-severity array and fails with numpy's own unhelpful "zero-size array to reduction
2070
+ # operation maximum which has no identity" rather than a message this package wrote itself.
2071
+ # Checked once, here, before either fitting path is chosen -- after
2072
+ # `_check_control_incompatible` so a control with failures still reports
2073
+ # CONTROL_INCOMPATIBLE, and before `_check_separation` so a control plus exactly one other
2074
+ # level still reaches that check and reports SEPARATION as before.
2075
+ if obs.axis.scale == "log" and not any(c.severity != 0.0 for c in cells):
2076
+ raise ValueError(
2077
+ "fit_dose_response: this observation has no severity level besides the "
2078
+ "zero-severity control; a dose-response curve needs at least one severity level "
2079
+ "besides the control"
2080
+ )
2081
+
2082
+ separation_warning = _check_separation(cells, axis=obs.axis)
2083
+ if separation_warning is not None:
2084
+ return _finalize(
2085
+ _status_only_fit(
2086
+ obs,
2087
+ cells=cells,
2088
+ cluster_ids=cluster_ids,
2089
+ link=link,
2090
+ status=Status.SEPARATION,
2091
+ fit_warnings=(separation_warning,),
2092
+ ),
2093
+ monotonicity=monotonicity,
2094
+ )
2095
+
2096
+ if (
2097
+ not upper_is_estimate
2098
+ and not lower_is_estimate
2099
+ and upper_value == 1.0
2100
+ and lower_value == 0.0
2101
+ ):
2102
+ return _finalize(
2103
+ _fit_glm(obs, cells=cells, cluster_ids=cluster_ids, link=link),
2104
+ monotonicity=monotonicity,
2105
+ )
2106
+
2107
+ return _finalize(
2108
+ _fit_generic(
2109
+ obs,
2110
+ cells=cells,
2111
+ cluster_ids=cluster_ids,
2112
+ link=link,
2113
+ upper_is_estimate=upper_is_estimate,
2114
+ upper_value=upper_value,
2115
+ lower_is_estimate=lower_is_estimate,
2116
+ lower_value=lower_value,
2117
+ ),
2118
+ monotonicity=monotonicity,
2119
+ )