robustmargin 0.2.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {robustmargin-0.2.0 → robustmargin-0.2.1}/PKG-INFO +4 -2
  2. {robustmargin-0.2.0 → robustmargin-0.2.1}/README.md +3 -1
  3. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/__init__.py +1 -1
  4. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/intervals.py +65 -9
  5. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/models.py +510 -44
  6. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/threshold.py +7 -5
  7. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/types.py +7 -1
  8. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin.egg-info/PKG-INFO +4 -2
  9. {robustmargin-0.2.0 → robustmargin-0.2.1}/LICENSE +0 -0
  10. {robustmargin-0.2.0 → robustmargin-0.2.1}/pyproject.toml +0 -0
  11. {robustmargin-0.2.0 → robustmargin-0.2.1}/setup.cfg +0 -0
  12. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/censoring.py +0 -0
  13. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/empirical.py +0 -0
  14. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/py.typed +0 -0
  15. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/ratio.py +0 -0
  16. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/report.py +0 -0
  17. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/schema/scorecard-v1.json +0 -0
  18. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/testing.py +0 -0
  19. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin/validation.py +0 -0
  20. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin.egg-info/SOURCES.txt +0 -0
  21. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin.egg-info/dependency_links.txt +0 -0
  22. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin.egg-info/requires.txt +0 -0
  23. {robustmargin-0.2.0 → robustmargin-0.2.1}/src/robustmargin.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: robustmargin
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: Robustness margins with confidence intervals, via dose-response threshold estimation.
5
5
  Author: robustmargin contributors
6
6
  License-Expression: Apache-2.0
@@ -34,9 +34,11 @@ Requires-Dist: jsonschema>=4.20; extra == "dev"
34
34
  Dynamic: license-file
35
35
 
36
36
  # robustmargin
37
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23585829.svg)](https://doi.org/10.5281/zenodo.23585829)
38
+
37
39
  Robustness margins with confidence intervals, via dose-response threshold estimation. Model-agnostic — no GPU, no ML framework required
38
40
 
39
- > **Status: latest release `0.2.0`.** Binomial dose-response fits, thresholds with profile-likelihood or delta
41
+ > **Status: latest release `0.2.1`.** Binomial dose-response fits, thresholds with profile-likelihood or delta
40
42
  > confidence intervals, censoring, ratios of two thresholds with Fieller or log-delta intervals,
41
43
  > fit diagnostics, and the grid break-point rule. Continuous outcomes, clustered and paired
42
44
  > inference, and design helpers are planned for `0.3`. While the version is below 1.0, a minor
@@ -1,7 +1,9 @@
1
1
  # robustmargin
2
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23585829.svg)](https://doi.org/10.5281/zenodo.23585829)
3
+
2
4
  Robustness margins with confidence intervals, via dose-response threshold estimation. Model-agnostic — no GPU, no ML framework required
3
5
 
4
- > **Status: latest release `0.2.0`.** Binomial dose-response fits, thresholds with profile-likelihood or delta
6
+ > **Status: latest release `0.2.1`.** Binomial dose-response fits, thresholds with profile-likelihood or delta
5
7
  > confidence intervals, censoring, ratios of two thresholds with Fieller or log-delta intervals,
6
8
  > fit diagnostics, and the grid break-point rule. Continuous outcomes, clustered and paired
7
9
  > inference, and design helpers are planned for `0.3`. While the version is below 1.0, a minor
@@ -56,7 +56,7 @@ from robustmargin.validation import (
56
56
  diagnose,
57
57
  )
58
58
 
59
- __version__ = "0.2.0"
59
+ __version__ = "0.2.1"
60
60
 
61
61
  __all__ = [
62
62
  "AdjacentPair",
@@ -40,7 +40,7 @@ from __future__ import annotations
40
40
  import math
41
41
  import warnings
42
42
  from collections.abc import Callable
43
- from dataclasses import dataclass
43
+ from dataclasses import dataclass, replace
44
44
 
45
45
  import numpy as np
46
46
  from numpy.typing import NDArray
@@ -54,10 +54,12 @@ from statsmodels.genmod.generalized_linear_model import GLM
54
54
  from statsmodels.tools.sm_exceptions import PerfectSeparationWarning
55
55
 
56
56
  from robustmargin.models import (
57
+ _NEWTON_DECREMENT_TOLERANCE,
57
58
  _STATSMODELS_LINKS,
58
59
  Fit,
59
60
  _cells_to_arrays,
60
61
  _full_binomial_log_likelihood,
62
+ _newton_decrement_half,
61
63
  _transform,
62
64
  )
63
65
  from robustmargin.types import Censoring, Definition, _check_finite_number
@@ -410,12 +412,19 @@ def _fixed_profile_llf(
410
412
  link: str,
411
413
  scale: str,
412
414
  z_star: float,
415
+ direct: Callable[[], float | None],
413
416
  ) -> float | None:
414
417
  """``ell_p(theta)`` for the fixed-asymptote path: fit the no-intercept offset GLM at this
415
418
  ``theta`` (an *optimiser*, per this module's docstring), then evaluate the full likelihood
416
- at the resulting ``(mu, s)``. Returns ``None`` (an inner-fit failure) when the GLM does not
417
- converge or its fitted slope is non-positive -- a wrong-sign slope means the profile value
418
- is the flat-curve boundary, so that side never closes there.
419
+ at the resulting ``(mu, s)``. Returns ``None`` (an inner-fit failure) when its fitted slope
420
+ is non-positive -- a wrong-sign slope means the profile value is the flat-curve boundary, so
421
+ that side never closes there.
422
+
423
+ The GLM's clipped iteratively reweighted least squares can stop short of the maximum, or
424
+ report non-convergence where a maximum exists. When it does not converge, or its slope
425
+ fails the Newton-decrement check on the stable log-likelihood (``lambda^2 / 2`` above the
426
+ shared tolerance), ``direct`` recomputes ``ell_p(theta)`` by direct constrained maximisation
427
+ of the same model.
419
428
  """
420
429
  covariate = (_transform(nonzero_severity, scale=scale) - theta).reshape(-1, 1)
421
430
  failures = nonzero_trials - nonzero_successes
@@ -439,14 +448,30 @@ def _fixed_profile_llf(
439
448
  except (np.linalg.LinAlgError, ValueError):
440
449
  return None
441
450
  if not bool(result.converged):
442
- return None
451
+ return direct()
443
452
  beta = float(result.params[0])
444
453
  if not beta > 0.0:
445
454
  return None
446
455
 
456
+ # The decrement is parametrisation-invariant; `log(beta)` keeps its finite-difference step
457
+ # relative, whatever the units of the severity axis make `beta` (the inverse slope scale).
458
+ def _log_beta_loglike(x: NDArray[np.float64]) -> float:
459
+ slope_scale = math.exp(-float(x[0]))
460
+ return _full_binomial_log_likelihood(
461
+ all_severity,
462
+ all_successes,
463
+ all_trials,
464
+ mu=theta - slope_scale * z_star,
465
+ s=slope_scale,
466
+ upper=upper,
467
+ lower=lower,
468
+ scale=scale,
469
+ link=link,
470
+ )
471
+
447
472
  s = 1.0 / beta
448
473
  mu = theta - s * z_star
449
- return _full_binomial_log_likelihood(
474
+ glm_llf = _full_binomial_log_likelihood(
450
475
  all_severity,
451
476
  all_successes,
452
477
  all_trials,
@@ -457,10 +482,20 @@ def _fixed_profile_llf(
457
482
  scale=scale,
458
483
  link=link,
459
484
  )
485
+ if _newton_decrement_half(_log_beta_loglike, np.array([math.log(beta)])) <= (
486
+ _NEWTON_DECREMENT_TOLERANCE
487
+ ):
488
+ return glm_llf
489
+ # The direct value is a maximum over the same set, so the larger of the two is the better
490
+ # estimate of the profile; a failed direct maximisation leaves the converged GLM value.
491
+ direct_llf = direct()
492
+ if direct_llf is None or not math.isfinite(glm_llf):
493
+ return direct_llf if direct_llf is not None else glm_llf
494
+ return max(glm_llf, direct_llf)
460
495
 
461
496
 
462
497
  def _profile_interval_fixed(
463
- fit: Fit, *, target: float, level: float, theta_hat: float
498
+ fit: Fit, *, definition: Definition, target: float, level: float, theta_hat: float
464
499
  ) -> IntervalResult:
465
500
  assert fit.params is not None
466
501
  upper = fit.params["upper"].value
@@ -477,7 +512,14 @@ def _profile_interval_fixed(
477
512
  llf_hat = _llf_hat(fit)
478
513
  chi2_target = float(chi2.ppf(level, 1))
479
514
 
515
+ replaced = False
516
+
480
517
  def g(theta: float) -> float | None:
518
+ def direct() -> float | None:
519
+ nonlocal replaced
520
+ replaced = True
521
+ return _estimated_profile_llf(theta, fit=fit, definition=definition)
522
+
481
523
  llf_p = _fixed_profile_llf(
482
524
  theta,
483
525
  nonzero_severity=nz_severity,
@@ -491,6 +533,7 @@ def _profile_interval_fixed(
491
533
  link=fit.link,
492
534
  scale=fit.axis.scale,
493
535
  z_star=z_star,
536
+ direct=direct,
494
537
  )
495
538
  if llf_p is None:
496
539
  return None
@@ -505,13 +548,24 @@ def _profile_interval_fixed(
505
548
  )
506
549
 
507
550
  lower_limit, upper_limit = _search_limits(fit)
508
- return _solve_profile_bracket(
551
+ result = _solve_profile_bracket(
509
552
  g,
510
553
  theta_hat=theta_hat,
511
554
  lower_limit=lower_limit,
512
555
  upper_limit=upper_limit,
513
556
  scale=fit.axis.scale,
514
557
  )
558
+ if not replaced:
559
+ return result
560
+ return replace(
561
+ result,
562
+ warnings=(
563
+ *result.warnings,
564
+ "intervals.profile_interval: the profile inner fit was replaced by direct "
565
+ "constrained maximisation of the same model at some theta, because the GLM did not "
566
+ "reach the maximum there",
567
+ ),
568
+ )
515
569
 
516
570
 
517
571
  # ------------------------------------------------------------------------------------------
@@ -731,7 +785,9 @@ def profile_interval(
731
785
  and fit.params["upper"].value == 1.0
732
786
  and fit.params["lower"].value == 0.0
733
787
  ):
734
- return _profile_interval_fixed(fit, target=target, level=level, theta_hat=theta_hat)
788
+ return _profile_interval_fixed(
789
+ fit, definition=definition, target=target, level=level, theta_hat=theta_hat
790
+ )
735
791
  return _profile_interval_estimated(fit, definition=definition, level=level, theta_hat=theta_hat)
736
792
 
737
793
 
@@ -9,10 +9,11 @@ interior maximum (wrong-sign slope, or an estimated asymptote pinned to a parame
9
9
  boundary) is reported as ``Status.NOT_CONVERGED`` with the reason in ``Fit.warnings``, never as
10
10
  a number that happens to be where the optimizer stopped.
11
11
 
12
- **Known limitation.** A near-step curve (``s -> 0``) with an estimated asymptote, or a fixed
13
- one other than ``1``/``0``, is not detected: it can return ``Status.OK`` with Wald standard
14
- errors that carry little information. With such asymptotes the pre-fit overlap check is
15
- necessary but not sufficient for an interior maximum; it is exact only for ``u = 1, l = 0``.
12
+ A curve that is a step between adjacent tested levels, with an estimated asymptote or a fixed
13
+ one other than ``1``/``0``, has no finite maximum: the log-likelihood keeps rising as ``s -> 0``.
14
+ Such a fit is reported as ``Status.NOT_CONVERGED``, found by comparing the fitted
15
+ log-likelihood with the closed-form supremum on the ``s -> 0`` boundary. The pre-fit overlap check
16
+ is exact only for ``u = 1, l = 0``.
16
17
 
17
18
  **v1 is binomial-only.** ``model`` accepts only ``"binomial"``; a new model family (``"ll4"``,
18
19
  ``"isotonic"``) is a later addition that requires a new ``schema_version``, not a silent
@@ -33,11 +34,11 @@ import math
33
34
  import warnings
34
35
  from collections.abc import Callable, Mapping
35
36
  from dataclasses import dataclass, field, replace
36
- from typing import Literal
37
+ from typing import Literal, NamedTuple
37
38
 
38
39
  import numpy as np
39
40
  from numpy.typing import ArrayLike, NDArray
40
- from scipy.special import expit, gammaln, xlogy
41
+ from scipy.special import expit, gammaln, log_expit, xlog1py, xlogy
41
42
  from scipy.special import logit as _logit
42
43
  from scipy.stats import norm
43
44
  from statsmodels.base.model import GenericLikelihoodModel
@@ -95,9 +96,10 @@ _ASYMPTOTE_PROBABILITY_EPS = 1e-6
95
96
  # statsmodels' PerfectSeparationWarning. That warning tests `allclose(fitted, observed)`, which on
96
97
  # grouped binomial data also fires on an exact fit -- every two-level design with overlap -- so it
97
98
  # is filtered to "ignore" around both GLM calls below. Every fit reaching those calls has already
98
- # passed the exact check. The check is exact for fixed u = 1, l = 0 (Albert & Anderson, 1984); with
99
- # estimated or other fixed asymptotes it is necessary but not sufficient, and a near-step curve
100
- # there is not yet detected (see the module docstring).
99
+ # passed the exact check. The check is exact for fixed u = 1, l = 0 (Albert & Anderson, 1984;
100
+ # Silvapulle, 1981, for the existence of a finite maximum under log-concave links); with
101
+ # estimated or other fixed asymptotes it is necessary but not sufficient, and a curve that is a
102
+ # step between adjacent levels there is detected after the fit (see the module docstring).
101
103
 
102
104
  # A fitted `s` greater than this factor times the span of the likelihood-contributing transformed
103
105
  # levels (the tested range on `log x` or `x`, excluding a log-axis zero-severity control, which
@@ -861,6 +863,82 @@ def _centering_and_scale(severity: NDArray[np.float64], *, scale: str) -> tuple[
861
863
  return m, d
862
864
 
863
865
 
866
+ def _f_log_cdf(z: NDArray[np.float64], link: str) -> NDArray[np.float64]:
867
+ """``log F(z)``, computed in log space so it stays finite and accurate where ``F`` is
868
+ far below the smallest double: ``norm.logcdf`` for probit, ``log_expit(z)`` for logit, and
869
+ ``log(-expm1(-exp(z)))`` for cloglog.
870
+ """
871
+ if link == "probit":
872
+ return np.asarray(norm.logcdf(z), dtype=np.float64)
873
+ if link == "logit":
874
+ return np.asarray(log_expit(z), dtype=np.float64)
875
+ # For z < -30, `-expm1(-exp(z))` is exp(z) to double precision and its log is z; evaluating
876
+ # the log directly would underflow to -inf near z = -745.
877
+ with np.errstate(over="ignore", divide="ignore"):
878
+ direct = np.log(-np.expm1(-np.exp(z)))
879
+ return np.asarray(np.where(z < -30.0, z, direct), dtype=np.float64)
880
+
881
+
882
+ def _f_log_survival(z: NDArray[np.float64], link: str) -> NDArray[np.float64]:
883
+ """``log(1 - F(z))``, the mirror of :func:`_f_log_cdf`: ``norm.logsf`` for probit,
884
+ ``log_expit(-z)`` for logit, and ``-exp(z)`` for cloglog.
885
+ """
886
+ if link == "probit":
887
+ return np.asarray(norm.logsf(z), dtype=np.float64)
888
+ if link == "logit":
889
+ return np.asarray(log_expit(-z), dtype=np.float64)
890
+ with np.errstate(over="ignore"):
891
+ return np.asarray(-np.exp(z), dtype=np.float64)
892
+
893
+
894
+ def _log_or_neg_inf(value: float) -> float:
895
+ return math.log(value) if value > 0.0 else -math.inf
896
+
897
+
898
+ def _curve_log_success_and_failure_probability(
899
+ severity: ArrayLike,
900
+ *,
901
+ mu: float,
902
+ s: float,
903
+ upper: float,
904
+ lower: float,
905
+ scale: str,
906
+ link: str,
907
+ center: float = 0.0,
908
+ spread: float = 1.0,
909
+ ) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
910
+ """``(log p, log(1 - p))`` for ``p = l + (u - l) * (1 - F(z))``, each combined in log space:
911
+ ``log p = logaddexp(log l, log(u - l) + log(1 - F))`` and
912
+ ``log(1 - p) = logaddexp(log(1 - u), log(u - l) + log F)``. Neither term is formed as the
913
+ complement of the other, so neither rounds to ``log(0)`` in a tail where the curve is
914
+ smaller than the smallest double. ``log 0`` is ``-inf``, which ``logaddexp`` absorbs.
915
+
916
+ Shares :func:`_curve_success_probability`'s zero-severity-control handling (``p(0) = u``)
917
+ and ``center``/``spread`` reparametrisation.
918
+ """
919
+ x = np.asarray(severity, dtype=np.float64)
920
+ log_p = np.empty_like(x)
921
+ log_q = np.empty_like(x)
922
+ zero_control = (scale == "log") & (x == 0.0)
923
+ nonzero = ~zero_control
924
+ log_lower = _log_or_neg_inf(lower)
925
+ log_one_minus_upper = _log_or_neg_inf(1.0 - upper)
926
+ # `l < u` holds only in exact arithmetic; a zero range is the constant curve `p = u`.
927
+ log_range = _log_or_neg_inf(upper - lower)
928
+ if np.any(nonzero):
929
+ t = (_transform(x[nonzero], scale=scale) - center) / spread
930
+ z = (t - mu) / s
931
+ # A non-finite `z` (an optimizer step that overflowed `s`) gives NaN, as it always did;
932
+ # the optimizers and the finiteness guard handle it, so the warning is not repeated here.
933
+ with np.errstate(invalid="ignore"):
934
+ log_p[nonzero] = np.logaddexp(log_lower, log_range + _f_log_survival(z, link))
935
+ log_q[nonzero] = np.logaddexp(log_one_minus_upper, log_range + _f_log_cdf(z, link))
936
+ if np.any(zero_control):
937
+ log_p[zero_control] = _log_or_neg_inf(upper)
938
+ log_q[zero_control] = log_one_minus_upper
939
+ return log_p, log_q
940
+
941
+
864
942
  def _full_binomial_log_likelihood(
865
943
  severity: NDArray[np.float64],
866
944
  successes: NDArray[np.float64],
@@ -878,13 +956,16 @@ def _full_binomial_log_likelihood(
878
956
  """The full binomial log-likelihood, including the ``lchoose`` combinatorial term, so it
879
957
  matches R ``drc``'s ``logLik()`` on both fitting paths (see
880
958
  https://github.com/robustmargin/robustmargin/blob/main/tools/drc_reference/README.md).
881
- :func:`scipy.special.xlogy` gives a 0-successes or 0-failures cell a term of exactly ``0.0``
882
- without ever evaluating ``log(0)``.
883
959
 
884
- ``center``/``spread`` are forwarded to :func:`_curve_success_probability` -- see there.
960
+ Evaluated in log space through :func:`_curve_log_success_and_failure_probability`, so a cell
961
+ deep in a tail keeps a finite, accurate term instead of the ``-inf`` that ``log`` of a
962
+ probability rounded to exactly ``0`` gives. A 0-successes or 0-failures cell has a term of
963
+ exactly ``0.0``, even where the matching log probability is ``-inf``.
964
+
965
+ ``center``/``spread`` are forwarded to the curve evaluation -- see there.
885
966
  Defaulted to the identity, so this still computes the reported ``Fit.log_likelihood`` on
886
967
  natural-scale ``mu``/``s`` against raw severity when a caller omits them (both fitting
887
- paths' final report does); only the estimated-asymptote optimiser's own iterations, which
968
+ paths' final report does); only the optimisers' own iterations, which
888
969
  work in centred-and-scaled units, pass them explicitly.
889
970
 
890
971
  Summed with the plain Python builtin ``sum`` over a fixed-order list, not ``numpy.sum``:
@@ -896,7 +977,7 @@ def _full_binomial_log_likelihood(
896
977
  sequential accumulation is a bit-exact no-op. This is what keeps the aggregation property
897
978
  test's "with control" and "without control" fits bit-identical, not merely close.
898
979
  """
899
- p = _curve_success_probability(
980
+ log_p, log_q = _curve_log_success_and_failure_probability(
900
981
  severity,
901
982
  mu=mu,
902
983
  s=s,
@@ -909,7 +990,10 @@ def _full_binomial_log_likelihood(
909
990
  )
910
991
  failures = trials - successes
911
992
  lchoose = gammaln(trials + 1.0) - gammaln(successes + 1.0) - gammaln(failures + 1.0)
912
- terms = lchoose + xlogy(successes, p) + xlogy(failures, 1.0 - p)
993
+ with np.errstate(invalid="ignore"):
994
+ success_terms = np.where(successes > 0.0, successes * log_p, 0.0)
995
+ failure_terms = np.where(failures > 0.0, failures * log_q, 0.0)
996
+ terms = lchoose + success_terms + failure_terms
913
997
  return float(sum(terms.tolist()))
914
998
 
915
999
 
@@ -1161,6 +1245,98 @@ class _GenericLogLikelihood:
1161
1245
  )
1162
1246
 
1163
1247
 
1248
+ def _pooled_rate(successes: NDArray[np.float64], trials: NDArray[np.float64]) -> float:
1249
+ return float(np.sum(successes) / np.sum(trials))
1250
+
1251
+
1252
+ def _binomial_group_log_likelihood(
1253
+ successes: NDArray[np.float64], trials: NDArray[np.float64], p: float
1254
+ ) -> float:
1255
+ """``sum k log p + (n - k) log(1 - p)`` with ``0 log 0 = 0`` (no binomial coefficient)."""
1256
+ return float(np.sum(xlogy(successes, p) + xlog1py(trials - successes, -p)))
1257
+
1258
+
1259
+ class _StepSupremum(NamedTuple):
1260
+ """The step supremum and whether its best configuration is the constant curve (``u == l``)."""
1261
+
1262
+ value: float
1263
+ degenerate: bool
1264
+
1265
+
1266
+ def _step_supremum_log_likelihood(
1267
+ severity: NDArray[np.float64],
1268
+ successes: NDArray[np.float64],
1269
+ trials: NDArray[np.float64],
1270
+ *,
1271
+ estimate_lower: bool,
1272
+ estimate_upper: bool,
1273
+ lower_value: float,
1274
+ upper_value: float,
1275
+ scale: str,
1276
+ ) -> _StepSupremum:
1277
+ """The supremum of the full binomial log-likelihood (binomial coefficients included) on the
1278
+ ``s -> 0`` boundary of the decreasing curve, where the curve is a step at ``mu``.
1279
+
1280
+ Levels are ordered by severity. For a step between two adjacent levels, the levels before it
1281
+ sit at the upper asymptote ``u`` and the levels after it at the lower asymptote ``l``. A single
1282
+ level may instead sit on the step itself, with the levels before it at ``u``, the levels after
1283
+ it at ``l``, and its own fitted probability its rate clipped to ``[l, u]``; an end level may do
1284
+ so only when the asymptote of its empty side is fixed, and a zero-severity control on a log
1285
+ axis never does (it always takes ``u``). An
1286
+ estimated asymptote is its side's pooled rate subject to ``l <= u`` (when the pooled rates
1287
+ violate it, both equal the pooled rate of the two sides together; with ``u`` fixed an
1288
+ estimated ``l`` is clipped into ``[0, u]``, and with ``l`` fixed an estimated ``u`` into
1289
+ ``[l, 1]``); a fixed asymptote keeps its value. The result is the maximum over all steps and
1290
+ free-level choices, with ``degenerate`` true when the best configuration has ``u == l`` (the
1291
+ constant curve); fewer than two levels give ``(-inf, False)``.
1292
+ """
1293
+ order = np.argsort(severity, kind="stable")
1294
+ sev = np.asarray(severity, dtype=np.float64)[order]
1295
+ k = np.asarray(successes, dtype=np.float64)[order]
1296
+ n = np.asarray(trials, dtype=np.float64)[order]
1297
+ count = k.size
1298
+ if count < 2:
1299
+ return _StepSupremum(-math.inf, False)
1300
+ coefficients = float(np.sum(gammaln(n + 1.0) - gammaln(k + 1.0) - gammaln(n - k + 1.0)))
1301
+ rates = k / n
1302
+ has_control = scale == "log" and bool(sev[0] == 0.0)
1303
+ everyone = np.arange(count)
1304
+
1305
+ # Each configuration is (baseline-side levels, free level or None, far-side levels). A free
1306
+ # level sits at the transition itself, so the sides are the levels before and after it.
1307
+ configurations: list[tuple[NDArray[np.intp], int | None, NDArray[np.intp]]] = [
1308
+ (everyone[:split], None, everyone[split:]) for split in range(1, count)
1309
+ ]
1310
+ for level in range(count):
1311
+ left, right = everyone[:level], everyone[level + 1 :]
1312
+ if has_control and level == 0:
1313
+ continue
1314
+ if (left.size == 0 and estimate_upper) or (right.size == 0 and estimate_lower):
1315
+ continue
1316
+ configurations.append((left, level, right))
1317
+
1318
+ best = _StepSupremum(-math.inf, False)
1319
+ for left, free, right in configurations:
1320
+ upper = _pooled_rate(k[left], n[left]) if estimate_upper else upper_value
1321
+ lower = _pooled_rate(k[right], n[right]) if estimate_lower else lower_value
1322
+ if estimate_upper and estimate_lower:
1323
+ if lower > upper:
1324
+ upper = lower = _pooled_rate(k, n)
1325
+ elif estimate_lower:
1326
+ lower = min(lower, upper)
1327
+ elif estimate_upper:
1328
+ upper = max(upper, lower)
1329
+ total = coefficients
1330
+ total += _binomial_group_log_likelihood(k[left], n[left], upper)
1331
+ total += _binomial_group_log_likelihood(k[right], n[right], lower)
1332
+ if free is not None:
1333
+ p_free = float(np.clip(rates[free], lower, upper))
1334
+ total += _binomial_group_log_likelihood(k[free : free + 1], n[free : free + 1], p_free)
1335
+ if total > best.value:
1336
+ best = _StepSupremum(total, abs(upper - lower) <= 1e-12)
1337
+ return best
1338
+
1339
+
1164
1340
  def _pilot_start(
1165
1341
  severity: NDArray[np.float64],
1166
1342
  successes: NDArray[np.float64],
@@ -1304,6 +1480,34 @@ def _boundary_warning(
1304
1480
  return None
1305
1481
 
1306
1482
 
1483
+ _NONFINITE_LOG_LIKELIHOOD_WARNING = (
1484
+ "fit_dose_response: the log-likelihood at the fitted parameters is not finite -- no "
1485
+ "estimate is reported"
1486
+ )
1487
+ _STEP_PLATEAU_WARNING = (
1488
+ "fit_dose_response: no fitted curve exceeded the log-likelihood of a step between adjacent "
1489
+ "tested levels (s -> 0) -- the data do not identify the slope"
1490
+ )
1491
+ _NONFINITE_ESTIMATE_WARNING = (
1492
+ "fit_dose_response: the optimizer's estimate or its log-likelihood is not finite -- no "
1493
+ "estimate is reported"
1494
+ )
1495
+ _REPLACED = "so it was replaced by direct likelihood maximisation of the same model"
1496
+ _REFIT_NOT_CONVERGED_WARNING = f"fit_dose_response: the GLM did not converge, {_REPLACED}"
1497
+ _REFIT_OPTIMALITY_WARNING = (
1498
+ f"fit_dose_response: the GLM estimate failed the optimality check (it is not a maximum of "
1499
+ f"the likelihood), {_REPLACED}"
1500
+ )
1501
+ _REFIT_INFORMATION_WARNING = (
1502
+ f"fit_dose_response: the GLM's information could not be evaluated, {_REPLACED}"
1503
+ )
1504
+ _INFORMATION_RECOVERED_WARNING = (
1505
+ "fit_dose_response: the GLM's observed information could not be evaluated (it is not "
1506
+ "finite at cells where the fitted probability is 0 or 1), so it was computed from the "
1507
+ "stable log-likelihood instead"
1508
+ )
1509
+
1510
+
1307
1511
  def _failure_warnings(reason: str, *, pilot_wrong_sign: bool) -> tuple[str, ...]:
1308
1512
  """The ``warnings`` tuple for a ``NOT_CONVERGED`` estimated-asymptote fit: ``reason`` alone,
1309
1513
  or ``reason`` plus the shared wrong-sign/flat reason when a pilot GLM slope on the (rescaled)
@@ -1315,6 +1519,16 @@ def _failure_warnings(reason: str, *, pilot_wrong_sign: bool) -> tuple[str, ...]
1315
1519
  return (reason,)
1316
1520
 
1317
1521
 
1522
+ def _step_plateau_reason(log_likelihood: float, step: _StepSupremum) -> str | None:
1523
+ """The reason when a fitted log-likelihood does not exceed the step supremum by more than the
1524
+ optimality tolerance, else ``None``. When the best step is the constant curve, the data are
1525
+ flat or rising and the reason is the shared wrong-sign/flat one.
1526
+ """
1527
+ if log_likelihood > step.value + _NEWTON_DECREMENT_TOLERANCE:
1528
+ return None
1529
+ return _WRONG_SIGN_OR_FLAT_WARNING if step.degenerate else _STEP_PLATEAU_WARNING
1530
+
1531
+
1318
1532
  def _fit_generic(
1319
1533
  obs: Observations,
1320
1534
  *,
@@ -1403,6 +1617,20 @@ def _fit_generic(
1403
1617
  exog = np.ones((severity.size, start_params.size))
1404
1618
  model = GenericLikelihoodModel(successes, exog, loglike=loglike_fn)
1405
1619
 
1620
+ # When the data are a step between adjacent levels the log-likelihood has no finite maximum
1621
+ # (it rises as s -> 0), whatever the optimizer reported: a fit that has not exceeded the
1622
+ # closed-form supremum on the s -> 0 boundary is a point on that plateau. The supremum is
1623
+ # taken over the same observations and fixed values the objective uses.
1624
+ step_supremum = _step_supremum_log_likelihood(
1625
+ severity,
1626
+ successes,
1627
+ trials,
1628
+ estimate_lower=lower_is_estimate,
1629
+ estimate_upper=upper_is_estimate,
1630
+ lower_value=lower_value,
1631
+ upper_value=upper_value,
1632
+ scale=axis.scale,
1633
+ )
1406
1634
  # The BFGS stage is only ever a starting point for the Newton polish: it routinely reports its
1407
1635
  # own `converged=False` / ConvergenceWarning even when it has landed close enough for Newton to
1408
1636
  # finish the job, so only the *final* (Newton) result's own convergence signal decides the
@@ -1429,7 +1657,7 @@ def _fit_generic(
1429
1657
  # stage already shows it, so the reason does not depend on which platform's linear
1430
1658
  # algebra happens to fail first.
1431
1659
  bfgs_params = np.asarray(bfgs_result.params, dtype=np.float64)
1432
- _, _, bfgs_lower, bfgs_upper = _unpack_optimizer_params(
1660
+ _, bfgs_s_scaled, bfgs_lower, bfgs_upper = _unpack_optimizer_params(
1433
1661
  bfgs_params,
1434
1662
  estimate_lower=lower_is_estimate,
1435
1663
  estimate_upper=upper_is_estimate,
@@ -1442,10 +1670,20 @@ def _fit_generic(
1442
1670
  estimate_upper=upper_is_estimate,
1443
1671
  lower=bfgs_lower,
1444
1672
  upper=bfgs_upper,
1445
- ) or (
1446
- "fit_dose_response: the optimizer's Hessian was singular while fitting -- "
1447
- "no interior maximum was found"
1448
1673
  )
1674
+ if (
1675
+ singular_reason is None
1676
+ and spread * bfgs_s_scaled
1677
+ > _FLAT_SLOPE_SPAN_FACTOR * _transformed_span(severity, scale=axis.scale)
1678
+ ):
1679
+ singular_reason = _WRONG_SIGN_OR_FLAT_WARNING
1680
+ if singular_reason is None:
1681
+ singular_reason = _step_plateau_reason(float(bfgs_result.llf), step_supremum)
1682
+ if singular_reason is None:
1683
+ singular_reason = (
1684
+ "fit_dose_response: the optimizer's Hessian was singular while fitting -- "
1685
+ "no interior maximum was found"
1686
+ )
1449
1687
  return _status_only_fit(
1450
1688
  obs,
1451
1689
  cells=cells,
@@ -1458,23 +1696,57 @@ def _fit_generic(
1458
1696
  converged_flag = bool(result.mle_retvals.get("converged", False))
1459
1697
  final_params = np.asarray(result.params, dtype=np.float64)
1460
1698
 
1461
- mu_scaled, s_scaled, lower, upper = _unpack_optimizer_params(
1462
- final_params,
1699
+ estimate_is_finite = bool(np.all(np.isfinite(final_params))) and math.isfinite(
1700
+ float(result.llf)
1701
+ )
1702
+ mu = s = lower = upper = math.nan
1703
+ if estimate_is_finite:
1704
+ mu_scaled, s_scaled, lower, upper = _unpack_optimizer_params(
1705
+ final_params,
1706
+ estimate_lower=lower_is_estimate,
1707
+ estimate_upper=upper_is_estimate,
1708
+ lower_value=lower_value,
1709
+ upper_value=upper_value,
1710
+ )
1711
+ # mu/s from the optimizer are on the centred-and-scaled covariate; map back to natural
1712
+ # units (the exact inverse of _centering_and_scale's reparametrisation). lower/upper are
1713
+ # already natural -- asymptotes are untouched by the covariate rescaling.
1714
+ mu = center + spread * mu_scaled
1715
+ s = spread * s_scaled
1716
+
1717
+ # An estimated asymptote at a parameter-space boundary keeps its own reason, even on data
1718
+ # that are a step. A non-finite Newton result is judged on the BFGS parameters instead.
1719
+ boundary_params = (
1720
+ final_params if estimate_is_finite else np.asarray(bfgs_result.params, dtype=np.float64)
1721
+ )
1722
+ _, _, boundary_lower, boundary_upper = _unpack_optimizer_params(
1723
+ boundary_params,
1463
1724
  estimate_lower=lower_is_estimate,
1464
1725
  estimate_upper=upper_is_estimate,
1465
1726
  lower_value=lower_value,
1466
1727
  upper_value=upper_value,
1467
1728
  )
1468
- # mu/s from the optimizer are on the centred-and-scaled covariate; map back to natural
1469
- # units (the exact inverse of _centering_and_scale's reparametrisation). lower/upper are
1470
- # already natural -- asymptotes are untouched by the covariate rescaling.
1471
- mu = center + spread * mu_scaled
1472
- s = spread * s_scaled
1729
+ boundary_message = _boundary_warning(
1730
+ boundary_params,
1731
+ estimate_lower=lower_is_estimate,
1732
+ estimate_upper=upper_is_estimate,
1733
+ lower=boundary_lower,
1734
+ upper=boundary_upper,
1735
+ )
1736
+ if boundary_message is not None:
1737
+ return _status_only_fit(
1738
+ obs,
1739
+ cells=cells,
1740
+ cluster_ids=cluster_ids,
1741
+ link=link,
1742
+ status=Status.NOT_CONVERGED,
1743
+ fit_warnings=_failure_warnings(boundary_message, pilot_wrong_sign=pilot_wrong_sign),
1744
+ )
1473
1745
 
1474
1746
  # A flat curve up to rounding, checked before anything that would build a covariance from a
1475
1747
  # near-degenerate slope.
1476
1748
  span = _transformed_span(severity, scale=axis.scale)
1477
- if s > _FLAT_SLOPE_SPAN_FACTOR * span:
1749
+ if estimate_is_finite and s > _FLAT_SLOPE_SPAN_FACTOR * span:
1478
1750
  return _status_only_fit(
1479
1751
  obs,
1480
1752
  cells=cells,
@@ -1486,21 +1758,33 @@ def _fit_generic(
1486
1758
  ),
1487
1759
  )
1488
1760
 
1489
- boundary_message = _boundary_warning(
1490
- final_params,
1491
- estimate_lower=lower_is_estimate,
1492
- estimate_upper=upper_is_estimate,
1493
- lower=lower,
1494
- upper=upper,
1761
+ # The best log-likelihood any stage reached: a stage that overshot to a non-finite value
1762
+ # does not count against the finite ones.
1763
+ best_llf = max(
1764
+ float(bfgs_result.llf),
1765
+ float(result.llf) if math.isfinite(float(result.llf)) else -math.inf,
1495
1766
  )
1496
- if boundary_message is not None:
1767
+ plateau_reason = _step_plateau_reason(best_llf, step_supremum)
1768
+ if plateau_reason is not None:
1497
1769
  return _status_only_fit(
1498
1770
  obs,
1499
1771
  cells=cells,
1500
1772
  cluster_ids=cluster_ids,
1501
1773
  link=link,
1502
1774
  status=Status.NOT_CONVERGED,
1503
- fit_warnings=_failure_warnings(boundary_message, pilot_wrong_sign=pilot_wrong_sign),
1775
+ fit_warnings=_failure_warnings(plateau_reason, pilot_wrong_sign=pilot_wrong_sign),
1776
+ )
1777
+
1778
+ if not estimate_is_finite:
1779
+ return _status_only_fit(
1780
+ obs,
1781
+ cells=cells,
1782
+ cluster_ids=cluster_ids,
1783
+ link=link,
1784
+ status=Status.NOT_CONVERGED,
1785
+ fit_warnings=_failure_warnings(
1786
+ _NONFINITE_ESTIMATE_WARNING, pilot_wrong_sign=pilot_wrong_sign
1787
+ ),
1504
1788
  )
1505
1789
 
1506
1790
  # Newton's own `converged` flag only bounds the last step, so the final log-likelihood must
@@ -1514,8 +1798,17 @@ def _fit_generic(
1514
1798
  score_tolerance = _SCORE_TOLERANCE_FACTOR * (1.0 + abs(float(result.llf)))
1515
1799
  score_not_near_zero = bool(np.any(np.abs(score) > score_tolerance))
1516
1800
 
1517
- not_converged_reason: str | None = None
1801
+ # Newton's step tolerance is below the roundoff of the numerical derivatives its step is
1802
+ # built from, so the step criterion can stall exactly at the maximum and report
1803
+ # non-convergence there. The Newton decrement of the same objective is scale-free, so it
1804
+ # decides instead.
1805
+ step_stall_at_maximum = False
1518
1806
  if not converged_flag or convergence_warned:
1807
+ decrement = _newton_decrement_half(loglike_fn, final_params)
1808
+ step_stall_at_maximum = decrement <= _NEWTON_DECREMENT_TOLERANCE
1809
+
1810
+ not_converged_reason: str | None = None
1811
+ if (not converged_flag or convergence_warned) and not step_stall_at_maximum:
1519
1812
  not_converged_reason = "fit_dose_response: the optimizer did not converge"
1520
1813
  elif llf_regressed:
1521
1814
  not_converged_reason = (
@@ -1616,6 +1909,17 @@ def _fit_generic(
1616
1909
  scale=axis.scale,
1617
1910
  link=link,
1618
1911
  )
1912
+ if not math.isfinite(log_likelihood):
1913
+ return _status_only_fit(
1914
+ obs,
1915
+ cells=cells,
1916
+ cluster_ids=cluster_ids,
1917
+ link=link,
1918
+ status=Status.NOT_CONVERGED,
1919
+ fit_warnings=_failure_warnings(
1920
+ _NONFINITE_LOG_LIKELIHOOD_WARNING, pilot_wrong_sign=pilot_wrong_sign
1921
+ ),
1922
+ )
1619
1923
 
1620
1924
  params = {
1621
1925
  "mu": Parameter(value=mu, fixed=False),
@@ -1649,6 +1953,112 @@ def _fit_generic(
1649
1953
  # ================================================================================================
1650
1954
 
1651
1955
 
1956
+ # The GLM path and the profile's inner fit accept an estimate only when its Newton decrement on
1957
+ # the stable log-likelihood, lambda^2 = g' (-H)^-1 g, satisfies lambda^2 / 2 <= this value: the
1958
+ # log-likelihood the estimate would still gain from a Newton step. Unlike a bound on the raw score,
1959
+ # it does not depend on the parametrisation or on the sample size.
1960
+ _NEWTON_DECREMENT_TOLERANCE = 1e-6
1961
+
1962
+
1963
+ _STANDARD_ERROR_STEP_FRACTION = 1e-2
1964
+
1965
+
1966
+ def _numerical_gradient_and_hessian(
1967
+ func: Callable[[NDArray[np.float64]], float],
1968
+ x: NDArray[np.float64],
1969
+ step: NDArray[np.float64] | None = None,
1970
+ ) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
1971
+ """The gradient and Hessian of ``func`` at ``x`` by central differences, with the given
1972
+ per-coordinate ``step`` or by default ``1e-4 * max(1, |x_j|)`` -- numerical differentiation
1973
+ of an already-fitted quantity, not a fitting procedure.
1974
+ """
1975
+ x = np.asarray(x, dtype=np.float64)
1976
+ size = x.size
1977
+ if step is None:
1978
+ step = 1e-4 * np.maximum(1.0, np.abs(x))
1979
+ f0 = func(x)
1980
+ gradient = np.zeros(size, dtype=np.float64)
1981
+ hessian = np.zeros((size, size), dtype=np.float64)
1982
+ for i in range(size):
1983
+ up = x.copy()
1984
+ up[i] += step[i]
1985
+ down = x.copy()
1986
+ down[i] -= step[i]
1987
+ f_up, f_down = func(up), func(down)
1988
+ gradient[i] = (f_up - f_down) / (2.0 * step[i])
1989
+ hessian[i, i] = (f_up - 2.0 * f0 + f_down) / step[i] ** 2
1990
+ for j in range(i + 1, size):
1991
+ corners = []
1992
+ for sign_i, sign_j in ((1, 1), (1, -1), (-1, 1), (-1, -1)):
1993
+ point = x.copy()
1994
+ point[i] += sign_i * step[i]
1995
+ point[j] += sign_j * step[j]
1996
+ corners.append(func(point))
1997
+ hessian[i, j] = hessian[j, i] = (corners[0] - corners[1] - corners[2] + corners[3]) / (
1998
+ 4.0 * step[i] * step[j]
1999
+ )
2000
+ return gradient, hessian
2001
+
2002
+
2003
+ def _newton_decrement_half(
2004
+ func: Callable[[NDArray[np.float64]], float], x: NDArray[np.float64]
2005
+ ) -> float:
2006
+ """``lambda^2 / 2`` for the log-likelihood ``func`` at ``x``, with ``lambda^2 = g' (-H)^-1 g``
2007
+ from the numerical score and observed information. ``inf`` when the information is not
2008
+ finite and positive-definite, which is itself a failed optimality check.
2009
+
2010
+ The finite-difference steps follow the curvature: a first pass with steps proportional to
2011
+ ``|x|`` gives the information, and the derivatives are then recomputed with each step a
2012
+ fixed small fraction of the coordinate's standard error, so truncation error stays small
2013
+ relative to the curvature on badly scaled functions. Steps are only ever reduced: enlarging
2014
+ them for a weakly identified coordinate would measure curvature far from the estimate.
2015
+ """
2016
+ gradient, hessian = _numerical_gradient_and_hessian(func, x)
2017
+ information = -(hessian + hessian.T) / 2.0
2018
+ if not bool(np.all(np.isfinite(information))) or not bool(np.all(np.isfinite(gradient))):
2019
+ return math.inf
2020
+ if float(np.min(np.linalg.eigvalsh(information))) <= 0.0:
2021
+ return math.inf
2022
+ variances = np.diag(np.linalg.inv(information))
2023
+ if bool(np.all(np.isfinite(variances))) and bool(np.all(variances > 0.0)):
2024
+ default_step = 1e-4 * np.maximum(1.0, np.abs(np.asarray(x, dtype=np.float64)))
2025
+ scaled_step = np.minimum(default_step, _STANDARD_ERROR_STEP_FRACTION * np.sqrt(variances))
2026
+ gradient, hessian = _numerical_gradient_and_hessian(func, x, step=scaled_step)
2027
+ information = -(hessian + hessian.T) / 2.0
2028
+ if not bool(np.all(np.isfinite(information))) or not bool(np.all(np.isfinite(gradient))):
2029
+ return math.inf
2030
+ if float(np.min(np.linalg.eigvalsh(information))) <= 0.0:
2031
+ return math.inf
2032
+ return float(gradient @ np.linalg.solve(information, gradient)) / 2.0
2033
+
2034
+
2035
+ def _refit_by_direct_likelihood(
2036
+ obs: Observations,
2037
+ *,
2038
+ cells: tuple[Cell, ...],
2039
+ cluster_ids: tuple[str | int, ...] | None,
2040
+ link: str,
2041
+ reason: str,
2042
+ ) -> Fit:
2043
+ """Refit the fixed ``upper=1, lower=0`` model by direct maximisation of its likelihood
2044
+ through the estimated-asymptote path with both asymptotes fixed: the same link, the same
2045
+ curve, a statsmodels optimizer. That path's own convergence, score and Hessian checks decide
2046
+ the status, and the result carries ``reason``, which states which check replaced the GLM
2047
+ estimate.
2048
+ """
2049
+ refit = _fit_generic(
2050
+ obs,
2051
+ cells=cells,
2052
+ cluster_ids=cluster_ids,
2053
+ link=link,
2054
+ upper_is_estimate=False,
2055
+ upper_value=1.0,
2056
+ lower_is_estimate=False,
2057
+ lower_value=0.0,
2058
+ )
2059
+ return replace(refit, warnings=(reason, *refit.warnings))
2060
+
2061
+
1652
2062
  def _fit_glm(
1653
2063
  obs: Observations,
1654
2064
  *,
@@ -1677,14 +2087,16 @@ def _fit_glm(
1677
2087
  glm_result = model.fit()
1678
2088
  convergence_warned = any(issubclass(w.category, ConvergenceWarning) for w in caught)
1679
2089
 
2090
+ # The GLM's clipped IRLS can report non-convergence, or converge to a point that is not the
2091
+ # maximum, on data whose maximum is finite and unique. Either case is refit as the same
2092
+ # model by direct likelihood maximisation, and the result says so.
1680
2093
  if convergence_warned or not bool(glm_result.converged):
1681
- return _status_only_fit(
2094
+ return _refit_by_direct_likelihood(
1682
2095
  obs,
1683
2096
  cells=cells,
1684
2097
  cluster_ids=cluster_ids,
1685
2098
  link=link,
1686
- status=Status.NOT_CONVERGED,
1687
- fit_warnings=("fit_dose_response: the GLM optimizer did not converge",),
2099
+ reason=_REFIT_NOT_CONVERGED_WARNING,
1688
2100
  )
1689
2101
 
1690
2102
  beta0 = float(glm_result.params[0])
@@ -1720,6 +2132,35 @@ def _fit_glm(
1720
2132
  fit_warnings=(_WRONG_SIGN_OR_FLAT_WARNING,),
1721
2133
  )
1722
2134
 
2135
+ # Optimality of the GLM estimate on the stable log-likelihood, in the GLM's own coefficients:
2136
+ # the Newton decrement is the same in any parametrisation.
2137
+ def _beta_loglike(beta: NDArray[np.float64]) -> float:
2138
+ if not float(beta[1]) > 0.0:
2139
+ return -math.inf
2140
+ return _full_binomial_log_likelihood(
2141
+ severity,
2142
+ successes,
2143
+ trials,
2144
+ mu=-float(beta[0]) / float(beta[1]),
2145
+ s=1.0 / float(beta[1]),
2146
+ upper=1.0,
2147
+ lower=0.0,
2148
+ scale=axis.scale,
2149
+ link=link,
2150
+ center=center,
2151
+ spread=spread,
2152
+ )
2153
+
2154
+ glm_beta = np.array([beta0, beta1], dtype=np.float64)
2155
+ if _newton_decrement_half(_beta_loglike, glm_beta) > _NEWTON_DECREMENT_TOLERANCE:
2156
+ return _refit_by_direct_likelihood(
2157
+ obs,
2158
+ cells=cells,
2159
+ cluster_ids=cluster_ids,
2160
+ link=link,
2161
+ reason=_REFIT_OPTIMALITY_WARNING,
2162
+ )
2163
+
1723
2164
  # Both fitting paths report covariance from the *observed* information
1724
2165
  # (the inverse of the negative Hessian of the log-likelihood at the MLE), not statsmodels'
1725
2166
  # own `cov_params()` (the *expected*/Fisher information), which for a non-canonical link
@@ -1738,6 +2179,21 @@ def _fit_glm(
1738
2179
  dtype=np.float64,
1739
2180
  )
1740
2181
  neg_observed_information = -(observed_hessian + observed_hessian.T) / 2.0
2182
+ ok_warnings: tuple[str, ...] = ()
2183
+ if not bool(np.all(np.isfinite(neg_observed_information))):
2184
+ # statsmodels divides by zero at a cell whose fitted probability is exactly 0 or 1; the
2185
+ # information of the stable log-likelihood is finite there.
2186
+ _, numerical_hessian = _numerical_gradient_and_hessian(_beta_loglike, glm_beta)
2187
+ neg_observed_information = -(numerical_hessian + numerical_hessian.T) / 2.0
2188
+ if not _is_positive_definite(neg_observed_information):
2189
+ return _refit_by_direct_likelihood(
2190
+ obs,
2191
+ cells=cells,
2192
+ cluster_ids=cluster_ids,
2193
+ link=link,
2194
+ reason=_REFIT_INFORMATION_WARNING,
2195
+ )
2196
+ ok_warnings = (_INFORMATION_RECOVERED_WARNING,)
1741
2197
  try:
1742
2198
  sigma_beta = np.linalg.inv(neg_observed_information)
1743
2199
  except np.linalg.LinAlgError:
@@ -1797,6 +2253,15 @@ def _fit_glm(
1797
2253
  scale=axis.scale,
1798
2254
  link=link,
1799
2255
  )
2256
+ if not math.isfinite(log_likelihood):
2257
+ return _status_only_fit(
2258
+ obs,
2259
+ cells=cells,
2260
+ cluster_ids=cluster_ids,
2261
+ link=link,
2262
+ status=Status.NOT_CONVERGED,
2263
+ fit_warnings=(_NONFINITE_LOG_LIKELIHOOD_WARNING,),
2264
+ )
1800
2265
 
1801
2266
  params = {
1802
2267
  "mu": Parameter(value=mu, fixed=False),
@@ -1820,7 +2285,7 @@ def _fit_glm(
1820
2285
  cells=cells,
1821
2286
  status=Status.OK,
1822
2287
  cluster_ids=cluster_ids,
1823
- warnings=(),
2288
+ warnings=ok_warnings,
1824
2289
  provenance={},
1825
2290
  )
1826
2291
 
@@ -1964,10 +2429,11 @@ def fit_dose_response(
1964
2429
  hand-rolled optimizer or IRLS loop -- for every other combination of fixed and estimated
1965
2430
  asymptotes. Both paths are checked for separation before fitting and for a non-interior maximum
1966
2431
  after fitting; either gives an explicit :class:`~robustmargin.Status` with no numbers, never a
1967
- value that merely happens to be where the optimizer stopped. One case is not yet detected: a
1968
- near-step curve (``s -> 0``) with an estimated asymptote, or a fixed one other than
1969
- ``1``/``0``, can return ``Status.OK`` with Wald standard errors that carry little
1970
- information. With such asymptotes the overlap check is necessary but not sufficient.
2432
+ value that merely happens to be where the optimizer stopped. Data that are a step between
2433
+ adjacent tested levels, with an estimated asymptote or a fixed one other than ``1``/``0``,
2434
+ have no finite maximum (the log-likelihood rises as ``s -> 0``) and give
2435
+ ``Status.NOT_CONVERGED``. With such asymptotes the overlap check alone is necessary but not
2436
+ sufficient.
1971
2437
 
1972
2438
  Before any of that, :func:`robustmargin.check_monotonicity` is run on ``obs`` directly, and
1973
2439
  its result is attached to the returned ``Fit`` as ``monotonicity`` on *every* return path
@@ -108,11 +108,13 @@ class Threshold:
108
108
  - trials are independent;
109
109
  - the target is fixed, as it is under ``absolute``.
110
110
 
111
- **A target taken from the control.** For a failed fit under ``baseline_fraction``, the
112
- target is the stated fraction of the control's observed rate, so it depends on the control's
113
- counts. The joint guarantee is not proved for that case. In simulation at ``level = 0.95``
114
- the rate of wrong exact results stayed below ``1 - level``: at most 1.4%, on a design built so
115
- that the control passes its exact test only when the target exceeds its true rate.
111
+ **A target taken from the control.** On a failed fit (``SEPARATION`` or ``NOT_CONVERGED``),
112
+ ``baseline_fraction(p)`` resolves its target from the control's observed rate, ``p * k0 / n0``
113
+ for ``k0`` successes in ``n0`` control trials. The exact claims are then tests against that
114
+ plug-in target. They carry no coverage guarantee for ``p`` times the true zero-severity rate,
115
+ which is the quantity the definition names, and the joint bound above is not proved for a
116
+ target that depends on the control's counts. For a covered claim on a failed fit, use
117
+ ``absolute(a)``.
116
118
 
117
119
  **Exact ties.** A claim is read non-strictly at the tested severities: a lower bound means the
118
120
  true rate there is at least the target, an upper bound that it is at most the target. If a
@@ -285,7 +285,13 @@ class Definition:
285
285
  @classmethod
286
286
  def baseline_fraction(cls, p: float, /) -> Definition:
287
287
  """Build a ``baseline_fraction(p)`` definition: performance falls to ``p`` times the
288
- zero-severity performance. Requires ``0 < p < 1``."""
288
+ zero-severity performance. Requires ``0 < p < 1``.
289
+
290
+ On a failed fit (``SEPARATION`` or ``NOT_CONVERGED``) the target is resolved from the
291
+ control's observed rate, ``p * k0 / n0``, and the exact claims are tests against that
292
+ plug-in target. They carry no coverage guarantee for ``p`` times the true zero-severity
293
+ rate. For a covered claim on a failed fit, use ``absolute(a)``.
294
+ """
289
295
  return cls(kind="baseline_fraction", value=p)
290
296
 
291
297
  @classmethod
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: robustmargin
3
- Version: 0.2.0
3
+ Version: 0.2.1
4
4
  Summary: Robustness margins with confidence intervals, via dose-response threshold estimation.
5
5
  Author: robustmargin contributors
6
6
  License-Expression: Apache-2.0
@@ -34,9 +34,11 @@ Requires-Dist: jsonschema>=4.20; extra == "dev"
34
34
  Dynamic: license-file
35
35
 
36
36
  # robustmargin
37
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.23585829.svg)](https://doi.org/10.5281/zenodo.23585829)
38
+
37
39
  Robustness margins with confidence intervals, via dose-response threshold estimation. Model-agnostic — no GPU, no ML framework required
38
40
 
39
- > **Status: latest release `0.2.0`.** Binomial dose-response fits, thresholds with profile-likelihood or delta
41
+ > **Status: latest release `0.2.1`.** Binomial dose-response fits, thresholds with profile-likelihood or delta
40
42
  > confidence intervals, censoring, ratios of two thresholds with Fieller or log-delta intervals,
41
43
  > fit diagnostics, and the grid break-point rule. Continuous outcomes, clustered and paired
42
44
  > inference, and design helpers are planned for `0.3`. While the version is below 1.0, a minor
File without changes
File without changes