binference 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- binference/__init__.py +85 -0
- binference/_kernels.py +252 -0
- binference/_quadrature.py +202 -0
- binference/_simplex.py +54 -0
- binference/_utils.py +139 -0
- binference/_version.py +1 -0
- binference/basis.py +314 -0
- binference/bins.py +645 -0
- binference/density.py +323 -0
- binference/diagnostics.py +400 -0
- binference/estimator.py +197 -0
- binference/inference.py +368 -0
- binference/io.py +309 -0
- binference/likelihoods.py +1684 -0
- binference/measure.py +246 -0
- binference/model.py +833 -0
- binference/plotting.py +538 -0
- binference/priors/__init__.py +47 -0
- binference/priors/base.py +248 -0
- binference/priors/dirichlet.py +640 -0
- binference/priors/hyper.py +240 -0
- binference/priors/logistic.py +676 -0
- binference/priors/rates.py +139 -0
- binference/response.py +539 -0
- binference/results.py +2271 -0
- binference/samplers/__init__.py +19 -0
- binference/samplers/_curvature.py +67 -0
- binference/samplers/_nuts_core.py +284 -0
- binference/samplers/base.py +118 -0
- binference/samplers/bootstrap.py +136 -0
- binference/samplers/gibbs.py +287 -0
- binference/samplers/laplace.py +200 -0
- binference/samplers/nuts.py +681 -0
- binference/samplers/optimize.py +488 -0
- binference/samplers/vb.py +118 -0
- binference/selection.py +621 -0
- binference/simulate.py +720 -0
- binference/validation.py +477 -0
- binference/warnings.py +48 -0
- binference-0.1.0.dist-info/METADATA +130 -0
- binference-0.1.0.dist-info/RECORD +43 -0
- binference-0.1.0.dist-info/WHEEL +4 -0
- binference-0.1.0.dist-info/licenses/LICENSE +21 -0
binference/selection.py
ADDED
|
@@ -0,0 +1,621 @@
|
|
|
1
|
+
r"""Selection functions (detection probabilities or exposures per bin).
|
|
2
|
+
|
|
3
|
+
Selection enters the likelihood in one of three ways, and choosing the wrong one
|
|
4
|
+
biases the population (Mandel, Farr & Gair 2019). binference therefore requires
|
|
5
|
+
the mode to be stated explicitly (DECISIONS #7, #8):
|
|
6
|
+
|
|
7
|
+
``mode="on_measurement"``
|
|
8
|
+
Objects are selected on their *measured data* (e.g. a flux or
|
|
9
|
+
signal-to-noise cut), or the per-object likelihoods already include the
|
|
10
|
+
detection probability (e.g. responses that fold in efficiencies). The
|
|
11
|
+
likelihood of each detected object is used as is, and only the
|
|
12
|
+
normalisation :math:`(S\cdot\rho)^{-W}` (fractions) or
|
|
13
|
+
:math:`e^{-S\cdot\lambda}` (rates) is added. Here :math:`S_k` is the
|
|
14
|
+
probability that an object in bin *k* would be selected.
|
|
15
|
+
|
|
16
|
+
``mode="on_truth"``
|
|
17
|
+
Objects are detected with probability :math:`S_k` that depends on their
|
|
18
|
+
*true* bin, independently of their measured data given the truth (e.g. a
|
|
19
|
+
completeness that depends on true planet radius). Each object's
|
|
20
|
+
likelihood row is multiplied by :math:`S_k` *and* the normalisation is
|
|
21
|
+
added.
|
|
22
|
+
|
|
23
|
+
``Selection.combined(on_truth=C, on_measurement=T)``
|
|
24
|
+
Both at once, :math:`P(\text{obs} \mid z, d) = C(z)\,T(d)`: rows are
|
|
25
|
+
multiplied by :math:`C_k` only, and the normalisation uses
|
|
26
|
+
:math:`S_k`, the bin average of :math:`C(z)\,P(d \in A \mid z)`.
|
|
27
|
+
|
|
28
|
+
Values are detection probabilities in (0, 1] by default; ``scale="exposure"``
|
|
29
|
+
allows any positive absolute values (rates), and ``scale="relative"`` declares
|
|
30
|
+
values known only up to a constant, which is enough for fractions but not for
|
|
31
|
+
totals or rates (DECISIONS #46).
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
from collections.abc import Callable
|
|
37
|
+
from typing import Any, NamedTuple
|
|
38
|
+
|
|
39
|
+
import numpy as np
|
|
40
|
+
from scipy import special
|
|
41
|
+
|
|
42
|
+
from ._quadrature import bin_log_average
|
|
43
|
+
from ._utils import as_generator, check_positive, frozen, warn
|
|
44
|
+
from .basis import Basis
|
|
45
|
+
from .bins import Bins
|
|
46
|
+
from .density import _log_nonneg
|
|
47
|
+
from .priors.logistic import _rw_precision
|
|
48
|
+
from .warnings import ConvergenceWarning, InputWarning
|
|
49
|
+
|
|
50
|
+
__all__ = ["SELECTION_MODES", "SELECTION_SCALES", "Selection", "as_selection"]
|
|
51
|
+
|
|
52
|
+
#: The values of ``Selection.mode``: how objects are selected.
|
|
53
|
+
SELECTION_MODES = ("on_truth", "on_measurement", "combined")
|
|
54
|
+
#: The values of ``Selection.scale``: what the selection values measure.
|
|
55
|
+
SELECTION_SCALES = ("probability", "exposure", "relative")
|
|
56
|
+
|
|
57
|
+
#: A selection that changes by more than this factor between neighbouring bins raises a
|
|
58
|
+
#: warning.
|
|
59
|
+
STEEP_RATIO = 2.0
|
|
60
|
+
|
|
61
|
+
# a Beta draw can underflow to zero, which would void its bin
|
|
62
|
+
#: Smallest detection probability a draw of an inferred selection takes.
|
|
63
|
+
S_FLOOR = 1e-12
|
|
64
|
+
|
|
65
|
+
_HELP = (
|
|
66
|
+
"A selection function needs an explicit selection_mode: 'on_measurement' if objects are "
|
|
67
|
+
"selected on their measured data (e.g. a flux limit) or the likelihoods already include "
|
|
68
|
+
"the detection probability; 'on_truth' if detection depends on the true bin independently "
|
|
69
|
+
"of the measured data; Selection.combined(...) for both. See the user guide on selection "
|
|
70
|
+
"effects."
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Selection:
|
|
75
|
+
"""Per-bin detection probability (fractions) or exposure (rates), with its mode.
|
|
76
|
+
|
|
77
|
+
Parameters
|
|
78
|
+
----------
|
|
79
|
+
values : array_like (K,)
|
|
80
|
+
Strictly positive detection probabilities or exposures per bin.
|
|
81
|
+
mode : {"on_measurement", "on_truth"}
|
|
82
|
+
How selection enters the likelihood (see the module documentation);
|
|
83
|
+
:meth:`combined` builds the third mode.
|
|
84
|
+
scale : {"probability", "exposure", "relative"}
|
|
85
|
+
Whether the values are detection probabilities, exposures or relative
|
|
86
|
+
values. ``"probability"`` (default): values are detection probabilities in
|
|
87
|
+
(0, 1]. ``"exposure"``: absolute positive values, such as the exposure
|
|
88
|
+
of a rate model. ``"relative"``: only ratios between bins are known,
|
|
89
|
+
which suffices for fractions but not for rates or totals.
|
|
90
|
+
|
|
91
|
+
Attributes
|
|
92
|
+
----------
|
|
93
|
+
values : ndarray (K,)
|
|
94
|
+
:math:`S_k`, the probability that an object in bin *k* is selected.
|
|
95
|
+
row : ndarray (K,) or None
|
|
96
|
+
The factor applied to the likelihood columns (:math:`S_k` on the
|
|
97
|
+
truth, :math:`C_k` combined, ``None`` on the measurement).
|
|
98
|
+
basis : Basis or None
|
|
99
|
+
Set when the values are per component of a :class:`~binference.Basis`
|
|
100
|
+
(:math:`\\phi_k`-weighted averages) rather than per bin.
|
|
101
|
+
injections : tuple or None
|
|
102
|
+
For a selection measured by injections (:meth:`from_injections`), a named
|
|
103
|
+
tuple ``(injected, recovered, prior, smooth)``: counts per bin, the Beta prior
|
|
104
|
+
and the step scale of the smooth model (or ``None``). The selection is
|
|
105
|
+
then uncertain, and inferred jointly with the population (DECISIONS #30,
|
|
106
|
+
#80); ``values`` are its posterior means given the injections alone
|
|
107
|
+
(independent Betas) or the mode of :math:`\\mathrm{logit}\\,S` (smooth).
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
__slots__ = ("basis", "injections", "row", "scale", "values")
|
|
111
|
+
|
|
112
|
+
def __init__(self, values: Any, mode: str, *, scale: str = "probability") -> None:
|
|
113
|
+
_check_mode(mode)
|
|
114
|
+
v = _check_values(values, scale, "selection")
|
|
115
|
+
self.values = frozen(v)
|
|
116
|
+
self.row = self.values if mode == "on_truth" else None
|
|
117
|
+
self.scale = scale
|
|
118
|
+
self.basis = None
|
|
119
|
+
self.injections = None
|
|
120
|
+
|
|
121
|
+
def __setattr__(self, name: str, value: Any) -> None:
|
|
122
|
+
if name in Selection.__slots__ and not hasattr(self, name):
|
|
123
|
+
object.__setattr__(self, name, value)
|
|
124
|
+
else:
|
|
125
|
+
raise AttributeError("Selection objects are immutable.")
|
|
126
|
+
|
|
127
|
+
def _parts(self) -> tuple:
|
|
128
|
+
# the arguments of _rebuild; row as "values" when it is the values (on the truth)
|
|
129
|
+
row = "values" if self.row is self.values else self.row
|
|
130
|
+
return self.values, row, self.scale, self.basis, self.injections
|
|
131
|
+
|
|
132
|
+
def __reduce__(self) -> tuple:
|
|
133
|
+
return _rebuild, self._parts()
|
|
134
|
+
|
|
135
|
+
@classmethod
|
|
136
|
+
def from_injections(
|
|
137
|
+
cls,
|
|
138
|
+
z: Any,
|
|
139
|
+
detected: Any,
|
|
140
|
+
bins: Bins,
|
|
141
|
+
mode: str,
|
|
142
|
+
*,
|
|
143
|
+
prior: tuple[float, float] = (1.0, 1.0),
|
|
144
|
+
smooth: float | None = None,
|
|
145
|
+
) -> Selection:
|
|
146
|
+
"""A selection measured by injecting objects of known true value ``z``.
|
|
147
|
+
|
|
148
|
+
Counts the injections per bin, :math:`n_k`, and those recovered,
|
|
149
|
+
:math:`d_k`; with a :math:`\\mathrm{Beta}(a, b)` prior each :math:`S_k` has the
|
|
150
|
+
posterior :math:`\\mathrm{Beta}(a + d_k, b + n_k - d_k)` given the
|
|
151
|
+
injections alone. With ``smooth`` the :math:`S_k` are tied together by a
|
|
152
|
+
second-order random walk on :math:`y_k = \\mathrm{logit}\\,S_k`, the log density
|
|
153
|
+
gaining :math:`-\\sum_k (y_{k-1} - 2y_k + y_{k+1})^2 / 2\\tau^2`, which lends
|
|
154
|
+
sparsely injected bins strength from their neighbours.
|
|
155
|
+
|
|
156
|
+
A selection on the truth is uncertain: NUTS, MAP (``space="unconstrained"``),
|
|
157
|
+
Laplace and, for independent Betas and a conjugate prior, Gibbs infer it
|
|
158
|
+
jointly with the population, where the catalogue informs the ratios
|
|
159
|
+
between the :math:`S_k` (their level comes from the injections; DECISIONS
|
|
160
|
+
#10, #30, #79, #80); :meth:`at_mean` gives a fixed selection at
|
|
161
|
+
:attr:`values`, ignoring the uncertainty. A selection on the measurement
|
|
162
|
+
cannot be inferred (the likelihoods' cut already fixes it), and neither can a
|
|
163
|
+
selection with a rates prior; pass :meth:`at_mean` for both.
|
|
164
|
+
|
|
165
|
+
Parameters
|
|
166
|
+
----------
|
|
167
|
+
z : array_like, shape (n,) or (n, ndim)
|
|
168
|
+
True values of the injected objects.
|
|
169
|
+
detected : array_like of bool, shape (n,)
|
|
170
|
+
Whether each injected object was recovered.
|
|
171
|
+
bins : Bins
|
|
172
|
+
The bins of the population.
|
|
173
|
+
mode : {"on_truth", "on_measurement"}
|
|
174
|
+
As for :class:`Selection`.
|
|
175
|
+
prior : (float, float)
|
|
176
|
+
Beta prior :math:`(a, b)` of every :math:`S_k`, both positive.
|
|
177
|
+
smooth : float, optional
|
|
178
|
+
Step scale :math:`\\tau` of the random walk, in logit per bin (1-D bins
|
|
179
|
+
only); ``None`` (default) keeps the :math:`S_k` independent.
|
|
180
|
+
"""
|
|
181
|
+
_check_mode(mode)
|
|
182
|
+
if isinstance(bins, Basis):
|
|
183
|
+
raise ValueError(
|
|
184
|
+
"from_injections needs plain Bins: a component of a Basis is not a set of "
|
|
185
|
+
"injections, so its Beta posterior would not be conjugate."
|
|
186
|
+
)
|
|
187
|
+
d = np.asarray(detected)
|
|
188
|
+
if d.ndim != 1 or not (d.dtype == bool or np.isin(d, (0, 1)).all()):
|
|
189
|
+
raise ValueError("detected must be a 1-D array of booleans (or zeros and ones).")
|
|
190
|
+
try:
|
|
191
|
+
k = bins.index(z)
|
|
192
|
+
except ValueError as err:
|
|
193
|
+
raise ValueError(f"z: {err}") from None
|
|
194
|
+
if k.shape != d.shape:
|
|
195
|
+
raise ValueError(f"z has {k.size} injections but detected has {d.size}.")
|
|
196
|
+
try:
|
|
197
|
+
a, b = (float(x) for x in prior)
|
|
198
|
+
except (TypeError, ValueError):
|
|
199
|
+
a = b = np.nan
|
|
200
|
+
if not (a > 0 and b > 0 and np.isfinite(a) and np.isfinite(b)):
|
|
201
|
+
raise ValueError(f"prior must be two positive numbers (a, b); got {prior!r}.")
|
|
202
|
+
if smooth is not None:
|
|
203
|
+
if bins.ndim != 1 or bins.is_categorical or bins.K < 3:
|
|
204
|
+
raise ValueError(
|
|
205
|
+
"smooth needs ordered 1-D bins, at least three: the random walk of "
|
|
206
|
+
"second differences runs along them."
|
|
207
|
+
)
|
|
208
|
+
smooth = float(check_positive(smooth, "smooth"))
|
|
209
|
+
inside = k >= 0
|
|
210
|
+
if not inside.all():
|
|
211
|
+
warn(
|
|
212
|
+
f"{int((~inside).sum())} injections fall outside the bins and are ignored.",
|
|
213
|
+
InputWarning,
|
|
214
|
+
)
|
|
215
|
+
n = np.bincount(k[inside], minlength=bins.K).astype(float)
|
|
216
|
+
r = np.bincount(k[inside], weights=d[inside].astype(float), minlength=bins.K)
|
|
217
|
+
injections = _Injections(frozen(n), frozen(r), (a, b), smooth)
|
|
218
|
+
s, f = _beta_posterior(injections)
|
|
219
|
+
S = s / (s + f) # the posterior means, where the smooth model's Newton steps start
|
|
220
|
+
if smooth is not None:
|
|
221
|
+
S = special.expit(_logit_mode(special.logit(S), injections))
|
|
222
|
+
sel = cls(S, mode)
|
|
223
|
+
object.__setattr__(sel, "injections", injections)
|
|
224
|
+
return sel
|
|
225
|
+
|
|
226
|
+
def draw(self, n: int, rng: Any = None) -> np.ndarray:
|
|
227
|
+
"""``(n, K)`` posterior draws of :math:`S` given the injections alone
|
|
228
|
+
(independent Betas), floored at :data:`S_FLOOR` so that none underflows to zero.
|
|
229
|
+
|
|
230
|
+
Parameters
|
|
231
|
+
----------
|
|
232
|
+
n : int
|
|
233
|
+
Number of draws.
|
|
234
|
+
rng : int, numpy.random.Generator, or None
|
|
235
|
+
Random number generator.
|
|
236
|
+
"""
|
|
237
|
+
if self.injections is None:
|
|
238
|
+
raise ValueError("Only a selection from injections has draws.")
|
|
239
|
+
if self.injections.smooth is not None:
|
|
240
|
+
raise ValueError("Draws are available for independent Betas only (smooth=None).")
|
|
241
|
+
s, f = _beta_posterior(self.injections)
|
|
242
|
+
return _draw_beta(s, f, as_generator(rng), size=(int(n), self.K))
|
|
243
|
+
|
|
244
|
+
def at_mean(self) -> Selection:
|
|
245
|
+
"""This selection fixed at its :attr:`values`, ignoring their uncertainty."""
|
|
246
|
+
return _rebuild(*self._parts()[:4])
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def mode(self) -> str:
|
|
250
|
+
"""``"on_measurement"``, ``"on_truth"`` or ``"combined"``, from :attr:`row`."""
|
|
251
|
+
if self.row is None:
|
|
252
|
+
return "on_measurement"
|
|
253
|
+
return "on_truth" if self.row is self.values else "combined"
|
|
254
|
+
|
|
255
|
+
@classmethod
|
|
256
|
+
def from_function(
|
|
257
|
+
cls,
|
|
258
|
+
function: Callable[[np.ndarray], Any],
|
|
259
|
+
bins: Bins,
|
|
260
|
+
mode: str,
|
|
261
|
+
*,
|
|
262
|
+
scale: str = "probability",
|
|
263
|
+
tol: float = 1e-6,
|
|
264
|
+
) -> Selection:
|
|
265
|
+
"""A selection given as a function of the true value, averaged over each bin.
|
|
266
|
+
|
|
267
|
+
``function(z)`` returns the detection probability (or exposure) at
|
|
268
|
+
points ``z`` (shape ``(M,)`` in 1-D, ``(M, ndim)`` on grids, labels for
|
|
269
|
+
categories). Each bin takes its average in the measure coordinate, the
|
|
270
|
+
within-bin shape of the population (DECISIONS #10, #47), by adaptive
|
|
271
|
+
quadrature to ``tol`` relative to the largest bin average. With a
|
|
272
|
+
:class:`~binference.Basis` for ``bins``, the averages are taken on its
|
|
273
|
+
quadrature cells and weighted by each component (DECISIONS #47, #76).
|
|
274
|
+
|
|
275
|
+
Parameters
|
|
276
|
+
----------
|
|
277
|
+
function : callable
|
|
278
|
+
Detection probability or exposure at points ``z``.
|
|
279
|
+
bins : Bins or Basis
|
|
280
|
+
Bins over which ``function`` is averaged.
|
|
281
|
+
mode : {"on_measurement", "on_truth"}
|
|
282
|
+
How selection enters the likelihood, as for :class:`Selection`.
|
|
283
|
+
scale : {"probability", "exposure", "relative"}, optional
|
|
284
|
+
Whether the values are detection probabilities, exposures or relative
|
|
285
|
+
values, as for :class:`Selection`.
|
|
286
|
+
tol : float, optional
|
|
287
|
+
Quadrature tolerance relative to the largest bin average.
|
|
288
|
+
"""
|
|
289
|
+
_check_mode(mode)
|
|
290
|
+
_check_values(np.ones(2), scale, "selection") # the scale, before any quadrature
|
|
291
|
+
if isinstance(bins, Basis):
|
|
292
|
+
sel = cls.from_function(function, bins.grid, mode, scale=scale, tol=tol)
|
|
293
|
+
return _on_basis(sel, bins, bins.grid_masses)
|
|
294
|
+
return cls(_bin_average(function, bins, tol=tol)[0], mode, scale=scale)
|
|
295
|
+
|
|
296
|
+
@classmethod
|
|
297
|
+
def combined(
|
|
298
|
+
cls,
|
|
299
|
+
on_truth: Any,
|
|
300
|
+
on_measurement: Any,
|
|
301
|
+
bins: Bins | None = None,
|
|
302
|
+
*,
|
|
303
|
+
scale: str = "probability",
|
|
304
|
+
tol: float = 1e-6,
|
|
305
|
+
) -> Selection:
|
|
306
|
+
"""Selection on the truth and on the measurement at once (DECISIONS #8).
|
|
307
|
+
|
|
308
|
+
Parameters
|
|
309
|
+
----------
|
|
310
|
+
on_truth : array_like (K,) or callable
|
|
311
|
+
:math:`C`, the detection probability given the true value.
|
|
312
|
+
on_measurement : array_like (K,) or callable
|
|
313
|
+
:math:`P(d \\in A \\mid z)`, the probability that the measured data
|
|
314
|
+
pass the cut given the true value.
|
|
315
|
+
bins : Bins or Basis, optional
|
|
316
|
+
Bins over which callable parts are averaged, in the measure coordinate;
|
|
317
|
+
needed when either part is a callable (when both are, the product is
|
|
318
|
+
averaged as a whole). With a :class:`~binference.Basis` the result
|
|
319
|
+
is per component, as for :meth:`from_function`.
|
|
320
|
+
scale : {"probability", "exposure", "relative"}, optional
|
|
321
|
+
Whether the values of both parts are detection probabilities, exposures
|
|
322
|
+
or relative values, as for :class:`Selection`.
|
|
323
|
+
tol : float
|
|
324
|
+
Quadrature tolerance for callable parts (see :meth:`from_function`).
|
|
325
|
+
"""
|
|
326
|
+
if isinstance(bins, Basis): # on the quadrature cells, then per component
|
|
327
|
+
parts = [
|
|
328
|
+
p if callable(p) else _per_bin(p, bins.K)[bins.grid_to_bins]
|
|
329
|
+
for p in (on_truth, on_measurement)
|
|
330
|
+
]
|
|
331
|
+
sel = cls.combined(*parts, bins.grid, scale=scale, tol=tol)
|
|
332
|
+
return _on_basis(sel, bins, bins.grid_masses)
|
|
333
|
+
if callable(on_truth) and callable(on_measurement):
|
|
334
|
+
if bins is None:
|
|
335
|
+
raise ValueError("Selection.combined needs bins when a part is a callable.")
|
|
336
|
+
C, CT = _bin_average(on_truth, bins, on_measurement, tol=tol)
|
|
337
|
+
C = _check_values(C, scale, "on_truth")
|
|
338
|
+
return _combined(C, CT / C, scale)
|
|
339
|
+
parts = []
|
|
340
|
+
for part in (on_truth, on_measurement):
|
|
341
|
+
if callable(part):
|
|
342
|
+
if bins is None:
|
|
343
|
+
raise ValueError("Selection.combined needs bins when a part is a callable.")
|
|
344
|
+
part = _bin_average(part, bins, tol=tol)[0] # the other is constant in bins
|
|
345
|
+
parts.append(part)
|
|
346
|
+
return _combined(*parts, scale)
|
|
347
|
+
|
|
348
|
+
@property
|
|
349
|
+
def K(self) -> int:
|
|
350
|
+
return int(self.values.size)
|
|
351
|
+
|
|
352
|
+
def __repr__(self) -> str:
|
|
353
|
+
return f"Selection(K={self.K}, mode={self.mode!r}, scale={self.scale!r})"
|
|
354
|
+
|
|
355
|
+
def __eq__(self, other: object) -> bool:
|
|
356
|
+
if not isinstance(other, Selection):
|
|
357
|
+
return NotImplemented
|
|
358
|
+
return (
|
|
359
|
+
(self.mode, self.scale) == (other.mode, other.scale)
|
|
360
|
+
and np.array_equal(self.values, other.values)
|
|
361
|
+
and (self.row is None or np.array_equal(self.row, other.row))
|
|
362
|
+
and _same_injections(self.injections, other.injections)
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
def __hash__(self) -> int:
|
|
366
|
+
return hash((self.mode, self.scale, self.values.tobytes()))
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _combined(on_truth: Any, on_measurement: Any, scale: str) -> Selection:
|
|
370
|
+
# rows scaled by the truth part, the normaliser by the product
|
|
371
|
+
C = _check_values(on_truth, scale, "on_truth")
|
|
372
|
+
T = _check_values(on_measurement, scale, "on_measurement")
|
|
373
|
+
if T.shape != C.shape:
|
|
374
|
+
raise ValueError("on_truth and on_measurement must have one value per bin.")
|
|
375
|
+
return _rebuild(C * T, C, scale)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _same_injections(a: Any, b: Any) -> bool:
|
|
379
|
+
if a is None or b is None:
|
|
380
|
+
return a is b
|
|
381
|
+
return np.array_equal(a[0], b[0]) and np.array_equal(a[1], b[1]) and a[2:] == b[2:]
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
class _Injections(NamedTuple):
|
|
385
|
+
"""Injection counts per bin and the model of :math:`S` given them."""
|
|
386
|
+
|
|
387
|
+
injected: np.ndarray
|
|
388
|
+
recovered: np.ndarray
|
|
389
|
+
prior: tuple[float, float]
|
|
390
|
+
smooth: float | None
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _beta_posterior(injections: _Injections) -> tuple[np.ndarray, np.ndarray]:
|
|
394
|
+
"""Parameters ``(s, f)`` of the Beta posteriors of :math:`S` given the injections
|
|
395
|
+
alone: :math:`s = a + d`, :math:`f = b + n - d`."""
|
|
396
|
+
n, r, (a, b), _ = injections
|
|
397
|
+
return a + r, b + n - r
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _draw_beta(s: Any, f: Any, rng: np.random.Generator, size: Any = None) -> np.ndarray:
|
|
401
|
+
r"""Draws of :math:`S \sim \mathrm{Beta}(s, f)`, floored at :data:`S_FLOOR`."""
|
|
402
|
+
return np.maximum(rng.beta(s, f, size=size), S_FLOOR)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _logit_density(y: np.ndarray, injections: _Injections) -> tuple[float, np.ndarray]:
|
|
406
|
+
r"""Log density of :math:`y = \mathrm{logit}\,S` given the injections alone, with
|
|
407
|
+
the Jacobian, and its gradient: :math:`\sum_k s_k\log S_k + f_k\log(1 - S_k)` with
|
|
408
|
+
:math:`s = a + d`, :math:`f = b + n - d`, plus the random walk of the smooth model."""
|
|
409
|
+
s, f = _beta_posterior(injections)
|
|
410
|
+
tau = injections.smooth
|
|
411
|
+
lp = float(s @ special.log_expit(y) + f @ special.log_expit(-y))
|
|
412
|
+
g = s - (s + f) * special.expit(y)
|
|
413
|
+
if tau is not None:
|
|
414
|
+
d2 = np.diff(y, 2)
|
|
415
|
+
lp -= 0.5 * float(d2 @ d2) / tau**2
|
|
416
|
+
g -= np.convolve(d2, [1.0, -2.0, 1.0]) / tau**2 # D2^T d2: the stencil is symmetric
|
|
417
|
+
return lp, g
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _logit_mode(y: np.ndarray, injections: _Injections) -> np.ndarray:
|
|
421
|
+
"""The mode of :func:`_logit_density` (strictly concave), by damped Newton steps from ``y``."""
|
|
422
|
+
s, f = _beta_posterior(injections)
|
|
423
|
+
Q = _rw_precision(s.size, 2) / injections.smooth**2
|
|
424
|
+
lp, g = _logit_density(y, injections)
|
|
425
|
+
for _ in range(100):
|
|
426
|
+
S = special.expit(y)
|
|
427
|
+
step = np.linalg.solve(np.diag((s + f) * S * (1.0 - S)) + Q, g)
|
|
428
|
+
t = 1.0
|
|
429
|
+
while t > 1e-10:
|
|
430
|
+
new_lp, new_g = _logit_density(y + t * step, injections)
|
|
431
|
+
if new_lp >= lp:
|
|
432
|
+
break
|
|
433
|
+
t /= 2
|
|
434
|
+
else:
|
|
435
|
+
break
|
|
436
|
+
y, lp, g = y + t * step, new_lp, new_g
|
|
437
|
+
if np.max(np.abs(t * step)) < 1e-10:
|
|
438
|
+
return y
|
|
439
|
+
warn(
|
|
440
|
+
f"The smooth selection's mode did not converge (gradient {np.max(np.abs(g)):.1e}).",
|
|
441
|
+
ConvergenceWarning,
|
|
442
|
+
)
|
|
443
|
+
return y
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _per_bin(values: Any, K: int) -> np.ndarray:
|
|
447
|
+
v = np.asarray(values, dtype=float).ravel()
|
|
448
|
+
if v.size != K:
|
|
449
|
+
raise ValueError(f"selection must have length K={K}; got {v.size}.")
|
|
450
|
+
return v
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _on_basis(sel: Selection, basis: Basis, M: Any) -> Selection:
|
|
454
|
+
"""``sel`` given per cell of a grid (the rows of ``M``), as averages per component."""
|
|
455
|
+
if sel.injections is not None:
|
|
456
|
+
raise ValueError(
|
|
457
|
+
"A selection from injections cannot be moved onto a Basis: its Beta posteriors "
|
|
458
|
+
"are per bin. Use plain bins, or sel.at_mean() for a fixed selection."
|
|
459
|
+
)
|
|
460
|
+
row = None if sel.row is None else "values" if sel.row is sel.values else M.T @ sel.row
|
|
461
|
+
return _rebuild(M.T @ sel.values, row, sel.scale, basis)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _rebuild(
|
|
465
|
+
values: np.ndarray, row: Any, scale: str, basis: Basis | None = None, injections: Any = None
|
|
466
|
+
) -> Selection:
|
|
467
|
+
# a Selection from its arrays; row is None, "values" (on the truth) or an array
|
|
468
|
+
out = Selection.__new__(Selection)
|
|
469
|
+
out.values = frozen(np.array(values, dtype=float))
|
|
470
|
+
out.row = (
|
|
471
|
+
out.values
|
|
472
|
+
if isinstance(row, str)
|
|
473
|
+
else None
|
|
474
|
+
if row is None
|
|
475
|
+
else frozen(np.array(row, dtype=float))
|
|
476
|
+
)
|
|
477
|
+
out.scale = scale
|
|
478
|
+
out.basis = basis
|
|
479
|
+
out.injections = (
|
|
480
|
+
None
|
|
481
|
+
if injections is None
|
|
482
|
+
else _Injections(
|
|
483
|
+
frozen(np.array(injections[0])),
|
|
484
|
+
frozen(np.array(injections[1])),
|
|
485
|
+
tuple(injections[2]),
|
|
486
|
+
injections[3],
|
|
487
|
+
)
|
|
488
|
+
)
|
|
489
|
+
return out
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _check_values(values: Any, scale: str, name: str) -> np.ndarray:
|
|
493
|
+
if scale not in SELECTION_SCALES:
|
|
494
|
+
raise ValueError(f"scale must be one of {SELECTION_SCALES}; got {scale!r}.")
|
|
495
|
+
v = np.array(values, dtype=float).ravel()
|
|
496
|
+
if v.size < 2:
|
|
497
|
+
raise ValueError(f"{name} needs one value per bin.")
|
|
498
|
+
if not np.all(np.isfinite(v)) or np.any(v <= 0):
|
|
499
|
+
raise ValueError(
|
|
500
|
+
f"{name} values must be finite and strictly positive (bins that can never be "
|
|
501
|
+
"observed are not identifiable; remove them from the bins)."
|
|
502
|
+
)
|
|
503
|
+
if scale == "probability" and np.any(v > 1.0 + 1e-12):
|
|
504
|
+
raise ValueError(
|
|
505
|
+
f"{name} values are detection probabilities and must not exceed one; for "
|
|
506
|
+
"absolute exposures pass Selection(S, mode, scale='exposure'), for values known "
|
|
507
|
+
"only up to a constant Selection(S, mode, scale='relative') (DECISIONS #46)."
|
|
508
|
+
)
|
|
509
|
+
return v
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _check_mode(mode: str) -> None:
|
|
513
|
+
if mode not in ("on_truth", "on_measurement"):
|
|
514
|
+
raise ValueError(
|
|
515
|
+
"selection mode must be 'on_measurement' or 'on_truth' (or use "
|
|
516
|
+
f"Selection.combined); got {mode!r}. {_HELP}"
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _bin_average(
|
|
521
|
+
function: Callable,
|
|
522
|
+
bins: Bins,
|
|
523
|
+
factor: Callable | None = None,
|
|
524
|
+
log_weight: Callable | None = None,
|
|
525
|
+
tol: float = 1e-6,
|
|
526
|
+
) -> np.ndarray:
|
|
527
|
+
"""Weighted bin averages of a non-negative ``function`` (and of ``function * factor``).
|
|
528
|
+
|
|
529
|
+
Returns an array with one row: the average of ``function`` over each bin,
|
|
530
|
+
and a second row with the average of ``function * factor`` when ``factor``
|
|
531
|
+
is given, both from one adaptive quadrature. The weight is uniform in the
|
|
532
|
+
measure coordinate (the within-bin population shape), or ``exp(log_weight(z))``
|
|
533
|
+
per unit ``z`` when given (a population density); a bin where that weight
|
|
534
|
+
vanishes takes the unweighted average.
|
|
535
|
+
"""
|
|
536
|
+
if bins.is_categorical:
|
|
537
|
+
labels = np.asarray(bins.labels, dtype=object)
|
|
538
|
+
f = np.asarray(function(labels), dtype=float).ravel()
|
|
539
|
+
return f[None] if factor is None else np.stack([f, f * np.ravel(factor(labels))])
|
|
540
|
+
|
|
541
|
+
def logs(f, z):
|
|
542
|
+
return _log_nonneg(np.ravel(f(z)), "The values of a selection function")
|
|
543
|
+
|
|
544
|
+
def logf(z):
|
|
545
|
+
lf = logs(function, z)
|
|
546
|
+
rows = [lf] if factor is None else [lf, lf + logs(factor, z)]
|
|
547
|
+
if log_weight is not None:
|
|
548
|
+
lw = np.asarray(log_weight(z), dtype=float).ravel()
|
|
549
|
+
rows = [r + lw for r in rows] + [lw]
|
|
550
|
+
return np.stack(rows)
|
|
551
|
+
|
|
552
|
+
if log_weight is None:
|
|
553
|
+
return np.exp(bin_log_average(logf, bins, tol=tol))
|
|
554
|
+
out = bin_log_average(logf, bins, coordinate="z", tol=tol)
|
|
555
|
+
with np.errstate(invalid="ignore"):
|
|
556
|
+
avg = np.exp(out[:-1] - out[-1]) # divide by the bin's weight
|
|
557
|
+
empty = ~np.isfinite(out[-1])
|
|
558
|
+
if empty.any(): # no population in the bin: fall back to the unweighted average
|
|
559
|
+
avg[:, empty] = _bin_average(function, bins, factor, tol=tol)[:, empty]
|
|
560
|
+
return avg
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def _warn_if_steep(values: np.ndarray, bins: Bins) -> None:
|
|
564
|
+
# within-bin variation of S biases the fractions when S changes steeply (DECISIONS #47)
|
|
565
|
+
if bins.is_categorical or bins.K < 2:
|
|
566
|
+
return
|
|
567
|
+
pairs = bins.neighbors()
|
|
568
|
+
worst = float(np.abs(np.diff(np.log(values)[pairs], axis=1)).max(initial=0.0))
|
|
569
|
+
if worst > np.log(STEEP_RATIO):
|
|
570
|
+
warn(
|
|
571
|
+
f"The selection changes by a factor of {np.exp(worst):.3g} between neighbouring "
|
|
572
|
+
"bins; its variation within a bin then biases the fractions there. Pass S as a "
|
|
573
|
+
"function of the true value (averaged within bins), use narrower bins, or pass a "
|
|
574
|
+
"Selection built from bin averages.",
|
|
575
|
+
InputWarning,
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def as_selection(selection: Any, mode: str | None, bins: Bins | Basis) -> Selection | None:
|
|
580
|
+
"""Normalise user input: ``None``, a :class:`Selection`, a callable or an array plus mode.
|
|
581
|
+
|
|
582
|
+
A raw array is checked for steep changes between neighbouring bins; a
|
|
583
|
+
:class:`Selection` or a callable is taken as already averaged within bins.
|
|
584
|
+
With a :class:`~binference.Basis`, arrays and selections given per bin are
|
|
585
|
+
mapped to the components (:math:`\\phi_k`-weighted averages) and callables are
|
|
586
|
+
averaged on the basis's quadrature cells.
|
|
587
|
+
|
|
588
|
+
Parameters
|
|
589
|
+
----------
|
|
590
|
+
selection : None, Selection, callable or array_like
|
|
591
|
+
The selection as the user gave it.
|
|
592
|
+
mode : str or None
|
|
593
|
+
``selection_mode``, required with a callable or an array.
|
|
594
|
+
bins : Bins or Basis
|
|
595
|
+
Bins of the model.
|
|
596
|
+
"""
|
|
597
|
+
if selection is None:
|
|
598
|
+
if mode is not None:
|
|
599
|
+
raise ValueError("selection_mode was given without a selection function.")
|
|
600
|
+
return None
|
|
601
|
+
basis = bins if isinstance(bins, Basis) else None
|
|
602
|
+
report = bins if basis is None else basis.bins
|
|
603
|
+
if isinstance(selection, Selection):
|
|
604
|
+
if mode is not None and mode != selection.mode:
|
|
605
|
+
raise ValueError("selection_mode conflicts with the mode of the Selection object.")
|
|
606
|
+
if selection.basis is not None:
|
|
607
|
+
if selection.basis != basis:
|
|
608
|
+
raise ValueError("The Selection was built for a different basis.")
|
|
609
|
+
return selection
|
|
610
|
+
sel = selection
|
|
611
|
+
elif mode is None:
|
|
612
|
+
raise ValueError(_HELP)
|
|
613
|
+
elif callable(selection):
|
|
614
|
+
return Selection.from_function(selection, bins, mode)
|
|
615
|
+
else:
|
|
616
|
+
sel = Selection(selection, mode)
|
|
617
|
+
if sel.K == report.K:
|
|
618
|
+
_warn_if_steep(sel.values, report)
|
|
619
|
+
if sel.K != report.K:
|
|
620
|
+
raise ValueError(f"selection must have length K={report.K}; got {sel.K}.")
|
|
621
|
+
return sel if basis is None else _on_basis(sel, basis, basis.matrix)
|