binference 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- binference/__init__.py +85 -0
- binference/_kernels.py +252 -0
- binference/_quadrature.py +202 -0
- binference/_simplex.py +54 -0
- binference/_utils.py +139 -0
- binference/_version.py +1 -0
- binference/basis.py +314 -0
- binference/bins.py +645 -0
- binference/density.py +323 -0
- binference/diagnostics.py +400 -0
- binference/estimator.py +197 -0
- binference/inference.py +368 -0
- binference/io.py +309 -0
- binference/likelihoods.py +1684 -0
- binference/measure.py +246 -0
- binference/model.py +833 -0
- binference/plotting.py +538 -0
- binference/priors/__init__.py +47 -0
- binference/priors/base.py +248 -0
- binference/priors/dirichlet.py +640 -0
- binference/priors/hyper.py +240 -0
- binference/priors/logistic.py +676 -0
- binference/priors/rates.py +139 -0
- binference/response.py +539 -0
- binference/results.py +2271 -0
- binference/samplers/__init__.py +19 -0
- binference/samplers/_curvature.py +67 -0
- binference/samplers/_nuts_core.py +284 -0
- binference/samplers/base.py +118 -0
- binference/samplers/bootstrap.py +136 -0
- binference/samplers/gibbs.py +287 -0
- binference/samplers/laplace.py +200 -0
- binference/samplers/nuts.py +681 -0
- binference/samplers/optimize.py +488 -0
- binference/samplers/vb.py +118 -0
- binference/selection.py +621 -0
- binference/simulate.py +720 -0
- binference/validation.py +477 -0
- binference/warnings.py +48 -0
- binference-0.1.0.dist-info/METADATA +130 -0
- binference-0.1.0.dist-info/RECORD +43 -0
- binference-0.1.0.dist-info/WHEEL +4 -0
- binference-0.1.0.dist-info/licenses/LICENSE +21 -0
binference/results.py
ADDED
|
@@ -0,0 +1,2271 @@
|
|
|
1
|
+
"""Result objects returned by every inference engine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import math
|
|
7
|
+
from collections.abc import Callable, Sequence
|
|
8
|
+
from functools import partial
|
|
9
|
+
from typing import TYPE_CHECKING, Any
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
from scipy import special
|
|
13
|
+
|
|
14
|
+
from . import _kernels as kern
|
|
15
|
+
from . import diagnostics as diag
|
|
16
|
+
from ._utils import RNGLike, as_generator, check_nonnegative, chunk_slices, is_integer_array, warn
|
|
17
|
+
from .bins import Bins
|
|
18
|
+
from .measure import Measure
|
|
19
|
+
from .warnings import ConvergenceWarning, DiagnosticsError, InputWarning
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
22
|
+
from .basis import Basis
|
|
23
|
+
from .likelihoods import Likelihoods
|
|
24
|
+
from .model import PopulationModel
|
|
25
|
+
from .selection import Selection
|
|
26
|
+
|
|
27
|
+
__all__ = ["PointEstimate", "PopulationResult", "Summary"]
|
|
28
|
+
|
|
29
|
+
# the per-object methods' outputs grow with draws x objects (audit O-18)
|
|
30
|
+
#: Population draws the per-object methods use by default.
|
|
31
|
+
N_DRAWS = 500
|
|
32
|
+
#: Largest split R-hat that :meth:`PopulationResult.check` accepts by default.
|
|
33
|
+
RHAT_MAX = 1.01
|
|
34
|
+
#: Smallest bulk and tail ESS per chain that :meth:`PopulationResult.check` accepts by
|
|
35
|
+
#: default.
|
|
36
|
+
ESS_MIN_PER_CHAIN = 100.0
|
|
37
|
+
#: Chains shorter than this many draws get no R-hat or ESS.
|
|
38
|
+
MIXING_MIN_DRAWS = 8
|
|
39
|
+
#: :meth:`PopulationResult.check` warns when a NUTS chain's E-BFMI is below this.
|
|
40
|
+
EBFMI_MIN = 0.3
|
|
41
|
+
#: :meth:`PopulationResult.check` warns when a larger share of NUTS iterations hit
|
|
42
|
+
#: ``max_depth``.
|
|
43
|
+
MAX_DEPTH_SHARE = 0.05
|
|
44
|
+
#: :meth:`PopulationResult.check` warns when the summed Monte Carlo variance of
|
|
45
|
+
#: likelihoods from samples exceeds this.
|
|
46
|
+
SAMPLE_VARIANCE_MAX = 1.0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ----------------------------------------------------------------------------
|
|
50
|
+
# Summary table
|
|
51
|
+
# ----------------------------------------------------------------------------
|
|
52
|
+
def _small_concentration(model: Any) -> bool:
|
|
53
|
+
"""Whether a conjugate prior's density is unbounded at the boundary (long log tails)."""
|
|
54
|
+
prior = getattr(model, "prior", None)
|
|
55
|
+
return bool(prior is not None and prior.conjugate and prior.unbounded_density())
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Summary:
|
|
59
|
+
"""A small per-bin summary table with a readable ``repr``.
|
|
60
|
+
|
|
61
|
+
Columns are available as attributes and via :meth:`to_dict` /
|
|
62
|
+
:meth:`to_pandas`.
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
columns : dict of str to ndarray
|
|
67
|
+
The table's columns, by name.
|
|
68
|
+
title : str
|
|
69
|
+
A heading shown above the table.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(self, columns: dict[str, np.ndarray], title: str = "") -> None:
|
|
73
|
+
self.columns = columns
|
|
74
|
+
self.title = title
|
|
75
|
+
|
|
76
|
+
def __getitem__(self, key: str) -> np.ndarray:
|
|
77
|
+
return self.columns[key]
|
|
78
|
+
|
|
79
|
+
def __getattr__(self, key: str) -> np.ndarray:
|
|
80
|
+
try:
|
|
81
|
+
return self.__dict__["columns"][key]
|
|
82
|
+
except KeyError:
|
|
83
|
+
raise AttributeError(key) from None
|
|
84
|
+
|
|
85
|
+
def to_dict(self) -> dict[str, np.ndarray]:
|
|
86
|
+
return dict(self.columns)
|
|
87
|
+
|
|
88
|
+
def to_pandas(self):
|
|
89
|
+
import pandas as pd
|
|
90
|
+
|
|
91
|
+
return pd.DataFrame(self.columns)
|
|
92
|
+
|
|
93
|
+
def __len__(self) -> int:
|
|
94
|
+
return len(next(iter(self.columns.values())))
|
|
95
|
+
|
|
96
|
+
def __repr__(self) -> str:
|
|
97
|
+
names = list(self.columns)
|
|
98
|
+
rows = []
|
|
99
|
+
for i in range(len(self)):
|
|
100
|
+
row = []
|
|
101
|
+
for n in names:
|
|
102
|
+
v = self.columns[n][i]
|
|
103
|
+
if isinstance(v, float | np.floating):
|
|
104
|
+
row.append(f"{v:.4g}") # inf and nan as themselves
|
|
105
|
+
else:
|
|
106
|
+
row.append(str(v))
|
|
107
|
+
rows.append(row)
|
|
108
|
+
widths = [max(len(n), *(len(r[j]) for r in rows)) for j, n in enumerate(names)]
|
|
109
|
+
lines = [self.title] if self.title else []
|
|
110
|
+
lines.append(" ".join(n.rjust(w) for n, w in zip(names, widths, strict=True)))
|
|
111
|
+
max_rows = 40
|
|
112
|
+
shown = rows if len(rows) <= max_rows else [*rows[:20], ["..."] * len(names), *rows[-19:]]
|
|
113
|
+
lines.extend(" ".join(c.rjust(w) for c, w in zip(r, widths, strict=True)) for r in shown)
|
|
114
|
+
return "\n".join(lines)
|
|
115
|
+
|
|
116
|
+
_repr_html_ = None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _bin_labels(bins: Bins) -> np.ndarray:
|
|
120
|
+
if bins.is_categorical:
|
|
121
|
+
return np.array([str(lab) for lab in bins.labels], dtype=object)
|
|
122
|
+
if bins.ndim == 1:
|
|
123
|
+
e = bins.edges[0]
|
|
124
|
+
j = bins.cell_index[0]
|
|
125
|
+
return np.array([f"[{e[k]:.4g}, {e[k + 1]:.4g})" for k in j], dtype=object)
|
|
126
|
+
idx = np.stack(bins.cell_index, axis=1)
|
|
127
|
+
return np.array([str(tuple(int(i) for i in row)) for row in idx], dtype=object)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _hdi(x: np.ndarray, level: float) -> tuple[float, float]:
|
|
131
|
+
s = np.sort(x)
|
|
132
|
+
n = s.size
|
|
133
|
+
m = int(np.ceil(level * n))
|
|
134
|
+
if m >= n:
|
|
135
|
+
return float(s[0]), float(s[-1])
|
|
136
|
+
widths = s[m - 1 :] - s[: n - m + 1]
|
|
137
|
+
j = int(np.argmin(widths))
|
|
138
|
+
return float(s[j]), float(s[j + m - 1])
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# ----------------------------------------------------------------------------
|
|
142
|
+
# shared individual-object helpers
|
|
143
|
+
# ----------------------------------------------------------------------------
|
|
144
|
+
class _IndividualMixin:
|
|
145
|
+
"""Methods for per-object outputs that need the likelihoods (via the model).
|
|
146
|
+
|
|
147
|
+
With a :class:`~binference.Basis` the model's columns are components: these
|
|
148
|
+
methods work on the component weights and report per bin, sharing each
|
|
149
|
+
component among the bins by its mass :math:`A_{jk}` (exact for flat
|
|
150
|
+
components; for hats the likelihood's variation within a component is
|
|
151
|
+
ignored in that share).
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
model: PopulationModel | None
|
|
155
|
+
basis: Basis | None
|
|
156
|
+
bins: Bins
|
|
157
|
+
kind: str
|
|
158
|
+
method: str
|
|
159
|
+
info: dict[str, Any]
|
|
160
|
+
|
|
161
|
+
def _require_model(self) -> PopulationModel:
|
|
162
|
+
if self.model is None:
|
|
163
|
+
raise RuntimeError(
|
|
164
|
+
"This result is not attached to its likelihoods; call result.attach(likelihoods)."
|
|
165
|
+
)
|
|
166
|
+
return self.model
|
|
167
|
+
|
|
168
|
+
def _rho_draws(self) -> np.ndarray: # (n, K) fractions in the bins
|
|
169
|
+
raise NotImplementedError
|
|
170
|
+
|
|
171
|
+
def _weight_draws(self) -> np.ndarray: # (n, C) fractions of the model's components
|
|
172
|
+
raise NotImplementedError
|
|
173
|
+
|
|
174
|
+
def _fractions(self, values: np.ndarray) -> np.ndarray:
|
|
175
|
+
"""Values ``(..., C)`` as fractions: rates normalised, fractions as they are."""
|
|
176
|
+
return values / values.sum(axis=-1, keepdims=True) if self.kind == "rates" else values
|
|
177
|
+
|
|
178
|
+
def plot_individual(self, i: int, **kwargs: Any):
|
|
179
|
+
"""Object ``i``'s own likelihood against its hierarchical posterior (see
|
|
180
|
+
:func:`binference.plotting.plot_individual`).
|
|
181
|
+
|
|
182
|
+
Parameters
|
|
183
|
+
----------
|
|
184
|
+
i : int
|
|
185
|
+
Object index.
|
|
186
|
+
**kwargs
|
|
187
|
+
Passed to :func:`binference.plotting.plot_individual`.
|
|
188
|
+
"""
|
|
189
|
+
from .plotting import plot_individual
|
|
190
|
+
|
|
191
|
+
return plot_individual(self, i, **kwargs)
|
|
192
|
+
|
|
193
|
+
def _select_draws(self, n_draws: int | None, components: bool = False) -> np.ndarray:
|
|
194
|
+
R = self._weight_draws() if components else self._rho_draws()
|
|
195
|
+
return R[_draw_index(R.shape[0], n_draws)]
|
|
196
|
+
|
|
197
|
+
def _inferred_selection(self) -> np.ndarray | None:
|
|
198
|
+
"""Per-draw selection ``(chain, draw, K)`` when a selection from injections or a
|
|
199
|
+
tallied response (its efficiencies) is inferred with the population."""
|
|
200
|
+
if "response_draws" in self.info:
|
|
201
|
+
return np.asarray(self.info["response_draws"]).sum(axis=-2)
|
|
202
|
+
return self.info.get("selection_draws")
|
|
203
|
+
|
|
204
|
+
def _joint_selection(self, n_draws: int | None) -> np.ndarray | None:
|
|
205
|
+
""":meth:`_inferred_selection` as ``(n_draws, K)``, thinned as the population."""
|
|
206
|
+
S = self._inferred_selection()
|
|
207
|
+
return None if S is None else _flat_draws(S, 1, n_draws)
|
|
208
|
+
|
|
209
|
+
def _response_rows(self, n_draws: int | None) -> np.ndarray | None:
|
|
210
|
+
"""An inferred response's draws ``(n_draws, J, K)``, thinned as the population."""
|
|
211
|
+
Rd = self.info.get("response_draws")
|
|
212
|
+
return None if Rd is None else _flat_draws(Rd, 2, n_draws)
|
|
213
|
+
|
|
214
|
+
def _object_draws(self, n_draws: int | None):
|
|
215
|
+
"""The likelihoods and per-draw column weights for per-object methods.
|
|
216
|
+
|
|
217
|
+
With a selection inferred jointly (DECISIONS #79) these are the bare
|
|
218
|
+
likelihoods and :math:`\\rho\\circ S` with each draw's own :math:`S`, which is
|
|
219
|
+
also returned; with a response inferred jointly (DECISIONS #81), each draw's
|
|
220
|
+
rows ``(n_draws, N, K)`` and :math:`\\rho`; otherwise the model's likelihoods
|
|
221
|
+
(selection on the truth applied) and :math:`\\rho`, and ``None``.
|
|
222
|
+
"""
|
|
223
|
+
model = self._require_model()
|
|
224
|
+
R = self._select_draws(n_draws, components=True)
|
|
225
|
+
Rd = self._response_rows(n_draws)
|
|
226
|
+
if Rd is not None:
|
|
227
|
+
return Rd[:, model.likelihoods.meta["measured_bin"]], R, None
|
|
228
|
+
S = self._joint_selection(n_draws) # a selection: no response was inferred
|
|
229
|
+
return *model._rows(R, S), S
|
|
230
|
+
|
|
231
|
+
def _to_bins(self, P: np.ndarray) -> np.ndarray:
|
|
232
|
+
"""Per-component values ``(..., C)`` summed into the bins ``(..., K)``."""
|
|
233
|
+
return P if self.basis is None else self.basis.to_bins(P)
|
|
234
|
+
|
|
235
|
+
def _labels_to_bins(self, labels: np.ndarray, gen: np.random.Generator) -> np.ndarray:
|
|
236
|
+
"""Component labels as bin labels, each drawn from its component's shares."""
|
|
237
|
+
if self.basis is None or self.basis.is_identity:
|
|
238
|
+
return labels
|
|
239
|
+
S = self.basis.shares # rows sum to one: invert their cumulative sums
|
|
240
|
+
cum = np.cumsum(S.data)
|
|
241
|
+
flat = labels.ravel()
|
|
242
|
+
first, last = S.indptr[flat], S.indptr[flat + 1] - 1
|
|
243
|
+
start = np.concatenate(([0.0], cum))[first]
|
|
244
|
+
pos = np.searchsorted(cum, start + gen.random(flat.size), side="right")
|
|
245
|
+
return S.indices[np.clip(pos, first, last)].reshape(labels.shape)
|
|
246
|
+
|
|
247
|
+
def _counts_to_bins(self, counts: np.ndarray, gen: np.random.Generator) -> np.ndarray:
|
|
248
|
+
"""Integer counts per component as counts per bin, shared multinomially."""
|
|
249
|
+
if self.basis is None or self.basis.is_identity:
|
|
250
|
+
return counts
|
|
251
|
+
S = self.basis.shares
|
|
252
|
+
n = kern._multinomial_csr(S, np.round(counts).astype(np.int64), np.ones(self.bins.K), gen)
|
|
253
|
+
return np.bincount(S.indices, weights=n, minlength=self.bins.K)
|
|
254
|
+
|
|
255
|
+
def _new_likelihoods(self, likelihoods: Any) -> Likelihoods:
|
|
256
|
+
from .likelihoods import Likelihoods
|
|
257
|
+
|
|
258
|
+
model = self._require_model()
|
|
259
|
+
basis = model.likelihoods.basis
|
|
260
|
+
lk = (
|
|
261
|
+
likelihoods
|
|
262
|
+
if isinstance(likelihoods, Likelihoods)
|
|
263
|
+
else Likelihoods(likelihoods, basis or model.bins)
|
|
264
|
+
)
|
|
265
|
+
if (
|
|
266
|
+
lk.K != model.K
|
|
267
|
+
or lk.basis != basis
|
|
268
|
+
or (basis is None and not _same_bins(lk.bins, model.bins))
|
|
269
|
+
):
|
|
270
|
+
raise ValueError("New likelihoods must use the same bins (and basis) as the fit.")
|
|
271
|
+
return lk
|
|
272
|
+
|
|
273
|
+
best: PointEstimate
|
|
274
|
+
|
|
275
|
+
def assignment_probabilities_loo(
|
|
276
|
+
self, idx: Any = None, n_draws: int | None = N_DRAWS, chunk: int = 4096
|
|
277
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
278
|
+
r"""Each object's posterior over bins under the population fitted without it,
|
|
279
|
+
:math:`P(z_i = k \mid D_{-i})`: leave-one-out, for calibration checks
|
|
280
|
+
(DECISIONS #52).
|
|
281
|
+
|
|
282
|
+
Pareto-smoothed importance sampling over the posterior draws (Vehtari, Gelman &
|
|
283
|
+
Gabry 2017): draw :math:`s` has the weight :math:`p(d_i \mid \rho_s)^{-w_i}`
|
|
284
|
+
(the whole row, with its multiplicity, is left out), smoothed per object by
|
|
285
|
+
:func:`~binference.diagnostics.psis`. Objects whose Pareto :math:`\hat k` is at
|
|
286
|
+
or above :func:`~binference.diagnostics.psis_threshold` are flagged with a
|
|
287
|
+
warning: their estimate is unreliable, and refitting without them is the remedy.
|
|
288
|
+
|
|
289
|
+
The draws must come from the posterior or an approximation of it (bootstrap
|
|
290
|
+
draws are refused). In a rate model an object's factor is
|
|
291
|
+
:math:`\Lambda\,L_i\cdot\rho`; with a selection in a fractions model,
|
|
292
|
+
:math:`L_i\cdot\rho / S\cdot\rho`. An object impossible under some draw gets
|
|
293
|
+
:math:`\hat k = \infty` and a zero row.
|
|
294
|
+
|
|
295
|
+
Returns ``(probabilities, k_hat)``: ``(n, K)`` and ``(n,)`` for the objects
|
|
296
|
+
``idx`` (all by default), ``chunk`` objects at a time.
|
|
297
|
+
|
|
298
|
+
Parameters
|
|
299
|
+
----------
|
|
300
|
+
idx : index, optional
|
|
301
|
+
Objects to use; ``None`` uses all of them.
|
|
302
|
+
n_draws : int, optional
|
|
303
|
+
Population draws used; ``None`` uses all of them.
|
|
304
|
+
chunk : int, optional
|
|
305
|
+
Objects processed per block, which bounds memory.
|
|
306
|
+
"""
|
|
307
|
+
if n_draws is not None and n_draws < 5:
|
|
308
|
+
raise ValueError(f"n_draws must be at least 5 for leave-one-out; got {n_draws}.")
|
|
309
|
+
if self.method == "bootstrap":
|
|
310
|
+
raise ValueError("Leave-one-out reweights posterior draws; bootstrap draws are not.")
|
|
311
|
+
model = self._require_model()
|
|
312
|
+
L, R, S = self._object_draws(n_draws)
|
|
313
|
+
m = R.shape[0]
|
|
314
|
+
if m < 5:
|
|
315
|
+
raise ValueError(f"Leave-one-out needs posterior draws (at least 5); this has {m}.")
|
|
316
|
+
if self.kind == "rates": # Poisson: the object's factor is Lambda (L_i . rho)
|
|
317
|
+
log_norm = -np.log(_flat_draws(self._params, 1, n_draws).sum(axis=1))
|
|
318
|
+
else:
|
|
319
|
+
norm = self._norms(R, S, n_draws)
|
|
320
|
+
log_norm = 0.0 if norm is None else np.log(norm)
|
|
321
|
+
rows = np.atleast_1d(np.arange(model.N)[slice(None) if idx is None else idx])
|
|
322
|
+
w, k_hat = model._w[rows], np.empty(rows.size)
|
|
323
|
+
|
|
324
|
+
def weigh(sl, T): # PSIS weights of the objects sl from p(d_i | rho_s) ~ T
|
|
325
|
+
with np.errstate(divide="ignore"):
|
|
326
|
+
lw = np.log(T)
|
|
327
|
+
lw -= log_norm
|
|
328
|
+
lw *= -w[sl, None]
|
|
329
|
+
bad = ~np.all(np.isfinite(lw), axis=1) # impossible under some draw
|
|
330
|
+
lw[bad] = 0.0
|
|
331
|
+
lw, k_hat[sl] = diag._psis_rows(lw)
|
|
332
|
+
k_hat[sl][bad], lw[bad] = np.inf, -np.inf
|
|
333
|
+
return np.exp(lw, out=lw)
|
|
334
|
+
|
|
335
|
+
out = _mean_responsibilities(_objects(L, model.N, idx), R, chunk, weigh)
|
|
336
|
+
bad = k_hat >= diag.psis_threshold(m)
|
|
337
|
+
if bad.any():
|
|
338
|
+
warn(
|
|
339
|
+
f"{int(bad.sum())} of {rows.size} objects have Pareto k-hat >= "
|
|
340
|
+
f"{diag.psis_threshold(m):.2f}: their leave-one-out posteriors are unreliable "
|
|
341
|
+
f"(first: {rows[bad][:5].tolist()}); refit without them, or use more draws.",
|
|
342
|
+
ConvergenceWarning,
|
|
343
|
+
)
|
|
344
|
+
return self._to_bins(out), k_hat
|
|
345
|
+
|
|
346
|
+
# per-object outputs at the best fit (DECISIONS #51)
|
|
347
|
+
def assignment_probabilities_best(self, idx: Any = None, **kwargs: Any) -> np.ndarray:
|
|
348
|
+
""":meth:`assignment_probabilities` at the best fit, :attr:`best`.
|
|
349
|
+
|
|
350
|
+
Parameters
|
|
351
|
+
----------
|
|
352
|
+
idx : index, optional
|
|
353
|
+
Objects to use; ``None`` uses all of them.
|
|
354
|
+
**kwargs
|
|
355
|
+
Further arguments of :meth:`assignment_probabilities`.
|
|
356
|
+
"""
|
|
357
|
+
return self.best.assignment_probabilities(idx, **kwargs)
|
|
358
|
+
|
|
359
|
+
def sample_assignments_best(self, idx: Any = None, **kwargs: Any) -> np.ndarray:
|
|
360
|
+
""":meth:`sample_assignments` at the best fit, :attr:`best`.
|
|
361
|
+
|
|
362
|
+
Parameters
|
|
363
|
+
----------
|
|
364
|
+
idx : index, optional
|
|
365
|
+
Objects to use; ``None`` uses all of them.
|
|
366
|
+
**kwargs
|
|
367
|
+
Further arguments of :meth:`sample_assignments`.
|
|
368
|
+
"""
|
|
369
|
+
return self.best.sample_assignments(idx, **kwargs)
|
|
370
|
+
|
|
371
|
+
def sample_values_best(self, idx: Any = None, **kwargs: Any) -> np.ndarray:
|
|
372
|
+
""":meth:`sample_values` at the best fit, :attr:`best`.
|
|
373
|
+
|
|
374
|
+
Parameters
|
|
375
|
+
----------
|
|
376
|
+
idx : index, optional
|
|
377
|
+
Objects to use; ``None`` uses all of them.
|
|
378
|
+
**kwargs
|
|
379
|
+
Further arguments of :meth:`sample_values`.
|
|
380
|
+
"""
|
|
381
|
+
return self.best.sample_values(idx, **kwargs)
|
|
382
|
+
|
|
383
|
+
def realized_counts_best(self, **kwargs: Any) -> np.ndarray:
|
|
384
|
+
""":meth:`realized_counts` at the best fit, :attr:`best`.
|
|
385
|
+
|
|
386
|
+
Parameters
|
|
387
|
+
----------
|
|
388
|
+
**kwargs
|
|
389
|
+
Passed to :meth:`realized_counts`.
|
|
390
|
+
"""
|
|
391
|
+
return self.best.realized_counts(**kwargs)
|
|
392
|
+
|
|
393
|
+
def predict_proba_best(self, likelihoods: Likelihoods | Any, **kwargs: Any) -> np.ndarray:
|
|
394
|
+
""":meth:`predict_proba` at the best fit, :attr:`best`.
|
|
395
|
+
|
|
396
|
+
Parameters
|
|
397
|
+
----------
|
|
398
|
+
likelihoods : Likelihoods or array_like (N, K)
|
|
399
|
+
Likelihoods of the objects.
|
|
400
|
+
**kwargs
|
|
401
|
+
Passed to :meth:`predict_proba`.
|
|
402
|
+
"""
|
|
403
|
+
return self.best.predict_proba(likelihoods, **kwargs)
|
|
404
|
+
|
|
405
|
+
def log_predictive_best(self, likelihoods: Likelihoods | Any, **kwargs: Any) -> np.ndarray:
|
|
406
|
+
""":meth:`log_predictive` at the best fit, :attr:`best`.
|
|
407
|
+
|
|
408
|
+
Parameters
|
|
409
|
+
----------
|
|
410
|
+
likelihoods : Likelihoods or array_like (N, K)
|
|
411
|
+
Likelihoods of the objects.
|
|
412
|
+
**kwargs
|
|
413
|
+
Passed to :meth:`log_predictive`.
|
|
414
|
+
"""
|
|
415
|
+
return self.best.log_predictive(likelihoods, **kwargs)
|
|
416
|
+
|
|
417
|
+
def assignment_probabilities(
|
|
418
|
+
self, idx: Any = None, n_draws: int | None = N_DRAWS, chunk: int = 4096
|
|
419
|
+
) -> np.ndarray:
|
|
420
|
+
r"""Posterior probability that each object lies in each bin, :math:`P(z_i = k \mid D)`.
|
|
421
|
+
|
|
422
|
+
Averages the per-object conditional posteriors
|
|
423
|
+
:math:`r_{ik}(\rho)` over (up to ``n_draws``) population draws: the
|
|
424
|
+
"hierarchical shrinkage" posteriors of the individual objects.
|
|
425
|
+
|
|
426
|
+
Returns an ``(n, K)`` array for the objects ``idx`` (all by default).
|
|
427
|
+
|
|
428
|
+
Parameters
|
|
429
|
+
----------
|
|
430
|
+
idx : index, optional
|
|
431
|
+
Objects to use; ``None`` uses all of them.
|
|
432
|
+
n_draws : int, optional
|
|
433
|
+
Population draws used; ``None`` uses all of them.
|
|
434
|
+
chunk : int, optional
|
|
435
|
+
Objects processed per block, which bounds memory.
|
|
436
|
+
"""
|
|
437
|
+
model = self._require_model()
|
|
438
|
+
L, R, _ = self._object_draws(n_draws)
|
|
439
|
+
return self._to_bins(_mean_responsibilities(_objects(L, model.N, idx), R, chunk))
|
|
440
|
+
|
|
441
|
+
def sample_assignments(
|
|
442
|
+
self, idx: Any = None, n_draws: int | None = N_DRAWS, rng: RNGLike = None
|
|
443
|
+
) -> np.ndarray:
|
|
444
|
+
"""Joint draws of the bin labels of objects ``idx`` (all by default).
|
|
445
|
+
|
|
446
|
+
For each of (up to ``n_draws``) population draws :math:`\\rho^{(s)}`, labels are
|
|
447
|
+
drawn from :math:`z_i \\sim \\mathrm{Cat}(r_i(\\rho^{(s)}))`; together these
|
|
448
|
+
are joint draws from :math:`p(\\rho, z \\mid D)`, exact given exact draws of
|
|
449
|
+
:math:`\\rho` (approximate engines carry their approximation over). Returns an
|
|
450
|
+
``(n_draws, n_objects)`` integer array.
|
|
451
|
+
|
|
452
|
+
Parameters
|
|
453
|
+
----------
|
|
454
|
+
idx : index, optional
|
|
455
|
+
Objects to use; ``None`` uses all of them.
|
|
456
|
+
n_draws : int, optional
|
|
457
|
+
Population draws used; ``None`` uses all of them.
|
|
458
|
+
rng : int, Generator or None
|
|
459
|
+
Random number generator.
|
|
460
|
+
"""
|
|
461
|
+
gen = as_generator(rng)
|
|
462
|
+
return self._labels_to_bins(self._component_labels(idx, n_draws, gen), gen)
|
|
463
|
+
|
|
464
|
+
def _component_labels(self, idx: Any, n_draws: int | None, gen: np.random.Generator):
|
|
465
|
+
model = self._require_model()
|
|
466
|
+
L, R, _ = self._object_draws(n_draws)
|
|
467
|
+
L = _objects(L, model.N, idx)
|
|
468
|
+
return np.stack(
|
|
469
|
+
[kern.sample_labels(_draw_rows(L, s), rho, gen) for s, rho in enumerate(R)]
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
def realized_counts(
|
|
473
|
+
self, n_draws: int | None = N_DRAWS, rng: RNGLike = None, full: bool = False
|
|
474
|
+
) -> np.ndarray:
|
|
475
|
+
"""Draws of the realised histogram of the observed objects' true bins.
|
|
476
|
+
|
|
477
|
+
Unlike the population fractions (which include the shot noise of
|
|
478
|
+
drawing ``N`` objects from a population), this is the distribution of
|
|
479
|
+
``n_k = sum_i w_i 1[z_i = k]`` for *these* objects. With ``full=True`` the
|
|
480
|
+
objects the selection missed are added, drawn for each draw as
|
|
481
|
+
:math:`\\mathrm{NegBin}(W, S\\cdot\\rho)` of them in bins
|
|
482
|
+
:math:`\\propto \\rho\\circ(1 - S)`: the counts of the full sample (DECISIONS #53).
|
|
483
|
+
Returns ``(n_draws, K)`` weighted counts.
|
|
484
|
+
|
|
485
|
+
Parameters
|
|
486
|
+
----------
|
|
487
|
+
n_draws : int, optional
|
|
488
|
+
Population draws used; ``None`` uses all of them.
|
|
489
|
+
rng : int, Generator or None
|
|
490
|
+
Random number generator.
|
|
491
|
+
full : bool, optional
|
|
492
|
+
Add the objects the selection missed.
|
|
493
|
+
"""
|
|
494
|
+
model = self._require_model()
|
|
495
|
+
gen = as_generator(rng)
|
|
496
|
+
S = self._missed_selection(n_draws) if full else None
|
|
497
|
+
L, R, _ = self._object_draws(n_draws)
|
|
498
|
+
comps = self._select_draws(n_draws, components=True) # R holds rho*S when S is inferred
|
|
499
|
+
w = model._w
|
|
500
|
+
unit = bool(np.all(w == 1))
|
|
501
|
+
integer = is_integer_array(w)
|
|
502
|
+
out = np.empty((R.shape[0], self.bins.K))
|
|
503
|
+
for s, rho in enumerate(R):
|
|
504
|
+
Ls = _draw_rows(L, s)
|
|
505
|
+
if integer:
|
|
506
|
+
n = kern.sample_counts(Ls, w, rho, gen, unit_weights=unit)[0]
|
|
507
|
+
out[s] = self._counts_to_bins(n, gen)
|
|
508
|
+
else:
|
|
509
|
+
labels = self._labels_to_bins(kern.sample_labels(Ls, rho, gen), gen)
|
|
510
|
+
out[s] = np.bincount(labels, weights=w, minlength=self.bins.K)
|
|
511
|
+
if full:
|
|
512
|
+
Ss = S if S.ndim == 1 else S[s]
|
|
513
|
+
missed = model._draw_missed(comps[s], np.zeros(comps.shape[1]), gen, S=Ss)
|
|
514
|
+
out[s] += self._counts_to_bins(missed, gen)
|
|
515
|
+
return out
|
|
516
|
+
|
|
517
|
+
def _missed_selection(self, n_draws: int | None) -> np.ndarray:
|
|
518
|
+
# the detection probabilities: one row, or one per draw when inferred
|
|
519
|
+
model = self._require_model()
|
|
520
|
+
sel = model.selection_spec
|
|
521
|
+
if self.kind == "rates" or (sel is not None and sel.scale == "exposure"):
|
|
522
|
+
raise ValueError("Full-sample counts need detection probabilities, not exposures.")
|
|
523
|
+
S = self._joint_selection(n_draws)
|
|
524
|
+
S = model.total_exposure if S is None else S
|
|
525
|
+
if S is None:
|
|
526
|
+
raise ValueError(
|
|
527
|
+
"Full-sample counts need an absolute selection; this one is relative, so "
|
|
528
|
+
"the number of missed objects is not identified (DECISIONS #46)."
|
|
529
|
+
)
|
|
530
|
+
return S
|
|
531
|
+
|
|
532
|
+
def _selected_of(self, W: np.ndarray) -> np.ndarray:
|
|
533
|
+
"""The selected population :math:`\\phi = S\\circ w / (S\\cdot w)` per bin, for rows of
|
|
534
|
+
component weights ``W`` (all draws, or the estimate), with each draw's own S
|
|
535
|
+
when the selection is inferred (DECISIONS #49)."""
|
|
536
|
+
S = self._joint_selection(None) # all draws, as W
|
|
537
|
+
if S is None and self.selection is not None:
|
|
538
|
+
S = self.selection.values
|
|
539
|
+
if S is None:
|
|
540
|
+
return self._to_bins(W)
|
|
541
|
+
P = W * S
|
|
542
|
+
return self._to_bins(P / P.sum(axis=-1, keepdims=True))
|
|
543
|
+
|
|
544
|
+
@property
|
|
545
|
+
def data_support(self) -> np.ndarray:
|
|
546
|
+
"""Objects' worth of likelihood in each bin, :math:`\\sum_i w_i \\ell_{ik}` with each
|
|
547
|
+
object's likelihood normalised over the bins: below about one, a bin's fraction
|
|
548
|
+
is set by the prior (DECISIONS #41)."""
|
|
549
|
+
if "data_support" not in self._cache:
|
|
550
|
+
model = self._require_model()
|
|
551
|
+
L = model._L # with a selection on the truth applied, as the fit sees them
|
|
552
|
+
tot = kern.matvec(L, np.ones(L.shape[1]))
|
|
553
|
+
w = np.divide(model._w, tot, out=np.zeros_like(tot), where=tot > 0)
|
|
554
|
+
self._cache["data_support"] = self._to_bins(kern.rmatvec(L, w))
|
|
555
|
+
return self._cache["data_support"]
|
|
556
|
+
|
|
557
|
+
def prior_information_ratio(self, h: float = 1e-4) -> float:
|
|
558
|
+
r"""Efron's (2016) ratio of prior to data information,
|
|
559
|
+
:math:`\mathrm{tr}(-H_\text{prior}) / \mathrm{tr}(-H_\text{data})`.
|
|
560
|
+
|
|
561
|
+
The Hessians are of the log densities in the prior's unconstrained coordinates
|
|
562
|
+
at the mean of the unconstrained draws (the optimum for point estimates; for
|
|
563
|
+
engines that keep no unconstrained draws, the image of the posterior mean, with
|
|
564
|
+
any jointly inferred selection or response at its injection or tally values),
|
|
565
|
+
by central differences of the gradient with step ``h``; the prior's is of its
|
|
566
|
+
density in those
|
|
567
|
+
coordinates, so a Dirichlet(:math:`\alpha`) counts as :math:`\alpha` objects
|
|
568
|
+
per bin. The traces depend on those coordinates, so compare ratios across fits
|
|
569
|
+
of one prior family, not across families. Small values mean the data dominate
|
|
570
|
+
(DECISIONS #41); NaN when the data show no curvature (e.g. an empty bin at a
|
|
571
|
+
boundary optimum). Costs :math:`4d` gradient evaluations for ``d`` prior
|
|
572
|
+
coordinates, half of them of the full log density.
|
|
573
|
+
|
|
574
|
+
Parameters
|
|
575
|
+
----------
|
|
576
|
+
h : float, optional
|
|
577
|
+
Finite-difference step of the Hessians.
|
|
578
|
+
"""
|
|
579
|
+
from .samplers._curvature import fd_hessian_diag
|
|
580
|
+
|
|
581
|
+
model = self._require_model()
|
|
582
|
+
x = self._center_x()
|
|
583
|
+
d = model.prior.dim
|
|
584
|
+
|
|
585
|
+
def prior_grad(xp):
|
|
586
|
+
eta, cache = model.prior.forward(xp)
|
|
587
|
+
v = model._value(eta)
|
|
588
|
+
return model.prior.log_prior_and_grad(xp, cache, v, np.zeros_like(v))
|
|
589
|
+
|
|
590
|
+
def total_grad(xp):
|
|
591
|
+
return model.log_density_unconstrained_and_grad(np.concatenate([xp, x[d:]]))
|
|
592
|
+
|
|
593
|
+
idx = range(d)
|
|
594
|
+
Hp = fd_hessian_diag(prior_grad, x[:d], idx, h=h)
|
|
595
|
+
Ht = fd_hessian_diag(total_grad, x[:d], idx, h=h)
|
|
596
|
+
data = float(np.sum(Hp - Ht))
|
|
597
|
+
return float(np.sum(-Hp) / data) if data > 0 else float("nan")
|
|
598
|
+
|
|
599
|
+
def sample_values(
|
|
600
|
+
self,
|
|
601
|
+
idx: Any = None,
|
|
602
|
+
n_draws: int | None = N_DRAWS,
|
|
603
|
+
rng: RNGLike = None,
|
|
604
|
+
loglike: Callable[[np.ndarray, int], np.ndarray] | None = None,
|
|
605
|
+
n_sub: int = 64,
|
|
606
|
+
n_sweeps: int = 20,
|
|
607
|
+
) -> np.ndarray:
|
|
608
|
+
r"""Joint draws of the continuous true values of objects ``idx``.
|
|
609
|
+
|
|
610
|
+
Labels are drawn as in :meth:`sample_assignments`. Within the drawn bin
|
|
611
|
+
the population is flat in the measure coordinate ``u`` (its density in
|
|
612
|
+
``z`` carries :math:`T'(z)`), so a true value follows the object's own
|
|
613
|
+
likelihood times that density, from each input's within-bin information
|
|
614
|
+
(DECISIONS #50):
|
|
615
|
+
|
|
616
|
+
* :meth:`Likelihoods.from_gaussian` with independent errors (``sigma``):
|
|
617
|
+
exactly (truncated Gaussians) on linear axes, else by inverse-CDF
|
|
618
|
+
sampling on ``n_sub`` sub-points per bin; with correlated errors
|
|
619
|
+
(``cov``), ``n_sweeps`` Gibbs sweeps over the axes' conditionals, started
|
|
620
|
+
from independent draws;
|
|
621
|
+
* :meth:`Likelihoods.from_density` on 1-D bins: the density (divided by
|
|
622
|
+
the interim prior for posteriors) on ``n_sub`` sub-points per bin;
|
|
623
|
+
* :meth:`Likelihoods.from_samples`: the object's samples inside the bin,
|
|
624
|
+
resampled with their weights :math:`w_s / \pi_u(u_s)` (flat in ``u``
|
|
625
|
+
in a bin no sample reached, as the ``floor`` is);
|
|
626
|
+
* ``loglike(z, i)``, if given, overrides these on 1-D bins: the
|
|
627
|
+
log-likelihood of object ``i`` (an index into the likelihoods) at the
|
|
628
|
+
points ``z``, on ``n_sub`` sub-points per bin;
|
|
629
|
+
* otherwise (tables, binned inputs, compressed rows, densities on grids,
|
|
630
|
+
samples not kept) flat in ``u`` within the bin, with a warning: the
|
|
631
|
+
top-hat approximation, accurate only when likelihoods are nearly flat
|
|
632
|
+
within bins.
|
|
633
|
+
|
|
634
|
+
With a :class:`~binference.Basis` the label is a component: flat
|
|
635
|
+
components are sampled as above on their cells; inside a hat the
|
|
636
|
+
population is the hat, so sub-point weights include its height, and
|
|
637
|
+
other draws on its support are kept with that height.
|
|
638
|
+
|
|
639
|
+
With weighted (compressed) rows each row is treated as one object.
|
|
640
|
+
Returns ``(n_draws, n_objects)`` (1-D) or ``(n_draws, n_objects, ndim)``.
|
|
641
|
+
|
|
642
|
+
Parameters
|
|
643
|
+
----------
|
|
644
|
+
idx : index, optional
|
|
645
|
+
Objects to use; ``None`` uses all of them.
|
|
646
|
+
n_draws : int, optional
|
|
647
|
+
Population draws used; ``None`` uses all of them.
|
|
648
|
+
rng : int, Generator or None
|
|
649
|
+
Random number generator.
|
|
650
|
+
loglike : callable, optional
|
|
651
|
+
Each object's own log-likelihood within bins, ``loglike(z, i)``; overrides
|
|
652
|
+
the stored information on 1-D bins.
|
|
653
|
+
n_sub : int, optional
|
|
654
|
+
Sub-points per bin for inverse-CDF draws.
|
|
655
|
+
n_sweeps : int, optional
|
|
656
|
+
Gibbs sweeps over the axes for correlated Gaussian errors.
|
|
657
|
+
"""
|
|
658
|
+
model = self._require_model()
|
|
659
|
+
bins = model.bins # the components' cells with a basis
|
|
660
|
+
if bins.is_categorical:
|
|
661
|
+
raise ValueError("Categorical bins have no continuous values; use sample_assignments.")
|
|
662
|
+
if loglike is not None and bins.ndim != 1:
|
|
663
|
+
raise ValueError("loglike-based sampling supports 1-D bins only.")
|
|
664
|
+
gen = as_generator(rng)
|
|
665
|
+
draw, informed = _value_drawer(model, loglike, n_sub, n_sweeps)
|
|
666
|
+
if not informed:
|
|
667
|
+
what = model.likelihoods.meta.get("source", "a table")
|
|
668
|
+
warn(
|
|
669
|
+
f"sample_values draws values from the population alone within each bin: these "
|
|
670
|
+
f"likelihoods ({what}) keep no information within bins; pass loglike= for each "
|
|
671
|
+
"object's own likelihood.",
|
|
672
|
+
InputWarning,
|
|
673
|
+
)
|
|
674
|
+
rows = np.atleast_1d(np.arange(model.N)[slice(None) if idx is None else idx])
|
|
675
|
+
labels = self._component_labels(rows, n_draws, gen)
|
|
676
|
+
basis = model.likelihoods.basis
|
|
677
|
+
hat = basis is not None and basis.order == 1
|
|
678
|
+
objects = np.broadcast_to(rows, labels.shape)
|
|
679
|
+
out = np.empty((*labels.shape, bins.ndim))
|
|
680
|
+
todo = np.ones(labels.shape, dtype=bool)
|
|
681
|
+
while todo.any(): # a single pass unless hats reject a proposal
|
|
682
|
+
lab = labels[todo]
|
|
683
|
+
if hat:
|
|
684
|
+
lo, hi = basis.support(lab)
|
|
685
|
+
else:
|
|
686
|
+
cells = list(zip(bins.u_edges, bins.grid_index(lab), strict=True))
|
|
687
|
+
lo = np.stack([u[j] for u, j in cells], axis=-1)
|
|
688
|
+
hi = np.stack([u[j + 1] for u, j in cells], axis=-1)
|
|
689
|
+
z, ok = draw(objects[todo], lab, lo, hi, gen, basis.height if hat else None)
|
|
690
|
+
at = np.flatnonzero(todo)[ok]
|
|
691
|
+
out.reshape(-1, bins.ndim)[at] = z[ok]
|
|
692
|
+
todo.ravel()[at] = False
|
|
693
|
+
return out[..., 0] if bins.ndim == 1 else out
|
|
694
|
+
|
|
695
|
+
def predict_proba(
|
|
696
|
+
self, likelihoods: Likelihoods | Any, n_draws: int | None = N_DRAWS, chunk: int = 4096
|
|
697
|
+
):
|
|
698
|
+
"""Posterior over bins for *new* objects under the inferred population.
|
|
699
|
+
|
|
700
|
+
Parameters
|
|
701
|
+
----------
|
|
702
|
+
likelihoods : Likelihoods or array_like (N, K)
|
|
703
|
+
Likelihoods of the new objects, on the bins of the fit.
|
|
704
|
+
n_draws : int, optional
|
|
705
|
+
Population draws used; ``None`` uses all of them.
|
|
706
|
+
chunk : int, optional
|
|
707
|
+
Objects processed per block, which bounds memory.
|
|
708
|
+
"""
|
|
709
|
+
L, R, _, _ = self._new_object_draws(self._new_likelihoods(likelihoods), n_draws)
|
|
710
|
+
return self._to_bins(_mean_responsibilities(L, R, chunk))
|
|
711
|
+
|
|
712
|
+
def log_predictive(
|
|
713
|
+
self, likelihoods: Likelihoods | Any, n_draws: int | None = N_DRAWS, chunk: int = 4096
|
|
714
|
+
):
|
|
715
|
+
r"""Log posterior-predictive density of new objects, :math:`\log E[L_i\cdot\rho]`.
|
|
716
|
+
|
|
717
|
+
Includes the per-object log-scales of the likelihoods, so values can be
|
|
718
|
+
compared across priors and binnings (e.g. for cross-validation). Works
|
|
719
|
+
through the objects ``chunk`` at a time.
|
|
720
|
+
|
|
721
|
+
Parameters
|
|
722
|
+
----------
|
|
723
|
+
likelihoods : Likelihoods or array_like (N, K)
|
|
724
|
+
Likelihoods of the new objects, on the bins of the fit.
|
|
725
|
+
n_draws : int, optional
|
|
726
|
+
Population draws used; ``None`` uses all of them.
|
|
727
|
+
chunk : int, optional
|
|
728
|
+
Objects processed per block, which bounds memory.
|
|
729
|
+
"""
|
|
730
|
+
# rates rescale out of the ratio
|
|
731
|
+
L, R, norm, log_scale = self._new_object_draws(self._new_likelihoods(likelihoods), n_draws)
|
|
732
|
+
n = L.shape[-2]
|
|
733
|
+
lp = np.empty(n)
|
|
734
|
+
for sl in chunk_slices(n, chunk):
|
|
735
|
+
T = _row_dots(L, R, sl)[1]
|
|
736
|
+
if norm is not None: # a new *detected* object: divide by S . rho
|
|
737
|
+
T = T / norm[None, :]
|
|
738
|
+
with np.errstate(divide="ignore"):
|
|
739
|
+
lp[sl] = special.logsumexp(np.log(T), axis=1)
|
|
740
|
+
return lp - np.log(R.shape[0]) + log_scale
|
|
741
|
+
|
|
742
|
+
def _new_object_draws(self, lk: Likelihoods, n_draws: int | None):
|
|
743
|
+
"""For new objects ``lk``: rows and column weights per draw as in
|
|
744
|
+
:meth:`_object_draws`, each draw's :math:`S\\cdot\\rho` (``None`` without a
|
|
745
|
+
selection) and the rows' log-scales. New binned objects with the response
|
|
746
|
+
inferred in this fit take its draws (DECISIONS #81)."""
|
|
747
|
+
model = self._require_model()
|
|
748
|
+
Rd = self._response_rows(n_draws)
|
|
749
|
+
if Rd is not None and lk.meta.get("response") is model.response:
|
|
750
|
+
R = self._select_draws(n_draws, components=True)
|
|
751
|
+
rows = Rd[:, lk.meta["measured_bin"]] # the response's own rows: scale one
|
|
752
|
+
return rows, R, self._norms(R, None, n_draws), np.zeros(lk.N)
|
|
753
|
+
_, R, S = self._object_draws(n_draws)
|
|
754
|
+
norm = self._norms(R, S, n_draws)
|
|
755
|
+
if S is None and model.selection_row is not None:
|
|
756
|
+
R = R * model.selection_row # L~ . rho = L . (S o rho): the rows are not copied
|
|
757
|
+
return lk.values, R, norm, lk.log_scale
|
|
758
|
+
|
|
759
|
+
def _norms(self, R: np.ndarray, S: np.ndarray | None, n_draws: int | None):
|
|
760
|
+
"""Each draw's normaliser of a detected object's likelihood, :math:`S\\cdot\\rho`
|
|
761
|
+
(the response's efficiencies with an inferred response), for the draws ``R`` and
|
|
762
|
+
inferred selection ``S`` of :meth:`_object_draws`; ``None`` without a selection."""
|
|
763
|
+
model = self._require_model()
|
|
764
|
+
if "response_draws" in self.info:
|
|
765
|
+
return np.sum(self._joint_selection(n_draws) * R, axis=1)
|
|
766
|
+
if model.selection is None:
|
|
767
|
+
return None
|
|
768
|
+
return R.sum(axis=1) if S is not None else R @ model.selection
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def _rebuild_gaussian(lk: Likelihoods, basis: Basis) -> Likelihoods:
|
|
772
|
+
"""Gaussian likelihoods built again on ``basis`` from their kept means and errors."""
|
|
773
|
+
from .likelihoods import Likelihoods
|
|
774
|
+
|
|
775
|
+
meta = lk.meta
|
|
776
|
+
if "gaussian_mean" not in meta:
|
|
777
|
+
raise ValueError(
|
|
778
|
+
"Only likelihoods from Likelihoods.from_gaussian rebuild themselves; pass "
|
|
779
|
+
"build(bins_or_basis) returning these data's likelihoods on a basis."
|
|
780
|
+
)
|
|
781
|
+
spread = (
|
|
782
|
+
{"cov": meta["gaussian_cov"]}
|
|
783
|
+
if "gaussian_cov" in meta
|
|
784
|
+
else {"sigma": meta["gaussian_sigma"]}
|
|
785
|
+
)
|
|
786
|
+
w = lk.weights
|
|
787
|
+
return Likelihoods.from_gaussian(
|
|
788
|
+
meta["gaussian_mean"],
|
|
789
|
+
bins=basis,
|
|
790
|
+
weights=w,
|
|
791
|
+
fractional_weights=not is_integer_array(w),
|
|
792
|
+
warn_outside=False,
|
|
793
|
+
**meta.get("gaussian_options", {}),
|
|
794
|
+
**spread,
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _mean_responsibilities(
|
|
799
|
+
L: Any, R: np.ndarray, chunk: int, weigh: Callable | None = None
|
|
800
|
+
) -> np.ndarray:
|
|
801
|
+
r"""Per-object posteriors over the columns, averaged over the draws of column weights
|
|
802
|
+
``R``, ``chunk`` objects at a time (``L`` may hold rows per draw, ``(n, N, K)``):
|
|
803
|
+
equally, or with the weights ``weigh(sl, T)`` returns for the objects ``sl`` from
|
|
804
|
+
their :math:`T_{is} = L_i\cdot R_s` (``(n_sl, draws)``, rows summing to one)."""
|
|
805
|
+
n, m = L.shape[-2], R.shape[0]
|
|
806
|
+
out = np.zeros((n, L.shape[-1]))
|
|
807
|
+
for sl in chunk_slices(n, chunk):
|
|
808
|
+
block, T = _row_dots(L, R, sl)
|
|
809
|
+
block = block.toarray() if hasattr(block, "toarray") else np.asarray(block)
|
|
810
|
+
rows = np.arange(sl.start, sl.stop)
|
|
811
|
+
W = np.full(T.shape, 1.0 / m) if weigh is None else weigh(sl, T)
|
|
812
|
+
if block.ndim == 2: # rows shared by the draws: sum_s w_s L R_s / (L.R_s), 2 products
|
|
813
|
+
tiny = np.finfo(float).tiny # 1/T overflows below: those rows go draw by draw
|
|
814
|
+
inv = np.divide(W, T, out=np.zeros_like(T), where=T > tiny) # no support: zero
|
|
815
|
+
out[sl] = block * (inv @ R)
|
|
816
|
+
slow = np.any((T > 0) & (T <= tiny), axis=1)
|
|
817
|
+
if not slow.any():
|
|
818
|
+
continue
|
|
819
|
+
rows, block = (
|
|
820
|
+
rows[slow],
|
|
821
|
+
np.broadcast_to(block[slow], (m, slow.sum(), out.shape[1])),
|
|
822
|
+
)
|
|
823
|
+
out[rows] = 0.0
|
|
824
|
+
for s, rho in enumerate(R): # draw by draw
|
|
825
|
+
P = block[s] * rho
|
|
826
|
+
tot = P.sum(axis=1, keepdims=True)
|
|
827
|
+
out[rows] += W[rows - sl.start, s, None] * P / np.where(tot > 0, tot, 1.0)
|
|
828
|
+
return out
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def _row_dots(L: Any, R: np.ndarray, sl: slice) -> tuple[Any, np.ndarray]:
|
|
832
|
+
# the rows of the objects sl (per draw when L is (n, N, K)) and their products with
|
|
833
|
+
# the draws R, T_is = L_i . R_s (n_sl, draws)
|
|
834
|
+
block = L[:, sl] if L.ndim == 3 else L[sl]
|
|
835
|
+
T = np.einsum("snk,sk->ns", block, R) if L.ndim == 3 else np.asarray(block @ R.T)
|
|
836
|
+
return block, T
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _objects(L: Any, N: int, idx: Any) -> Any:
|
|
840
|
+
# the rows of objects idx (all by default), of per-draw rows (n_draws, N, K) too
|
|
841
|
+
if idx is None:
|
|
842
|
+
return L
|
|
843
|
+
rows = np.atleast_1d(np.arange(N)[idx])
|
|
844
|
+
return L[:, rows] if L.ndim == 3 else L[rows]
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def _draw_rows(L: Any, s: int) -> Any:
|
|
848
|
+
# draw s's rows when they differ by draw (an inferred response), else the rows
|
|
849
|
+
return L[s] if L.ndim == 3 else L
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _flat_draws(A: Any, n_tail: int, n_draws: int | None) -> np.ndarray:
|
|
853
|
+
# per-draw arrays (chain, draw, *tail) as (n, *tail), up to n_draws spread evenly
|
|
854
|
+
A = np.asarray(A)
|
|
855
|
+
A = A.reshape(-1, *A.shape[A.ndim - n_tail :])
|
|
856
|
+
return A[_draw_index(len(A), n_draws)]
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
def _central(X: np.ndarray, level: float, axis: int = 0) -> np.ndarray:
|
|
860
|
+
"""Bounds ``(lo, hi)`` of the central interval of probability ``level`` along ``axis``."""
|
|
861
|
+
return np.quantile(X, [0.5 - level / 2, 0.5 + level / 2], axis=axis)
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
#: the symbol of each quantity in labels and names (rates as in audit G-10)
|
|
865
|
+
_SYMBOL = {"density": "density", "fractions": "rho", "rates": "lambda", "selected": "phi"}
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def _which_name(which: str) -> str:
|
|
869
|
+
"""``"fractions"``, ``"rates"``, ``"density"`` or ``"selected"``, singular spellings
|
|
870
|
+
accepted (O-12)."""
|
|
871
|
+
name = {"fraction": "fractions", "rate": "rates"}.get(which, which)
|
|
872
|
+
if name not in ("fractions", "rates", "density", "selected"):
|
|
873
|
+
raise ValueError(
|
|
874
|
+
f"which must be 'fractions', 'rates', 'density' or 'selected'; got {which!r}."
|
|
875
|
+
)
|
|
876
|
+
return name
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def _mixing(C: np.ndarray, tail: bool = False) -> dict[str, np.ndarray]:
|
|
880
|
+
"""Bulk (and tail) ESS and split R-hat of each column of draws ``(chain, draw, K)``;
|
|
881
|
+
none below ``MIXING_MIN_DRAWS`` per chain (split R-hat also for one chain, audit O-7)."""
|
|
882
|
+
if C.shape[1] < MIXING_MIN_DRAWS:
|
|
883
|
+
return {}
|
|
884
|
+
cols = range(C.shape[-1])
|
|
885
|
+
out = {"ess_bulk": np.array([diag.ess(C[:, :, k], "bulk") for k in cols])}
|
|
886
|
+
if tail:
|
|
887
|
+
out["ess_tail"] = np.array([diag.ess(C[:, :, k], "tail") for k in cols])
|
|
888
|
+
out["rhat"] = np.array([diag.rhat(C[:, :, k]) for k in cols])
|
|
889
|
+
return out
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def _draw_index(n_total: int, n_draws: int | None) -> np.ndarray | slice:
|
|
893
|
+
# up to n_draws draws spread evenly over all of them
|
|
894
|
+
if n_draws is None or n_draws >= n_total:
|
|
895
|
+
return slice(None)
|
|
896
|
+
return np.linspace(0, n_total - 1, int(n_draws)).round().astype(int)
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def _draws_rng(draws: np.ndarray) -> np.random.Generator:
|
|
900
|
+
# a stream seeded by a hash of the draws: reproducible, different for different draws,
|
|
901
|
+
# and independent of any generator a caller holds (DECISIONS #78)
|
|
902
|
+
digest = hashlib.sha256(np.ascontiguousarray(draws).tobytes()).digest()
|
|
903
|
+
return np.random.default_rng(np.frombuffer(digest, dtype=np.uint32))
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def _report(groups: list[tuple[list[str], list[str], type]], strict: bool) -> None:
|
|
907
|
+
# warn once per (messages, hints, category) group, or raise all of them together
|
|
908
|
+
texts = []
|
|
909
|
+
for found, why, category in groups:
|
|
910
|
+
if found:
|
|
911
|
+
text = "; ".join(found) + (". Hint: " + "; ".join(why) if why else "")
|
|
912
|
+
texts.append(text)
|
|
913
|
+
if not strict:
|
|
914
|
+
warn(f"Diagnostics: {text}.", category)
|
|
915
|
+
if strict and texts:
|
|
916
|
+
raise DiagnosticsError(f"Diagnostics failed: {'; '.join(texts)}")
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def _same_bins(a: Bins, b: Bins) -> bool:
|
|
920
|
+
# equal bins up to the identity of their measures, which do not survive save and load
|
|
921
|
+
# (a transform axis loads as a table)
|
|
922
|
+
if a.is_categorical or b.is_categorical:
|
|
923
|
+
return a == b
|
|
924
|
+
return (
|
|
925
|
+
a.shape == b.shape
|
|
926
|
+
and all(np.array_equal(x, y) for x, y in zip(a.edges, b.edges, strict=True))
|
|
927
|
+
and [m.name for m in a.measures] == [m.name for m in b.measures]
|
|
928
|
+
and a.is_masked == b.is_masked
|
|
929
|
+
and (not a.is_masked or np.array_equal(a.mask, b.mask))
|
|
930
|
+
)
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def _same_layout(a: Basis | None, b: Basis | None) -> bool:
|
|
934
|
+
# equal bases up to the identity of their measures, which do not survive save and load
|
|
935
|
+
if a is None or b is None:
|
|
936
|
+
return a is b
|
|
937
|
+
return (a.order, a.resolution, a.oversample, a.K, a.C) == (
|
|
938
|
+
b.order,
|
|
939
|
+
b.resolution,
|
|
940
|
+
b.oversample,
|
|
941
|
+
b.K,
|
|
942
|
+
b.C,
|
|
943
|
+
) and all(np.allclose(x, y) for x, y in zip(a.bins.edges, b.bins.edges, strict=True))
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def _value_drawer(model, loglike, n_sub: int, n_sweeps: int) -> tuple[Callable, bool]:
|
|
947
|
+
"""``draw(objects, labels, lo, hi, gen, height) -> (z, keep)``: values in the boxes
|
|
948
|
+
``[lo, hi]`` (in ``u``, ``(n, ndim)``) from the input's own likelihood times the
|
|
949
|
+
population, flat in ``u`` times the hat's ``height(labels, d, u)`` per axis (``None``
|
|
950
|
+
when flat); and whether the input has within-bin information (DECISIONS #50)."""
|
|
951
|
+
meta = model.likelihoods.meta
|
|
952
|
+
measures = model.bins.measures
|
|
953
|
+
flat = partial(_draw_subpoints, measures, n_sub, None)
|
|
954
|
+
if loglike is not None:
|
|
955
|
+
return partial(_draw_subpoints, measures, n_sub, _per_object(loglike)), True
|
|
956
|
+
if "gaussian_mean" in meta:
|
|
957
|
+
return partial(_draw_gauss, measures, n_sub, n_sweeps, meta), True
|
|
958
|
+
within = meta.get("within")
|
|
959
|
+
if within is None or within.data is None or "value_row" not in meta:
|
|
960
|
+
return flat, False # e.g. rows merged by compress, which drops value_row
|
|
961
|
+
kind, *data = within.data
|
|
962
|
+
row = np.asarray(meta["value_row"])
|
|
963
|
+
if kind == "samples":
|
|
964
|
+
direct = model.likelihoods.basis is None # labels are the bins the samples are in
|
|
965
|
+
return partial(_draw_samples, measures, flat, direct, row, *data), True
|
|
966
|
+
evaluate = partial(_dists_logl if kind == "dists" else _rows_logl, row, *data)
|
|
967
|
+
return partial(_draw_subpoints, measures, n_sub, evaluate), True
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
#: Most values a density function returns per call when values are drawn (2**24 floats
|
|
971
|
+
#: are 128 MB).
|
|
972
|
+
EVAL_BUDGET = 2**24
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def _dists_logl(row, dists, interim, objects, labels, zz):
|
|
976
|
+
# distributions of from_density: one vectorised call for the rows' own points
|
|
977
|
+
from .likelihoods import _rows_logpdf, log_interim
|
|
978
|
+
|
|
979
|
+
with np.errstate(invalid="ignore"): # -inf - -inf where the interim vanishes: NaN
|
|
980
|
+
return _rows_logpdf(dists, 1)(zz, row[objects]) - log_interim(interim, zz)
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
def _rows_logl(row, n_in, fn, log, interim, objects, labels, zz):
|
|
984
|
+
# a density function of from_density returns every input row at common points: it
|
|
985
|
+
# is called on the sub-points of a few bins at a time, at most EVAL_BUDGET values
|
|
986
|
+
from .density import _log_nonneg
|
|
987
|
+
from .likelihoods import log_interim
|
|
988
|
+
|
|
989
|
+
out = np.empty(zz.shape)
|
|
990
|
+
_, first, inverse = np.unique(labels, return_index=True, return_inverse=True)
|
|
991
|
+
inverse = inverse.ravel()
|
|
992
|
+
per = max(1, EVAL_BUDGET // (n_in * zz.shape[1]))
|
|
993
|
+
for c in range(0, len(first), per):
|
|
994
|
+
at = np.flatnonzero((inverse >= c) & (inverse < c + per))
|
|
995
|
+
pts = zz[first[c : c + per]]
|
|
996
|
+
v = fn(pts.ravel())
|
|
997
|
+
v = np.asarray(v if log else _log_nonneg(v), dtype=float).reshape(n_in, *pts.shape)
|
|
998
|
+
out[at] = v[row[objects[at]], inverse[at] - c]
|
|
999
|
+
with np.errstate(invalid="ignore"): # -inf - -inf where the interim vanishes: NaN
|
|
1000
|
+
return out - log_interim(interim, zz)
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
def _per_object(loglike: Callable[[np.ndarray, int], np.ndarray]) -> Callable:
|
|
1004
|
+
"""An evaluator ``(objects, labels, zz) -> log-likelihoods`` calling ``loglike(z, i)``
|
|
1005
|
+
once per object, at the sub-points of all its bins."""
|
|
1006
|
+
|
|
1007
|
+
def evaluate(objects, labels, zz):
|
|
1008
|
+
key = objects.astype(np.int64) * (int(labels.max()) + 1) + labels # by object
|
|
1009
|
+
_, first, inverse = np.unique(key, return_index=True, return_inverse=True)
|
|
1010
|
+
vals = np.empty((len(first), zz.shape[1]))
|
|
1011
|
+
ids, starts = np.unique(objects[first], return_index=True)
|
|
1012
|
+
for i, a, b in zip(ids, starts, [*starts[1:], len(first)], strict=True):
|
|
1013
|
+
pts = zz[first[a:b]]
|
|
1014
|
+
vals[a:b] = np.asarray(loglike(pts.ravel(), int(i)), dtype=float).reshape(pts.shape)
|
|
1015
|
+
return vals[inverse.ravel()]
|
|
1016
|
+
|
|
1017
|
+
return evaluate
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
def _subpoints(zlo, zhi, n_sub):
|
|
1021
|
+
# midpoints of n_sub equal cells in z of each box, (n, n_sub)
|
|
1022
|
+
return zlo[:, None] + (zhi - zlo)[:, None] * ((np.arange(n_sub) + 0.5) / n_sub)
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def _subpoint_draw(zz, width, logw, gen, objects) -> np.ndarray:
|
|
1026
|
+
"""One value per row by inverse-CDF sampling on the sub-points ``zz`` (cells of
|
|
1027
|
+
``width / n_sub``) with log-weights ``logw``."""
|
|
1028
|
+
n, n_sub = logw.shape
|
|
1029
|
+
logw = np.where(np.isnan(logw), -np.inf, logw) # e.g. a posterior over a zero prior
|
|
1030
|
+
empty = ~np.isfinite(logw).any(axis=1)
|
|
1031
|
+
if empty.any(): # it drew from the box's first sub-point before (audit O-11)
|
|
1032
|
+
raise ValueError(
|
|
1033
|
+
f"The likelihood is zero at all {n_sub} sub-points of a bin with posterior "
|
|
1034
|
+
f"mass (object {int(objects[empty][0])}); raise n_sub or check loglike."
|
|
1035
|
+
)
|
|
1036
|
+
cell = kern._sample_labels_dense(np.exp(logw - logw.max(axis=1, keepdims=True)), 1.0, gen)
|
|
1037
|
+
return zz[np.arange(n), cell] + width / n_sub * (gen.random(n) - 0.5)
|
|
1038
|
+
|
|
1039
|
+
|
|
1040
|
+
def _base_logw(m, zz, labels, d, height):
|
|
1041
|
+
# flat in u (the Jacobian in z) times the hat's height along axis d
|
|
1042
|
+
with np.errstate(divide="ignore"):
|
|
1043
|
+
return m.log_jacobian(zz) + (
|
|
1044
|
+
0.0 if height is None else np.log(height(labels, d, m.forward(zz)))
|
|
1045
|
+
)
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
def _draw_subpoints(measures, n_sub, evaluate, objects, labels, lo, hi, gen, height):
|
|
1049
|
+
# each axis on n_sub sub-points: the population, times evaluate(objects, labels, zz)
|
|
1050
|
+
z = np.empty(lo.shape)
|
|
1051
|
+
for d, m in enumerate(measures):
|
|
1052
|
+
zlo, zhi = m.inverse(lo[:, d]), m.inverse(hi[:, d])
|
|
1053
|
+
zz = _subpoints(zlo, zhi, n_sub)
|
|
1054
|
+
logw = _base_logw(m, zz, labels, d, height)
|
|
1055
|
+
if evaluate is not None:
|
|
1056
|
+
logw = logw + evaluate(objects, labels, zz)
|
|
1057
|
+
z[:, d] = _subpoint_draw(zz, zhi - zlo, logw, gen, objects)
|
|
1058
|
+
return z, np.ones(len(z), dtype=bool)
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _truncnorm(a, b, gen) -> np.ndarray:
|
|
1062
|
+
"""Standard normal draws truncated to ``[a, b]``, by the inverse CDF in log space
|
|
1063
|
+
(intervals above zero are reflected, so tails keep their precision)."""
|
|
1064
|
+
flip = a > 0
|
|
1065
|
+
lo, hi = np.where(flip, -b, a), np.where(flip, -a, b)
|
|
1066
|
+
la, lb = special.log_ndtr(lo), special.log_ndtr(hi)
|
|
1067
|
+
u = gen.random(np.shape(lo))
|
|
1068
|
+
with np.errstate(divide="ignore"): # u = 0 at a lower bound of -inf
|
|
1069
|
+
x = special.ndtri_exp(lb + np.log(u + (1 - u) * np.exp(la - lb)))
|
|
1070
|
+
x = np.clip(x, lo, hi)
|
|
1071
|
+
return np.where(flip, -x, x)
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
def _draw_gauss(measures, n_sub, n_sweeps, meta, objects, labels, lo, hi, gen, height):
|
|
1075
|
+
"""Gaussian likelihoods: exact truncated draws per axis on linear axes (the hat's
|
|
1076
|
+
height as an acceptance test), sub-points elsewhere; with a covariance, ``n_sweeps``
|
|
1077
|
+
Gibbs sweeps over the axes' conditionals from that start (sub-points inside hats)."""
|
|
1078
|
+
n, nd = lo.shape
|
|
1079
|
+
mu = np.asarray(meta["gaussian_mean"])[objects]
|
|
1080
|
+
P = None
|
|
1081
|
+
if "gaussian_cov" in meta:
|
|
1082
|
+
ids, inverse = np.unique(objects, return_inverse=True) # one inverse per object
|
|
1083
|
+
C = np.asarray(meta["gaussian_cov"])[ids]
|
|
1084
|
+
P = np.linalg.inv(C)[inverse]
|
|
1085
|
+
sig = np.sqrt(np.diagonal(C, axis1=1, axis2=2))[inverse]
|
|
1086
|
+
else:
|
|
1087
|
+
sig = np.asarray(meta["gaussian_sigma"])[objects]
|
|
1088
|
+
zlo = np.stack([m.inverse(lo[:, d]) for d, m in enumerate(measures)], axis=1)
|
|
1089
|
+
zhi = np.stack([m.inverse(hi[:, d]) for d, m in enumerate(measures)], axis=1)
|
|
1090
|
+
keep = np.ones(n, dtype=bool)
|
|
1091
|
+
|
|
1092
|
+
def axis(d, m, mu_d, sig_d, reject):
|
|
1093
|
+
if m == Measure.linear() and (height is None or reject):
|
|
1094
|
+
x = mu_d + sig_d * _truncnorm(
|
|
1095
|
+
(zlo[:, d] - mu_d) / sig_d, (zhi[:, d] - mu_d) / sig_d, gen
|
|
1096
|
+
)
|
|
1097
|
+
if height is not None:
|
|
1098
|
+
keep[:] &= gen.random(n) < height(labels, d, m.forward(x))
|
|
1099
|
+
return x
|
|
1100
|
+
zz = _subpoints(zlo[:, d], zhi[:, d], n_sub)
|
|
1101
|
+
logw = (
|
|
1102
|
+
_base_logw(m, zz, labels, d, height)
|
|
1103
|
+
- 0.5 * ((zz - mu_d[:, None]) / sig_d[:, None]) ** 2
|
|
1104
|
+
)
|
|
1105
|
+
return _subpoint_draw(zz, zhi[:, d] - zlo[:, d], logw, gen, objects)
|
|
1106
|
+
|
|
1107
|
+
z = np.empty((n, nd))
|
|
1108
|
+
for d, m in enumerate(measures):
|
|
1109
|
+
z[:, d] = axis(d, m, mu[:, d], sig[:, d], reject=P is None)
|
|
1110
|
+
for _ in range(n_sweeps if P is not None else 0):
|
|
1111
|
+
for d, m in enumerate(measures):
|
|
1112
|
+
s2 = 1.0 / P[:, d, d]
|
|
1113
|
+
rest = np.einsum("nj,nj->n", P[:, d], z - mu) - P[:, d, d] * (z[:, d] - mu[:, d])
|
|
1114
|
+
z[:, d] = axis(d, m, mu[:, d] - s2 * rest, np.sqrt(s2), reject=False)
|
|
1115
|
+
return z, keep
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def _draw_samples(
|
|
1119
|
+
measures, flat, direct, row, K, key, zs, G, objects, labels, lo, hi, gen, height
|
|
1120
|
+
):
|
|
1121
|
+
"""An object's kept samples inside the box, drawn with their weights (``G`` their
|
|
1122
|
+
cumulative sum, each row's summing to one; ``key`` = row * K + bin, sorted); in
|
|
1123
|
+
a box no sample reached, flat in ``u`` as the ``floor`` is."""
|
|
1124
|
+
n, nd = lo.shape
|
|
1125
|
+
z, keep = np.empty((n, nd)), np.ones(n, dtype=bool)
|
|
1126
|
+
r = row[objects]
|
|
1127
|
+
G0 = np.concatenate(([0.0], G))
|
|
1128
|
+
if direct: # the label is the bin each sample was counted in: one segment each
|
|
1129
|
+
s0 = np.searchsorted(key, r * K + labels, side="left")
|
|
1130
|
+
s1 = np.searchsorted(key, r * K + labels, side="right")
|
|
1131
|
+
a, b = G0[s0], G0[s1]
|
|
1132
|
+
hit = b > a # samples with weight in the bin
|
|
1133
|
+
u = a[hit] + (1.0 - gen.random(hit.sum())) * (b - a)[hit]
|
|
1134
|
+
z[hit] = zs[np.clip(np.searchsorted(G, u, side="left"), s0[hit], s1[hit] - 1)]
|
|
1135
|
+
else: # components: the samples of the row inside the box, times the hat's height
|
|
1136
|
+
hit = np.zeros(n, dtype=bool)
|
|
1137
|
+
s0 = np.searchsorted(key, r * K, side="left")
|
|
1138
|
+
s1 = np.searchsorted(key, (r + 1) * K, side="left")
|
|
1139
|
+
order = np.lexsort((labels, objects))
|
|
1140
|
+
cut = np.flatnonzero(np.diff(objects[order]) | np.diff(labels[order])) + 1
|
|
1141
|
+
last = -1
|
|
1142
|
+
for at in np.split(order, cut): # the draws of one object in one box
|
|
1143
|
+
i = at[0]
|
|
1144
|
+
seg = slice(s0[i], s1[i])
|
|
1145
|
+
if objects[i] != last: # the object's samples in u, once for all its boxes
|
|
1146
|
+
us = np.stack([m.forward(zs[seg, d]) for d, m in enumerate(measures)], axis=1)
|
|
1147
|
+
last = objects[i]
|
|
1148
|
+
w = np.diff(G0[s0[i] : s1[i] + 1]) * np.all((us >= lo[i]) & (us <= hi[i]), axis=1)
|
|
1149
|
+
for d in range(nd if height is not None else 0):
|
|
1150
|
+
w = w * height(labels[i], d, us[:, d])
|
|
1151
|
+
c = np.cumsum(w)
|
|
1152
|
+
if c.size and c[-1] > 0:
|
|
1153
|
+
z[at] = zs[seg][np.searchsorted(c, (1.0 - gen.random(at.size)) * c[-1])]
|
|
1154
|
+
hit[at] = True
|
|
1155
|
+
miss = ~hit
|
|
1156
|
+
if miss.any():
|
|
1157
|
+
z[miss], keep[miss] = flat(objects[miss], labels[miss], lo[miss], hi[miss], gen, height)
|
|
1158
|
+
return z, keep
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
# ----------------------------------------------------------------------------
|
|
1162
|
+
# PopulationResult
|
|
1163
|
+
# ----------------------------------------------------------------------------
|
|
1164
|
+
class PopulationResult(_IndividualMixin):
|
|
1165
|
+
"""Posterior draws of a population (fractions and, for rate models, rates).
|
|
1166
|
+
|
|
1167
|
+
Every inference engine returns this object (``MAP`` returns a
|
|
1168
|
+
:class:`PointEstimate`). Draws are stored per chain; flattened views are
|
|
1169
|
+
available as :attr:`samples` (fractions) and :attr:`rates`.
|
|
1170
|
+
|
|
1171
|
+
Attributes
|
|
1172
|
+
----------
|
|
1173
|
+
method : str
|
|
1174
|
+
Engine that produced the draws.
|
|
1175
|
+
approximate : bool
|
|
1176
|
+
True for approximations: VB, bootstrap, and Laplace draws that are
|
|
1177
|
+
uncorrected, fail their Pareto k-hat check, or (under
|
|
1178
|
+
``method="auto"``) have an importance ESS below ``target_ess``.
|
|
1179
|
+
info : dict
|
|
1180
|
+
Engine statistics (e.g. NUTS tree depths, divergences, step sizes).
|
|
1181
|
+
labels : ndarray or None
|
|
1182
|
+
Joint label draws stored by Gibbs, ``(chains, draws, objects)`` for the
|
|
1183
|
+
objects in ``label_index``: bin indices, or basis-component indices when
|
|
1184
|
+
the likelihoods use a :class:`~binference.Basis`;
|
|
1185
|
+
:meth:`sample_assignments` returns bins either way.
|
|
1186
|
+
"""
|
|
1187
|
+
|
|
1188
|
+
def __init__(
|
|
1189
|
+
self,
|
|
1190
|
+
model: PopulationModel | None,
|
|
1191
|
+
draws: np.ndarray,
|
|
1192
|
+
*,
|
|
1193
|
+
method: str,
|
|
1194
|
+
kind: str | None = None,
|
|
1195
|
+
bins: Bins | None = None,
|
|
1196
|
+
x: np.ndarray | None = None,
|
|
1197
|
+
log_density_unconstrained: np.ndarray | None = None,
|
|
1198
|
+
approximate: bool = False,
|
|
1199
|
+
info: dict[str, Any] | None = None,
|
|
1200
|
+
labels: np.ndarray | None = None,
|
|
1201
|
+
label_index: np.ndarray | None = None,
|
|
1202
|
+
prior_repr: str | None = None,
|
|
1203
|
+
prior_key: str | None = None,
|
|
1204
|
+
basis: Basis | None = None,
|
|
1205
|
+
total: np.ndarray | None = None,
|
|
1206
|
+
selection: Selection | None = None,
|
|
1207
|
+
) -> None:
|
|
1208
|
+
d = np.asarray(draws, dtype=float)
|
|
1209
|
+
if d.ndim == 2:
|
|
1210
|
+
d = d[None]
|
|
1211
|
+
if d.ndim != 3:
|
|
1212
|
+
raise ValueError("draws must have shape (n_chains, n_draws, K) or (n_draws, K).")
|
|
1213
|
+
self.model = model
|
|
1214
|
+
if basis is None and model is not None:
|
|
1215
|
+
basis = model.likelihoods.basis
|
|
1216
|
+
#: the :class:`~binference.Basis` of the model's components, or ``None``
|
|
1217
|
+
self.basis = basis
|
|
1218
|
+
self._params = d # the model's columns: component weights with a basis
|
|
1219
|
+
self._draws = self._to_bins(d) # per bin (the same array without a basis)
|
|
1220
|
+
self.kind = kind = kind or (model.kind if model is not None else "fractions")
|
|
1221
|
+
if kind == "rates":
|
|
1222
|
+
total = self._draws.sum(axis=-1)
|
|
1223
|
+
# for fractions, drawn on first use (see total)
|
|
1224
|
+
self._total = (
|
|
1225
|
+
None if total is None else np.asarray(total, dtype=float).reshape(d.shape[:2])
|
|
1226
|
+
)
|
|
1227
|
+
self.bins = (
|
|
1228
|
+
bins
|
|
1229
|
+
if bins is not None
|
|
1230
|
+
else basis.bins
|
|
1231
|
+
if basis is not None
|
|
1232
|
+
else model.bins
|
|
1233
|
+
if model is not None
|
|
1234
|
+
else Bins.categories(d.shape[-1])
|
|
1235
|
+
)
|
|
1236
|
+
self.x = x
|
|
1237
|
+
#: log posterior density of each draw in the unconstrained coordinates
|
|
1238
|
+
#: (with the Jacobian; DECISIONS #15), for samplers that compute it
|
|
1239
|
+
self.log_density_unconstrained = log_density_unconstrained
|
|
1240
|
+
self.method = method
|
|
1241
|
+
self.approximate = approximate
|
|
1242
|
+
self.info = dict(info or {})
|
|
1243
|
+
self.labels = labels
|
|
1244
|
+
self.label_index = label_index
|
|
1245
|
+
self.prior_repr = prior_repr or (repr(model.prior_spec) if model is not None else None)
|
|
1246
|
+
#: a digest of the prior's arguments in full, which attach compares (the repr
|
|
1247
|
+
#: shortens arrays)
|
|
1248
|
+
self.prior_key = prior_key or (
|
|
1249
|
+
model.prior_spec._fingerprint() if model is not None else None
|
|
1250
|
+
)
|
|
1251
|
+
# the fit's selection, kept for attach after a load (audit O-1)
|
|
1252
|
+
self._selection = selection if model is None else model.selection_spec
|
|
1253
|
+
self._cache: dict[str, Any] = {}
|
|
1254
|
+
|
|
1255
|
+
# ------------------------------------------------------------- creation
|
|
1256
|
+
@classmethod
|
|
1257
|
+
def from_samples(
|
|
1258
|
+
cls,
|
|
1259
|
+
model: PopulationModel,
|
|
1260
|
+
samples: Any,
|
|
1261
|
+
*,
|
|
1262
|
+
method: str = "external",
|
|
1263
|
+
approximate: bool = False,
|
|
1264
|
+
info: dict[str, Any] | None = None,
|
|
1265
|
+
) -> PopulationResult:
|
|
1266
|
+
"""Wrap draws from an external sampler, ``(n_chains, n_draws, K)`` or ``(n_draws, K)``:
|
|
1267
|
+
fractions for a fractions model (rows summing to one), rates for a rate model.
|
|
1268
|
+
|
|
1269
|
+
A model that infers its selection or response with the population needs their
|
|
1270
|
+
draws too, as ``info["selection_draws"]`` or ``info["response_draws"]`` (the
|
|
1271
|
+
layout the engines store).
|
|
1272
|
+
|
|
1273
|
+
Parameters
|
|
1274
|
+
----------
|
|
1275
|
+
model : PopulationModel
|
|
1276
|
+
The model the draws are of.
|
|
1277
|
+
samples : array_like (n_chains, n_draws, K) or (n_draws, K)
|
|
1278
|
+
The draws.
|
|
1279
|
+
method : str, optional
|
|
1280
|
+
Engine name recorded in the result.
|
|
1281
|
+
approximate : bool, optional
|
|
1282
|
+
Flag the draws as approximate.
|
|
1283
|
+
info : dict, optional
|
|
1284
|
+
Extra information stored with the draws.
|
|
1285
|
+
"""
|
|
1286
|
+
key = "selection_draws" if model.selection_uncertain else "response_draws"
|
|
1287
|
+
if (model.selection_uncertain or model.response is not None) and key not in (info or {}):
|
|
1288
|
+
raise ValueError(
|
|
1289
|
+
f"This model infers its {key.split('_')[0]} with the population: pass its "
|
|
1290
|
+
f'draws as info["{key}"], or fix it (`.fixed()` or `.at_mean()`) first.'
|
|
1291
|
+
)
|
|
1292
|
+
X = np.asarray(samples, dtype=float)
|
|
1293
|
+
if X.ndim not in (2, 3) or X.shape[-1] != model.K:
|
|
1294
|
+
raise ValueError(
|
|
1295
|
+
f"samples must have shape (n_chains, n_draws, {model.K}) or (n_draws, "
|
|
1296
|
+
f"{model.K}); got {X.shape}."
|
|
1297
|
+
)
|
|
1298
|
+
check_nonnegative(X, "samples")
|
|
1299
|
+
if model.kind == "fractions":
|
|
1300
|
+
sums = X.sum(axis=-1)
|
|
1301
|
+
if not np.allclose(sums, 1.0, atol=1e-6):
|
|
1302
|
+
raise ValueError(
|
|
1303
|
+
"samples of a fractions model must sum to one per draw; rates need a rate "
|
|
1304
|
+
"model (a GammaRates prior)."
|
|
1305
|
+
)
|
|
1306
|
+
X = X / sums[..., None]
|
|
1307
|
+
if (
|
|
1308
|
+
key in (info or {})
|
|
1309
|
+
and np.shape(info[key])[:2] != (X[None] if X.ndim == 2 else X).shape[:2]
|
|
1310
|
+
):
|
|
1311
|
+
raise ValueError(
|
|
1312
|
+
f'info["{key}"] must have one draw per sample, (n_chains, n_draws, ...).'
|
|
1313
|
+
)
|
|
1314
|
+
return cls(model, X, method=method, approximate=approximate, info=info)
|
|
1315
|
+
|
|
1316
|
+
@property
|
|
1317
|
+
def selection(self) -> Selection | None:
|
|
1318
|
+
"""The selection function of the fit, kept through :meth:`save` and :func:`load`."""
|
|
1319
|
+
return self._selection
|
|
1320
|
+
|
|
1321
|
+
_fit_model = True # whether the model is the fit's own (not bare likelihoods attached)
|
|
1322
|
+
|
|
1323
|
+
def _fit_model_only(self, what: str) -> PopulationModel:
|
|
1324
|
+
# the fit's model, refused when bare likelihoods were attached after load
|
|
1325
|
+
if not self._fit_model:
|
|
1326
|
+
raise ValueError(
|
|
1327
|
+
f"{what} needs the fit's prior: attach a PopulationModel built with it "
|
|
1328
|
+
"(bare likelihoods get the default prior)."
|
|
1329
|
+
)
|
|
1330
|
+
return self._require_model()
|
|
1331
|
+
|
|
1332
|
+
@property
|
|
1333
|
+
def best(self) -> PointEstimate:
|
|
1334
|
+
"""The best fit: :class:`~binference.samplers.MAP` with its defaults, which is the
|
|
1335
|
+
mode of :math:`\\rho` on the simplex for conjugate priors whose density is bounded
|
|
1336
|
+
there (e.g. Dirichlet concentrations of at least one), and the mode in the
|
|
1337
|
+
unconstrained coordinates otherwise, and always with a jointly inferred selection
|
|
1338
|
+
or response (DECISIONS #15, #35). Computed on first use
|
|
1339
|
+
and kept; per-object outputs at it carry a ``_best`` suffix (DECISIONS #51).
|
|
1340
|
+
After :func:`load`, it needs the fit's prior: attach a model, not bare likelihoods."""
|
|
1341
|
+
if "best" not in self._cache:
|
|
1342
|
+
from .samplers import MAP
|
|
1343
|
+
|
|
1344
|
+
self._cache["best"] = MAP(self._fit_model_only("The best fit")).run()
|
|
1345
|
+
return self._cache["best"]
|
|
1346
|
+
|
|
1347
|
+
def attach(self, likelihoods_or_model: Any) -> PopulationResult:
|
|
1348
|
+
"""Attach likelihoods (or a model) to a loaded result to enable per-object methods.
|
|
1349
|
+
|
|
1350
|
+
Bare likelihoods take the fit's selection; likelihoods or a model with another
|
|
1351
|
+
selection, or on other bins, are refused (audit O-1, O-2).
|
|
1352
|
+
|
|
1353
|
+
Parameters
|
|
1354
|
+
----------
|
|
1355
|
+
likelihoods_or_model : Likelihoods or PopulationModel
|
|
1356
|
+
The fit's likelihoods, or a model built on them.
|
|
1357
|
+
"""
|
|
1358
|
+
from .likelihoods import Likelihoods
|
|
1359
|
+
from .model import PopulationModel
|
|
1360
|
+
from .selection import _rebuild as _rebuild_selection
|
|
1361
|
+
|
|
1362
|
+
m, saved = likelihoods_or_model, self.selection
|
|
1363
|
+
if not isinstance(m, PopulationModel):
|
|
1364
|
+
lk = m if isinstance(m, Likelihoods) else Likelihoods(m, self.basis or self.bins)
|
|
1365
|
+
sel = None if lk.selection is not None else saved
|
|
1366
|
+
if sel is not None and sel.basis is not None and _same_layout(lk.basis, sel.basis):
|
|
1367
|
+
# the same layout; its measure did not survive saving
|
|
1368
|
+
sel = _rebuild_selection(*sel._parts()[:3], lk.basis, sel.injections)
|
|
1369
|
+
m = PopulationModel(lk, selection=sel)
|
|
1370
|
+
if m.selection_spec != saved:
|
|
1371
|
+
raise ValueError("The likelihoods (or model) carry another selection than this fit.")
|
|
1372
|
+
if not _same_layout(m.likelihoods.basis, self.basis):
|
|
1373
|
+
raise ValueError("The likelihoods use a different basis than this result.")
|
|
1374
|
+
if self.basis is None and not _same_bins(m.bins, self.bins):
|
|
1375
|
+
raise ValueError(f"The likelihoods are on other bins ({m.bins}) than this result.")
|
|
1376
|
+
# bare likelihoods get the default prior; a model counts as the fit's with its prior
|
|
1377
|
+
self._fit_model = isinstance(m, PopulationModel) and (
|
|
1378
|
+
self.prior_key is None or m.prior_spec._fingerprint() == self.prior_key
|
|
1379
|
+
)
|
|
1380
|
+
self.model = m
|
|
1381
|
+
self._selection = m.selection_spec
|
|
1382
|
+
# everything computed from the previous likelihoods: the cache, the missing
|
|
1383
|
+
# information and a fractions model's total (drawn with its W)
|
|
1384
|
+
self._cache.clear()
|
|
1385
|
+
self.info.pop("missing_information", None)
|
|
1386
|
+
if self.kind == "fractions":
|
|
1387
|
+
self._total = None
|
|
1388
|
+
return self
|
|
1389
|
+
|
|
1390
|
+
# ------------------------------------------------------------ properties
|
|
1391
|
+
@property
|
|
1392
|
+
def n_chains(self) -> int:
|
|
1393
|
+
return int(self._draws.shape[0])
|
|
1394
|
+
|
|
1395
|
+
@property
|
|
1396
|
+
def n_draws(self) -> int:
|
|
1397
|
+
"""Total number of draws (over all chains)."""
|
|
1398
|
+
return int(self._draws.shape[0] * self._draws.shape[1])
|
|
1399
|
+
|
|
1400
|
+
@property
|
|
1401
|
+
def K(self) -> int:
|
|
1402
|
+
return int(self._draws.shape[-1])
|
|
1403
|
+
|
|
1404
|
+
@property
|
|
1405
|
+
def chains(self) -> np.ndarray:
|
|
1406
|
+
"""Fraction draws per chain, shape ``(n_chains, n_per_chain, K)``."""
|
|
1407
|
+
return self._fractions(self._draws)
|
|
1408
|
+
|
|
1409
|
+
@property
|
|
1410
|
+
def samples(self) -> np.ndarray:
|
|
1411
|
+
"""Population fractions, shape ``(n_draws, K)`` (chains concatenated)."""
|
|
1412
|
+
return _flat_draws(self.chains, 1, None)
|
|
1413
|
+
|
|
1414
|
+
@property
|
|
1415
|
+
def total(self) -> np.ndarray | None:
|
|
1416
|
+
r"""Draws of the total rate :math:`\Lambda` ``(n_draws,)``, or ``None``.
|
|
1417
|
+
|
|
1418
|
+
For fractions, each draw is :math:`\Lambda \sim \mathrm{Gamma}(W, S\cdot\rho)`
|
|
1419
|
+
given its :math:`\rho`: the exact posterior of the total under the prior
|
|
1420
|
+
:math:`p(\Lambda) \propto 1/\Lambda`, with :math:`W` the number of objects
|
|
1421
|
+
and :math:`S` the detection probabilities or exposures (one without a
|
|
1422
|
+
selection). ``None`` when the selection is only relative (DECISIONS #9,
|
|
1423
|
+
#46, #78) or the result was loaded without its total. For rate models,
|
|
1424
|
+
the sum of the rates.
|
|
1425
|
+
|
|
1426
|
+
The Gamma variates come from a stream seeded by the fraction draws
|
|
1427
|
+
themselves, so the totals are reproducible for any result (also wrapped
|
|
1428
|
+
external draws, and loaded results after :meth:`attach`), independent
|
|
1429
|
+
across different draws, and leave the caller's generator untouched.
|
|
1430
|
+
"""
|
|
1431
|
+
if self._total is None and self.model is not None:
|
|
1432
|
+
S = self._inferred_selection() # per draw, when inferred with the population
|
|
1433
|
+
self._total = self.model.draw_total(self._params, _draws_rng(self._params), S)
|
|
1434
|
+
return None if self._total is None else self._total.ravel()
|
|
1435
|
+
|
|
1436
|
+
@property
|
|
1437
|
+
def rates(self) -> np.ndarray | None:
|
|
1438
|
+
r"""Rate draws ``(n_draws, K)``: :math:`\lambda = \Lambda\rho` per draw for
|
|
1439
|
+
fractions (see :attr:`total`), the sampled rates for rate models; ``None``
|
|
1440
|
+
when the total is not available."""
|
|
1441
|
+
if self.kind == "rates":
|
|
1442
|
+
return _flat_draws(self._draws, 1, None)
|
|
1443
|
+
total = self.total
|
|
1444
|
+
return None if total is None else total[:, None] * self.samples
|
|
1445
|
+
|
|
1446
|
+
def _rho_draws(self) -> np.ndarray:
|
|
1447
|
+
return self.samples
|
|
1448
|
+
|
|
1449
|
+
@property
|
|
1450
|
+
def component_draws(self) -> np.ndarray:
|
|
1451
|
+
"""Draws of the model's parameters, ``(n_chains, n_per_chain, C)``: the component
|
|
1452
|
+
weights with a :class:`~binference.Basis`, else the same as the per-bin draws."""
|
|
1453
|
+
return self._params
|
|
1454
|
+
|
|
1455
|
+
def _weight_draws(self) -> np.ndarray:
|
|
1456
|
+
return self._fractions(_flat_draws(self._params, 1, None))
|
|
1457
|
+
|
|
1458
|
+
def _which(self, which: str) -> np.ndarray:
|
|
1459
|
+
which = _which_name(which)
|
|
1460
|
+
if which == "fractions":
|
|
1461
|
+
return self.samples
|
|
1462
|
+
if which == "rates":
|
|
1463
|
+
rates = self.rates
|
|
1464
|
+
if rates is None:
|
|
1465
|
+
raise ValueError(
|
|
1466
|
+
"This result has no rates: its selection is only relative, so the total "
|
|
1467
|
+
"is not identified (DECISIONS #46), or it has no model (attach the "
|
|
1468
|
+
"likelihoods)."
|
|
1469
|
+
)
|
|
1470
|
+
return rates
|
|
1471
|
+
if which == "selected":
|
|
1472
|
+
return self.selected
|
|
1473
|
+
return self.density()
|
|
1474
|
+
|
|
1475
|
+
@property
|
|
1476
|
+
def selected(self) -> np.ndarray:
|
|
1477
|
+
"""Draws ``(n_draws, K)`` of the selected population
|
|
1478
|
+
:math:`\\phi = S\\circ\\rho / (S\\cdot\\rho)`, the bins of the observed objects;
|
|
1479
|
+
:math:`\\rho` itself without a selection (DECISIONS #49)."""
|
|
1480
|
+
if "selected" not in self._cache:
|
|
1481
|
+
self._cache["selected"] = self._selected_of(self._weight_draws())
|
|
1482
|
+
return self._cache["selected"]
|
|
1483
|
+
|
|
1484
|
+
def _center_x(self) -> np.ndarray:
|
|
1485
|
+
# the mean of the unconstrained draws, else the image of the mean columns
|
|
1486
|
+
model = self._require_model()
|
|
1487
|
+
if self.x is not None and self.x.shape[-1] == model.dim:
|
|
1488
|
+
return _flat_draws(self.x, 1, None).mean(axis=0)
|
|
1489
|
+
return model.from_value(_flat_draws(self._params, 1, None).mean(axis=0))
|
|
1490
|
+
|
|
1491
|
+
@property
|
|
1492
|
+
def mean(self) -> np.ndarray:
|
|
1493
|
+
"""Posterior mean fractions ``(K,)``."""
|
|
1494
|
+
return self.samples.mean(axis=0)
|
|
1495
|
+
|
|
1496
|
+
@property
|
|
1497
|
+
def median(self) -> np.ndarray:
|
|
1498
|
+
"""Posterior median fractions ``(K,)``."""
|
|
1499
|
+
return np.median(self.samples, axis=0)
|
|
1500
|
+
|
|
1501
|
+
@property
|
|
1502
|
+
def std(self) -> np.ndarray:
|
|
1503
|
+
"""Posterior standard deviation of the fractions ``(K,)``."""
|
|
1504
|
+
return self.samples.std(axis=0, ddof=1)
|
|
1505
|
+
|
|
1506
|
+
def cov(self, which: str = "fractions") -> np.ndarray:
|
|
1507
|
+
"""Posterior covariance ``(K, K)`` of the fractions (or ``which="rates"``, ...).
|
|
1508
|
+
|
|
1509
|
+
Parameters
|
|
1510
|
+
----------
|
|
1511
|
+
which : {"fractions", "rates", "density", "selected"}, optional
|
|
1512
|
+
Quantity used: fractions, rates, densities or the selected population.
|
|
1513
|
+
"""
|
|
1514
|
+
return np.cov(self._which(which), rowvar=False)
|
|
1515
|
+
|
|
1516
|
+
def corr(self, which: str = "fractions") -> np.ndarray:
|
|
1517
|
+
"""Posterior correlation matrix ``(K, K)`` of the fractions (or ``which=...``).
|
|
1518
|
+
|
|
1519
|
+
Parameters
|
|
1520
|
+
----------
|
|
1521
|
+
which : {"fractions", "rates", "density", "selected"}, optional
|
|
1522
|
+
Quantity used: fractions, rates, densities or the selected population.
|
|
1523
|
+
"""
|
|
1524
|
+
with np.errstate(invalid="ignore", divide="ignore"):
|
|
1525
|
+
return np.corrcoef(self._which(which), rowvar=False)
|
|
1526
|
+
|
|
1527
|
+
def quantile(self, q: Any, which: str = "fractions") -> np.ndarray:
|
|
1528
|
+
"""Posterior quantiles, shape ``(*np.shape(q), K)``.
|
|
1529
|
+
|
|
1530
|
+
Parameters
|
|
1531
|
+
----------
|
|
1532
|
+
q : float or array_like
|
|
1533
|
+
Quantiles, in ``[0, 1]``.
|
|
1534
|
+
which : {"fractions", "rates", "density", "selected"}, optional
|
|
1535
|
+
Quantity used: fractions, rates, densities or the selected population.
|
|
1536
|
+
"""
|
|
1537
|
+
return np.quantile(self._which(which), q, axis=0)
|
|
1538
|
+
|
|
1539
|
+
def interval(
|
|
1540
|
+
self, level: float = 0.68, kind: str = "central", which: str = "fractions"
|
|
1541
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
1542
|
+
"""Per-bin credible intervals ``(lo, hi)``, each ``(K,)``.
|
|
1543
|
+
|
|
1544
|
+
Parameters
|
|
1545
|
+
----------
|
|
1546
|
+
level : float, optional
|
|
1547
|
+
Probability inside each interval.
|
|
1548
|
+
kind : {"central", "hdi"}, optional
|
|
1549
|
+
Equal-tailed (``"central"``) or highest-density (``"hdi"``) intervals.
|
|
1550
|
+
which : {"fractions", "rates", "density", "selected"}, optional
|
|
1551
|
+
Quantity used: fractions, rates, densities or the selected population.
|
|
1552
|
+
"""
|
|
1553
|
+
if not 0 < level < 1:
|
|
1554
|
+
raise ValueError("level must be in (0, 1).")
|
|
1555
|
+
X = self._which(which)
|
|
1556
|
+
if kind == "central":
|
|
1557
|
+
lo, hi = _central(X, level)
|
|
1558
|
+
return lo, hi
|
|
1559
|
+
if kind == "hdi":
|
|
1560
|
+
bounds = np.array([_hdi(X[:, k], level) for k in range(X.shape[1])])
|
|
1561
|
+
return bounds[:, 0], bounds[:, 1]
|
|
1562
|
+
raise ValueError("kind must be 'central' or 'hdi'.")
|
|
1563
|
+
|
|
1564
|
+
def density(self) -> np.ndarray:
|
|
1565
|
+
"""Draws of the population density per unit bin volume, ``(n_draws, K)``.
|
|
1566
|
+
|
|
1567
|
+
Fractions divided by bin volumes (rates divided by volumes for rate
|
|
1568
|
+
models): the quantity to plot as n(z).
|
|
1569
|
+
"""
|
|
1570
|
+
base = self.rates if self.kind == "rates" else self.samples
|
|
1571
|
+
return base / self.bins.volumes
|
|
1572
|
+
|
|
1573
|
+
def functional(
|
|
1574
|
+
self, func: Callable[[np.ndarray], Any], vectorized: bool = False, which: str = "fractions"
|
|
1575
|
+
):
|
|
1576
|
+
"""Draws of a derived quantity ``func(rho)`` (e.g. the mean redshift), or of
|
|
1577
|
+
the rates or densities with ``which``.
|
|
1578
|
+
|
|
1579
|
+
Parameters
|
|
1580
|
+
----------
|
|
1581
|
+
func : callable
|
|
1582
|
+
Function of one draw ``(K,)``.
|
|
1583
|
+
vectorized : bool, optional
|
|
1584
|
+
Pass all draws to ``func`` at once, as an ``(n_draws, K)`` array.
|
|
1585
|
+
which : {"fractions", "rates", "density", "selected"}, optional
|
|
1586
|
+
Quantity used: fractions, rates, densities or the selected population.
|
|
1587
|
+
|
|
1588
|
+
Examples
|
|
1589
|
+
--------
|
|
1590
|
+
>>> zbar = result.functional(lambda r: r @ result.bins.centers) # doctest: +SKIP
|
|
1591
|
+
"""
|
|
1592
|
+
X = self._which(which)
|
|
1593
|
+
if vectorized:
|
|
1594
|
+
return np.asarray(func(X))
|
|
1595
|
+
return np.array([func(r) for r in X])
|
|
1596
|
+
|
|
1597
|
+
def rebin(self, groups: Any, which: str = "fractions") -> np.ndarray:
|
|
1598
|
+
"""Fractions (or rates, with ``which="rates"``) aggregated into coarser groups.
|
|
1599
|
+
|
|
1600
|
+
``groups`` is either a length-``K`` array of integers mapping bins to groups
|
|
1601
|
+
(e.g. from :meth:`Bins.coarsen`) or a sequence of lists of bin indices.
|
|
1602
|
+
Returns ``(n_draws, n_groups)``.
|
|
1603
|
+
|
|
1604
|
+
Parameters
|
|
1605
|
+
----------
|
|
1606
|
+
groups : array_like (K,) or sequence of lists
|
|
1607
|
+
Group of each bin, or the bins of each group.
|
|
1608
|
+
which : {"fractions", "rates", "density", "selected"}, optional
|
|
1609
|
+
Quantity used: fractions, rates, densities or the selected population.
|
|
1610
|
+
"""
|
|
1611
|
+
if _which_name(which) == "density":
|
|
1612
|
+
raise ValueError("Densities per unit volume do not add across bins; rebin fractions.")
|
|
1613
|
+
X = self._which(which)
|
|
1614
|
+
try:
|
|
1615
|
+
arr = np.asarray(groups)
|
|
1616
|
+
except ValueError: # ragged sequence of groups
|
|
1617
|
+
arr = None
|
|
1618
|
+
if (
|
|
1619
|
+
arr is not None
|
|
1620
|
+
and arr.ndim == 1
|
|
1621
|
+
and arr.shape[0] == self.K
|
|
1622
|
+
and arr.dtype.kind in "iuf"
|
|
1623
|
+
):
|
|
1624
|
+
if not is_integer_array(arr) or arr.min() < 0:
|
|
1625
|
+
raise ValueError("groups must map each bin to a non-negative integer group.")
|
|
1626
|
+
arr = arr.astype(np.intp)
|
|
1627
|
+
one_hot = np.zeros((self.K, int(arr.max()) + 1))
|
|
1628
|
+
one_hot[np.arange(self.K), arr] = 1.0
|
|
1629
|
+
return X @ one_hot
|
|
1630
|
+
return np.stack([X[:, list(grp)].sum(axis=1) for grp in groups], axis=1)
|
|
1631
|
+
|
|
1632
|
+
# ---------------------------------------------------------- diagnostics
|
|
1633
|
+
def _mixing_of(self, which: str) -> dict[str, np.ndarray]:
|
|
1634
|
+
"""Bulk and tail ESS and split R-hat of the fractions or the rates, cached; a
|
|
1635
|
+
density shares its base's (they ignore a positive scale per bin)."""
|
|
1636
|
+
base = self.kind if which == "density" else which
|
|
1637
|
+
if ("mixing", base) not in self._cache:
|
|
1638
|
+
X = self._which(base).reshape(self.n_chains, -1, self.K)
|
|
1639
|
+
self._cache["mixing", base] = _mixing(X, tail=True)
|
|
1640
|
+
return self._cache["mixing", base]
|
|
1641
|
+
|
|
1642
|
+
@property
|
|
1643
|
+
def diagnostics(self) -> dict[str, Any]:
|
|
1644
|
+
"""Convergence diagnostics (computed lazily and cached)."""
|
|
1645
|
+
if "diagnostics" in self._cache:
|
|
1646
|
+
return self._cache["diagnostics"]
|
|
1647
|
+
out: dict[str, Any] = {}
|
|
1648
|
+
# for rate models diagnose the rates, which also carry the total
|
|
1649
|
+
out.update(self._mixing_of(self.kind))
|
|
1650
|
+
Sd = self._inferred_selection() # a selection, or a response's efficiencies
|
|
1651
|
+
if Sd is not None:
|
|
1652
|
+
name = "response" if "response_draws" in self.info else "selection"
|
|
1653
|
+
out.update({f"{k}_{name}": v for k, v in _mixing(Sd).items()})
|
|
1654
|
+
for key in (
|
|
1655
|
+
"n_divergent",
|
|
1656
|
+
"k_hat",
|
|
1657
|
+
"k_hat_threshold",
|
|
1658
|
+
"kkt_gap",
|
|
1659
|
+
"max_depth_fraction",
|
|
1660
|
+
"missing_information",
|
|
1661
|
+
"converged",
|
|
1662
|
+
"ebfmi",
|
|
1663
|
+
):
|
|
1664
|
+
if key in self.info:
|
|
1665
|
+
out[key] = self.info[key]
|
|
1666
|
+
self._cache["diagnostics"] = out
|
|
1667
|
+
return out
|
|
1668
|
+
|
|
1669
|
+
def _missing_information(self) -> float | None:
|
|
1670
|
+
"""Fraction of missing information at the posterior mean (cached in ``info``)."""
|
|
1671
|
+
if (
|
|
1672
|
+
"missing_information" not in self.info
|
|
1673
|
+
and self.model is not None
|
|
1674
|
+
and self.model.K <= 500
|
|
1675
|
+
):
|
|
1676
|
+
value = _flat_draws(self._params, 1, None).mean(axis=0)
|
|
1677
|
+
try:
|
|
1678
|
+
self.info["missing_information"] = self.model.missing_information(value)
|
|
1679
|
+
except (np.linalg.LinAlgError, ValueError):
|
|
1680
|
+
return None
|
|
1681
|
+
self._cache.pop("diagnostics", None)
|
|
1682
|
+
return self.info.get("missing_information")
|
|
1683
|
+
|
|
1684
|
+
def _mixing_problems(
|
|
1685
|
+
self, rhat_max: float = RHAT_MAX, ess_min: float = ESS_MIN_PER_CHAIN
|
|
1686
|
+
) -> list[str]:
|
|
1687
|
+
"""R-hat above ``rhat_max`` and bulk or tail ESS below ``ess_min`` per chain, of
|
|
1688
|
+
the population and of an inferred selection or response, for Markov chains."""
|
|
1689
|
+
d, msgs = self.diagnostics, []
|
|
1690
|
+
# Markov chains (and external draws), whose draws are correlated; independent
|
|
1691
|
+
# draws have no chain for R-hat and ESS to describe
|
|
1692
|
+
from .inference import METHODS
|
|
1693
|
+
|
|
1694
|
+
engine = METHODS.get(self.method)
|
|
1695
|
+
if self.approximate or (engine is not None and not engine.markov):
|
|
1696
|
+
return msgs
|
|
1697
|
+
for suffix, what in (
|
|
1698
|
+
("", ""),
|
|
1699
|
+
("_selection", " of the selection"),
|
|
1700
|
+
("_response", " of the response's efficiencies"),
|
|
1701
|
+
):
|
|
1702
|
+
if "rhat" + suffix in d and np.nanmax(d["rhat" + suffix]) > rhat_max:
|
|
1703
|
+
msgs.append(f"max R-hat{what} = {np.nanmax(d['rhat' + suffix]):.3f} > {rhat_max}")
|
|
1704
|
+
for kind in ("bulk", "tail"):
|
|
1705
|
+
key = f"ess_{kind}{suffix}"
|
|
1706
|
+
if key in d:
|
|
1707
|
+
ess = np.nanmin(d[key])
|
|
1708
|
+
per_chain = ess / max(1, self.n_chains)
|
|
1709
|
+
if per_chain < ess_min:
|
|
1710
|
+
# rounded down, so the message never shows the limit as reached
|
|
1711
|
+
msgs.append(
|
|
1712
|
+
f"min {kind} ESS{what} = {math.floor(ess)} "
|
|
1713
|
+
f"({math.floor(per_chain * 10) / 10:.1f} per chain < {ess_min:g})"
|
|
1714
|
+
)
|
|
1715
|
+
return msgs
|
|
1716
|
+
|
|
1717
|
+
def check(
|
|
1718
|
+
self,
|
|
1719
|
+
strict: bool = False,
|
|
1720
|
+
rhat_max: float = RHAT_MAX,
|
|
1721
|
+
ess_min: float = ESS_MIN_PER_CHAIN,
|
|
1722
|
+
predictive_p_min: float = 1e-3,
|
|
1723
|
+
predictive_draws: int = 200,
|
|
1724
|
+
support_min: float | None = None,
|
|
1725
|
+
):
|
|
1726
|
+
"""Check diagnostics and warn (or raise with ``strict=True``) about problems.
|
|
1727
|
+
|
|
1728
|
+
Sampler problems raise a :class:`~binference.ConvergenceWarning`; problems
|
|
1729
|
+
with the inputs (Monte Carlo noise of sample likelihoods, binned counts
|
|
1730
|
+
the response does not describe) an :class:`~binference.InputWarning`.
|
|
1731
|
+
For binned inputs with more measured than truth bins, the counts are
|
|
1732
|
+
tested with :meth:`Response.predictive_pvalue` over ``predictive_draws``
|
|
1733
|
+
draws, and flagged below ``predictive_p_min``. With ``support_min``, bins whose
|
|
1734
|
+
:attr:`data_support` is below it (in objects) are flagged as set by the prior.
|
|
1735
|
+
|
|
1736
|
+
Returns a list of messages (empty if everything looks fine).
|
|
1737
|
+
|
|
1738
|
+
Parameters
|
|
1739
|
+
----------
|
|
1740
|
+
strict : bool, optional
|
|
1741
|
+
Raise :class:`~binference.DiagnosticsError` instead of warning.
|
|
1742
|
+
rhat_max : float, optional
|
|
1743
|
+
Largest R-hat that passes.
|
|
1744
|
+
ess_min : float, optional
|
|
1745
|
+
Smallest bulk or tail ESS per chain that passes.
|
|
1746
|
+
predictive_p_min : float, optional
|
|
1747
|
+
Smallest posterior predictive p-value of binned counts that passes.
|
|
1748
|
+
predictive_draws : int, optional
|
|
1749
|
+
Draws used for the predictive p-value.
|
|
1750
|
+
support_min : float, optional
|
|
1751
|
+
Flag bins with less data support than this; ``None`` skips the test.
|
|
1752
|
+
"""
|
|
1753
|
+
d = self.diagnostics
|
|
1754
|
+
msgs = self._mixing_problems(rhat_max, ess_min)
|
|
1755
|
+
hints = []
|
|
1756
|
+
if msgs and self.method == "gibbs":
|
|
1757
|
+
hint = "Gibbs mixes slowly when likelihoods are broad compared with the bins"
|
|
1758
|
+
f = self._missing_information()
|
|
1759
|
+
if f is not None:
|
|
1760
|
+
tau = (1.0 + f) / max(1.0 - f, 1e-12)
|
|
1761
|
+
tau_text = f"~{tau:.0f}" if tau < 1000 else "over 1000"
|
|
1762
|
+
hint += f" (missing information {f:.2f}: autocorrelation time {tau_text})"
|
|
1763
|
+
hints.append(hint + "; run longer or use method='nuts'")
|
|
1764
|
+
if d.get("n_divergent", 0):
|
|
1765
|
+
n_div = int(d["n_divergent"])
|
|
1766
|
+
share = 100.0 * n_div / max(self.n_draws, 1)
|
|
1767
|
+
msgs.append(f"{n_div} divergent transitions ({share:.2g}% of draws)")
|
|
1768
|
+
hints.append(
|
|
1769
|
+
"try a higher target_accept, e.g. 0.95; divergences that persist mark a "
|
|
1770
|
+
"region the sampler cannot explore (see the checking guide: diagnostics)"
|
|
1771
|
+
)
|
|
1772
|
+
if "ebfmi" in d and np.nanmin(d["ebfmi"]) < EBFMI_MIN:
|
|
1773
|
+
msgs.append(f"E-BFMI = {np.nanmin(d['ebfmi']):.2f} < {EBFMI_MIN:g} in some chain")
|
|
1774
|
+
hints.append(
|
|
1775
|
+
"NUTS explores the energy distribution poorly (heavy tails or a funnel); "
|
|
1776
|
+
"a longer warm-up or another prior may help"
|
|
1777
|
+
)
|
|
1778
|
+
if d.get("converged") is False:
|
|
1779
|
+
msgs.append("the optimiser did not converge")
|
|
1780
|
+
threshold = d.get("k_hat_threshold", diag.K_HAT_MAX)
|
|
1781
|
+
if d.get("k_hat", 0) >= threshold:
|
|
1782
|
+
msgs.append(
|
|
1783
|
+
f"Pareto k-hat = {d['k_hat']:.2f} >= {threshold:.2f} (importance sampling "
|
|
1784
|
+
"unreliable)"
|
|
1785
|
+
)
|
|
1786
|
+
if d.get("max_depth_fraction", 0) > MAX_DEPTH_SHARE:
|
|
1787
|
+
msgs.append(
|
|
1788
|
+
f"{100 * d['max_depth_fraction']:.0f}% of NUTS iterations hit the maximum tree "
|
|
1789
|
+
"depth"
|
|
1790
|
+
)
|
|
1791
|
+
if _small_concentration(self.model):
|
|
1792
|
+
hints.append(
|
|
1793
|
+
"this prior gives nearly empty bins very long tails in log space "
|
|
1794
|
+
"(e.g. concentrations below one); method='gibbs' samples such "
|
|
1795
|
+
"conjugate priors efficiently"
|
|
1796
|
+
)
|
|
1797
|
+
lk = self.model.likelihoods if self.model is not None else None
|
|
1798
|
+
inputs, input_hints = [], []
|
|
1799
|
+
if lk is not None and "sample_sum" in lk.meta:
|
|
1800
|
+
# the population the objects are drawn from, after selection
|
|
1801
|
+
rho = self._weight_draws().mean(axis=0)
|
|
1802
|
+
if self.model.selection_row is not None:
|
|
1803
|
+
rho = rho * self.model.selection_row
|
|
1804
|
+
var, _ = lk.sample_variance(rho)
|
|
1805
|
+
total = float(np.sum(lk.weights * var))
|
|
1806
|
+
if total > SAMPLE_VARIANCE_MAX:
|
|
1807
|
+
inputs.append(
|
|
1808
|
+
f"likelihoods from samples carry Monte Carlo variance sum_i V_i = {total:.2g} "
|
|
1809
|
+
f"> {SAMPLE_VARIANCE_MAX:g} at the fitted population"
|
|
1810
|
+
)
|
|
1811
|
+
input_hints.append("draw more samples per object, or use wider bins or floor=")
|
|
1812
|
+
if lk is not None and support_min is not None:
|
|
1813
|
+
low = np.flatnonzero(self.data_support < support_min)
|
|
1814
|
+
if low.size:
|
|
1815
|
+
inputs.append(
|
|
1816
|
+
f"{low.size} bins have data support below {support_min:g} objects "
|
|
1817
|
+
f"(first: {low[:5].tolist()}): their fractions are set by the prior"
|
|
1818
|
+
)
|
|
1819
|
+
input_hints.append("merge sparse bins, or read their fractions as the prior's")
|
|
1820
|
+
if lk is not None and "measured_bin" in lk.meta:
|
|
1821
|
+
ppp = self._predictive_pvalue(lk, predictive_draws)
|
|
1822
|
+
if ppp is not None and ppp < predictive_p_min:
|
|
1823
|
+
inputs.append(
|
|
1824
|
+
"the measured counts disagree with the fitted population and the response "
|
|
1825
|
+
f"(posterior predictive p = {ppp:.1g})"
|
|
1826
|
+
)
|
|
1827
|
+
input_hints.append(
|
|
1828
|
+
"the response may not describe these data; for a classifier, the score "
|
|
1829
|
+
"distribution of a class differs between the validation and the target set "
|
|
1830
|
+
"(label shift violated)"
|
|
1831
|
+
)
|
|
1832
|
+
_report([(msgs, hints, ConvergenceWarning), (inputs, input_hints, InputWarning)], strict)
|
|
1833
|
+
return msgs + inputs
|
|
1834
|
+
|
|
1835
|
+
def _predictive_pvalue(self, lk: Any, n_draws: int) -> float | None:
|
|
1836
|
+
# label-shift check of binned inputs (DECISIONS #68); needs J > K
|
|
1837
|
+
key = ("predictive_p", int(n_draws))
|
|
1838
|
+
if key not in self._cache:
|
|
1839
|
+
resp = lk.meta["response"]
|
|
1840
|
+
J, K = resp.matrix.shape
|
|
1841
|
+
n = np.bincount(lk.meta["measured_bin"], weights=lk.weights, minlength=J)
|
|
1842
|
+
Rd = self._response_rows(n_draws) # an inferred response, drawn with rho
|
|
1843
|
+
self._cache[key] = (
|
|
1844
|
+
resp.predictive_pvalue(n, self._select_draws(n_draws), Rd) if J > K else None
|
|
1845
|
+
)
|
|
1846
|
+
return self._cache[key]
|
|
1847
|
+
|
|
1848
|
+
def summary(self, level: float = 0.68, which: str = "fractions") -> Summary:
|
|
1849
|
+
"""Per-bin posterior summary table (mean, sd, median, interval, ESS, R-hat).
|
|
1850
|
+
|
|
1851
|
+
Parameters
|
|
1852
|
+
----------
|
|
1853
|
+
level : float, optional
|
|
1854
|
+
Probability inside the central intervals shown.
|
|
1855
|
+
which : {"fractions", "rates", "density", "selected"}, optional
|
|
1856
|
+
Quantity used: fractions, rates, densities or the selected population.
|
|
1857
|
+
"""
|
|
1858
|
+
which = _which_name(which)
|
|
1859
|
+
X = self._which(which)
|
|
1860
|
+
lo, hi = _central(X, level)
|
|
1861
|
+
cols: dict[str, np.ndarray] = {
|
|
1862
|
+
"bin": _bin_labels(self.bins),
|
|
1863
|
+
"mean": X.mean(axis=0),
|
|
1864
|
+
"sd": X.std(axis=0, ddof=1) if X.shape[0] > 1 else np.zeros(self.K),
|
|
1865
|
+
"median": np.median(X, axis=0),
|
|
1866
|
+
f"q{100 * (0.5 - level / 2):g}": lo,
|
|
1867
|
+
f"q{100 * (0.5 + level / 2):g}": hi,
|
|
1868
|
+
}
|
|
1869
|
+
mixing = self._mixing_of(which) # of the quantity summarised (audit O-8)
|
|
1870
|
+
cols.update({k: v for k, v in mixing.items() if k != "ess_tail"})
|
|
1871
|
+
if self.model is not None: # objects' worth of likelihood per bin (DECISIONS #41)
|
|
1872
|
+
cols["support"] = self.data_support
|
|
1873
|
+
flag = " [approximate]" if self.approximate else ""
|
|
1874
|
+
source = None if self.model is None else self.model.likelihoods.meta.get("source")
|
|
1875
|
+
inputs = "" if source is None else f"; inputs: {source}"
|
|
1876
|
+
title = f"binference {which} posterior ({self.method}{flag}, {self.n_draws} draws{inputs})"
|
|
1877
|
+
return Summary(cols, title)
|
|
1878
|
+
|
|
1879
|
+
def binning_bias(
|
|
1880
|
+
self,
|
|
1881
|
+
build: Callable[[Any], Likelihoods] | None = None,
|
|
1882
|
+
*,
|
|
1883
|
+
order: int = 1,
|
|
1884
|
+
resolution: int = 1,
|
|
1885
|
+
threshold: float = 2.0,
|
|
1886
|
+
prior: Any = None,
|
|
1887
|
+
method: str | None = None,
|
|
1888
|
+
n_samples: int | None = None,
|
|
1889
|
+
rng: RNGLike = None,
|
|
1890
|
+
**engine_kwargs: Any,
|
|
1891
|
+
) -> Summary:
|
|
1892
|
+
r"""Check the flat-within-bin assumption: refit the same data on a B-spline basis
|
|
1893
|
+
over these bins and compare the bin masses (DECISIONS #6, #20).
|
|
1894
|
+
|
|
1895
|
+
``build(bins_or_basis)`` returns the likelihoods of the fit's data on what it is
|
|
1896
|
+
given, e.g. ``lambda b: Likelihoods.from_gaussian(mean, sigma, b)``; likelihoods
|
|
1897
|
+
from :meth:`Likelihoods.from_gaussian` rebuild themselves without it. The
|
|
1898
|
+
reference is fitted on ``Basis(self.bins, order, resolution)`` with this fit's
|
|
1899
|
+
prior (or ``prior``), selection, engine (or ``method``), chains and draws (or
|
|
1900
|
+
``n_samples``), and a Gibbs fit's warm-up and thinning when the engine is the
|
|
1901
|
+
same; ``engine_kwargs`` go to the engine.
|
|
1902
|
+
|
|
1903
|
+
The fit's selection keeps only its bin averages, so the reference sees it flat
|
|
1904
|
+
within bins; to test a selection's shape within bins as well, attach it to the
|
|
1905
|
+
likelihoods ``build`` returns, which then take precedence (a selection from
|
|
1906
|
+
injections must be attached that way, fixed).
|
|
1907
|
+
|
|
1908
|
+
Returns a table per bin: ``shift``, the reference's mean fraction minus this
|
|
1909
|
+
fit's; ``sd``, this fit's posterior s.d.; ``z = shift / sd`` (NaN where the
|
|
1910
|
+
fit has no spread); and, where
|
|
1911
|
+
:math:`|z|` exceeds ``threshold``, ``n_crit`` :math:`= W/z^2`, the number of
|
|
1912
|
+
objects at which the shift equals the statistical error (which falls as
|
|
1913
|
+
:math:`1/\sqrt{W}`): from a pilot fit, the size above which these bins bias
|
|
1914
|
+
the result (``inf`` elsewhere). Warns when :math:`|z|` exceeds ``threshold`` in
|
|
1915
|
+
any bin: two models of the same data differ by up to about one s.d. even when
|
|
1916
|
+
the population is flat within bins (DECISIONS #86), so the default is 2.
|
|
1917
|
+
|
|
1918
|
+
Parameters
|
|
1919
|
+
----------
|
|
1920
|
+
build : callable, optional
|
|
1921
|
+
Function that rebuilds the likelihoods on a basis; ``None`` works for
|
|
1922
|
+
Gaussian inputs.
|
|
1923
|
+
order : int, optional
|
|
1924
|
+
B-spline order of the reference fit's basis.
|
|
1925
|
+
resolution : int, optional
|
|
1926
|
+
Components per bin of the reference fit's basis.
|
|
1927
|
+
threshold : float, optional
|
|
1928
|
+
Shift, in posterior standard deviations, above which a bin is flagged.
|
|
1929
|
+
prior : PopulationPrior, optional
|
|
1930
|
+
Prior of the reference fit; ``None`` reuses this fit's.
|
|
1931
|
+
method : str, optional
|
|
1932
|
+
Engine of the reference fit; ``None`` reuses this fit's.
|
|
1933
|
+
n_samples : int, optional
|
|
1934
|
+
Draws of the reference fit; ``None`` reuses this fit's, except that
|
|
1935
|
+
``method="auto"`` sets its own run length.
|
|
1936
|
+
rng : int, Generator or None
|
|
1937
|
+
Random number generator.
|
|
1938
|
+
**engine_kwargs
|
|
1939
|
+
Options for the reference fit's engine.
|
|
1940
|
+
"""
|
|
1941
|
+
from .basis import Basis
|
|
1942
|
+
from .inference import METHODS, infer
|
|
1943
|
+
|
|
1944
|
+
model = self._fit_model_only("binning_bias")
|
|
1945
|
+
if self.basis is not None or self.bins.is_categorical:
|
|
1946
|
+
raise ValueError("binning_bias compares a fit on grid bins with a basis over them.")
|
|
1947
|
+
key = str(self.method if method is None else method).lower()
|
|
1948
|
+
if key not in ("auto", "nuts", "gibbs", "laplace", "vb", "bootstrap"):
|
|
1949
|
+
raise ValueError(
|
|
1950
|
+
f"method must be an engine that draws ('auto', 'nuts', 'gibbs', 'laplace', "
|
|
1951
|
+
f"'vb' or 'bootstrap'); got {key!r}."
|
|
1952
|
+
)
|
|
1953
|
+
basis = Basis(self.bins, order=order, resolution=resolution)
|
|
1954
|
+
if basis.is_identity:
|
|
1955
|
+
raise ValueError(
|
|
1956
|
+
"order=0 with resolution=1 is the bins themselves: nothing to compare."
|
|
1957
|
+
)
|
|
1958
|
+
if build is None:
|
|
1959
|
+
lk = _rebuild_gaussian(model.likelihoods, basis)
|
|
1960
|
+
else:
|
|
1961
|
+
lk = build(basis)
|
|
1962
|
+
if lk.basis != basis or lk.N != model.N or not np.array_equal(lk.weights, model._w):
|
|
1963
|
+
raise ValueError(
|
|
1964
|
+
"build(basis) must return the fit's objects (same rows and weights) on "
|
|
1965
|
+
"the basis it is given."
|
|
1966
|
+
)
|
|
1967
|
+
selection = self.selection if lk.selection is None else None # build's own wins
|
|
1968
|
+
if selection is not None and selection.injections is not None:
|
|
1969
|
+
raise ValueError(
|
|
1970
|
+
"A selection from injections does not move onto a basis: attach a fixed one "
|
|
1971
|
+
"(selection.at_mean()) to the likelihoods build returns."
|
|
1972
|
+
)
|
|
1973
|
+
same = key == self.method
|
|
1974
|
+
if METHODS[key].markov: # draws per chain, as the fit had them
|
|
1975
|
+
engine_kwargs.setdefault("n_chains", self.n_chains)
|
|
1976
|
+
for name in ("n_warmup", "thin") if same and key == "gibbs" else ():
|
|
1977
|
+
if name in self.info:
|
|
1978
|
+
engine_kwargs.setdefault(name, self.info[name])
|
|
1979
|
+
n = self.n_draws // self.n_chains
|
|
1980
|
+
elif key == "auto": # its own defaults, as for a new fit
|
|
1981
|
+
n = None
|
|
1982
|
+
else:
|
|
1983
|
+
n = self.n_draws
|
|
1984
|
+
ref = infer(
|
|
1985
|
+
lk,
|
|
1986
|
+
model.prior_spec if prior is None else prior,
|
|
1987
|
+
selection=selection,
|
|
1988
|
+
method=key,
|
|
1989
|
+
n_samples=n if n_samples is None else n_samples,
|
|
1990
|
+
rng=rng,
|
|
1991
|
+
check=False,
|
|
1992
|
+
**engine_kwargs,
|
|
1993
|
+
)
|
|
1994
|
+
sd, shift = self.std, ref.mean - self.mean
|
|
1995
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
1996
|
+
z = np.where(sd > 0, shift / sd, np.nan) # no spread: nothing to compare with
|
|
1997
|
+
big = np.abs(z) > threshold
|
|
1998
|
+
n_crit = np.where(big, model.W / np.where(big, z, 1.0) ** 2, np.inf)
|
|
1999
|
+
if big.any():
|
|
2000
|
+
warn(
|
|
2001
|
+
f"Binning bias: {big.sum()} bins move by more than {threshold:g} posterior s.d. "
|
|
2002
|
+
f"on a basis of order {order} (first: {np.flatnonzero(big)[:5].tolist()}); "
|
|
2003
|
+
f"fit with binference.Basis(bins, order={order}) or narrower bins.",
|
|
2004
|
+
InputWarning,
|
|
2005
|
+
)
|
|
2006
|
+
cols = {"bin": _bin_labels(self.bins), "shift": shift, "sd": sd, "z": z, "n_crit": n_crit}
|
|
2007
|
+
return Summary(cols, f"binference binning bias (basis of order {order}, {key})")
|
|
2008
|
+
|
|
2009
|
+
def __repr__(self) -> str:
|
|
2010
|
+
flag = ", approximate" if self.approximate else ""
|
|
2011
|
+
return (
|
|
2012
|
+
f"PopulationResult(method={self.method!r}, kind={self.kind!r}, K={self.K}, "
|
|
2013
|
+
f"n_chains={self.n_chains}, n_draws={self.n_draws}{flag})"
|
|
2014
|
+
)
|
|
2015
|
+
|
|
2016
|
+
# ------------------------------------------------------------- plotting
|
|
2017
|
+
def plot(self, **kwargs: Any):
|
|
2018
|
+
"""Plot the population posterior (see :func:`binference.plotting.plot_population`).
|
|
2019
|
+
|
|
2020
|
+
Parameters
|
|
2021
|
+
----------
|
|
2022
|
+
**kwargs
|
|
2023
|
+
Passed to :func:`binference.plotting.plot_population`.
|
|
2024
|
+
"""
|
|
2025
|
+
from .plotting import plot_population
|
|
2026
|
+
|
|
2027
|
+
return plot_population(self, **kwargs)
|
|
2028
|
+
|
|
2029
|
+
def plot_corner(self, idx: Sequence[int] | None = None, **kwargs: Any):
|
|
2030
|
+
"""Corner plot of selected bins (see :func:`binference.plotting.plot_corner`).
|
|
2031
|
+
|
|
2032
|
+
Parameters
|
|
2033
|
+
----------
|
|
2034
|
+
idx : sequence of int, optional
|
|
2035
|
+
Bins to show; ``None`` chooses the most populated.
|
|
2036
|
+
**kwargs
|
|
2037
|
+
Further arguments of :func:`~binference.plotting.plot_corner`.
|
|
2038
|
+
"""
|
|
2039
|
+
from .plotting import plot_corner
|
|
2040
|
+
|
|
2041
|
+
return plot_corner(self, idx=idx, **kwargs)
|
|
2042
|
+
|
|
2043
|
+
def plot_trace(self, idx: Sequence[int] | None = None, **kwargs: Any):
|
|
2044
|
+
"""Trace plots of selected bins (see :func:`binference.plotting.plot_trace`).
|
|
2045
|
+
|
|
2046
|
+
Parameters
|
|
2047
|
+
----------
|
|
2048
|
+
idx : sequence of int, optional
|
|
2049
|
+
Bins to show; ``None`` chooses the most populated.
|
|
2050
|
+
**kwargs
|
|
2051
|
+
Further arguments of :func:`~binference.plotting.plot_trace`.
|
|
2052
|
+
"""
|
|
2053
|
+
from .plotting import plot_trace
|
|
2054
|
+
|
|
2055
|
+
return plot_trace(self, idx=idx, **kwargs)
|
|
2056
|
+
|
|
2057
|
+
# ---------------------------------------------------------- persistence
|
|
2058
|
+
def save(self, path: str) -> None:
|
|
2059
|
+
"""Save draws, metadata and bins to a compressed ``.npz`` file.
|
|
2060
|
+
|
|
2061
|
+
Parameters
|
|
2062
|
+
----------
|
|
2063
|
+
path : str
|
|
2064
|
+
File to write.
|
|
2065
|
+
"""
|
|
2066
|
+
from .io import save_result
|
|
2067
|
+
|
|
2068
|
+
save_result(self, path)
|
|
2069
|
+
|
|
2070
|
+
@classmethod
|
|
2071
|
+
def load(cls, path: str) -> PopulationResult:
|
|
2072
|
+
"""Load a result saved with :meth:`save`.
|
|
2073
|
+
|
|
2074
|
+
Re-:meth:`attach` the likelihoods to use the per-object methods.
|
|
2075
|
+
|
|
2076
|
+
Parameters
|
|
2077
|
+
----------
|
|
2078
|
+
path : str
|
|
2079
|
+
File written by :meth:`save`.
|
|
2080
|
+
"""
|
|
2081
|
+
from .io import load_result
|
|
2082
|
+
|
|
2083
|
+
return load_result(path)
|
|
2084
|
+
|
|
2085
|
+
def to_dict(self) -> dict[str, Any]:
|
|
2086
|
+
"""Plain-dictionary representation (arrays and metadata)."""
|
|
2087
|
+
from .io import result_to_dict
|
|
2088
|
+
|
|
2089
|
+
return result_to_dict(self)
|
|
2090
|
+
|
|
2091
|
+
def to_datatree(self):
|
|
2092
|
+
"""Convert to an :class:`xarray.DataTree` (ArviZ >= 1.0 layout; needs xarray)."""
|
|
2093
|
+
from .io import to_datatree
|
|
2094
|
+
|
|
2095
|
+
return to_datatree(self)
|
|
2096
|
+
|
|
2097
|
+
|
|
2098
|
+
# ----------------------------------------------------------------------------
|
|
2099
|
+
# PointEstimate
|
|
2100
|
+
# ----------------------------------------------------------------------------
|
|
2101
|
+
class PointEstimate(_IndividualMixin):
|
|
2102
|
+
"""A MAP / maximum-likelihood estimate of the population.
|
|
2103
|
+
|
|
2104
|
+
Attributes
|
|
2105
|
+
----------
|
|
2106
|
+
value : (K,) array
|
|
2107
|
+
Fractions (or rates for rate models) at the optimum; with a
|
|
2108
|
+
:class:`~binference.Basis`, the component weights (:attr:`rho` and
|
|
2109
|
+
:attr:`rates` are per bin).
|
|
2110
|
+
x : (dim,) array or None
|
|
2111
|
+
Unconstrained coordinates (when optimised there).
|
|
2112
|
+
kkt_gap : float
|
|
2113
|
+
Optimality certificate (zero at the exact optimum on the simplex).
|
|
2114
|
+
"""
|
|
2115
|
+
|
|
2116
|
+
def __init__(
|
|
2117
|
+
self,
|
|
2118
|
+
model: PopulationModel | None,
|
|
2119
|
+
value: np.ndarray,
|
|
2120
|
+
*,
|
|
2121
|
+
method: str = "map",
|
|
2122
|
+
x: np.ndarray | None = None,
|
|
2123
|
+
log_posterior: float = np.nan,
|
|
2124
|
+
log_likelihood: float = np.nan,
|
|
2125
|
+
kkt_gap: float = np.nan,
|
|
2126
|
+
n_iter: int = 0,
|
|
2127
|
+
converged: bool = True,
|
|
2128
|
+
info: dict[str, Any] | None = None,
|
|
2129
|
+
) -> None:
|
|
2130
|
+
self.model = model
|
|
2131
|
+
self.value = np.asarray(value, dtype=float)
|
|
2132
|
+
self.method = method
|
|
2133
|
+
self.x = x
|
|
2134
|
+
self.log_posterior = float(log_posterior)
|
|
2135
|
+
self.log_likelihood = float(log_likelihood)
|
|
2136
|
+
self.kkt_gap = float(kkt_gap)
|
|
2137
|
+
self.n_iter = int(n_iter)
|
|
2138
|
+
self.converged = bool(converged)
|
|
2139
|
+
self.info = dict(info or {})
|
|
2140
|
+
self._cache: dict[str, Any] = {}
|
|
2141
|
+
self.kind = model.kind if model is not None else "fractions"
|
|
2142
|
+
self.basis = model.likelihoods.basis if model is not None else None
|
|
2143
|
+
self.bins = (
|
|
2144
|
+
self.basis.bins
|
|
2145
|
+
if self.basis is not None
|
|
2146
|
+
else model.bins
|
|
2147
|
+
if model is not None
|
|
2148
|
+
else Bins.categories(self.value.size)
|
|
2149
|
+
)
|
|
2150
|
+
|
|
2151
|
+
@property
|
|
2152
|
+
def rho(self) -> np.ndarray:
|
|
2153
|
+
"""Population fractions at the optimum (per bin)."""
|
|
2154
|
+
return self._fractions(self._to_bins(self.value))
|
|
2155
|
+
|
|
2156
|
+
@property
|
|
2157
|
+
def rates(self) -> np.ndarray | None:
|
|
2158
|
+
"""Rates at the optimum (per bin): for fractions :math:`\\hat\\Lambda\\rho` with
|
|
2159
|
+
:math:`\\hat\\Lambda = W / (S\\cdot\\rho)` (``None`` with a relative selection)."""
|
|
2160
|
+
if self.kind == "rates":
|
|
2161
|
+
return self._to_bins(self.value)
|
|
2162
|
+
total = self.total
|
|
2163
|
+
return None if total is None else total * self.rho
|
|
2164
|
+
|
|
2165
|
+
@property
|
|
2166
|
+
def total(self) -> float | None:
|
|
2167
|
+
"""Total rate at the optimum (see :attr:`rates`)."""
|
|
2168
|
+
if self.kind == "rates":
|
|
2169
|
+
return float(self.value.sum())
|
|
2170
|
+
if self.model is None:
|
|
2171
|
+
return None
|
|
2172
|
+
S = self._inferred_selection() # inferred, at the mode
|
|
2173
|
+
S = self.model.total_exposure if S is None else np.ravel(S)
|
|
2174
|
+
return None if S is None else float(self.model.W / (S @ self.value))
|
|
2175
|
+
|
|
2176
|
+
def _rho_draws(self) -> np.ndarray:
|
|
2177
|
+
return self.rho[None, :]
|
|
2178
|
+
|
|
2179
|
+
def _which(self, which: str) -> np.ndarray:
|
|
2180
|
+
"""The estimate of ``which`` as one draw ``(1, K)``, as for posterior draws."""
|
|
2181
|
+
which = _which_name(which)
|
|
2182
|
+
if which == "density":
|
|
2183
|
+
return self.density()[None, :]
|
|
2184
|
+
if which == "rates" and self.rates is None:
|
|
2185
|
+
raise ValueError("No rates in this result (a relative selection, or no model).")
|
|
2186
|
+
if which == "selected":
|
|
2187
|
+
return self.selected[None, :]
|
|
2188
|
+
return (self.rho if which == "fractions" else self.rates)[None, :]
|
|
2189
|
+
|
|
2190
|
+
@property
|
|
2191
|
+
def selection(self) -> Selection | None:
|
|
2192
|
+
"""The selection function of the fit (a point estimate is not saved, so it is
|
|
2193
|
+
always the model's)."""
|
|
2194
|
+
return None if self.model is None else self.model.selection_spec
|
|
2195
|
+
|
|
2196
|
+
@property
|
|
2197
|
+
def best(self) -> PointEstimate:
|
|
2198
|
+
"""The best fit of a point estimate is itself."""
|
|
2199
|
+
return self
|
|
2200
|
+
|
|
2201
|
+
@property
|
|
2202
|
+
def selected(self) -> np.ndarray:
|
|
2203
|
+
"""The selected population at the estimate (see :attr:`PopulationResult.selected`)."""
|
|
2204
|
+
return self._selected_of(self._weight_draws())[0]
|
|
2205
|
+
|
|
2206
|
+
def _center_x(self) -> np.ndarray:
|
|
2207
|
+
model = self._require_model()
|
|
2208
|
+
if self.x is not None and np.shape(self.x)[-1] == model.dim:
|
|
2209
|
+
return np.asarray(self.x, dtype=float).ravel()
|
|
2210
|
+
return model.from_value(self.value)
|
|
2211
|
+
|
|
2212
|
+
def check(self, kkt_max: float = 1e-4, strict: bool = False) -> list[str]:
|
|
2213
|
+
"""Warn (or raise with ``strict=True``) when the optimiser did not converge or the
|
|
2214
|
+
KKT gap (per object; see :meth:`PopulationModel.kkt_gap`) exceeds ``kkt_max``.
|
|
2215
|
+
|
|
2216
|
+
Estimates found by L-BFGS in unconstrained coordinates have no KKT gap (NaN); for
|
|
2217
|
+
them the check is ``converged`` (see ``tol`` of :class:`~binference.samplers.MAP`).
|
|
2218
|
+
|
|
2219
|
+
The default is ten times the round-off floor at which MAP stops (DECISIONS #12).
|
|
2220
|
+
Returns the messages (empty if the estimate looks fine).
|
|
2221
|
+
|
|
2222
|
+
Parameters
|
|
2223
|
+
----------
|
|
2224
|
+
kkt_max : float, optional
|
|
2225
|
+
Largest KKT gap that passes.
|
|
2226
|
+
strict : bool, optional
|
|
2227
|
+
Raise :class:`~binference.DiagnosticsError` instead of warning.
|
|
2228
|
+
"""
|
|
2229
|
+
msgs = []
|
|
2230
|
+
if not self.converged:
|
|
2231
|
+
msgs.append("the optimiser did not converge")
|
|
2232
|
+
if self.kkt_gap > kkt_max: # inf (an impossible object) fails; NaN passes
|
|
2233
|
+
msgs.append(f"KKT gap {self.kkt_gap:.1e} > {kkt_max:.0e}: not at the optimum")
|
|
2234
|
+
_report([(msgs, [], ConvergenceWarning)], strict)
|
|
2235
|
+
return msgs
|
|
2236
|
+
|
|
2237
|
+
def _weight_draws(self) -> np.ndarray:
|
|
2238
|
+
return self._fractions(self.value)[None, :]
|
|
2239
|
+
|
|
2240
|
+
def density(self) -> np.ndarray:
|
|
2241
|
+
"""Population density per unit bin volume at the optimum."""
|
|
2242
|
+
base = self.rates if self.kind == "rates" else self.rho
|
|
2243
|
+
return base / self.bins.volumes
|
|
2244
|
+
|
|
2245
|
+
def laplace(self, n_samples: int = 4000, rng: RNGLike = None, **kwargs: Any):
|
|
2246
|
+
"""Laplace approximation around the mode (see :class:`binference.samplers.Laplace`).
|
|
2247
|
+
|
|
2248
|
+
Parameters
|
|
2249
|
+
----------
|
|
2250
|
+
n_samples : int, optional
|
|
2251
|
+
Draws of the Laplace approximation.
|
|
2252
|
+
rng : int, Generator or None
|
|
2253
|
+
Random number generator.
|
|
2254
|
+
**kwargs
|
|
2255
|
+
Options of :class:`~binference.samplers.Laplace`.
|
|
2256
|
+
"""
|
|
2257
|
+
from .samplers import Laplace
|
|
2258
|
+
|
|
2259
|
+
return Laplace(self._require_model(), rng=rng, **kwargs).run(n_samples)
|
|
2260
|
+
|
|
2261
|
+
def __repr__(self) -> str:
|
|
2262
|
+
return (
|
|
2263
|
+
f"PointEstimate(method={self.method!r}, K={self.value.size}, "
|
|
2264
|
+
f"log_posterior={self.log_posterior:.6g}, kkt_gap={self.kkt_gap:.2e}, "
|
|
2265
|
+
f"converged={self.converged})"
|
|
2266
|
+
)
|
|
2267
|
+
|
|
2268
|
+
def plot(self, **kwargs: Any):
|
|
2269
|
+
from .plotting import plot_population
|
|
2270
|
+
|
|
2271
|
+
return plot_population(self, **kwargs)
|