binference 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. binference/__init__.py +85 -0
  2. binference/_kernels.py +252 -0
  3. binference/_quadrature.py +202 -0
  4. binference/_simplex.py +54 -0
  5. binference/_utils.py +139 -0
  6. binference/_version.py +1 -0
  7. binference/basis.py +314 -0
  8. binference/bins.py +645 -0
  9. binference/density.py +323 -0
  10. binference/diagnostics.py +400 -0
  11. binference/estimator.py +197 -0
  12. binference/inference.py +368 -0
  13. binference/io.py +309 -0
  14. binference/likelihoods.py +1684 -0
  15. binference/measure.py +246 -0
  16. binference/model.py +833 -0
  17. binference/plotting.py +538 -0
  18. binference/priors/__init__.py +47 -0
  19. binference/priors/base.py +248 -0
  20. binference/priors/dirichlet.py +640 -0
  21. binference/priors/hyper.py +240 -0
  22. binference/priors/logistic.py +676 -0
  23. binference/priors/rates.py +139 -0
  24. binference/response.py +539 -0
  25. binference/results.py +2271 -0
  26. binference/samplers/__init__.py +19 -0
  27. binference/samplers/_curvature.py +67 -0
  28. binference/samplers/_nuts_core.py +284 -0
  29. binference/samplers/base.py +118 -0
  30. binference/samplers/bootstrap.py +136 -0
  31. binference/samplers/gibbs.py +287 -0
  32. binference/samplers/laplace.py +200 -0
  33. binference/samplers/nuts.py +681 -0
  34. binference/samplers/optimize.py +488 -0
  35. binference/samplers/vb.py +118 -0
  36. binference/selection.py +621 -0
  37. binference/simulate.py +720 -0
  38. binference/validation.py +477 -0
  39. binference/warnings.py +48 -0
  40. binference-0.1.0.dist-info/METADATA +130 -0
  41. binference-0.1.0.dist-info/RECORD +43 -0
  42. binference-0.1.0.dist-info/WHEEL +4 -0
  43. binference-0.1.0.dist-info/licenses/LICENSE +21 -0
binference/results.py ADDED
@@ -0,0 +1,2271 @@
1
+ """Result objects returned by every inference engine."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import math
7
+ from collections.abc import Callable, Sequence
8
+ from functools import partial
9
+ from typing import TYPE_CHECKING, Any
10
+
11
+ import numpy as np
12
+ from scipy import special
13
+
14
+ from . import _kernels as kern
15
+ from . import diagnostics as diag
16
+ from ._utils import RNGLike, as_generator, check_nonnegative, chunk_slices, is_integer_array, warn
17
+ from .bins import Bins
18
+ from .measure import Measure
19
+ from .warnings import ConvergenceWarning, DiagnosticsError, InputWarning
20
+
21
+ if TYPE_CHECKING: # pragma: no cover
22
+ from .basis import Basis
23
+ from .likelihoods import Likelihoods
24
+ from .model import PopulationModel
25
+ from .selection import Selection
26
+
27
+ __all__ = ["PointEstimate", "PopulationResult", "Summary"]
28
+
29
+ # the per-object methods' outputs grow with draws x objects (audit O-18)
30
+ #: Population draws the per-object methods use by default.
31
+ N_DRAWS = 500
32
+ #: Largest split R-hat that :meth:`PopulationResult.check` accepts by default.
33
+ RHAT_MAX = 1.01
34
+ #: Smallest bulk and tail ESS per chain that :meth:`PopulationResult.check` accepts by
35
+ #: default.
36
+ ESS_MIN_PER_CHAIN = 100.0
37
+ #: Chains shorter than this many draws get no R-hat or ESS.
38
+ MIXING_MIN_DRAWS = 8
39
+ #: :meth:`PopulationResult.check` warns when a NUTS chain's E-BFMI is below this.
40
+ EBFMI_MIN = 0.3
41
+ #: :meth:`PopulationResult.check` warns when a larger share of NUTS iterations hit
42
+ #: ``max_depth``.
43
+ MAX_DEPTH_SHARE = 0.05
44
+ #: :meth:`PopulationResult.check` warns when the summed Monte Carlo variance of
45
+ #: likelihoods from samples exceeds this.
46
+ SAMPLE_VARIANCE_MAX = 1.0
47
+
48
+
49
+ # ----------------------------------------------------------------------------
50
+ # Summary table
51
+ # ----------------------------------------------------------------------------
52
+ def _small_concentration(model: Any) -> bool:
53
+ """Whether a conjugate prior's density is unbounded at the boundary (long log tails)."""
54
+ prior = getattr(model, "prior", None)
55
+ return bool(prior is not None and prior.conjugate and prior.unbounded_density())
56
+
57
+
58
+ class Summary:
59
+ """A small per-bin summary table with a readable ``repr``.
60
+
61
+ Columns are available as attributes and via :meth:`to_dict` /
62
+ :meth:`to_pandas`.
63
+
64
+ Parameters
65
+ ----------
66
+ columns : dict of str to ndarray
67
+ The table's columns, by name.
68
+ title : str
69
+ A heading shown above the table.
70
+ """
71
+
72
+ def __init__(self, columns: dict[str, np.ndarray], title: str = "") -> None:
73
+ self.columns = columns
74
+ self.title = title
75
+
76
+ def __getitem__(self, key: str) -> np.ndarray:
77
+ return self.columns[key]
78
+
79
+ def __getattr__(self, key: str) -> np.ndarray:
80
+ try:
81
+ return self.__dict__["columns"][key]
82
+ except KeyError:
83
+ raise AttributeError(key) from None
84
+
85
+ def to_dict(self) -> dict[str, np.ndarray]:
86
+ return dict(self.columns)
87
+
88
+ def to_pandas(self):
89
+ import pandas as pd
90
+
91
+ return pd.DataFrame(self.columns)
92
+
93
+ def __len__(self) -> int:
94
+ return len(next(iter(self.columns.values())))
95
+
96
+ def __repr__(self) -> str:
97
+ names = list(self.columns)
98
+ rows = []
99
+ for i in range(len(self)):
100
+ row = []
101
+ for n in names:
102
+ v = self.columns[n][i]
103
+ if isinstance(v, float | np.floating):
104
+ row.append(f"{v:.4g}") # inf and nan as themselves
105
+ else:
106
+ row.append(str(v))
107
+ rows.append(row)
108
+ widths = [max(len(n), *(len(r[j]) for r in rows)) for j, n in enumerate(names)]
109
+ lines = [self.title] if self.title else []
110
+ lines.append(" ".join(n.rjust(w) for n, w in zip(names, widths, strict=True)))
111
+ max_rows = 40
112
+ shown = rows if len(rows) <= max_rows else [*rows[:20], ["..."] * len(names), *rows[-19:]]
113
+ lines.extend(" ".join(c.rjust(w) for c, w in zip(r, widths, strict=True)) for r in shown)
114
+ return "\n".join(lines)
115
+
116
+ _repr_html_ = None
117
+
118
+
119
+ def _bin_labels(bins: Bins) -> np.ndarray:
120
+ if bins.is_categorical:
121
+ return np.array([str(lab) for lab in bins.labels], dtype=object)
122
+ if bins.ndim == 1:
123
+ e = bins.edges[0]
124
+ j = bins.cell_index[0]
125
+ return np.array([f"[{e[k]:.4g}, {e[k + 1]:.4g})" for k in j], dtype=object)
126
+ idx = np.stack(bins.cell_index, axis=1)
127
+ return np.array([str(tuple(int(i) for i in row)) for row in idx], dtype=object)
128
+
129
+
130
+ def _hdi(x: np.ndarray, level: float) -> tuple[float, float]:
131
+ s = np.sort(x)
132
+ n = s.size
133
+ m = int(np.ceil(level * n))
134
+ if m >= n:
135
+ return float(s[0]), float(s[-1])
136
+ widths = s[m - 1 :] - s[: n - m + 1]
137
+ j = int(np.argmin(widths))
138
+ return float(s[j]), float(s[j + m - 1])
139
+
140
+
141
+ # ----------------------------------------------------------------------------
142
+ # shared individual-object helpers
143
+ # ----------------------------------------------------------------------------
144
+ class _IndividualMixin:
145
+ """Methods for per-object outputs that need the likelihoods (via the model).
146
+
147
+ With a :class:`~binference.Basis` the model's columns are components: these
148
+ methods work on the component weights and report per bin, sharing each
149
+ component among the bins by its mass :math:`A_{jk}` (exact for flat
150
+ components; for hats the likelihood's variation within a component is
151
+ ignored in that share).
152
+ """
153
+
154
+ model: PopulationModel | None
155
+ basis: Basis | None
156
+ bins: Bins
157
+ kind: str
158
+ method: str
159
+ info: dict[str, Any]
160
+
161
+ def _require_model(self) -> PopulationModel:
162
+ if self.model is None:
163
+ raise RuntimeError(
164
+ "This result is not attached to its likelihoods; call result.attach(likelihoods)."
165
+ )
166
+ return self.model
167
+
168
+ def _rho_draws(self) -> np.ndarray: # (n, K) fractions in the bins
169
+ raise NotImplementedError
170
+
171
+ def _weight_draws(self) -> np.ndarray: # (n, C) fractions of the model's components
172
+ raise NotImplementedError
173
+
174
+ def _fractions(self, values: np.ndarray) -> np.ndarray:
175
+ """Values ``(..., C)`` as fractions: rates normalised, fractions as they are."""
176
+ return values / values.sum(axis=-1, keepdims=True) if self.kind == "rates" else values
177
+
178
+ def plot_individual(self, i: int, **kwargs: Any):
179
+ """Object ``i``'s own likelihood against its hierarchical posterior (see
180
+ :func:`binference.plotting.plot_individual`).
181
+
182
+ Parameters
183
+ ----------
184
+ i : int
185
+ Object index.
186
+ **kwargs
187
+ Passed to :func:`binference.plotting.plot_individual`.
188
+ """
189
+ from .plotting import plot_individual
190
+
191
+ return plot_individual(self, i, **kwargs)
192
+
193
+ def _select_draws(self, n_draws: int | None, components: bool = False) -> np.ndarray:
194
+ R = self._weight_draws() if components else self._rho_draws()
195
+ return R[_draw_index(R.shape[0], n_draws)]
196
+
197
+ def _inferred_selection(self) -> np.ndarray | None:
198
+ """Per-draw selection ``(chain, draw, K)`` when a selection from injections or a
199
+ tallied response (its efficiencies) is inferred with the population."""
200
+ if "response_draws" in self.info:
201
+ return np.asarray(self.info["response_draws"]).sum(axis=-2)
202
+ return self.info.get("selection_draws")
203
+
204
+ def _joint_selection(self, n_draws: int | None) -> np.ndarray | None:
205
+ """:meth:`_inferred_selection` as ``(n_draws, K)``, thinned as the population."""
206
+ S = self._inferred_selection()
207
+ return None if S is None else _flat_draws(S, 1, n_draws)
208
+
209
+ def _response_rows(self, n_draws: int | None) -> np.ndarray | None:
210
+ """An inferred response's draws ``(n_draws, J, K)``, thinned as the population."""
211
+ Rd = self.info.get("response_draws")
212
+ return None if Rd is None else _flat_draws(Rd, 2, n_draws)
213
+
214
+ def _object_draws(self, n_draws: int | None):
215
+ """The likelihoods and per-draw column weights for per-object methods.
216
+
217
+ With a selection inferred jointly (DECISIONS #79) these are the bare
218
+ likelihoods and :math:`\\rho\\circ S` with each draw's own :math:`S`, which is
219
+ also returned; with a response inferred jointly (DECISIONS #81), each draw's
220
+ rows ``(n_draws, N, K)`` and :math:`\\rho`; otherwise the model's likelihoods
221
+ (selection on the truth applied) and :math:`\\rho`, and ``None``.
222
+ """
223
+ model = self._require_model()
224
+ R = self._select_draws(n_draws, components=True)
225
+ Rd = self._response_rows(n_draws)
226
+ if Rd is not None:
227
+ return Rd[:, model.likelihoods.meta["measured_bin"]], R, None
228
+ S = self._joint_selection(n_draws) # a selection: no response was inferred
229
+ return *model._rows(R, S), S
230
+
231
+ def _to_bins(self, P: np.ndarray) -> np.ndarray:
232
+ """Per-component values ``(..., C)`` summed into the bins ``(..., K)``."""
233
+ return P if self.basis is None else self.basis.to_bins(P)
234
+
235
+ def _labels_to_bins(self, labels: np.ndarray, gen: np.random.Generator) -> np.ndarray:
236
+ """Component labels as bin labels, each drawn from its component's shares."""
237
+ if self.basis is None or self.basis.is_identity:
238
+ return labels
239
+ S = self.basis.shares # rows sum to one: invert their cumulative sums
240
+ cum = np.cumsum(S.data)
241
+ flat = labels.ravel()
242
+ first, last = S.indptr[flat], S.indptr[flat + 1] - 1
243
+ start = np.concatenate(([0.0], cum))[first]
244
+ pos = np.searchsorted(cum, start + gen.random(flat.size), side="right")
245
+ return S.indices[np.clip(pos, first, last)].reshape(labels.shape)
246
+
247
+ def _counts_to_bins(self, counts: np.ndarray, gen: np.random.Generator) -> np.ndarray:
248
+ """Integer counts per component as counts per bin, shared multinomially."""
249
+ if self.basis is None or self.basis.is_identity:
250
+ return counts
251
+ S = self.basis.shares
252
+ n = kern._multinomial_csr(S, np.round(counts).astype(np.int64), np.ones(self.bins.K), gen)
253
+ return np.bincount(S.indices, weights=n, minlength=self.bins.K)
254
+
255
+ def _new_likelihoods(self, likelihoods: Any) -> Likelihoods:
256
+ from .likelihoods import Likelihoods
257
+
258
+ model = self._require_model()
259
+ basis = model.likelihoods.basis
260
+ lk = (
261
+ likelihoods
262
+ if isinstance(likelihoods, Likelihoods)
263
+ else Likelihoods(likelihoods, basis or model.bins)
264
+ )
265
+ if (
266
+ lk.K != model.K
267
+ or lk.basis != basis
268
+ or (basis is None and not _same_bins(lk.bins, model.bins))
269
+ ):
270
+ raise ValueError("New likelihoods must use the same bins (and basis) as the fit.")
271
+ return lk
272
+
273
+ best: PointEstimate
274
+
275
+ def assignment_probabilities_loo(
276
+ self, idx: Any = None, n_draws: int | None = N_DRAWS, chunk: int = 4096
277
+ ) -> tuple[np.ndarray, np.ndarray]:
278
+ r"""Each object's posterior over bins under the population fitted without it,
279
+ :math:`P(z_i = k \mid D_{-i})`: leave-one-out, for calibration checks
280
+ (DECISIONS #52).
281
+
282
+ Pareto-smoothed importance sampling over the posterior draws (Vehtari, Gelman &
283
+ Gabry 2017): draw :math:`s` has the weight :math:`p(d_i \mid \rho_s)^{-w_i}`
284
+ (the whole row, with its multiplicity, is left out), smoothed per object by
285
+ :func:`~binference.diagnostics.psis`. Objects whose Pareto :math:`\hat k` is at
286
+ or above :func:`~binference.diagnostics.psis_threshold` are flagged with a
287
+ warning: their estimate is unreliable, and refitting without them is the remedy.
288
+
289
+ The draws must come from the posterior or an approximation of it (bootstrap
290
+ draws are refused). In a rate model an object's factor is
291
+ :math:`\Lambda\,L_i\cdot\rho`; with a selection in a fractions model,
292
+ :math:`L_i\cdot\rho / S\cdot\rho`. An object impossible under some draw gets
293
+ :math:`\hat k = \infty` and a zero row.
294
+
295
+ Returns ``(probabilities, k_hat)``: ``(n, K)`` and ``(n,)`` for the objects
296
+ ``idx`` (all by default), ``chunk`` objects at a time.
297
+
298
+ Parameters
299
+ ----------
300
+ idx : index, optional
301
+ Objects to use; ``None`` uses all of them.
302
+ n_draws : int, optional
303
+ Population draws used; ``None`` uses all of them.
304
+ chunk : int, optional
305
+ Objects processed per block, which bounds memory.
306
+ """
307
+ if n_draws is not None and n_draws < 5:
308
+ raise ValueError(f"n_draws must be at least 5 for leave-one-out; got {n_draws}.")
309
+ if self.method == "bootstrap":
310
+ raise ValueError("Leave-one-out reweights posterior draws; bootstrap draws are not.")
311
+ model = self._require_model()
312
+ L, R, S = self._object_draws(n_draws)
313
+ m = R.shape[0]
314
+ if m < 5:
315
+ raise ValueError(f"Leave-one-out needs posterior draws (at least 5); this has {m}.")
316
+ if self.kind == "rates": # Poisson: the object's factor is Lambda (L_i . rho)
317
+ log_norm = -np.log(_flat_draws(self._params, 1, n_draws).sum(axis=1))
318
+ else:
319
+ norm = self._norms(R, S, n_draws)
320
+ log_norm = 0.0 if norm is None else np.log(norm)
321
+ rows = np.atleast_1d(np.arange(model.N)[slice(None) if idx is None else idx])
322
+ w, k_hat = model._w[rows], np.empty(rows.size)
323
+
324
+ def weigh(sl, T): # PSIS weights of the objects sl from p(d_i | rho_s) ~ T
325
+ with np.errstate(divide="ignore"):
326
+ lw = np.log(T)
327
+ lw -= log_norm
328
+ lw *= -w[sl, None]
329
+ bad = ~np.all(np.isfinite(lw), axis=1) # impossible under some draw
330
+ lw[bad] = 0.0
331
+ lw, k_hat[sl] = diag._psis_rows(lw)
332
+ k_hat[sl][bad], lw[bad] = np.inf, -np.inf
333
+ return np.exp(lw, out=lw)
334
+
335
+ out = _mean_responsibilities(_objects(L, model.N, idx), R, chunk, weigh)
336
+ bad = k_hat >= diag.psis_threshold(m)
337
+ if bad.any():
338
+ warn(
339
+ f"{int(bad.sum())} of {rows.size} objects have Pareto k-hat >= "
340
+ f"{diag.psis_threshold(m):.2f}: their leave-one-out posteriors are unreliable "
341
+ f"(first: {rows[bad][:5].tolist()}); refit without them, or use more draws.",
342
+ ConvergenceWarning,
343
+ )
344
+ return self._to_bins(out), k_hat
345
+
346
+ # per-object outputs at the best fit (DECISIONS #51)
347
+ def assignment_probabilities_best(self, idx: Any = None, **kwargs: Any) -> np.ndarray:
348
+ """:meth:`assignment_probabilities` at the best fit, :attr:`best`.
349
+
350
+ Parameters
351
+ ----------
352
+ idx : index, optional
353
+ Objects to use; ``None`` uses all of them.
354
+ **kwargs
355
+ Further arguments of :meth:`assignment_probabilities`.
356
+ """
357
+ return self.best.assignment_probabilities(idx, **kwargs)
358
+
359
+ def sample_assignments_best(self, idx: Any = None, **kwargs: Any) -> np.ndarray:
360
+ """:meth:`sample_assignments` at the best fit, :attr:`best`.
361
+
362
+ Parameters
363
+ ----------
364
+ idx : index, optional
365
+ Objects to use; ``None`` uses all of them.
366
+ **kwargs
367
+ Further arguments of :meth:`sample_assignments`.
368
+ """
369
+ return self.best.sample_assignments(idx, **kwargs)
370
+
371
+ def sample_values_best(self, idx: Any = None, **kwargs: Any) -> np.ndarray:
372
+ """:meth:`sample_values` at the best fit, :attr:`best`.
373
+
374
+ Parameters
375
+ ----------
376
+ idx : index, optional
377
+ Objects to use; ``None`` uses all of them.
378
+ **kwargs
379
+ Further arguments of :meth:`sample_values`.
380
+ """
381
+ return self.best.sample_values(idx, **kwargs)
382
+
383
+ def realized_counts_best(self, **kwargs: Any) -> np.ndarray:
384
+ """:meth:`realized_counts` at the best fit, :attr:`best`.
385
+
386
+ Parameters
387
+ ----------
388
+ **kwargs
389
+ Passed to :meth:`realized_counts`.
390
+ """
391
+ return self.best.realized_counts(**kwargs)
392
+
393
+ def predict_proba_best(self, likelihoods: Likelihoods | Any, **kwargs: Any) -> np.ndarray:
394
+ """:meth:`predict_proba` at the best fit, :attr:`best`.
395
+
396
+ Parameters
397
+ ----------
398
+ likelihoods : Likelihoods or array_like (N, K)
399
+ Likelihoods of the objects.
400
+ **kwargs
401
+ Passed to :meth:`predict_proba`.
402
+ """
403
+ return self.best.predict_proba(likelihoods, **kwargs)
404
+
405
+ def log_predictive_best(self, likelihoods: Likelihoods | Any, **kwargs: Any) -> np.ndarray:
406
+ """:meth:`log_predictive` at the best fit, :attr:`best`.
407
+
408
+ Parameters
409
+ ----------
410
+ likelihoods : Likelihoods or array_like (N, K)
411
+ Likelihoods of the objects.
412
+ **kwargs
413
+ Passed to :meth:`log_predictive`.
414
+ """
415
+ return self.best.log_predictive(likelihoods, **kwargs)
416
+
417
+ def assignment_probabilities(
418
+ self, idx: Any = None, n_draws: int | None = N_DRAWS, chunk: int = 4096
419
+ ) -> np.ndarray:
420
+ r"""Posterior probability that each object lies in each bin, :math:`P(z_i = k \mid D)`.
421
+
422
+ Averages the per-object conditional posteriors
423
+ :math:`r_{ik}(\rho)` over (up to ``n_draws``) population draws: the
424
+ "hierarchical shrinkage" posteriors of the individual objects.
425
+
426
+ Returns an ``(n, K)`` array for the objects ``idx`` (all by default).
427
+
428
+ Parameters
429
+ ----------
430
+ idx : index, optional
431
+ Objects to use; ``None`` uses all of them.
432
+ n_draws : int, optional
433
+ Population draws used; ``None`` uses all of them.
434
+ chunk : int, optional
435
+ Objects processed per block, which bounds memory.
436
+ """
437
+ model = self._require_model()
438
+ L, R, _ = self._object_draws(n_draws)
439
+ return self._to_bins(_mean_responsibilities(_objects(L, model.N, idx), R, chunk))
440
+
441
+ def sample_assignments(
442
+ self, idx: Any = None, n_draws: int | None = N_DRAWS, rng: RNGLike = None
443
+ ) -> np.ndarray:
444
+ """Joint draws of the bin labels of objects ``idx`` (all by default).
445
+
446
+ For each of (up to ``n_draws``) population draws :math:`\\rho^{(s)}`, labels are
447
+ drawn from :math:`z_i \\sim \\mathrm{Cat}(r_i(\\rho^{(s)}))`; together these
448
+ are joint draws from :math:`p(\\rho, z \\mid D)`, exact given exact draws of
449
+ :math:`\\rho` (approximate engines carry their approximation over). Returns an
450
+ ``(n_draws, n_objects)`` integer array.
451
+
452
+ Parameters
453
+ ----------
454
+ idx : index, optional
455
+ Objects to use; ``None`` uses all of them.
456
+ n_draws : int, optional
457
+ Population draws used; ``None`` uses all of them.
458
+ rng : int, Generator or None
459
+ Random number generator.
460
+ """
461
+ gen = as_generator(rng)
462
+ return self._labels_to_bins(self._component_labels(idx, n_draws, gen), gen)
463
+
464
+ def _component_labels(self, idx: Any, n_draws: int | None, gen: np.random.Generator):
465
+ model = self._require_model()
466
+ L, R, _ = self._object_draws(n_draws)
467
+ L = _objects(L, model.N, idx)
468
+ return np.stack(
469
+ [kern.sample_labels(_draw_rows(L, s), rho, gen) for s, rho in enumerate(R)]
470
+ )
471
+
472
+ def realized_counts(
473
+ self, n_draws: int | None = N_DRAWS, rng: RNGLike = None, full: bool = False
474
+ ) -> np.ndarray:
475
+ """Draws of the realised histogram of the observed objects' true bins.
476
+
477
+ Unlike the population fractions (which include the shot noise of
478
+ drawing ``N`` objects from a population), this is the distribution of
479
+ ``n_k = sum_i w_i 1[z_i = k]`` for *these* objects. With ``full=True`` the
480
+ objects the selection missed are added, drawn for each draw as
481
+ :math:`\\mathrm{NegBin}(W, S\\cdot\\rho)` of them in bins
482
+ :math:`\\propto \\rho\\circ(1 - S)`: the counts of the full sample (DECISIONS #53).
483
+ Returns ``(n_draws, K)`` weighted counts.
484
+
485
+ Parameters
486
+ ----------
487
+ n_draws : int, optional
488
+ Population draws used; ``None`` uses all of them.
489
+ rng : int, Generator or None
490
+ Random number generator.
491
+ full : bool, optional
492
+ Add the objects the selection missed.
493
+ """
494
+ model = self._require_model()
495
+ gen = as_generator(rng)
496
+ S = self._missed_selection(n_draws) if full else None
497
+ L, R, _ = self._object_draws(n_draws)
498
+ comps = self._select_draws(n_draws, components=True) # R holds rho*S when S is inferred
499
+ w = model._w
500
+ unit = bool(np.all(w == 1))
501
+ integer = is_integer_array(w)
502
+ out = np.empty((R.shape[0], self.bins.K))
503
+ for s, rho in enumerate(R):
504
+ Ls = _draw_rows(L, s)
505
+ if integer:
506
+ n = kern.sample_counts(Ls, w, rho, gen, unit_weights=unit)[0]
507
+ out[s] = self._counts_to_bins(n, gen)
508
+ else:
509
+ labels = self._labels_to_bins(kern.sample_labels(Ls, rho, gen), gen)
510
+ out[s] = np.bincount(labels, weights=w, minlength=self.bins.K)
511
+ if full:
512
+ Ss = S if S.ndim == 1 else S[s]
513
+ missed = model._draw_missed(comps[s], np.zeros(comps.shape[1]), gen, S=Ss)
514
+ out[s] += self._counts_to_bins(missed, gen)
515
+ return out
516
+
517
+ def _missed_selection(self, n_draws: int | None) -> np.ndarray:
518
+ # the detection probabilities: one row, or one per draw when inferred
519
+ model = self._require_model()
520
+ sel = model.selection_spec
521
+ if self.kind == "rates" or (sel is not None and sel.scale == "exposure"):
522
+ raise ValueError("Full-sample counts need detection probabilities, not exposures.")
523
+ S = self._joint_selection(n_draws)
524
+ S = model.total_exposure if S is None else S
525
+ if S is None:
526
+ raise ValueError(
527
+ "Full-sample counts need an absolute selection; this one is relative, so "
528
+ "the number of missed objects is not identified (DECISIONS #46)."
529
+ )
530
+ return S
531
+
532
+ def _selected_of(self, W: np.ndarray) -> np.ndarray:
533
+ """The selected population :math:`\\phi = S\\circ w / (S\\cdot w)` per bin, for rows of
534
+ component weights ``W`` (all draws, or the estimate), with each draw's own S
535
+ when the selection is inferred (DECISIONS #49)."""
536
+ S = self._joint_selection(None) # all draws, as W
537
+ if S is None and self.selection is not None:
538
+ S = self.selection.values
539
+ if S is None:
540
+ return self._to_bins(W)
541
+ P = W * S
542
+ return self._to_bins(P / P.sum(axis=-1, keepdims=True))
543
+
544
+ @property
545
+ def data_support(self) -> np.ndarray:
546
+ """Objects' worth of likelihood in each bin, :math:`\\sum_i w_i \\ell_{ik}` with each
547
+ object's likelihood normalised over the bins: below about one, a bin's fraction
548
+ is set by the prior (DECISIONS #41)."""
549
+ if "data_support" not in self._cache:
550
+ model = self._require_model()
551
+ L = model._L # with a selection on the truth applied, as the fit sees them
552
+ tot = kern.matvec(L, np.ones(L.shape[1]))
553
+ w = np.divide(model._w, tot, out=np.zeros_like(tot), where=tot > 0)
554
+ self._cache["data_support"] = self._to_bins(kern.rmatvec(L, w))
555
+ return self._cache["data_support"]
556
+
557
+ def prior_information_ratio(self, h: float = 1e-4) -> float:
558
+ r"""Efron's (2016) ratio of prior to data information,
559
+ :math:`\mathrm{tr}(-H_\text{prior}) / \mathrm{tr}(-H_\text{data})`.
560
+
561
+ The Hessians are of the log densities in the prior's unconstrained coordinates
562
+ at the mean of the unconstrained draws (the optimum for point estimates; for
563
+ engines that keep no unconstrained draws, the image of the posterior mean, with
564
+ any jointly inferred selection or response at its injection or tally values),
565
+ by central differences of the gradient with step ``h``; the prior's is of its
566
+ density in those
567
+ coordinates, so a Dirichlet(:math:`\alpha`) counts as :math:`\alpha` objects
568
+ per bin. The traces depend on those coordinates, so compare ratios across fits
569
+ of one prior family, not across families. Small values mean the data dominate
570
+ (DECISIONS #41); NaN when the data show no curvature (e.g. an empty bin at a
571
+ boundary optimum). Costs :math:`4d` gradient evaluations for ``d`` prior
572
+ coordinates, half of them of the full log density.
573
+
574
+ Parameters
575
+ ----------
576
+ h : float, optional
577
+ Finite-difference step of the Hessians.
578
+ """
579
+ from .samplers._curvature import fd_hessian_diag
580
+
581
+ model = self._require_model()
582
+ x = self._center_x()
583
+ d = model.prior.dim
584
+
585
+ def prior_grad(xp):
586
+ eta, cache = model.prior.forward(xp)
587
+ v = model._value(eta)
588
+ return model.prior.log_prior_and_grad(xp, cache, v, np.zeros_like(v))
589
+
590
+ def total_grad(xp):
591
+ return model.log_density_unconstrained_and_grad(np.concatenate([xp, x[d:]]))
592
+
593
+ idx = range(d)
594
+ Hp = fd_hessian_diag(prior_grad, x[:d], idx, h=h)
595
+ Ht = fd_hessian_diag(total_grad, x[:d], idx, h=h)
596
+ data = float(np.sum(Hp - Ht))
597
+ return float(np.sum(-Hp) / data) if data > 0 else float("nan")
598
+
599
+ def sample_values(
600
+ self,
601
+ idx: Any = None,
602
+ n_draws: int | None = N_DRAWS,
603
+ rng: RNGLike = None,
604
+ loglike: Callable[[np.ndarray, int], np.ndarray] | None = None,
605
+ n_sub: int = 64,
606
+ n_sweeps: int = 20,
607
+ ) -> np.ndarray:
608
+ r"""Joint draws of the continuous true values of objects ``idx``.
609
+
610
+ Labels are drawn as in :meth:`sample_assignments`. Within the drawn bin
611
+ the population is flat in the measure coordinate ``u`` (its density in
612
+ ``z`` carries :math:`T'(z)`), so a true value follows the object's own
613
+ likelihood times that density, from each input's within-bin information
614
+ (DECISIONS #50):
615
+
616
+ * :meth:`Likelihoods.from_gaussian` with independent errors (``sigma``):
617
+ exactly (truncated Gaussians) on linear axes, else by inverse-CDF
618
+ sampling on ``n_sub`` sub-points per bin; with correlated errors
619
+ (``cov``), ``n_sweeps`` Gibbs sweeps over the axes' conditionals, started
620
+ from independent draws;
621
+ * :meth:`Likelihoods.from_density` on 1-D bins: the density (divided by
622
+ the interim prior for posteriors) on ``n_sub`` sub-points per bin;
623
+ * :meth:`Likelihoods.from_samples`: the object's samples inside the bin,
624
+ resampled with their weights :math:`w_s / \pi_u(u_s)` (flat in ``u``
625
+ in a bin no sample reached, as the ``floor`` is);
626
+ * ``loglike(z, i)``, if given, overrides these on 1-D bins: the
627
+ log-likelihood of object ``i`` (an index into the likelihoods) at the
628
+ points ``z``, on ``n_sub`` sub-points per bin;
629
+ * otherwise (tables, binned inputs, compressed rows, densities on grids,
630
+ samples not kept) flat in ``u`` within the bin, with a warning: the
631
+ top-hat approximation, accurate only when likelihoods are nearly flat
632
+ within bins.
633
+
634
+ With a :class:`~binference.Basis` the label is a component: flat
635
+ components are sampled as above on their cells; inside a hat the
636
+ population is the hat, so sub-point weights include its height, and
637
+ other draws on its support are kept with that height.
638
+
639
+ With weighted (compressed) rows each row is treated as one object.
640
+ Returns ``(n_draws, n_objects)`` (1-D) or ``(n_draws, n_objects, ndim)``.
641
+
642
+ Parameters
643
+ ----------
644
+ idx : index, optional
645
+ Objects to use; ``None`` uses all of them.
646
+ n_draws : int, optional
647
+ Population draws used; ``None`` uses all of them.
648
+ rng : int, Generator or None
649
+ Random number generator.
650
+ loglike : callable, optional
651
+ Each object's own log-likelihood within bins, ``loglike(z, i)``; overrides
652
+ the stored information on 1-D bins.
653
+ n_sub : int, optional
654
+ Sub-points per bin for inverse-CDF draws.
655
+ n_sweeps : int, optional
656
+ Gibbs sweeps over the axes for correlated Gaussian errors.
657
+ """
658
+ model = self._require_model()
659
+ bins = model.bins # the components' cells with a basis
660
+ if bins.is_categorical:
661
+ raise ValueError("Categorical bins have no continuous values; use sample_assignments.")
662
+ if loglike is not None and bins.ndim != 1:
663
+ raise ValueError("loglike-based sampling supports 1-D bins only.")
664
+ gen = as_generator(rng)
665
+ draw, informed = _value_drawer(model, loglike, n_sub, n_sweeps)
666
+ if not informed:
667
+ what = model.likelihoods.meta.get("source", "a table")
668
+ warn(
669
+ f"sample_values draws values from the population alone within each bin: these "
670
+ f"likelihoods ({what}) keep no information within bins; pass loglike= for each "
671
+ "object's own likelihood.",
672
+ InputWarning,
673
+ )
674
+ rows = np.atleast_1d(np.arange(model.N)[slice(None) if idx is None else idx])
675
+ labels = self._component_labels(rows, n_draws, gen)
676
+ basis = model.likelihoods.basis
677
+ hat = basis is not None and basis.order == 1
678
+ objects = np.broadcast_to(rows, labels.shape)
679
+ out = np.empty((*labels.shape, bins.ndim))
680
+ todo = np.ones(labels.shape, dtype=bool)
681
+ while todo.any(): # a single pass unless hats reject a proposal
682
+ lab = labels[todo]
683
+ if hat:
684
+ lo, hi = basis.support(lab)
685
+ else:
686
+ cells = list(zip(bins.u_edges, bins.grid_index(lab), strict=True))
687
+ lo = np.stack([u[j] for u, j in cells], axis=-1)
688
+ hi = np.stack([u[j + 1] for u, j in cells], axis=-1)
689
+ z, ok = draw(objects[todo], lab, lo, hi, gen, basis.height if hat else None)
690
+ at = np.flatnonzero(todo)[ok]
691
+ out.reshape(-1, bins.ndim)[at] = z[ok]
692
+ todo.ravel()[at] = False
693
+ return out[..., 0] if bins.ndim == 1 else out
694
+
695
+ def predict_proba(
696
+ self, likelihoods: Likelihoods | Any, n_draws: int | None = N_DRAWS, chunk: int = 4096
697
+ ):
698
+ """Posterior over bins for *new* objects under the inferred population.
699
+
700
+ Parameters
701
+ ----------
702
+ likelihoods : Likelihoods or array_like (N, K)
703
+ Likelihoods of the new objects, on the bins of the fit.
704
+ n_draws : int, optional
705
+ Population draws used; ``None`` uses all of them.
706
+ chunk : int, optional
707
+ Objects processed per block, which bounds memory.
708
+ """
709
+ L, R, _, _ = self._new_object_draws(self._new_likelihoods(likelihoods), n_draws)
710
+ return self._to_bins(_mean_responsibilities(L, R, chunk))
711
+
712
+ def log_predictive(
713
+ self, likelihoods: Likelihoods | Any, n_draws: int | None = N_DRAWS, chunk: int = 4096
714
+ ):
715
+ r"""Log posterior-predictive density of new objects, :math:`\log E[L_i\cdot\rho]`.
716
+
717
+ Includes the per-object log-scales of the likelihoods, so values can be
718
+ compared across priors and binnings (e.g. for cross-validation). Works
719
+ through the objects ``chunk`` at a time.
720
+
721
+ Parameters
722
+ ----------
723
+ likelihoods : Likelihoods or array_like (N, K)
724
+ Likelihoods of the new objects, on the bins of the fit.
725
+ n_draws : int, optional
726
+ Population draws used; ``None`` uses all of them.
727
+ chunk : int, optional
728
+ Objects processed per block, which bounds memory.
729
+ """
730
+ # rates rescale out of the ratio
731
+ L, R, norm, log_scale = self._new_object_draws(self._new_likelihoods(likelihoods), n_draws)
732
+ n = L.shape[-2]
733
+ lp = np.empty(n)
734
+ for sl in chunk_slices(n, chunk):
735
+ T = _row_dots(L, R, sl)[1]
736
+ if norm is not None: # a new *detected* object: divide by S . rho
737
+ T = T / norm[None, :]
738
+ with np.errstate(divide="ignore"):
739
+ lp[sl] = special.logsumexp(np.log(T), axis=1)
740
+ return lp - np.log(R.shape[0]) + log_scale
741
+
742
+ def _new_object_draws(self, lk: Likelihoods, n_draws: int | None):
743
+ """For new objects ``lk``: rows and column weights per draw as in
744
+ :meth:`_object_draws`, each draw's :math:`S\\cdot\\rho` (``None`` without a
745
+ selection) and the rows' log-scales. New binned objects with the response
746
+ inferred in this fit take its draws (DECISIONS #81)."""
747
+ model = self._require_model()
748
+ Rd = self._response_rows(n_draws)
749
+ if Rd is not None and lk.meta.get("response") is model.response:
750
+ R = self._select_draws(n_draws, components=True)
751
+ rows = Rd[:, lk.meta["measured_bin"]] # the response's own rows: scale one
752
+ return rows, R, self._norms(R, None, n_draws), np.zeros(lk.N)
753
+ _, R, S = self._object_draws(n_draws)
754
+ norm = self._norms(R, S, n_draws)
755
+ if S is None and model.selection_row is not None:
756
+ R = R * model.selection_row # L~ . rho = L . (S o rho): the rows are not copied
757
+ return lk.values, R, norm, lk.log_scale
758
+
759
+ def _norms(self, R: np.ndarray, S: np.ndarray | None, n_draws: int | None):
760
+ """Each draw's normaliser of a detected object's likelihood, :math:`S\\cdot\\rho`
761
+ (the response's efficiencies with an inferred response), for the draws ``R`` and
762
+ inferred selection ``S`` of :meth:`_object_draws`; ``None`` without a selection."""
763
+ model = self._require_model()
764
+ if "response_draws" in self.info:
765
+ return np.sum(self._joint_selection(n_draws) * R, axis=1)
766
+ if model.selection is None:
767
+ return None
768
+ return R.sum(axis=1) if S is not None else R @ model.selection
769
+
770
+
771
+ def _rebuild_gaussian(lk: Likelihoods, basis: Basis) -> Likelihoods:
772
+ """Gaussian likelihoods built again on ``basis`` from their kept means and errors."""
773
+ from .likelihoods import Likelihoods
774
+
775
+ meta = lk.meta
776
+ if "gaussian_mean" not in meta:
777
+ raise ValueError(
778
+ "Only likelihoods from Likelihoods.from_gaussian rebuild themselves; pass "
779
+ "build(bins_or_basis) returning these data's likelihoods on a basis."
780
+ )
781
+ spread = (
782
+ {"cov": meta["gaussian_cov"]}
783
+ if "gaussian_cov" in meta
784
+ else {"sigma": meta["gaussian_sigma"]}
785
+ )
786
+ w = lk.weights
787
+ return Likelihoods.from_gaussian(
788
+ meta["gaussian_mean"],
789
+ bins=basis,
790
+ weights=w,
791
+ fractional_weights=not is_integer_array(w),
792
+ warn_outside=False,
793
+ **meta.get("gaussian_options", {}),
794
+ **spread,
795
+ )
796
+
797
+
798
+ def _mean_responsibilities(
799
+ L: Any, R: np.ndarray, chunk: int, weigh: Callable | None = None
800
+ ) -> np.ndarray:
801
+ r"""Per-object posteriors over the columns, averaged over the draws of column weights
802
+ ``R``, ``chunk`` objects at a time (``L`` may hold rows per draw, ``(n, N, K)``):
803
+ equally, or with the weights ``weigh(sl, T)`` returns for the objects ``sl`` from
804
+ their :math:`T_{is} = L_i\cdot R_s` (``(n_sl, draws)``, rows summing to one)."""
805
+ n, m = L.shape[-2], R.shape[0]
806
+ out = np.zeros((n, L.shape[-1]))
807
+ for sl in chunk_slices(n, chunk):
808
+ block, T = _row_dots(L, R, sl)
809
+ block = block.toarray() if hasattr(block, "toarray") else np.asarray(block)
810
+ rows = np.arange(sl.start, sl.stop)
811
+ W = np.full(T.shape, 1.0 / m) if weigh is None else weigh(sl, T)
812
+ if block.ndim == 2: # rows shared by the draws: sum_s w_s L R_s / (L.R_s), 2 products
813
+ tiny = np.finfo(float).tiny # 1/T overflows below: those rows go draw by draw
814
+ inv = np.divide(W, T, out=np.zeros_like(T), where=T > tiny) # no support: zero
815
+ out[sl] = block * (inv @ R)
816
+ slow = np.any((T > 0) & (T <= tiny), axis=1)
817
+ if not slow.any():
818
+ continue
819
+ rows, block = (
820
+ rows[slow],
821
+ np.broadcast_to(block[slow], (m, slow.sum(), out.shape[1])),
822
+ )
823
+ out[rows] = 0.0
824
+ for s, rho in enumerate(R): # draw by draw
825
+ P = block[s] * rho
826
+ tot = P.sum(axis=1, keepdims=True)
827
+ out[rows] += W[rows - sl.start, s, None] * P / np.where(tot > 0, tot, 1.0)
828
+ return out
829
+
830
+
831
+ def _row_dots(L: Any, R: np.ndarray, sl: slice) -> tuple[Any, np.ndarray]:
832
+ # the rows of the objects sl (per draw when L is (n, N, K)) and their products with
833
+ # the draws R, T_is = L_i . R_s (n_sl, draws)
834
+ block = L[:, sl] if L.ndim == 3 else L[sl]
835
+ T = np.einsum("snk,sk->ns", block, R) if L.ndim == 3 else np.asarray(block @ R.T)
836
+ return block, T
837
+
838
+
839
+ def _objects(L: Any, N: int, idx: Any) -> Any:
840
+ # the rows of objects idx (all by default), of per-draw rows (n_draws, N, K) too
841
+ if idx is None:
842
+ return L
843
+ rows = np.atleast_1d(np.arange(N)[idx])
844
+ return L[:, rows] if L.ndim == 3 else L[rows]
845
+
846
+
847
+ def _draw_rows(L: Any, s: int) -> Any:
848
+ # draw s's rows when they differ by draw (an inferred response), else the rows
849
+ return L[s] if L.ndim == 3 else L
850
+
851
+
852
+ def _flat_draws(A: Any, n_tail: int, n_draws: int | None) -> np.ndarray:
853
+ # per-draw arrays (chain, draw, *tail) as (n, *tail), up to n_draws spread evenly
854
+ A = np.asarray(A)
855
+ A = A.reshape(-1, *A.shape[A.ndim - n_tail :])
856
+ return A[_draw_index(len(A), n_draws)]
857
+
858
+
859
+ def _central(X: np.ndarray, level: float, axis: int = 0) -> np.ndarray:
860
+ """Bounds ``(lo, hi)`` of the central interval of probability ``level`` along ``axis``."""
861
+ return np.quantile(X, [0.5 - level / 2, 0.5 + level / 2], axis=axis)
862
+
863
+
864
+ #: the symbol of each quantity in labels and names (rates as in audit G-10)
865
+ _SYMBOL = {"density": "density", "fractions": "rho", "rates": "lambda", "selected": "phi"}
866
+
867
+
868
+ def _which_name(which: str) -> str:
869
+ """``"fractions"``, ``"rates"``, ``"density"`` or ``"selected"``, singular spellings
870
+ accepted (O-12)."""
871
+ name = {"fraction": "fractions", "rate": "rates"}.get(which, which)
872
+ if name not in ("fractions", "rates", "density", "selected"):
873
+ raise ValueError(
874
+ f"which must be 'fractions', 'rates', 'density' or 'selected'; got {which!r}."
875
+ )
876
+ return name
877
+
878
+
879
+ def _mixing(C: np.ndarray, tail: bool = False) -> dict[str, np.ndarray]:
880
+ """Bulk (and tail) ESS and split R-hat of each column of draws ``(chain, draw, K)``;
881
+ none below ``MIXING_MIN_DRAWS`` per chain (split R-hat also for one chain, audit O-7)."""
882
+ if C.shape[1] < MIXING_MIN_DRAWS:
883
+ return {}
884
+ cols = range(C.shape[-1])
885
+ out = {"ess_bulk": np.array([diag.ess(C[:, :, k], "bulk") for k in cols])}
886
+ if tail:
887
+ out["ess_tail"] = np.array([diag.ess(C[:, :, k], "tail") for k in cols])
888
+ out["rhat"] = np.array([diag.rhat(C[:, :, k]) for k in cols])
889
+ return out
890
+
891
+
892
+ def _draw_index(n_total: int, n_draws: int | None) -> np.ndarray | slice:
893
+ # up to n_draws draws spread evenly over all of them
894
+ if n_draws is None or n_draws >= n_total:
895
+ return slice(None)
896
+ return np.linspace(0, n_total - 1, int(n_draws)).round().astype(int)
897
+
898
+
899
+ def _draws_rng(draws: np.ndarray) -> np.random.Generator:
900
+ # a stream seeded by a hash of the draws: reproducible, different for different draws,
901
+ # and independent of any generator a caller holds (DECISIONS #78)
902
+ digest = hashlib.sha256(np.ascontiguousarray(draws).tobytes()).digest()
903
+ return np.random.default_rng(np.frombuffer(digest, dtype=np.uint32))
904
+
905
+
906
+ def _report(groups: list[tuple[list[str], list[str], type]], strict: bool) -> None:
907
+ # warn once per (messages, hints, category) group, or raise all of them together
908
+ texts = []
909
+ for found, why, category in groups:
910
+ if found:
911
+ text = "; ".join(found) + (". Hint: " + "; ".join(why) if why else "")
912
+ texts.append(text)
913
+ if not strict:
914
+ warn(f"Diagnostics: {text}.", category)
915
+ if strict and texts:
916
+ raise DiagnosticsError(f"Diagnostics failed: {'; '.join(texts)}")
917
+
918
+
919
+ def _same_bins(a: Bins, b: Bins) -> bool:
920
+ # equal bins up to the identity of their measures, which do not survive save and load
921
+ # (a transform axis loads as a table)
922
+ if a.is_categorical or b.is_categorical:
923
+ return a == b
924
+ return (
925
+ a.shape == b.shape
926
+ and all(np.array_equal(x, y) for x, y in zip(a.edges, b.edges, strict=True))
927
+ and [m.name for m in a.measures] == [m.name for m in b.measures]
928
+ and a.is_masked == b.is_masked
929
+ and (not a.is_masked or np.array_equal(a.mask, b.mask))
930
+ )
931
+
932
+
933
+ def _same_layout(a: Basis | None, b: Basis | None) -> bool:
934
+ # equal bases up to the identity of their measures, which do not survive save and load
935
+ if a is None or b is None:
936
+ return a is b
937
+ return (a.order, a.resolution, a.oversample, a.K, a.C) == (
938
+ b.order,
939
+ b.resolution,
940
+ b.oversample,
941
+ b.K,
942
+ b.C,
943
+ ) and all(np.allclose(x, y) for x, y in zip(a.bins.edges, b.bins.edges, strict=True))
944
+
945
+
946
+ def _value_drawer(model, loglike, n_sub: int, n_sweeps: int) -> tuple[Callable, bool]:
947
+ """``draw(objects, labels, lo, hi, gen, height) -> (z, keep)``: values in the boxes
948
+ ``[lo, hi]`` (in ``u``, ``(n, ndim)``) from the input's own likelihood times the
949
+ population, flat in ``u`` times the hat's ``height(labels, d, u)`` per axis (``None``
950
+ when flat); and whether the input has within-bin information (DECISIONS #50)."""
951
+ meta = model.likelihoods.meta
952
+ measures = model.bins.measures
953
+ flat = partial(_draw_subpoints, measures, n_sub, None)
954
+ if loglike is not None:
955
+ return partial(_draw_subpoints, measures, n_sub, _per_object(loglike)), True
956
+ if "gaussian_mean" in meta:
957
+ return partial(_draw_gauss, measures, n_sub, n_sweeps, meta), True
958
+ within = meta.get("within")
959
+ if within is None or within.data is None or "value_row" not in meta:
960
+ return flat, False # e.g. rows merged by compress, which drops value_row
961
+ kind, *data = within.data
962
+ row = np.asarray(meta["value_row"])
963
+ if kind == "samples":
964
+ direct = model.likelihoods.basis is None # labels are the bins the samples are in
965
+ return partial(_draw_samples, measures, flat, direct, row, *data), True
966
+ evaluate = partial(_dists_logl if kind == "dists" else _rows_logl, row, *data)
967
+ return partial(_draw_subpoints, measures, n_sub, evaluate), True
968
+
969
+
970
+ #: Most values a density function returns per call when values are drawn (2**24 floats
971
+ #: are 128 MB).
972
+ EVAL_BUDGET = 2**24
973
+
974
+
975
+ def _dists_logl(row, dists, interim, objects, labels, zz):
976
+ # distributions of from_density: one vectorised call for the rows' own points
977
+ from .likelihoods import _rows_logpdf, log_interim
978
+
979
+ with np.errstate(invalid="ignore"): # -inf - -inf where the interim vanishes: NaN
980
+ return _rows_logpdf(dists, 1)(zz, row[objects]) - log_interim(interim, zz)
981
+
982
+
983
+ def _rows_logl(row, n_in, fn, log, interim, objects, labels, zz):
984
+ # a density function of from_density returns every input row at common points: it
985
+ # is called on the sub-points of a few bins at a time, at most EVAL_BUDGET values
986
+ from .density import _log_nonneg
987
+ from .likelihoods import log_interim
988
+
989
+ out = np.empty(zz.shape)
990
+ _, first, inverse = np.unique(labels, return_index=True, return_inverse=True)
991
+ inverse = inverse.ravel()
992
+ per = max(1, EVAL_BUDGET // (n_in * zz.shape[1]))
993
+ for c in range(0, len(first), per):
994
+ at = np.flatnonzero((inverse >= c) & (inverse < c + per))
995
+ pts = zz[first[c : c + per]]
996
+ v = fn(pts.ravel())
997
+ v = np.asarray(v if log else _log_nonneg(v), dtype=float).reshape(n_in, *pts.shape)
998
+ out[at] = v[row[objects[at]], inverse[at] - c]
999
+ with np.errstate(invalid="ignore"): # -inf - -inf where the interim vanishes: NaN
1000
+ return out - log_interim(interim, zz)
1001
+
1002
+
1003
+ def _per_object(loglike: Callable[[np.ndarray, int], np.ndarray]) -> Callable:
1004
+ """An evaluator ``(objects, labels, zz) -> log-likelihoods`` calling ``loglike(z, i)``
1005
+ once per object, at the sub-points of all its bins."""
1006
+
1007
+ def evaluate(objects, labels, zz):
1008
+ key = objects.astype(np.int64) * (int(labels.max()) + 1) + labels # by object
1009
+ _, first, inverse = np.unique(key, return_index=True, return_inverse=True)
1010
+ vals = np.empty((len(first), zz.shape[1]))
1011
+ ids, starts = np.unique(objects[first], return_index=True)
1012
+ for i, a, b in zip(ids, starts, [*starts[1:], len(first)], strict=True):
1013
+ pts = zz[first[a:b]]
1014
+ vals[a:b] = np.asarray(loglike(pts.ravel(), int(i)), dtype=float).reshape(pts.shape)
1015
+ return vals[inverse.ravel()]
1016
+
1017
+ return evaluate
1018
+
1019
+
1020
+ def _subpoints(zlo, zhi, n_sub):
1021
+ # midpoints of n_sub equal cells in z of each box, (n, n_sub)
1022
+ return zlo[:, None] + (zhi - zlo)[:, None] * ((np.arange(n_sub) + 0.5) / n_sub)
1023
+
1024
+
1025
+ def _subpoint_draw(zz, width, logw, gen, objects) -> np.ndarray:
1026
+ """One value per row by inverse-CDF sampling on the sub-points ``zz`` (cells of
1027
+ ``width / n_sub``) with log-weights ``logw``."""
1028
+ n, n_sub = logw.shape
1029
+ logw = np.where(np.isnan(logw), -np.inf, logw) # e.g. a posterior over a zero prior
1030
+ empty = ~np.isfinite(logw).any(axis=1)
1031
+ if empty.any(): # it drew from the box's first sub-point before (audit O-11)
1032
+ raise ValueError(
1033
+ f"The likelihood is zero at all {n_sub} sub-points of a bin with posterior "
1034
+ f"mass (object {int(objects[empty][0])}); raise n_sub or check loglike."
1035
+ )
1036
+ cell = kern._sample_labels_dense(np.exp(logw - logw.max(axis=1, keepdims=True)), 1.0, gen)
1037
+ return zz[np.arange(n), cell] + width / n_sub * (gen.random(n) - 0.5)
1038
+
1039
+
1040
+ def _base_logw(m, zz, labels, d, height):
1041
+ # flat in u (the Jacobian in z) times the hat's height along axis d
1042
+ with np.errstate(divide="ignore"):
1043
+ return m.log_jacobian(zz) + (
1044
+ 0.0 if height is None else np.log(height(labels, d, m.forward(zz)))
1045
+ )
1046
+
1047
+
1048
+ def _draw_subpoints(measures, n_sub, evaluate, objects, labels, lo, hi, gen, height):
1049
+ # each axis on n_sub sub-points: the population, times evaluate(objects, labels, zz)
1050
+ z = np.empty(lo.shape)
1051
+ for d, m in enumerate(measures):
1052
+ zlo, zhi = m.inverse(lo[:, d]), m.inverse(hi[:, d])
1053
+ zz = _subpoints(zlo, zhi, n_sub)
1054
+ logw = _base_logw(m, zz, labels, d, height)
1055
+ if evaluate is not None:
1056
+ logw = logw + evaluate(objects, labels, zz)
1057
+ z[:, d] = _subpoint_draw(zz, zhi - zlo, logw, gen, objects)
1058
+ return z, np.ones(len(z), dtype=bool)
1059
+
1060
+
1061
+ def _truncnorm(a, b, gen) -> np.ndarray:
1062
+ """Standard normal draws truncated to ``[a, b]``, by the inverse CDF in log space
1063
+ (intervals above zero are reflected, so tails keep their precision)."""
1064
+ flip = a > 0
1065
+ lo, hi = np.where(flip, -b, a), np.where(flip, -a, b)
1066
+ la, lb = special.log_ndtr(lo), special.log_ndtr(hi)
1067
+ u = gen.random(np.shape(lo))
1068
+ with np.errstate(divide="ignore"): # u = 0 at a lower bound of -inf
1069
+ x = special.ndtri_exp(lb + np.log(u + (1 - u) * np.exp(la - lb)))
1070
+ x = np.clip(x, lo, hi)
1071
+ return np.where(flip, -x, x)
1072
+
1073
+
1074
+ def _draw_gauss(measures, n_sub, n_sweeps, meta, objects, labels, lo, hi, gen, height):
1075
+ """Gaussian likelihoods: exact truncated draws per axis on linear axes (the hat's
1076
+ height as an acceptance test), sub-points elsewhere; with a covariance, ``n_sweeps``
1077
+ Gibbs sweeps over the axes' conditionals from that start (sub-points inside hats)."""
1078
+ n, nd = lo.shape
1079
+ mu = np.asarray(meta["gaussian_mean"])[objects]
1080
+ P = None
1081
+ if "gaussian_cov" in meta:
1082
+ ids, inverse = np.unique(objects, return_inverse=True) # one inverse per object
1083
+ C = np.asarray(meta["gaussian_cov"])[ids]
1084
+ P = np.linalg.inv(C)[inverse]
1085
+ sig = np.sqrt(np.diagonal(C, axis1=1, axis2=2))[inverse]
1086
+ else:
1087
+ sig = np.asarray(meta["gaussian_sigma"])[objects]
1088
+ zlo = np.stack([m.inverse(lo[:, d]) for d, m in enumerate(measures)], axis=1)
1089
+ zhi = np.stack([m.inverse(hi[:, d]) for d, m in enumerate(measures)], axis=1)
1090
+ keep = np.ones(n, dtype=bool)
1091
+
1092
+ def axis(d, m, mu_d, sig_d, reject):
1093
+ if m == Measure.linear() and (height is None or reject):
1094
+ x = mu_d + sig_d * _truncnorm(
1095
+ (zlo[:, d] - mu_d) / sig_d, (zhi[:, d] - mu_d) / sig_d, gen
1096
+ )
1097
+ if height is not None:
1098
+ keep[:] &= gen.random(n) < height(labels, d, m.forward(x))
1099
+ return x
1100
+ zz = _subpoints(zlo[:, d], zhi[:, d], n_sub)
1101
+ logw = (
1102
+ _base_logw(m, zz, labels, d, height)
1103
+ - 0.5 * ((zz - mu_d[:, None]) / sig_d[:, None]) ** 2
1104
+ )
1105
+ return _subpoint_draw(zz, zhi[:, d] - zlo[:, d], logw, gen, objects)
1106
+
1107
+ z = np.empty((n, nd))
1108
+ for d, m in enumerate(measures):
1109
+ z[:, d] = axis(d, m, mu[:, d], sig[:, d], reject=P is None)
1110
+ for _ in range(n_sweeps if P is not None else 0):
1111
+ for d, m in enumerate(measures):
1112
+ s2 = 1.0 / P[:, d, d]
1113
+ rest = np.einsum("nj,nj->n", P[:, d], z - mu) - P[:, d, d] * (z[:, d] - mu[:, d])
1114
+ z[:, d] = axis(d, m, mu[:, d] - s2 * rest, np.sqrt(s2), reject=False)
1115
+ return z, keep
1116
+
1117
+
1118
+ def _draw_samples(
1119
+ measures, flat, direct, row, K, key, zs, G, objects, labels, lo, hi, gen, height
1120
+ ):
1121
+ """An object's kept samples inside the box, drawn with their weights (``G`` their
1122
+ cumulative sum, each row's summing to one; ``key`` = row * K + bin, sorted); in
1123
+ a box no sample reached, flat in ``u`` as the ``floor`` is."""
1124
+ n, nd = lo.shape
1125
+ z, keep = np.empty((n, nd)), np.ones(n, dtype=bool)
1126
+ r = row[objects]
1127
+ G0 = np.concatenate(([0.0], G))
1128
+ if direct: # the label is the bin each sample was counted in: one segment each
1129
+ s0 = np.searchsorted(key, r * K + labels, side="left")
1130
+ s1 = np.searchsorted(key, r * K + labels, side="right")
1131
+ a, b = G0[s0], G0[s1]
1132
+ hit = b > a # samples with weight in the bin
1133
+ u = a[hit] + (1.0 - gen.random(hit.sum())) * (b - a)[hit]
1134
+ z[hit] = zs[np.clip(np.searchsorted(G, u, side="left"), s0[hit], s1[hit] - 1)]
1135
+ else: # components: the samples of the row inside the box, times the hat's height
1136
+ hit = np.zeros(n, dtype=bool)
1137
+ s0 = np.searchsorted(key, r * K, side="left")
1138
+ s1 = np.searchsorted(key, (r + 1) * K, side="left")
1139
+ order = np.lexsort((labels, objects))
1140
+ cut = np.flatnonzero(np.diff(objects[order]) | np.diff(labels[order])) + 1
1141
+ last = -1
1142
+ for at in np.split(order, cut): # the draws of one object in one box
1143
+ i = at[0]
1144
+ seg = slice(s0[i], s1[i])
1145
+ if objects[i] != last: # the object's samples in u, once for all its boxes
1146
+ us = np.stack([m.forward(zs[seg, d]) for d, m in enumerate(measures)], axis=1)
1147
+ last = objects[i]
1148
+ w = np.diff(G0[s0[i] : s1[i] + 1]) * np.all((us >= lo[i]) & (us <= hi[i]), axis=1)
1149
+ for d in range(nd if height is not None else 0):
1150
+ w = w * height(labels[i], d, us[:, d])
1151
+ c = np.cumsum(w)
1152
+ if c.size and c[-1] > 0:
1153
+ z[at] = zs[seg][np.searchsorted(c, (1.0 - gen.random(at.size)) * c[-1])]
1154
+ hit[at] = True
1155
+ miss = ~hit
1156
+ if miss.any():
1157
+ z[miss], keep[miss] = flat(objects[miss], labels[miss], lo[miss], hi[miss], gen, height)
1158
+ return z, keep
1159
+
1160
+
1161
+ # ----------------------------------------------------------------------------
1162
+ # PopulationResult
1163
+ # ----------------------------------------------------------------------------
1164
+ class PopulationResult(_IndividualMixin):
1165
+ """Posterior draws of a population (fractions and, for rate models, rates).
1166
+
1167
+ Every inference engine returns this object (``MAP`` returns a
1168
+ :class:`PointEstimate`). Draws are stored per chain; flattened views are
1169
+ available as :attr:`samples` (fractions) and :attr:`rates`.
1170
+
1171
+ Attributes
1172
+ ----------
1173
+ method : str
1174
+ Engine that produced the draws.
1175
+ approximate : bool
1176
+ True for approximations: VB, bootstrap, and Laplace draws that are
1177
+ uncorrected, fail their Pareto k-hat check, or (under
1178
+ ``method="auto"``) have an importance ESS below ``target_ess``.
1179
+ info : dict
1180
+ Engine statistics (e.g. NUTS tree depths, divergences, step sizes).
1181
+ labels : ndarray or None
1182
+ Joint label draws stored by Gibbs, ``(chains, draws, objects)`` for the
1183
+ objects in ``label_index``: bin indices, or basis-component indices when
1184
+ the likelihoods use a :class:`~binference.Basis`;
1185
+ :meth:`sample_assignments` returns bins either way.
1186
+ """
1187
+
1188
+ def __init__(
1189
+ self,
1190
+ model: PopulationModel | None,
1191
+ draws: np.ndarray,
1192
+ *,
1193
+ method: str,
1194
+ kind: str | None = None,
1195
+ bins: Bins | None = None,
1196
+ x: np.ndarray | None = None,
1197
+ log_density_unconstrained: np.ndarray | None = None,
1198
+ approximate: bool = False,
1199
+ info: dict[str, Any] | None = None,
1200
+ labels: np.ndarray | None = None,
1201
+ label_index: np.ndarray | None = None,
1202
+ prior_repr: str | None = None,
1203
+ prior_key: str | None = None,
1204
+ basis: Basis | None = None,
1205
+ total: np.ndarray | None = None,
1206
+ selection: Selection | None = None,
1207
+ ) -> None:
1208
+ d = np.asarray(draws, dtype=float)
1209
+ if d.ndim == 2:
1210
+ d = d[None]
1211
+ if d.ndim != 3:
1212
+ raise ValueError("draws must have shape (n_chains, n_draws, K) or (n_draws, K).")
1213
+ self.model = model
1214
+ if basis is None and model is not None:
1215
+ basis = model.likelihoods.basis
1216
+ #: the :class:`~binference.Basis` of the model's components, or ``None``
1217
+ self.basis = basis
1218
+ self._params = d # the model's columns: component weights with a basis
1219
+ self._draws = self._to_bins(d) # per bin (the same array without a basis)
1220
+ self.kind = kind = kind or (model.kind if model is not None else "fractions")
1221
+ if kind == "rates":
1222
+ total = self._draws.sum(axis=-1)
1223
+ # for fractions, drawn on first use (see total)
1224
+ self._total = (
1225
+ None if total is None else np.asarray(total, dtype=float).reshape(d.shape[:2])
1226
+ )
1227
+ self.bins = (
1228
+ bins
1229
+ if bins is not None
1230
+ else basis.bins
1231
+ if basis is not None
1232
+ else model.bins
1233
+ if model is not None
1234
+ else Bins.categories(d.shape[-1])
1235
+ )
1236
+ self.x = x
1237
+ #: log posterior density of each draw in the unconstrained coordinates
1238
+ #: (with the Jacobian; DECISIONS #15), for samplers that compute it
1239
+ self.log_density_unconstrained = log_density_unconstrained
1240
+ self.method = method
1241
+ self.approximate = approximate
1242
+ self.info = dict(info or {})
1243
+ self.labels = labels
1244
+ self.label_index = label_index
1245
+ self.prior_repr = prior_repr or (repr(model.prior_spec) if model is not None else None)
1246
+ #: a digest of the prior's arguments in full, which attach compares (the repr
1247
+ #: shortens arrays)
1248
+ self.prior_key = prior_key or (
1249
+ model.prior_spec._fingerprint() if model is not None else None
1250
+ )
1251
+ # the fit's selection, kept for attach after a load (audit O-1)
1252
+ self._selection = selection if model is None else model.selection_spec
1253
+ self._cache: dict[str, Any] = {}
1254
+
1255
+ # ------------------------------------------------------------- creation
1256
+ @classmethod
1257
+ def from_samples(
1258
+ cls,
1259
+ model: PopulationModel,
1260
+ samples: Any,
1261
+ *,
1262
+ method: str = "external",
1263
+ approximate: bool = False,
1264
+ info: dict[str, Any] | None = None,
1265
+ ) -> PopulationResult:
1266
+ """Wrap draws from an external sampler, ``(n_chains, n_draws, K)`` or ``(n_draws, K)``:
1267
+ fractions for a fractions model (rows summing to one), rates for a rate model.
1268
+
1269
+ A model that infers its selection or response with the population needs their
1270
+ draws too, as ``info["selection_draws"]`` or ``info["response_draws"]`` (the
1271
+ layout the engines store).
1272
+
1273
+ Parameters
1274
+ ----------
1275
+ model : PopulationModel
1276
+ The model the draws are of.
1277
+ samples : array_like (n_chains, n_draws, K) or (n_draws, K)
1278
+ The draws.
1279
+ method : str, optional
1280
+ Engine name recorded in the result.
1281
+ approximate : bool, optional
1282
+ Flag the draws as approximate.
1283
+ info : dict, optional
1284
+ Extra information stored with the draws.
1285
+ """
1286
+ key = "selection_draws" if model.selection_uncertain else "response_draws"
1287
+ if (model.selection_uncertain or model.response is not None) and key not in (info or {}):
1288
+ raise ValueError(
1289
+ f"This model infers its {key.split('_')[0]} with the population: pass its "
1290
+ f'draws as info["{key}"], or fix it (`.fixed()` or `.at_mean()`) first.'
1291
+ )
1292
+ X = np.asarray(samples, dtype=float)
1293
+ if X.ndim not in (2, 3) or X.shape[-1] != model.K:
1294
+ raise ValueError(
1295
+ f"samples must have shape (n_chains, n_draws, {model.K}) or (n_draws, "
1296
+ f"{model.K}); got {X.shape}."
1297
+ )
1298
+ check_nonnegative(X, "samples")
1299
+ if model.kind == "fractions":
1300
+ sums = X.sum(axis=-1)
1301
+ if not np.allclose(sums, 1.0, atol=1e-6):
1302
+ raise ValueError(
1303
+ "samples of a fractions model must sum to one per draw; rates need a rate "
1304
+ "model (a GammaRates prior)."
1305
+ )
1306
+ X = X / sums[..., None]
1307
+ if (
1308
+ key in (info or {})
1309
+ and np.shape(info[key])[:2] != (X[None] if X.ndim == 2 else X).shape[:2]
1310
+ ):
1311
+ raise ValueError(
1312
+ f'info["{key}"] must have one draw per sample, (n_chains, n_draws, ...).'
1313
+ )
1314
+ return cls(model, X, method=method, approximate=approximate, info=info)
1315
+
1316
+ @property
1317
+ def selection(self) -> Selection | None:
1318
+ """The selection function of the fit, kept through :meth:`save` and :func:`load`."""
1319
+ return self._selection
1320
+
1321
+ _fit_model = True # whether the model is the fit's own (not bare likelihoods attached)
1322
+
1323
+ def _fit_model_only(self, what: str) -> PopulationModel:
1324
+ # the fit's model, refused when bare likelihoods were attached after load
1325
+ if not self._fit_model:
1326
+ raise ValueError(
1327
+ f"{what} needs the fit's prior: attach a PopulationModel built with it "
1328
+ "(bare likelihoods get the default prior)."
1329
+ )
1330
+ return self._require_model()
1331
+
1332
+ @property
1333
+ def best(self) -> PointEstimate:
1334
+ """The best fit: :class:`~binference.samplers.MAP` with its defaults, which is the
1335
+ mode of :math:`\\rho` on the simplex for conjugate priors whose density is bounded
1336
+ there (e.g. Dirichlet concentrations of at least one), and the mode in the
1337
+ unconstrained coordinates otherwise, and always with a jointly inferred selection
1338
+ or response (DECISIONS #15, #35). Computed on first use
1339
+ and kept; per-object outputs at it carry a ``_best`` suffix (DECISIONS #51).
1340
+ After :func:`load`, it needs the fit's prior: attach a model, not bare likelihoods."""
1341
+ if "best" not in self._cache:
1342
+ from .samplers import MAP
1343
+
1344
+ self._cache["best"] = MAP(self._fit_model_only("The best fit")).run()
1345
+ return self._cache["best"]
1346
+
1347
+ def attach(self, likelihoods_or_model: Any) -> PopulationResult:
1348
+ """Attach likelihoods (or a model) to a loaded result to enable per-object methods.
1349
+
1350
+ Bare likelihoods take the fit's selection; likelihoods or a model with another
1351
+ selection, or on other bins, are refused (audit O-1, O-2).
1352
+
1353
+ Parameters
1354
+ ----------
1355
+ likelihoods_or_model : Likelihoods or PopulationModel
1356
+ The fit's likelihoods, or a model built on them.
1357
+ """
1358
+ from .likelihoods import Likelihoods
1359
+ from .model import PopulationModel
1360
+ from .selection import _rebuild as _rebuild_selection
1361
+
1362
+ m, saved = likelihoods_or_model, self.selection
1363
+ if not isinstance(m, PopulationModel):
1364
+ lk = m if isinstance(m, Likelihoods) else Likelihoods(m, self.basis or self.bins)
1365
+ sel = None if lk.selection is not None else saved
1366
+ if sel is not None and sel.basis is not None and _same_layout(lk.basis, sel.basis):
1367
+ # the same layout; its measure did not survive saving
1368
+ sel = _rebuild_selection(*sel._parts()[:3], lk.basis, sel.injections)
1369
+ m = PopulationModel(lk, selection=sel)
1370
+ if m.selection_spec != saved:
1371
+ raise ValueError("The likelihoods (or model) carry another selection than this fit.")
1372
+ if not _same_layout(m.likelihoods.basis, self.basis):
1373
+ raise ValueError("The likelihoods use a different basis than this result.")
1374
+ if self.basis is None and not _same_bins(m.bins, self.bins):
1375
+ raise ValueError(f"The likelihoods are on other bins ({m.bins}) than this result.")
1376
+ # bare likelihoods get the default prior; a model counts as the fit's with its prior
1377
+ self._fit_model = isinstance(m, PopulationModel) and (
1378
+ self.prior_key is None or m.prior_spec._fingerprint() == self.prior_key
1379
+ )
1380
+ self.model = m
1381
+ self._selection = m.selection_spec
1382
+ # everything computed from the previous likelihoods: the cache, the missing
1383
+ # information and a fractions model's total (drawn with its W)
1384
+ self._cache.clear()
1385
+ self.info.pop("missing_information", None)
1386
+ if self.kind == "fractions":
1387
+ self._total = None
1388
+ return self
1389
+
1390
+ # ------------------------------------------------------------ properties
1391
+ @property
1392
+ def n_chains(self) -> int:
1393
+ return int(self._draws.shape[0])
1394
+
1395
+ @property
1396
+ def n_draws(self) -> int:
1397
+ """Total number of draws (over all chains)."""
1398
+ return int(self._draws.shape[0] * self._draws.shape[1])
1399
+
1400
+ @property
1401
+ def K(self) -> int:
1402
+ return int(self._draws.shape[-1])
1403
+
1404
+ @property
1405
+ def chains(self) -> np.ndarray:
1406
+ """Fraction draws per chain, shape ``(n_chains, n_per_chain, K)``."""
1407
+ return self._fractions(self._draws)
1408
+
1409
+ @property
1410
+ def samples(self) -> np.ndarray:
1411
+ """Population fractions, shape ``(n_draws, K)`` (chains concatenated)."""
1412
+ return _flat_draws(self.chains, 1, None)
1413
+
1414
+ @property
1415
+ def total(self) -> np.ndarray | None:
1416
+ r"""Draws of the total rate :math:`\Lambda` ``(n_draws,)``, or ``None``.
1417
+
1418
+ For fractions, each draw is :math:`\Lambda \sim \mathrm{Gamma}(W, S\cdot\rho)`
1419
+ given its :math:`\rho`: the exact posterior of the total under the prior
1420
+ :math:`p(\Lambda) \propto 1/\Lambda`, with :math:`W` the number of objects
1421
+ and :math:`S` the detection probabilities or exposures (one without a
1422
+ selection). ``None`` when the selection is only relative (DECISIONS #9,
1423
+ #46, #78) or the result was loaded without its total. For rate models,
1424
+ the sum of the rates.
1425
+
1426
+ The Gamma variates come from a stream seeded by the fraction draws
1427
+ themselves, so the totals are reproducible for any result (also wrapped
1428
+ external draws, and loaded results after :meth:`attach`), independent
1429
+ across different draws, and leave the caller's generator untouched.
1430
+ """
1431
+ if self._total is None and self.model is not None:
1432
+ S = self._inferred_selection() # per draw, when inferred with the population
1433
+ self._total = self.model.draw_total(self._params, _draws_rng(self._params), S)
1434
+ return None if self._total is None else self._total.ravel()
1435
+
1436
+ @property
1437
+ def rates(self) -> np.ndarray | None:
1438
+ r"""Rate draws ``(n_draws, K)``: :math:`\lambda = \Lambda\rho` per draw for
1439
+ fractions (see :attr:`total`), the sampled rates for rate models; ``None``
1440
+ when the total is not available."""
1441
+ if self.kind == "rates":
1442
+ return _flat_draws(self._draws, 1, None)
1443
+ total = self.total
1444
+ return None if total is None else total[:, None] * self.samples
1445
+
1446
+ def _rho_draws(self) -> np.ndarray:
1447
+ return self.samples
1448
+
1449
+ @property
1450
+ def component_draws(self) -> np.ndarray:
1451
+ """Draws of the model's parameters, ``(n_chains, n_per_chain, C)``: the component
1452
+ weights with a :class:`~binference.Basis`, else the same as the per-bin draws."""
1453
+ return self._params
1454
+
1455
+ def _weight_draws(self) -> np.ndarray:
1456
+ return self._fractions(_flat_draws(self._params, 1, None))
1457
+
1458
+ def _which(self, which: str) -> np.ndarray:
1459
+ which = _which_name(which)
1460
+ if which == "fractions":
1461
+ return self.samples
1462
+ if which == "rates":
1463
+ rates = self.rates
1464
+ if rates is None:
1465
+ raise ValueError(
1466
+ "This result has no rates: its selection is only relative, so the total "
1467
+ "is not identified (DECISIONS #46), or it has no model (attach the "
1468
+ "likelihoods)."
1469
+ )
1470
+ return rates
1471
+ if which == "selected":
1472
+ return self.selected
1473
+ return self.density()
1474
+
1475
+ @property
1476
+ def selected(self) -> np.ndarray:
1477
+ """Draws ``(n_draws, K)`` of the selected population
1478
+ :math:`\\phi = S\\circ\\rho / (S\\cdot\\rho)`, the bins of the observed objects;
1479
+ :math:`\\rho` itself without a selection (DECISIONS #49)."""
1480
+ if "selected" not in self._cache:
1481
+ self._cache["selected"] = self._selected_of(self._weight_draws())
1482
+ return self._cache["selected"]
1483
+
1484
+ def _center_x(self) -> np.ndarray:
1485
+ # the mean of the unconstrained draws, else the image of the mean columns
1486
+ model = self._require_model()
1487
+ if self.x is not None and self.x.shape[-1] == model.dim:
1488
+ return _flat_draws(self.x, 1, None).mean(axis=0)
1489
+ return model.from_value(_flat_draws(self._params, 1, None).mean(axis=0))
1490
+
1491
+ @property
1492
+ def mean(self) -> np.ndarray:
1493
+ """Posterior mean fractions ``(K,)``."""
1494
+ return self.samples.mean(axis=0)
1495
+
1496
+ @property
1497
+ def median(self) -> np.ndarray:
1498
+ """Posterior median fractions ``(K,)``."""
1499
+ return np.median(self.samples, axis=0)
1500
+
1501
+ @property
1502
+ def std(self) -> np.ndarray:
1503
+ """Posterior standard deviation of the fractions ``(K,)``."""
1504
+ return self.samples.std(axis=0, ddof=1)
1505
+
1506
+ def cov(self, which: str = "fractions") -> np.ndarray:
1507
+ """Posterior covariance ``(K, K)`` of the fractions (or ``which="rates"``, ...).
1508
+
1509
+ Parameters
1510
+ ----------
1511
+ which : {"fractions", "rates", "density", "selected"}, optional
1512
+ Quantity used: fractions, rates, densities or the selected population.
1513
+ """
1514
+ return np.cov(self._which(which), rowvar=False)
1515
+
1516
+ def corr(self, which: str = "fractions") -> np.ndarray:
1517
+ """Posterior correlation matrix ``(K, K)`` of the fractions (or ``which=...``).
1518
+
1519
+ Parameters
1520
+ ----------
1521
+ which : {"fractions", "rates", "density", "selected"}, optional
1522
+ Quantity used: fractions, rates, densities or the selected population.
1523
+ """
1524
+ with np.errstate(invalid="ignore", divide="ignore"):
1525
+ return np.corrcoef(self._which(which), rowvar=False)
1526
+
1527
+ def quantile(self, q: Any, which: str = "fractions") -> np.ndarray:
1528
+ """Posterior quantiles, shape ``(*np.shape(q), K)``.
1529
+
1530
+ Parameters
1531
+ ----------
1532
+ q : float or array_like
1533
+ Quantiles, in ``[0, 1]``.
1534
+ which : {"fractions", "rates", "density", "selected"}, optional
1535
+ Quantity used: fractions, rates, densities or the selected population.
1536
+ """
1537
+ return np.quantile(self._which(which), q, axis=0)
1538
+
1539
+ def interval(
1540
+ self, level: float = 0.68, kind: str = "central", which: str = "fractions"
1541
+ ) -> tuple[np.ndarray, np.ndarray]:
1542
+ """Per-bin credible intervals ``(lo, hi)``, each ``(K,)``.
1543
+
1544
+ Parameters
1545
+ ----------
1546
+ level : float, optional
1547
+ Probability inside each interval.
1548
+ kind : {"central", "hdi"}, optional
1549
+ Equal-tailed (``"central"``) or highest-density (``"hdi"``) intervals.
1550
+ which : {"fractions", "rates", "density", "selected"}, optional
1551
+ Quantity used: fractions, rates, densities or the selected population.
1552
+ """
1553
+ if not 0 < level < 1:
1554
+ raise ValueError("level must be in (0, 1).")
1555
+ X = self._which(which)
1556
+ if kind == "central":
1557
+ lo, hi = _central(X, level)
1558
+ return lo, hi
1559
+ if kind == "hdi":
1560
+ bounds = np.array([_hdi(X[:, k], level) for k in range(X.shape[1])])
1561
+ return bounds[:, 0], bounds[:, 1]
1562
+ raise ValueError("kind must be 'central' or 'hdi'.")
1563
+
1564
+ def density(self) -> np.ndarray:
1565
+ """Draws of the population density per unit bin volume, ``(n_draws, K)``.
1566
+
1567
+ Fractions divided by bin volumes (rates divided by volumes for rate
1568
+ models): the quantity to plot as n(z).
1569
+ """
1570
+ base = self.rates if self.kind == "rates" else self.samples
1571
+ return base / self.bins.volumes
1572
+
1573
+ def functional(
1574
+ self, func: Callable[[np.ndarray], Any], vectorized: bool = False, which: str = "fractions"
1575
+ ):
1576
+ """Draws of a derived quantity ``func(rho)`` (e.g. the mean redshift), or of
1577
+ the rates or densities with ``which``.
1578
+
1579
+ Parameters
1580
+ ----------
1581
+ func : callable
1582
+ Function of one draw ``(K,)``.
1583
+ vectorized : bool, optional
1584
+ Pass all draws to ``func`` at once, as an ``(n_draws, K)`` array.
1585
+ which : {"fractions", "rates", "density", "selected"}, optional
1586
+ Quantity used: fractions, rates, densities or the selected population.
1587
+
1588
+ Examples
1589
+ --------
1590
+ >>> zbar = result.functional(lambda r: r @ result.bins.centers) # doctest: +SKIP
1591
+ """
1592
+ X = self._which(which)
1593
+ if vectorized:
1594
+ return np.asarray(func(X))
1595
+ return np.array([func(r) for r in X])
1596
+
1597
+ def rebin(self, groups: Any, which: str = "fractions") -> np.ndarray:
1598
+ """Fractions (or rates, with ``which="rates"``) aggregated into coarser groups.
1599
+
1600
+ ``groups`` is either a length-``K`` array of integers mapping bins to groups
1601
+ (e.g. from :meth:`Bins.coarsen`) or a sequence of lists of bin indices.
1602
+ Returns ``(n_draws, n_groups)``.
1603
+
1604
+ Parameters
1605
+ ----------
1606
+ groups : array_like (K,) or sequence of lists
1607
+ Group of each bin, or the bins of each group.
1608
+ which : {"fractions", "rates", "density", "selected"}, optional
1609
+ Quantity used: fractions, rates, densities or the selected population.
1610
+ """
1611
+ if _which_name(which) == "density":
1612
+ raise ValueError("Densities per unit volume do not add across bins; rebin fractions.")
1613
+ X = self._which(which)
1614
+ try:
1615
+ arr = np.asarray(groups)
1616
+ except ValueError: # ragged sequence of groups
1617
+ arr = None
1618
+ if (
1619
+ arr is not None
1620
+ and arr.ndim == 1
1621
+ and arr.shape[0] == self.K
1622
+ and arr.dtype.kind in "iuf"
1623
+ ):
1624
+ if not is_integer_array(arr) or arr.min() < 0:
1625
+ raise ValueError("groups must map each bin to a non-negative integer group.")
1626
+ arr = arr.astype(np.intp)
1627
+ one_hot = np.zeros((self.K, int(arr.max()) + 1))
1628
+ one_hot[np.arange(self.K), arr] = 1.0
1629
+ return X @ one_hot
1630
+ return np.stack([X[:, list(grp)].sum(axis=1) for grp in groups], axis=1)
1631
+
1632
+ # ---------------------------------------------------------- diagnostics
1633
+ def _mixing_of(self, which: str) -> dict[str, np.ndarray]:
1634
+ """Bulk and tail ESS and split R-hat of the fractions or the rates, cached; a
1635
+ density shares its base's (they ignore a positive scale per bin)."""
1636
+ base = self.kind if which == "density" else which
1637
+ if ("mixing", base) not in self._cache:
1638
+ X = self._which(base).reshape(self.n_chains, -1, self.K)
1639
+ self._cache["mixing", base] = _mixing(X, tail=True)
1640
+ return self._cache["mixing", base]
1641
+
1642
+ @property
1643
+ def diagnostics(self) -> dict[str, Any]:
1644
+ """Convergence diagnostics (computed lazily and cached)."""
1645
+ if "diagnostics" in self._cache:
1646
+ return self._cache["diagnostics"]
1647
+ out: dict[str, Any] = {}
1648
+ # for rate models diagnose the rates, which also carry the total
1649
+ out.update(self._mixing_of(self.kind))
1650
+ Sd = self._inferred_selection() # a selection, or a response's efficiencies
1651
+ if Sd is not None:
1652
+ name = "response" if "response_draws" in self.info else "selection"
1653
+ out.update({f"{k}_{name}": v for k, v in _mixing(Sd).items()})
1654
+ for key in (
1655
+ "n_divergent",
1656
+ "k_hat",
1657
+ "k_hat_threshold",
1658
+ "kkt_gap",
1659
+ "max_depth_fraction",
1660
+ "missing_information",
1661
+ "converged",
1662
+ "ebfmi",
1663
+ ):
1664
+ if key in self.info:
1665
+ out[key] = self.info[key]
1666
+ self._cache["diagnostics"] = out
1667
+ return out
1668
+
1669
+ def _missing_information(self) -> float | None:
1670
+ """Fraction of missing information at the posterior mean (cached in ``info``)."""
1671
+ if (
1672
+ "missing_information" not in self.info
1673
+ and self.model is not None
1674
+ and self.model.K <= 500
1675
+ ):
1676
+ value = _flat_draws(self._params, 1, None).mean(axis=0)
1677
+ try:
1678
+ self.info["missing_information"] = self.model.missing_information(value)
1679
+ except (np.linalg.LinAlgError, ValueError):
1680
+ return None
1681
+ self._cache.pop("diagnostics", None)
1682
+ return self.info.get("missing_information")
1683
+
1684
+ def _mixing_problems(
1685
+ self, rhat_max: float = RHAT_MAX, ess_min: float = ESS_MIN_PER_CHAIN
1686
+ ) -> list[str]:
1687
+ """R-hat above ``rhat_max`` and bulk or tail ESS below ``ess_min`` per chain, of
1688
+ the population and of an inferred selection or response, for Markov chains."""
1689
+ d, msgs = self.diagnostics, []
1690
+ # Markov chains (and external draws), whose draws are correlated; independent
1691
+ # draws have no chain for R-hat and ESS to describe
1692
+ from .inference import METHODS
1693
+
1694
+ engine = METHODS.get(self.method)
1695
+ if self.approximate or (engine is not None and not engine.markov):
1696
+ return msgs
1697
+ for suffix, what in (
1698
+ ("", ""),
1699
+ ("_selection", " of the selection"),
1700
+ ("_response", " of the response's efficiencies"),
1701
+ ):
1702
+ if "rhat" + suffix in d and np.nanmax(d["rhat" + suffix]) > rhat_max:
1703
+ msgs.append(f"max R-hat{what} = {np.nanmax(d['rhat' + suffix]):.3f} > {rhat_max}")
1704
+ for kind in ("bulk", "tail"):
1705
+ key = f"ess_{kind}{suffix}"
1706
+ if key in d:
1707
+ ess = np.nanmin(d[key])
1708
+ per_chain = ess / max(1, self.n_chains)
1709
+ if per_chain < ess_min:
1710
+ # rounded down, so the message never shows the limit as reached
1711
+ msgs.append(
1712
+ f"min {kind} ESS{what} = {math.floor(ess)} "
1713
+ f"({math.floor(per_chain * 10) / 10:.1f} per chain < {ess_min:g})"
1714
+ )
1715
+ return msgs
1716
+
1717
+ def check(
1718
+ self,
1719
+ strict: bool = False,
1720
+ rhat_max: float = RHAT_MAX,
1721
+ ess_min: float = ESS_MIN_PER_CHAIN,
1722
+ predictive_p_min: float = 1e-3,
1723
+ predictive_draws: int = 200,
1724
+ support_min: float | None = None,
1725
+ ):
1726
+ """Check diagnostics and warn (or raise with ``strict=True``) about problems.
1727
+
1728
+ Sampler problems raise a :class:`~binference.ConvergenceWarning`; problems
1729
+ with the inputs (Monte Carlo noise of sample likelihoods, binned counts
1730
+ the response does not describe) an :class:`~binference.InputWarning`.
1731
+ For binned inputs with more measured than truth bins, the counts are
1732
+ tested with :meth:`Response.predictive_pvalue` over ``predictive_draws``
1733
+ draws, and flagged below ``predictive_p_min``. With ``support_min``, bins whose
1734
+ :attr:`data_support` is below it (in objects) are flagged as set by the prior.
1735
+
1736
+ Returns a list of messages (empty if everything looks fine).
1737
+
1738
+ Parameters
1739
+ ----------
1740
+ strict : bool, optional
1741
+ Raise :class:`~binference.DiagnosticsError` instead of warning.
1742
+ rhat_max : float, optional
1743
+ Largest R-hat that passes.
1744
+ ess_min : float, optional
1745
+ Smallest bulk or tail ESS per chain that passes.
1746
+ predictive_p_min : float, optional
1747
+ Smallest posterior predictive p-value of binned counts that passes.
1748
+ predictive_draws : int, optional
1749
+ Draws used for the predictive p-value.
1750
+ support_min : float, optional
1751
+ Flag bins with less data support than this; ``None`` skips the test.
1752
+ """
1753
+ d = self.diagnostics
1754
+ msgs = self._mixing_problems(rhat_max, ess_min)
1755
+ hints = []
1756
+ if msgs and self.method == "gibbs":
1757
+ hint = "Gibbs mixes slowly when likelihoods are broad compared with the bins"
1758
+ f = self._missing_information()
1759
+ if f is not None:
1760
+ tau = (1.0 + f) / max(1.0 - f, 1e-12)
1761
+ tau_text = f"~{tau:.0f}" if tau < 1000 else "over 1000"
1762
+ hint += f" (missing information {f:.2f}: autocorrelation time {tau_text})"
1763
+ hints.append(hint + "; run longer or use method='nuts'")
1764
+ if d.get("n_divergent", 0):
1765
+ n_div = int(d["n_divergent"])
1766
+ share = 100.0 * n_div / max(self.n_draws, 1)
1767
+ msgs.append(f"{n_div} divergent transitions ({share:.2g}% of draws)")
1768
+ hints.append(
1769
+ "try a higher target_accept, e.g. 0.95; divergences that persist mark a "
1770
+ "region the sampler cannot explore (see the checking guide: diagnostics)"
1771
+ )
1772
+ if "ebfmi" in d and np.nanmin(d["ebfmi"]) < EBFMI_MIN:
1773
+ msgs.append(f"E-BFMI = {np.nanmin(d['ebfmi']):.2f} < {EBFMI_MIN:g} in some chain")
1774
+ hints.append(
1775
+ "NUTS explores the energy distribution poorly (heavy tails or a funnel); "
1776
+ "a longer warm-up or another prior may help"
1777
+ )
1778
+ if d.get("converged") is False:
1779
+ msgs.append("the optimiser did not converge")
1780
+ threshold = d.get("k_hat_threshold", diag.K_HAT_MAX)
1781
+ if d.get("k_hat", 0) >= threshold:
1782
+ msgs.append(
1783
+ f"Pareto k-hat = {d['k_hat']:.2f} >= {threshold:.2f} (importance sampling "
1784
+ "unreliable)"
1785
+ )
1786
+ if d.get("max_depth_fraction", 0) > MAX_DEPTH_SHARE:
1787
+ msgs.append(
1788
+ f"{100 * d['max_depth_fraction']:.0f}% of NUTS iterations hit the maximum tree "
1789
+ "depth"
1790
+ )
1791
+ if _small_concentration(self.model):
1792
+ hints.append(
1793
+ "this prior gives nearly empty bins very long tails in log space "
1794
+ "(e.g. concentrations below one); method='gibbs' samples such "
1795
+ "conjugate priors efficiently"
1796
+ )
1797
+ lk = self.model.likelihoods if self.model is not None else None
1798
+ inputs, input_hints = [], []
1799
+ if lk is not None and "sample_sum" in lk.meta:
1800
+ # the population the objects are drawn from, after selection
1801
+ rho = self._weight_draws().mean(axis=0)
1802
+ if self.model.selection_row is not None:
1803
+ rho = rho * self.model.selection_row
1804
+ var, _ = lk.sample_variance(rho)
1805
+ total = float(np.sum(lk.weights * var))
1806
+ if total > SAMPLE_VARIANCE_MAX:
1807
+ inputs.append(
1808
+ f"likelihoods from samples carry Monte Carlo variance sum_i V_i = {total:.2g} "
1809
+ f"> {SAMPLE_VARIANCE_MAX:g} at the fitted population"
1810
+ )
1811
+ input_hints.append("draw more samples per object, or use wider bins or floor=")
1812
+ if lk is not None and support_min is not None:
1813
+ low = np.flatnonzero(self.data_support < support_min)
1814
+ if low.size:
1815
+ inputs.append(
1816
+ f"{low.size} bins have data support below {support_min:g} objects "
1817
+ f"(first: {low[:5].tolist()}): their fractions are set by the prior"
1818
+ )
1819
+ input_hints.append("merge sparse bins, or read their fractions as the prior's")
1820
+ if lk is not None and "measured_bin" in lk.meta:
1821
+ ppp = self._predictive_pvalue(lk, predictive_draws)
1822
+ if ppp is not None and ppp < predictive_p_min:
1823
+ inputs.append(
1824
+ "the measured counts disagree with the fitted population and the response "
1825
+ f"(posterior predictive p = {ppp:.1g})"
1826
+ )
1827
+ input_hints.append(
1828
+ "the response may not describe these data; for a classifier, the score "
1829
+ "distribution of a class differs between the validation and the target set "
1830
+ "(label shift violated)"
1831
+ )
1832
+ _report([(msgs, hints, ConvergenceWarning), (inputs, input_hints, InputWarning)], strict)
1833
+ return msgs + inputs
1834
+
1835
+ def _predictive_pvalue(self, lk: Any, n_draws: int) -> float | None:
1836
+ # label-shift check of binned inputs (DECISIONS #68); needs J > K
1837
+ key = ("predictive_p", int(n_draws))
1838
+ if key not in self._cache:
1839
+ resp = lk.meta["response"]
1840
+ J, K = resp.matrix.shape
1841
+ n = np.bincount(lk.meta["measured_bin"], weights=lk.weights, minlength=J)
1842
+ Rd = self._response_rows(n_draws) # an inferred response, drawn with rho
1843
+ self._cache[key] = (
1844
+ resp.predictive_pvalue(n, self._select_draws(n_draws), Rd) if J > K else None
1845
+ )
1846
+ return self._cache[key]
1847
+
1848
+ def summary(self, level: float = 0.68, which: str = "fractions") -> Summary:
1849
+ """Per-bin posterior summary table (mean, sd, median, interval, ESS, R-hat).
1850
+
1851
+ Parameters
1852
+ ----------
1853
+ level : float, optional
1854
+ Probability inside the central intervals shown.
1855
+ which : {"fractions", "rates", "density", "selected"}, optional
1856
+ Quantity used: fractions, rates, densities or the selected population.
1857
+ """
1858
+ which = _which_name(which)
1859
+ X = self._which(which)
1860
+ lo, hi = _central(X, level)
1861
+ cols: dict[str, np.ndarray] = {
1862
+ "bin": _bin_labels(self.bins),
1863
+ "mean": X.mean(axis=0),
1864
+ "sd": X.std(axis=0, ddof=1) if X.shape[0] > 1 else np.zeros(self.K),
1865
+ "median": np.median(X, axis=0),
1866
+ f"q{100 * (0.5 - level / 2):g}": lo,
1867
+ f"q{100 * (0.5 + level / 2):g}": hi,
1868
+ }
1869
+ mixing = self._mixing_of(which) # of the quantity summarised (audit O-8)
1870
+ cols.update({k: v for k, v in mixing.items() if k != "ess_tail"})
1871
+ if self.model is not None: # objects' worth of likelihood per bin (DECISIONS #41)
1872
+ cols["support"] = self.data_support
1873
+ flag = " [approximate]" if self.approximate else ""
1874
+ source = None if self.model is None else self.model.likelihoods.meta.get("source")
1875
+ inputs = "" if source is None else f"; inputs: {source}"
1876
+ title = f"binference {which} posterior ({self.method}{flag}, {self.n_draws} draws{inputs})"
1877
+ return Summary(cols, title)
1878
+
1879
+ def binning_bias(
1880
+ self,
1881
+ build: Callable[[Any], Likelihoods] | None = None,
1882
+ *,
1883
+ order: int = 1,
1884
+ resolution: int = 1,
1885
+ threshold: float = 2.0,
1886
+ prior: Any = None,
1887
+ method: str | None = None,
1888
+ n_samples: int | None = None,
1889
+ rng: RNGLike = None,
1890
+ **engine_kwargs: Any,
1891
+ ) -> Summary:
1892
+ r"""Check the flat-within-bin assumption: refit the same data on a B-spline basis
1893
+ over these bins and compare the bin masses (DECISIONS #6, #20).
1894
+
1895
+ ``build(bins_or_basis)`` returns the likelihoods of the fit's data on what it is
1896
+ given, e.g. ``lambda b: Likelihoods.from_gaussian(mean, sigma, b)``; likelihoods
1897
+ from :meth:`Likelihoods.from_gaussian` rebuild themselves without it. The
1898
+ reference is fitted on ``Basis(self.bins, order, resolution)`` with this fit's
1899
+ prior (or ``prior``), selection, engine (or ``method``), chains and draws (or
1900
+ ``n_samples``), and a Gibbs fit's warm-up and thinning when the engine is the
1901
+ same; ``engine_kwargs`` go to the engine.
1902
+
1903
+ The fit's selection keeps only its bin averages, so the reference sees it flat
1904
+ within bins; to test a selection's shape within bins as well, attach it to the
1905
+ likelihoods ``build`` returns, which then take precedence (a selection from
1906
+ injections must be attached that way, fixed).
1907
+
1908
+ Returns a table per bin: ``shift``, the reference's mean fraction minus this
1909
+ fit's; ``sd``, this fit's posterior s.d.; ``z = shift / sd`` (NaN where the
1910
+ fit has no spread); and, where
1911
+ :math:`|z|` exceeds ``threshold``, ``n_crit`` :math:`= W/z^2`, the number of
1912
+ objects at which the shift equals the statistical error (which falls as
1913
+ :math:`1/\sqrt{W}`): from a pilot fit, the size above which these bins bias
1914
+ the result (``inf`` elsewhere). Warns when :math:`|z|` exceeds ``threshold`` in
1915
+ any bin: two models of the same data differ by up to about one s.d. even when
1916
+ the population is flat within bins (DECISIONS #86), so the default is 2.
1917
+
1918
+ Parameters
1919
+ ----------
1920
+ build : callable, optional
1921
+ Function that rebuilds the likelihoods on a basis; ``None`` works for
1922
+ Gaussian inputs.
1923
+ order : int, optional
1924
+ B-spline order of the reference fit's basis.
1925
+ resolution : int, optional
1926
+ Components per bin of the reference fit's basis.
1927
+ threshold : float, optional
1928
+ Shift, in posterior standard deviations, above which a bin is flagged.
1929
+ prior : PopulationPrior, optional
1930
+ Prior of the reference fit; ``None`` reuses this fit's.
1931
+ method : str, optional
1932
+ Engine of the reference fit; ``None`` reuses this fit's.
1933
+ n_samples : int, optional
1934
+ Draws of the reference fit; ``None`` reuses this fit's, except that
1935
+ ``method="auto"`` sets its own run length.
1936
+ rng : int, Generator or None
1937
+ Random number generator.
1938
+ **engine_kwargs
1939
+ Options for the reference fit's engine.
1940
+ """
1941
+ from .basis import Basis
1942
+ from .inference import METHODS, infer
1943
+
1944
+ model = self._fit_model_only("binning_bias")
1945
+ if self.basis is not None or self.bins.is_categorical:
1946
+ raise ValueError("binning_bias compares a fit on grid bins with a basis over them.")
1947
+ key = str(self.method if method is None else method).lower()
1948
+ if key not in ("auto", "nuts", "gibbs", "laplace", "vb", "bootstrap"):
1949
+ raise ValueError(
1950
+ f"method must be an engine that draws ('auto', 'nuts', 'gibbs', 'laplace', "
1951
+ f"'vb' or 'bootstrap'); got {key!r}."
1952
+ )
1953
+ basis = Basis(self.bins, order=order, resolution=resolution)
1954
+ if basis.is_identity:
1955
+ raise ValueError(
1956
+ "order=0 with resolution=1 is the bins themselves: nothing to compare."
1957
+ )
1958
+ if build is None:
1959
+ lk = _rebuild_gaussian(model.likelihoods, basis)
1960
+ else:
1961
+ lk = build(basis)
1962
+ if lk.basis != basis or lk.N != model.N or not np.array_equal(lk.weights, model._w):
1963
+ raise ValueError(
1964
+ "build(basis) must return the fit's objects (same rows and weights) on "
1965
+ "the basis it is given."
1966
+ )
1967
+ selection = self.selection if lk.selection is None else None # build's own wins
1968
+ if selection is not None and selection.injections is not None:
1969
+ raise ValueError(
1970
+ "A selection from injections does not move onto a basis: attach a fixed one "
1971
+ "(selection.at_mean()) to the likelihoods build returns."
1972
+ )
1973
+ same = key == self.method
1974
+ if METHODS[key].markov: # draws per chain, as the fit had them
1975
+ engine_kwargs.setdefault("n_chains", self.n_chains)
1976
+ for name in ("n_warmup", "thin") if same and key == "gibbs" else ():
1977
+ if name in self.info:
1978
+ engine_kwargs.setdefault(name, self.info[name])
1979
+ n = self.n_draws // self.n_chains
1980
+ elif key == "auto": # its own defaults, as for a new fit
1981
+ n = None
1982
+ else:
1983
+ n = self.n_draws
1984
+ ref = infer(
1985
+ lk,
1986
+ model.prior_spec if prior is None else prior,
1987
+ selection=selection,
1988
+ method=key,
1989
+ n_samples=n if n_samples is None else n_samples,
1990
+ rng=rng,
1991
+ check=False,
1992
+ **engine_kwargs,
1993
+ )
1994
+ sd, shift = self.std, ref.mean - self.mean
1995
+ with np.errstate(divide="ignore", invalid="ignore"):
1996
+ z = np.where(sd > 0, shift / sd, np.nan) # no spread: nothing to compare with
1997
+ big = np.abs(z) > threshold
1998
+ n_crit = np.where(big, model.W / np.where(big, z, 1.0) ** 2, np.inf)
1999
+ if big.any():
2000
+ warn(
2001
+ f"Binning bias: {big.sum()} bins move by more than {threshold:g} posterior s.d. "
2002
+ f"on a basis of order {order} (first: {np.flatnonzero(big)[:5].tolist()}); "
2003
+ f"fit with binference.Basis(bins, order={order}) or narrower bins.",
2004
+ InputWarning,
2005
+ )
2006
+ cols = {"bin": _bin_labels(self.bins), "shift": shift, "sd": sd, "z": z, "n_crit": n_crit}
2007
+ return Summary(cols, f"binference binning bias (basis of order {order}, {key})")
2008
+
2009
+ def __repr__(self) -> str:
2010
+ flag = ", approximate" if self.approximate else ""
2011
+ return (
2012
+ f"PopulationResult(method={self.method!r}, kind={self.kind!r}, K={self.K}, "
2013
+ f"n_chains={self.n_chains}, n_draws={self.n_draws}{flag})"
2014
+ )
2015
+
2016
+ # ------------------------------------------------------------- plotting
2017
+ def plot(self, **kwargs: Any):
2018
+ """Plot the population posterior (see :func:`binference.plotting.plot_population`).
2019
+
2020
+ Parameters
2021
+ ----------
2022
+ **kwargs
2023
+ Passed to :func:`binference.plotting.plot_population`.
2024
+ """
2025
+ from .plotting import plot_population
2026
+
2027
+ return plot_population(self, **kwargs)
2028
+
2029
+ def plot_corner(self, idx: Sequence[int] | None = None, **kwargs: Any):
2030
+ """Corner plot of selected bins (see :func:`binference.plotting.plot_corner`).
2031
+
2032
+ Parameters
2033
+ ----------
2034
+ idx : sequence of int, optional
2035
+ Bins to show; ``None`` chooses the most populated.
2036
+ **kwargs
2037
+ Further arguments of :func:`~binference.plotting.plot_corner`.
2038
+ """
2039
+ from .plotting import plot_corner
2040
+
2041
+ return plot_corner(self, idx=idx, **kwargs)
2042
+
2043
+ def plot_trace(self, idx: Sequence[int] | None = None, **kwargs: Any):
2044
+ """Trace plots of selected bins (see :func:`binference.plotting.plot_trace`).
2045
+
2046
+ Parameters
2047
+ ----------
2048
+ idx : sequence of int, optional
2049
+ Bins to show; ``None`` chooses the most populated.
2050
+ **kwargs
2051
+ Further arguments of :func:`~binference.plotting.plot_trace`.
2052
+ """
2053
+ from .plotting import plot_trace
2054
+
2055
+ return plot_trace(self, idx=idx, **kwargs)
2056
+
2057
+ # ---------------------------------------------------------- persistence
2058
+ def save(self, path: str) -> None:
2059
+ """Save draws, metadata and bins to a compressed ``.npz`` file.
2060
+
2061
+ Parameters
2062
+ ----------
2063
+ path : str
2064
+ File to write.
2065
+ """
2066
+ from .io import save_result
2067
+
2068
+ save_result(self, path)
2069
+
2070
+ @classmethod
2071
+ def load(cls, path: str) -> PopulationResult:
2072
+ """Load a result saved with :meth:`save`.
2073
+
2074
+ Re-:meth:`attach` the likelihoods to use the per-object methods.
2075
+
2076
+ Parameters
2077
+ ----------
2078
+ path : str
2079
+ File written by :meth:`save`.
2080
+ """
2081
+ from .io import load_result
2082
+
2083
+ return load_result(path)
2084
+
2085
+ def to_dict(self) -> dict[str, Any]:
2086
+ """Plain-dictionary representation (arrays and metadata)."""
2087
+ from .io import result_to_dict
2088
+
2089
+ return result_to_dict(self)
2090
+
2091
+ def to_datatree(self):
2092
+ """Convert to an :class:`xarray.DataTree` (ArviZ >= 1.0 layout; needs xarray)."""
2093
+ from .io import to_datatree
2094
+
2095
+ return to_datatree(self)
2096
+
2097
+
2098
+ # ----------------------------------------------------------------------------
2099
+ # PointEstimate
2100
+ # ----------------------------------------------------------------------------
2101
+ class PointEstimate(_IndividualMixin):
2102
+ """A MAP / maximum-likelihood estimate of the population.
2103
+
2104
+ Attributes
2105
+ ----------
2106
+ value : (K,) array
2107
+ Fractions (or rates for rate models) at the optimum; with a
2108
+ :class:`~binference.Basis`, the component weights (:attr:`rho` and
2109
+ :attr:`rates` are per bin).
2110
+ x : (dim,) array or None
2111
+ Unconstrained coordinates (when optimised there).
2112
+ kkt_gap : float
2113
+ Optimality certificate (zero at the exact optimum on the simplex).
2114
+ """
2115
+
2116
+ def __init__(
2117
+ self,
2118
+ model: PopulationModel | None,
2119
+ value: np.ndarray,
2120
+ *,
2121
+ method: str = "map",
2122
+ x: np.ndarray | None = None,
2123
+ log_posterior: float = np.nan,
2124
+ log_likelihood: float = np.nan,
2125
+ kkt_gap: float = np.nan,
2126
+ n_iter: int = 0,
2127
+ converged: bool = True,
2128
+ info: dict[str, Any] | None = None,
2129
+ ) -> None:
2130
+ self.model = model
2131
+ self.value = np.asarray(value, dtype=float)
2132
+ self.method = method
2133
+ self.x = x
2134
+ self.log_posterior = float(log_posterior)
2135
+ self.log_likelihood = float(log_likelihood)
2136
+ self.kkt_gap = float(kkt_gap)
2137
+ self.n_iter = int(n_iter)
2138
+ self.converged = bool(converged)
2139
+ self.info = dict(info or {})
2140
+ self._cache: dict[str, Any] = {}
2141
+ self.kind = model.kind if model is not None else "fractions"
2142
+ self.basis = model.likelihoods.basis if model is not None else None
2143
+ self.bins = (
2144
+ self.basis.bins
2145
+ if self.basis is not None
2146
+ else model.bins
2147
+ if model is not None
2148
+ else Bins.categories(self.value.size)
2149
+ )
2150
+
2151
+ @property
2152
+ def rho(self) -> np.ndarray:
2153
+ """Population fractions at the optimum (per bin)."""
2154
+ return self._fractions(self._to_bins(self.value))
2155
+
2156
+ @property
2157
+ def rates(self) -> np.ndarray | None:
2158
+ """Rates at the optimum (per bin): for fractions :math:`\\hat\\Lambda\\rho` with
2159
+ :math:`\\hat\\Lambda = W / (S\\cdot\\rho)` (``None`` with a relative selection)."""
2160
+ if self.kind == "rates":
2161
+ return self._to_bins(self.value)
2162
+ total = self.total
2163
+ return None if total is None else total * self.rho
2164
+
2165
+ @property
2166
+ def total(self) -> float | None:
2167
+ """Total rate at the optimum (see :attr:`rates`)."""
2168
+ if self.kind == "rates":
2169
+ return float(self.value.sum())
2170
+ if self.model is None:
2171
+ return None
2172
+ S = self._inferred_selection() # inferred, at the mode
2173
+ S = self.model.total_exposure if S is None else np.ravel(S)
2174
+ return None if S is None else float(self.model.W / (S @ self.value))
2175
+
2176
+ def _rho_draws(self) -> np.ndarray:
2177
+ return self.rho[None, :]
2178
+
2179
+ def _which(self, which: str) -> np.ndarray:
2180
+ """The estimate of ``which`` as one draw ``(1, K)``, as for posterior draws."""
2181
+ which = _which_name(which)
2182
+ if which == "density":
2183
+ return self.density()[None, :]
2184
+ if which == "rates" and self.rates is None:
2185
+ raise ValueError("No rates in this result (a relative selection, or no model).")
2186
+ if which == "selected":
2187
+ return self.selected[None, :]
2188
+ return (self.rho if which == "fractions" else self.rates)[None, :]
2189
+
2190
+ @property
2191
+ def selection(self) -> Selection | None:
2192
+ """The selection function of the fit (a point estimate is not saved, so it is
2193
+ always the model's)."""
2194
+ return None if self.model is None else self.model.selection_spec
2195
+
2196
+ @property
2197
+ def best(self) -> PointEstimate:
2198
+ """The best fit of a point estimate is itself."""
2199
+ return self
2200
+
2201
+ @property
2202
+ def selected(self) -> np.ndarray:
2203
+ """The selected population at the estimate (see :attr:`PopulationResult.selected`)."""
2204
+ return self._selected_of(self._weight_draws())[0]
2205
+
2206
+ def _center_x(self) -> np.ndarray:
2207
+ model = self._require_model()
2208
+ if self.x is not None and np.shape(self.x)[-1] == model.dim:
2209
+ return np.asarray(self.x, dtype=float).ravel()
2210
+ return model.from_value(self.value)
2211
+
2212
+ def check(self, kkt_max: float = 1e-4, strict: bool = False) -> list[str]:
2213
+ """Warn (or raise with ``strict=True``) when the optimiser did not converge or the
2214
+ KKT gap (per object; see :meth:`PopulationModel.kkt_gap`) exceeds ``kkt_max``.
2215
+
2216
+ Estimates found by L-BFGS in unconstrained coordinates have no KKT gap (NaN); for
2217
+ them the check is ``converged`` (see ``tol`` of :class:`~binference.samplers.MAP`).
2218
+
2219
+ The default is ten times the round-off floor at which MAP stops (DECISIONS #12).
2220
+ Returns the messages (empty if the estimate looks fine).
2221
+
2222
+ Parameters
2223
+ ----------
2224
+ kkt_max : float, optional
2225
+ Largest KKT gap that passes.
2226
+ strict : bool, optional
2227
+ Raise :class:`~binference.DiagnosticsError` instead of warning.
2228
+ """
2229
+ msgs = []
2230
+ if not self.converged:
2231
+ msgs.append("the optimiser did not converge")
2232
+ if self.kkt_gap > kkt_max: # inf (an impossible object) fails; NaN passes
2233
+ msgs.append(f"KKT gap {self.kkt_gap:.1e} > {kkt_max:.0e}: not at the optimum")
2234
+ _report([(msgs, [], ConvergenceWarning)], strict)
2235
+ return msgs
2236
+
2237
+ def _weight_draws(self) -> np.ndarray:
2238
+ return self._fractions(self.value)[None, :]
2239
+
2240
+ def density(self) -> np.ndarray:
2241
+ """Population density per unit bin volume at the optimum."""
2242
+ base = self.rates if self.kind == "rates" else self.rho
2243
+ return base / self.bins.volumes
2244
+
2245
+ def laplace(self, n_samples: int = 4000, rng: RNGLike = None, **kwargs: Any):
2246
+ """Laplace approximation around the mode (see :class:`binference.samplers.Laplace`).
2247
+
2248
+ Parameters
2249
+ ----------
2250
+ n_samples : int, optional
2251
+ Draws of the Laplace approximation.
2252
+ rng : int, Generator or None
2253
+ Random number generator.
2254
+ **kwargs
2255
+ Options of :class:`~binference.samplers.Laplace`.
2256
+ """
2257
+ from .samplers import Laplace
2258
+
2259
+ return Laplace(self._require_model(), rng=rng, **kwargs).run(n_samples)
2260
+
2261
+ def __repr__(self) -> str:
2262
+ return (
2263
+ f"PointEstimate(method={self.method!r}, K={self.value.size}, "
2264
+ f"log_posterior={self.log_posterior:.6g}, kkt_gap={self.kkt_gap:.2e}, "
2265
+ f"converged={self.converged})"
2266
+ )
2267
+
2268
+ def plot(self, **kwargs: Any):
2269
+ from .plotting import plot_population
2270
+
2271
+ return plot_population(self, **kwargs)