binference 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. binference/__init__.py +85 -0
  2. binference/_kernels.py +252 -0
  3. binference/_quadrature.py +202 -0
  4. binference/_simplex.py +54 -0
  5. binference/_utils.py +139 -0
  6. binference/_version.py +1 -0
  7. binference/basis.py +314 -0
  8. binference/bins.py +645 -0
  9. binference/density.py +323 -0
  10. binference/diagnostics.py +400 -0
  11. binference/estimator.py +197 -0
  12. binference/inference.py +368 -0
  13. binference/io.py +309 -0
  14. binference/likelihoods.py +1684 -0
  15. binference/measure.py +246 -0
  16. binference/model.py +833 -0
  17. binference/plotting.py +538 -0
  18. binference/priors/__init__.py +47 -0
  19. binference/priors/base.py +248 -0
  20. binference/priors/dirichlet.py +640 -0
  21. binference/priors/hyper.py +240 -0
  22. binference/priors/logistic.py +676 -0
  23. binference/priors/rates.py +139 -0
  24. binference/response.py +539 -0
  25. binference/results.py +2271 -0
  26. binference/samplers/__init__.py +19 -0
  27. binference/samplers/_curvature.py +67 -0
  28. binference/samplers/_nuts_core.py +284 -0
  29. binference/samplers/base.py +118 -0
  30. binference/samplers/bootstrap.py +136 -0
  31. binference/samplers/gibbs.py +287 -0
  32. binference/samplers/laplace.py +200 -0
  33. binference/samplers/nuts.py +681 -0
  34. binference/samplers/optimize.py +488 -0
  35. binference/samplers/vb.py +118 -0
  36. binference/selection.py +621 -0
  37. binference/simulate.py +720 -0
  38. binference/validation.py +477 -0
  39. binference/warnings.py +48 -0
  40. binference-0.1.0.dist-info/METADATA +130 -0
  41. binference-0.1.0.dist-info/RECORD +43 -0
  42. binference-0.1.0.dist-info/WHEEL +4 -0
  43. binference-0.1.0.dist-info/licenses/LICENSE +21 -0
binference/simulate.py ADDED
@@ -0,0 +1,720 @@
1
+ """Mock catalogues for tests, tutorials and simulation-based calibration.
2
+
3
+ Each ``mock_*`` function returns a :class:`Mock` bundling the bins, the
4
+ correct per-object :class:`~binference.Likelihoods`, the :class:`Truth`, and
5
+ (optionally) posteriors under an interim prior and posterior samples, so that
6
+ every input convention can be exercised. ``Mock`` unpacks as
7
+ ``likelihoods, truth = mock``.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from collections.abc import Callable, Iterator
13
+ from dataclasses import dataclass, field
14
+ from typing import Any
15
+
16
+ import numpy as np
17
+ from scipy import special, stats
18
+
19
+ from ._utils import RNGLike, as_generator
20
+ from .bins import Bins
21
+ from .density import Density
22
+ from .likelihoods import Likelihoods
23
+ from .priors import PopulationPrior
24
+ from .response import Response
25
+ from .selection import Selection, _bin_average
26
+
27
+ __all__ = [
28
+ "Mock",
29
+ "Truth",
30
+ "mock_1d",
31
+ "mock_2d",
32
+ "mock_categorical",
33
+ "mock_unfolding",
34
+ "population",
35
+ "prior_predictive",
36
+ ]
37
+
38
+
39
+ @dataclass
40
+ class Truth:
41
+ """The true population and objects behind a mock catalogue.
42
+
43
+ Attributes
44
+ ----------
45
+ rho : (K,) array
46
+ Population fractions the objects were drawn from.
47
+ labels : (N,) int array
48
+ True bin of every observed object.
49
+ values : array or None
50
+ True continuous values of the observed objects (``None`` for categories).
51
+ counts : (K,) array
52
+ Realised histogram of the observed objects' true bins.
53
+ n_drawn : int
54
+ Objects drawn before selection.
55
+ """
56
+
57
+ rho: np.ndarray
58
+ labels: np.ndarray
59
+ values: np.ndarray | None
60
+ counts: np.ndarray
61
+ n_drawn: int
62
+ extra: dict[str, Any] = field(default_factory=dict)
63
+
64
+
65
+ @dataclass
66
+ class Mock:
67
+ """A mock catalogue: bins, likelihoods, truth and alternative input forms."""
68
+
69
+ bins: Bins
70
+ likelihoods: Likelihoods
71
+ truth: Truth
72
+ measurements: dict[str, np.ndarray] = field(default_factory=dict)
73
+ posteriors: np.ndarray | None = None
74
+ interim: np.ndarray | None = None
75
+ samples: np.ndarray | None = None
76
+ response: Response | None = None
77
+
78
+ def __iter__(self) -> Iterator[Any]:
79
+ yield self.likelihoods
80
+ yield self.truth
81
+
82
+
83
+ # ----------------------------------------------------------------------------
84
+ # densities and populations
85
+ # ----------------------------------------------------------------------------
86
+ def _density_1d(name: str, lo: float, hi: float) -> Callable[[np.ndarray], np.ndarray]:
87
+ span = hi - lo
88
+
89
+ def unit(z):
90
+ return (np.asarray(z, dtype=float) - lo) / span
91
+
92
+ if name == "bimodal":
93
+ return lambda z: (
94
+ 0.6 * stats.beta.pdf(unit(z), 3.0, 9.0) + 0.4 * stats.norm.pdf(unit(z), 0.55, 0.1)
95
+ )
96
+ if name == "gaussian":
97
+ return lambda z: stats.norm.pdf(unit(z), 0.5, 0.15)
98
+ if name == "uniform":
99
+ return lambda z: np.ones_like(np.asarray(z, dtype=float))
100
+ if name == "steep":
101
+ return lambda z: np.exp(-4.0 * unit(z))
102
+ raise ValueError(f"Unknown density {name!r}; use 'bimodal', 'gaussian', 'uniform' or 'steep'.")
103
+
104
+
105
+ def _density_2d(name: str, bins: Bins) -> Callable[[np.ndarray], np.ndarray]:
106
+ (lx, hx), (ly, hy) = bins.range
107
+
108
+ def f(p):
109
+ p = np.asarray(p, dtype=float)
110
+ u = (p[..., 0] - lx) / (hx - lx)
111
+ v = (p[..., 1] - ly) / (hy - ly)
112
+ if name == "banana":
113
+ ridge = 0.2 + 0.55 * u + 0.12 * np.sin(3.0 * u)
114
+ return stats.beta.pdf(u, 2.0, 2.5) * np.exp(-0.5 * ((v - ridge) / 0.06) ** 2)
115
+ if name == "gaussian":
116
+ return np.exp(-0.5 * (((u - 0.5) / 0.15) ** 2 + ((v - 0.5) / 0.2) ** 2))
117
+ raise ValueError(f"Unknown 2-D density {name!r}; use 'banana' or 'gaussian'.")
118
+
119
+ return f
120
+
121
+
122
+ def _as_density(density: Any, bins: Bins) -> Density:
123
+ # a Density from a mock density name, a Density, a callable, a distribution or masses
124
+ if isinstance(density, str) and density != "flat":
125
+ if bins.ndim == 1:
126
+ lo, hi = bins.range[0]
127
+ return Density.from_function(_density_1d(density, lo, hi))
128
+ return Density.from_function(_density_2d(density, bins))
129
+ return Density.coerce(density, "mass", name="density")
130
+
131
+
132
+ def population(bins: Bins, density: Any = "bimodal", tol: float = 1e-6) -> np.ndarray:
133
+ """Population fractions per bin from a density name, :class:`Density`, callable or masses.
134
+
135
+ Parameters
136
+ ----------
137
+ bins : Bins
138
+ The bins.
139
+ density : str, Density, callable or array_like (K,), optional
140
+ A named density, a :class:`~binference.Density`, a callable or masses. The
141
+ names are ``"bimodal"``, ``"gaussian"``, ``"uniform"`` and ``"steep"`` on
142
+ 1-D bins, ``"banana"`` and ``"gaussian"`` on 2-D grids.
143
+ tol : float, optional
144
+ Tolerance of the adaptive quadrature over each bin.
145
+ """
146
+ return _as_density(density, bins).masses(bins, tol=tol)
147
+
148
+
149
+ def _draw_values(bins: Bins, density: Any, n: int, rng: np.random.Generator):
150
+ """True values (continuous) and labels for n objects."""
151
+ f = _as_density(density, bins)._log_pointwise()
152
+ if f is None:
153
+ rho = population(bins, density)
154
+ labels = rng.choice(bins.K, size=n, p=rho)
155
+ return bins.sample_within(labels, rng), labels
156
+ # inverse-CDF sampling on every bin split into 20 equal steps in u (the mask repeated)
157
+ fb, coarse = bins.refine(20)
158
+ dens = np.exp(np.asarray(f(fb.centers), dtype=float).ravel()) * fb.volumes
159
+ if not dens.sum() > 0:
160
+ raise ValueError("density has no mass on the active bins.")
161
+ cell = rng.choice(fb.K, size=n, p=dens / dens.sum())
162
+ return fb.sample_within(cell, rng), coarse[cell]
163
+
164
+
165
+ def _noise(noise: Any, n: int, rng: np.random.Generator, shape=()) -> np.ndarray:
166
+ if np.ndim(noise) == 0:
167
+ return np.full((n, *shape), float(noise))
168
+ lo, hi = noise
169
+ return rng.uniform(lo, hi, size=(n, *shape))
170
+
171
+
172
+ def _measured_pass(z, noise, lo, hi):
173
+ # P(lo <= z + sigma * e <= hi), averaged over the noise distribution
174
+ if np.ndim(noise) == 0:
175
+ sig, wts = np.array([float(noise)]), np.array([1.0])
176
+ else:
177
+ x, w = np.polynomial.legendre.leggauss(16)
178
+ sig = noise[0] + (noise[1] - noise[0]) * 0.5 * (x + 1.0)
179
+ wts = 0.5 * w
180
+ zz = np.asarray(z, dtype=float)[..., None]
181
+ a, b = (lo - zz) / sig, (hi - zz) / sig
182
+ # take the difference in the tail where it keeps its precision
183
+ p = np.where(a > 0, special.ndtr(-a) - special.ndtr(-b), special.ndtr(b) - special.ndtr(a))
184
+ return p @ wts
185
+
186
+
187
+ # ----------------------------------------------------------------------------
188
+ # mock catalogues
189
+ # ----------------------------------------------------------------------------
190
+ def mock_1d(
191
+ n_objects: int = 2000,
192
+ bins: Bins | None = None,
193
+ *,
194
+ density: Any = "bimodal",
195
+ noise: Any = (0.05, 0.2),
196
+ interim: Any = None,
197
+ n_samples: int = 0,
198
+ selection: Any = None,
199
+ measured_range: tuple[float | None, float | None] | None = None,
200
+ outlier_fraction: float = 0.0,
201
+ rng: RNGLike = None,
202
+ ) -> Mock:
203
+ """A 1-D mock catalogue with heteroscedastic Gaussian measurement errors.
204
+
205
+ Parameters
206
+ ----------
207
+ n_objects : int
208
+ Number of *observed* objects (after selection).
209
+ bins : Bins, optional
210
+ Bins of the mock; ``None`` gives 30 equal bins on [0, 3].
211
+ density : str, callable, or array_like (K,)
212
+ True population: a named density ("bimodal", "gaussian", "uniform",
213
+ "steep"), a callable density, or bin fractions (values then uniform
214
+ within bins).
215
+ noise : float or (lo, hi)
216
+ Measurement standard deviation, fixed or drawn uniformly per object.
217
+ interim : str, callable or array_like, optional
218
+ If given, also return per-object posteriors on the bins computed under
219
+ this interim prior (``"gaussian"`` gives a broad Gaussian), as
220
+ ``mock.posteriors`` with masses ``mock.interim``.
221
+ n_samples : int
222
+ If positive, also return ``n_samples`` posterior samples per object
223
+ (under the interim prior, flat if not given) as ``mock.samples``.
224
+ selection : array_like (K,), optional
225
+ Per-bin detection probability depending on the *true* bin (selection on
226
+ the truth): objects are thinned accordingly.
227
+ measured_range : (lo, hi), optional
228
+ Selection on the measurement: only objects whose measured value lies in
229
+ ``[lo, hi]`` are observed (either end may be ``None``).
230
+ outlier_fraction : float
231
+ Fraction of measurements replaced by uniform draws over the range
232
+ (the likelihoods ignore this: a misspecification test).
233
+ rng : int, Generator or None
234
+
235
+ Notes
236
+ -----
237
+ The matching :class:`Selection` is attached to ``mock.likelihoods``:
238
+ ``"on_truth"`` with ``selection``, ``"on_measurement"`` with
239
+ ``measured_range`` (the population-weighted bin average of the probability
240
+ that the measured value passes, over the noise distribution), and
241
+ :meth:`Selection.combined` with both (DECISIONS #48).
242
+
243
+ With a named or callable density the true values follow that continuous
244
+ density, whereas the histogram model assumes the population is flat within
245
+ each bin. For coarse bins, measurement errors comparable to the bin width
246
+ and many objects, this *binning bias* can exceed the statistical errors.
247
+ Pass bin fractions (e.g. ``density=population(bins, "bimodal")``) to draw
248
+ values uniformly within bins, for which the model is exactly right.
249
+ """
250
+ gen = as_generator(rng)
251
+ bins = Bins.linear(0.0, 3.0, 30) if bins is None else bins
252
+ if bins.ndim != 1:
253
+ raise ValueError("mock_1d needs one-dimensional bins.")
254
+ mock = _gaussian_mock(
255
+ bins, density, n_objects, noise, selection, measured_range, outlier_fraction, gen
256
+ )
257
+ lk, obs, sigma = mock.likelihoods, mock.measurements["mean"], mock.measurements["sigma"]
258
+ log_pi = None
259
+ if interim is not None:
260
+ if isinstance(interim, str) and interim == "gaussian":
261
+ lo, hi = bins.range[0]
262
+ interim = stats.norm(0.4 * lo + 0.6 * hi, 0.3 * (hi - lo))
263
+ interim = _as_density(interim, bins)
264
+ log_pi = interim._log_pointwise()
265
+ Pi = interim.masses(bins, tol=1e-6)
266
+ P = lk.toarray() * Pi[None, :] # row scales are irrelevant after normalising
267
+ mock.posteriors = P / P.sum(axis=1, keepdims=True)
268
+ mock.interim = Pi
269
+ if n_samples > 0:
270
+ mock.samples = _posterior_samples_1d(
271
+ obs, sigma, bins, log_pi, mock.interim, n_samples, gen
272
+ )
273
+ if mock.interim is None:
274
+ mock.interim = population(bins, "uniform")
275
+ return mock
276
+
277
+
278
+ def _gaussian_mock(
279
+ bins,
280
+ density,
281
+ n_objects,
282
+ noise,
283
+ on_truth,
284
+ measured_range,
285
+ outliers,
286
+ gen,
287
+ *,
288
+ measured_keep=None,
289
+ selection=None,
290
+ ):
291
+ # draw objects until n_objects pass the selection, measure them, and build the mock
292
+ # with the Selection that describes how they were selected (DECISIONS #48);
293
+ # measured_keep thins on the bin of the measured value (the end bins extended
294
+ # outward) and `selection` replaces the Selection attached to the likelihoods
295
+ sel = None if on_truth is None else Selection(on_truth, "on_truth")
296
+ if measured_range is not None:
297
+ if len(measured_range) != 2:
298
+ raise ValueError("measured_range must be (lo, hi); either may be None.")
299
+ lo = -np.inf if measured_range[0] is None else float(measured_range[0])
300
+ hi = np.inf if measured_range[1] is None else float(measured_range[1])
301
+ if not lo < hi:
302
+ raise ValueError(f"measured_range must have lo < hi; got {measured_range}.")
303
+ # S_k: the population-weighted bin average of the probability of passing the cut
304
+ log_pop = _as_density(density, bins)._log_pointwise()
305
+ if log_pop is None: # flat in the measure coordinate within bins
306
+ log_pop = lambda z: bins.measures[0].log_jacobian(z) # noqa: E731
307
+ passes = _bin_average(lambda z: _measured_pass(z, noise, lo, hi), bins, log_weight=log_pop)
308
+ sel = (
309
+ Selection(passes[0], "on_measurement")
310
+ if on_truth is None
311
+ else Selection.combined(on_truth, passes[0])
312
+ )
313
+ S = None if sel is None else sel.row # thinning on the true bin
314
+ shape = () if bins.ndim == 1 else (bins.ndim,)
315
+ parts: list[tuple[np.ndarray, ...]] = []
316
+ n_drawn, need = 0, n_objects
317
+ while need > 0:
318
+ batch = max(2 * need, 1000)
319
+ v, lab = _draw_values(bins, density, batch, gen)
320
+ sigma = _noise(noise, batch, gen, shape)
321
+ obs = v + sigma * gen.standard_normal(v.shape)
322
+ if outliers > 0: # misspecification the likelihoods and the selection ignore
323
+ bad = gen.random(batch) < outliers
324
+ obs[bad] = gen.uniform(*bins.range[0], size=int(bad.sum()))
325
+ keep = lab >= 0
326
+ if S is not None:
327
+ keep &= gen.random(batch) < S[np.maximum(lab, 0)]
328
+ if measured_range is not None:
329
+ keep &= (obs >= lo) & (obs <= hi)
330
+ if measured_keep is not None:
331
+ j = np.clip(np.searchsorted(bins.edges[0], obs, side="right") - 1, 0, bins.K - 1)
332
+ keep &= gen.random(batch) < measured_keep[j]
333
+ idx = np.flatnonzero(keep)[:need]
334
+ # count only the draws needed to reach n_objects observed objects
335
+ n_drawn += batch if idx.size < need else int(idx[-1]) + 1
336
+ parts.append((v[idx], lab[idx], sigma[idx], obs[idx]))
337
+ need -= idx.size
338
+ values, labels, sigma, obs = (np.concatenate(a) for a in zip(*parts, strict=True))
339
+ lk = Likelihoods.from_gaussian(
340
+ obs,
341
+ sigma,
342
+ bins,
343
+ selection=sel if selection is None else selection,
344
+ warn_outside=False, # observations near the edges spill past them by construction
345
+ )
346
+ counts = np.bincount(labels, minlength=bins.K).astype(float)
347
+ rho = population(bins, density)
348
+ truth = Truth(rho=rho, labels=labels, values=values, counts=counts, n_drawn=n_drawn)
349
+ return Mock(bins, lk, truth, {"mean": obs, "sigma": sigma})
350
+
351
+
352
+ def _posterior_samples_1d(obs, sigma, bins, log_pi, pi_masses, n_samples, gen):
353
+ grid = bins.refine(50)[0].edges[0] # 50 steps per bin, so each step lies inside one bin
354
+ mid = 0.5 * (grid[1:] + grid[:-1])
355
+ k = bins.index(mid)
356
+ with np.errstate(divide="ignore"):
357
+ if log_pi is not None:
358
+ log_prior = np.asarray(log_pi(mid), dtype=float)
359
+ elif pi_masses is not None:
360
+ log_prior = np.log(pi_masses[k] / bins.volumes[k])
361
+ else:
362
+ log_prior = np.zeros_like(mid)
363
+ # the prior mass of each step (steps differ in width between bins); none in masked bins
364
+ log_prior = np.where(k >= 0, log_prior + np.log(np.diff(grid)), -np.inf)
365
+ out = np.empty((obs.size, n_samples))
366
+ for i in range(obs.size):
367
+ logp = -0.5 * ((mid - obs[i]) / sigma[i]) ** 2 + log_prior
368
+ p = np.exp(logp - logp.max())
369
+ cdf = np.concatenate(([0.0], np.cumsum(p)))
370
+ cdf /= cdf[-1]
371
+ out[i] = np.interp(gen.random(n_samples), cdf, grid)
372
+ return out
373
+
374
+
375
+ def mock_2d(
376
+ n_objects: int = 5000,
377
+ bins: Bins | None = None,
378
+ *,
379
+ density: Any = "banana",
380
+ noise: Any = (0.02, 0.06),
381
+ selection: Any = None,
382
+ rng: RNGLike = None,
383
+ ) -> Mock:
384
+ """A 2-D mock catalogue (e.g. a colour-magnitude-like ridge) with Gaussian errors.
385
+
386
+ Parameters
387
+ ----------
388
+ n_objects : int, optional
389
+ Number of *observed* objects (after selection).
390
+ bins : Bins, optional
391
+ Grid of the mock; ``None`` gives a 20 x 20 grid on the unit square.
392
+ density : str, callable or array_like (K,), optional
393
+ True population: a named density, a callable density or bin fractions,
394
+ as for :func:`mock_1d`.
395
+ noise : float or (lo, hi), optional
396
+ Measurement standard deviation per axis, fixed or drawn uniformly per object
397
+ and axis.
398
+ selection : array_like (K,), optional
399
+ Per-bin detection probability depending on the *true* bin.
400
+ rng : int, Generator or None
401
+ Random number generator.
402
+ """
403
+ gen = as_generator(rng)
404
+ if bins is None:
405
+ bins = Bins.grid(np.linspace(0, 1, 21), np.linspace(0, 1, 21))
406
+ if bins.ndim != 2:
407
+ raise ValueError("mock_2d needs a two-dimensional grid.")
408
+ return _gaussian_mock(bins, density, n_objects, noise, selection, None, 0.0, gen)
409
+
410
+
411
+ def mock_categorical(
412
+ n_objects: int = 2000,
413
+ n_classes: int = 4,
414
+ *,
415
+ prevalence: Any = None,
416
+ train_prior: Any = None,
417
+ separation: float = 1.5,
418
+ temperature: float = 1.0,
419
+ n_validation: int = 500,
420
+ rng: RNGLike = None,
421
+ ) -> Mock:
422
+ """A quantification problem: classifier outputs over unordered classes.
423
+
424
+ Each object has a one-dimensional feature drawn from a unit-variance
425
+ Gaussian whose mean depends on its class (means ``separation`` apart). A
426
+ classifier trained with class prior ``train_prior`` (uniform by default)
427
+ reports posteriors ``mock.posteriors``, calibrated when ``temperature`` is
428
+ one (its log-likelihoods are divided by ``temperature``);
429
+ ``mock.likelihoods`` divides them by the training prior (route A,
430
+ :meth:`Likelihoods.from_table`). ``mock.response`` is tallied with
431
+ :meth:`Response.from_labelled` from a validation set of ``n_validation``
432
+ objects drawn with the training prior, scored by their feature; with
433
+ ``mock.measurements["feature"]`` it gives route C
434
+ (:meth:`Likelihoods.from_binned`). The test-set prevalence ``prevalence``
435
+ (default: a skewed vector) differs from the training prior (label shift).
436
+
437
+ Parameters
438
+ ----------
439
+ n_objects : int, optional
440
+ Number of objects in the target set.
441
+ n_classes : int, optional
442
+ Number of classes.
443
+ prevalence : array_like (n_classes,), optional
444
+ Class fractions of the target set; ``None`` gives fractions rising as the
445
+ square of the class index.
446
+ train_prior : array_like (n_classes,), optional
447
+ Class prior the classifier was trained with; ``None`` is uniform.
448
+ separation : float, optional
449
+ Distance between the class means of the feature.
450
+ temperature : float, optional
451
+ Divisor of the classifier's log-likelihoods; one is calibrated.
452
+ n_validation : int, optional
453
+ Objects in the labelled validation set.
454
+ rng : int, Generator or None
455
+ Random number generator.
456
+ """
457
+ if not temperature > 0:
458
+ raise ValueError(f"temperature must be positive; got {temperature}.")
459
+ if int(n_validation) < 1:
460
+ raise ValueError(f"n_validation must be at least 1; got {n_validation}.")
461
+ gen = as_generator(rng)
462
+ C = int(n_classes)
463
+ prev = (
464
+ (np.arange(1, C + 1, dtype=float) ** 2)
465
+ if prevalence is None
466
+ else np.asarray(prevalence, dtype=float)
467
+ )
468
+ prev = prev / prev.sum()
469
+ pi = np.full(C, 1.0 / C) if train_prior is None else np.asarray(train_prior, dtype=float)
470
+ pi = pi / pi.sum()
471
+ means = separation * np.arange(C)
472
+ labels = gen.choice(C, size=n_objects, p=prev)
473
+ x = means[labels] + gen.standard_normal(n_objects)
474
+ val_labels = gen.choice(C, size=int(n_validation), p=pi)
475
+ val_x = means[val_labels] + gen.standard_normal(val_labels.size)
476
+ logpost = -0.5 * (x[:, None] - means[None, :]) ** 2 / temperature + np.log(pi)[None, :]
477
+ post = np.exp(logpost - special.logsumexp(logpost, axis=1, keepdims=True))
478
+ bins = Bins.categories([f"class_{c}" for c in range(C)])
479
+ lk = Likelihoods.from_table(post, bins, interim=pi)
480
+ counts = np.bincount(labels, minlength=C).astype(float)
481
+ truth = Truth(rho=prev, labels=labels, values=None, counts=counts, n_drawn=n_objects)
482
+ missing = np.setdiff1d(np.arange(C), val_labels)
483
+ if missing.size:
484
+ raise ValueError(
485
+ f"The validation set has no objects of class(es) {missing.tolist()}; raise "
486
+ "n_validation or the training prior of those classes."
487
+ )
488
+ response = Response.from_labelled(
489
+ val_x, np.asarray(bins.labels, dtype=object)[val_labels], classes=bins.labels
490
+ )
491
+ return Mock(bins, lk, truth, {"feature": x}, posteriors=post, interim=pi, response=response)
492
+
493
+
494
+ def mock_unfolding(
495
+ n_objects: int = 5000,
496
+ n_true: int = 20,
497
+ n_reco: int = 25,
498
+ *,
499
+ slope: float = 3.0,
500
+ resolution: float = 0.08,
501
+ efficiency: tuple[float, float] = (0.6, 0.95),
502
+ rng: RNGLike = None,
503
+ ) -> Mock:
504
+ """An unfolding problem: an exponential spectrum smeared by a detector.
505
+
506
+ Unlike in the other mocks, ``n_objects`` counts the objects (events)
507
+ *before* detection, so the number observed is random
508
+ (``mock.truth.counts.sum()``). True values on [0, 1] follow
509
+ ``exp(-slope * z)``; events are detected with an efficiency rising linearly
510
+ across the range, smeared with Gaussian ``resolution`` and histogrammed in
511
+ ``n_reco`` bins on [-0.1, 1.1]. The response matrix is computed for a
512
+ population uniform within each true bin, ``mock.response`` is a
513
+ :class:`Response` and ``mock.likelihoods`` is built with
514
+ :meth:`Likelihoods.from_binned` (its efficiency becomes the selection function).
515
+
516
+ Parameters
517
+ ----------
518
+ n_objects : int, optional
519
+ Number of events before detection.
520
+ n_true : int, optional
521
+ Number of true bins.
522
+ n_reco : int, optional
523
+ Number of measured bins.
524
+ slope : float, optional
525
+ Decay rate of the exponential spectrum.
526
+ resolution : float, optional
527
+ Standard deviation of the Gaussian smearing.
528
+ efficiency : (float, float), optional
529
+ Detection efficiency at the two ends of the range.
530
+ rng : int, Generator or None
531
+ Random number generator.
532
+ """
533
+ gen = as_generator(rng)
534
+ tb = Bins.linear(0.0, 1.0, n_true)
535
+ re = np.linspace(-0.1, 1.1, n_reco + 1)
536
+ density = lambda z: np.exp(-slope * np.asarray(z)) # noqa: E731
537
+ rho = population(tb, density)
538
+ eff = lambda z: efficiency[0] + (efficiency[1] - efficiency[0]) * np.asarray(z) # noqa: E731
539
+ # response R[j, k] = P(reco j, detected | true bin k), population uniform within bin k
540
+ u, wq = np.polynomial.legendre.leggauss(16)
541
+ e = tb.edges[0]
542
+ R = np.zeros((n_reco, n_true))
543
+ for k in range(n_true):
544
+ zk = e[k] + (e[k + 1] - e[k]) * 0.5 * (u + 1.0)
545
+ wk = 0.5 * wq
546
+ cdf = stats.norm.cdf((re[None, :] - zk[:, None]) / resolution)
547
+ R[:, k] = (np.diff(cdf, axis=1) * (wk * eff(zk))[:, None]).sum(axis=0)
548
+ # simulate events
549
+ z, _ = _draw_values(tb, density, n_objects, gen)
550
+ det = gen.random(n_objects) < eff(z)
551
+ reco = z + resolution * gen.standard_normal(n_objects)
552
+ observed = det & (reco >= re[0]) & (reco <= re[-1])
553
+ response = Response(R, tb, Bins(re))
554
+ lk = Likelihoods.from_binned(response, values=reco[observed])
555
+ counts = np.bincount(lk.meta["measured_bin"], weights=lk.weights, minlength=n_reco)
556
+ true_lab = tb.index(z[observed])
557
+ truth = Truth(
558
+ rho=rho,
559
+ labels=true_lab,
560
+ values=z[observed],
561
+ counts=np.bincount(true_lab, minlength=n_true).astype(float),
562
+ n_drawn=n_objects,
563
+ )
564
+ return Mock(tb, lk, truth, {"reco_counts": counts}, response=response)
565
+
566
+
567
+ # ----------------------------------------------------------------------------
568
+ # prior predictive simulator (for SBC)
569
+ # ----------------------------------------------------------------------------
570
+ def prior_predictive(
571
+ prior: PopulationPrior,
572
+ bins: Bins,
573
+ n_objects: int | None = None,
574
+ *,
575
+ noise: Any = (0.05, 0.2),
576
+ selection: Any = None,
577
+ selection_mode: str | None = None,
578
+ max_objects: float = 1e6,
579
+ ) -> Callable[[np.random.Generator], tuple[Likelihoods, np.ndarray]]:
580
+ r"""A simulator ``simulate(rng) -> (likelihoods, truth)`` drawing populations from a prior.
581
+
582
+ Each call draws a population from ``prior``, objects uniformly within bins
583
+ from it, and Gaussian measurements with the given ``noise`` (in units of the
584
+ bin coordinate), and returns the likelihoods with the selection attached and
585
+ the true fractions :math:`\rho` (or rates :math:`\lambda` for a rate
586
+ prior). Intended for :func:`binference.validation.sbc` and
587
+ :func:`binference.validation.coverage`. One-dimensional bins only.
588
+
589
+ Parameters
590
+ ----------
591
+ prior : PopulationPrior
592
+ Prior on fractions, or on rates (e.g.
593
+ :class:`~binference.priors.GammaRates`, which then needs an informative
594
+ ``rate``).
595
+ bins : Bins
596
+ One-dimensional bins.
597
+ n_objects : int, optional
598
+ Number of observed objects (after selection), for priors on fractions
599
+ (default 200). For rates the number is drawn: :math:`N \sim
600
+ \mathrm{Poisson}(S\cdot\lambda)`, i.e. :math:`\Lambda\,S\cdot\rho`, so
601
+ ``n_objects`` must not be given. A draw with no object is discarded and
602
+ the population drawn again, which conditions the joint draw on
603
+ :math:`N \ge 1` and leaves the ranks of SBC uniform.
604
+ noise : float or (lo, hi)
605
+ Measurement standard deviation, fixed or drawn uniformly per object.
606
+ selection : array_like (K,), optional
607
+ Selection per bin: detection probabilities in (0, 1] for fractions,
608
+ exposures (any positive values) for rates. Without it, every object is
609
+ observed (unit exposure for rates).
610
+ selection_mode : {"on_truth", "on_measurement"}
611
+ Required with ``selection``. ``"on_truth"``: an object is kept with a
612
+ probability proportional to ``selection`` of its *true* bin, and the
613
+ likelihoods carry ``Selection(selection, "on_truth")``.
614
+ ``"on_measurement"``: it is kept with a probability proportional to
615
+ ``selection`` of the bin its *measured* value falls in (values beyond
616
+ the range count in the end bins), and the likelihoods carry an
617
+ ``"on_measurement"`` selection whose :math:`S_k` is the bin average of
618
+ :math:`\sum_j T_j P(d \in j \mid z)` over the noise distribution, with
619
+ :math:`T` the given values.
620
+ max_objects : float
621
+ For rates, the largest expected number of objects :math:`S\cdot\lambda`
622
+ a draw may have. Every object is simulated, so a draw beyond it raises
623
+ (as the vague default :class:`~binference.priors.GammaRates` does).
624
+
625
+ Returns
626
+ -------
627
+ callable
628
+ ``simulate(rng)`` returning ``(likelihoods, truth)``.
629
+
630
+ Notes
631
+ -----
632
+ Objects are kept with probability ``selection / max(selection)`` until
633
+ enough are observed; only the ratios matter for which objects are kept,
634
+ while the absolute values set the selection attached to the likelihoods
635
+ and, for rates, the expected number of objects.
636
+ """
637
+ if not isinstance(bins, Bins) or bins.is_categorical or bins.ndim != 1:
638
+ raise ValueError("prior_predictive needs one-dimensional Bins.")
639
+ bound = prior.bind(bins)
640
+ rates = bound.kind == "rates"
641
+ if bound.kind not in ("fractions", "rates"):
642
+ raise ValueError("prior_predictive supports priors on fractions or rates.")
643
+ if rates and n_objects is not None:
644
+ raise ValueError(
645
+ "n_objects must not be given for a rate prior: the number of objects is drawn "
646
+ "from Poisson(S . lambda)."
647
+ )
648
+ n = 200 if n_objects is None else n_objects
649
+ if not rates and (int(n) != n or n < 1):
650
+ raise ValueError(f"n_objects must be a positive integer; got {n_objects}.")
651
+ n = int(n)
652
+ if not max_objects > 0:
653
+ raise ValueError(f"max_objects must be positive; got {max_objects}.")
654
+ sel = keep_truth = keep_measured = None
655
+ S = np.ones(bins.K)
656
+ if selection is None:
657
+ if selection_mode is not None:
658
+ raise ValueError("selection_mode was given without a selection.")
659
+ else:
660
+ scale = "exposure" if rates else "probability"
661
+ sel = Selection(selection, selection_mode, scale=scale) # checks the mode
662
+ T = sel.values
663
+ if T.size != bins.K:
664
+ raise ValueError(f"selection needs one value per bin ({bins.K}); got {T.size}.")
665
+ if selection_mode == "on_truth":
666
+ keep_truth = T / T.max()
667
+ else:
668
+ if bins.is_masked:
669
+ raise ValueError("selection_mode='on_measurement' needs unmasked bins.")
670
+ e = np.r_[-np.inf, bins.edges[0][1:-1], np.inf]
671
+ nz = np.asarray(noise, dtype=float)
672
+
673
+ def passes(z):
674
+ return sum(T[j] * _measured_pass(z, nz, e[j], e[j + 1]) for j in range(bins.K))
675
+
676
+ sel = Selection.from_function(passes, bins, selection_mode, scale=scale)
677
+ keep_measured = T / T.max()
678
+ S = sel.values
679
+
680
+ def draw(rng: np.random.Generator) -> tuple[np.ndarray, int]:
681
+ value = bound.sample(1, rng)[0]
682
+ if not rates:
683
+ return value, n
684
+ # N ~ Poisson(S . lambda)
685
+ mean = float(S @ value)
686
+ if not mean <= max_objects:
687
+ raise ValueError(
688
+ f"A rate drawn from the prior expects {mean:.3g} objects, above max_objects="
689
+ f"{max_objects:.3g}; give the rate prior an informative scale (e.g. "
690
+ "GammaRates(shape, rate) with a larger rate)."
691
+ )
692
+ return value, int(rng.poisson(mean))
693
+
694
+ def simulate(rng: np.random.Generator) -> tuple[Likelihoods, np.ndarray]:
695
+ for _ in range(1000): # condition on N >= 1 (rates only)
696
+ value, count = draw(rng)
697
+ if count:
698
+ break
699
+ else:
700
+ raise ValueError(
701
+ "1000 populations drawn from the rate prior in a row had no object; give "
702
+ "the rate prior an informative scale or the selection larger exposures."
703
+ )
704
+ rho = np.maximum(value, 0.0)
705
+ rho = rho / rho.sum()
706
+ mock = _gaussian_mock(
707
+ bins,
708
+ density=rho,
709
+ n_objects=count,
710
+ noise=noise,
711
+ on_truth=keep_truth,
712
+ measured_range=None,
713
+ outliers=0.0,
714
+ gen=rng,
715
+ measured_keep=keep_measured,
716
+ selection=sel,
717
+ )
718
+ return mock.likelihoods, value if rates else rho
719
+
720
+ return simulate