polyphys 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. polyphys/__init__.py +7 -0
  2. polyphys/__version__.py +22 -0
  3. polyphys/analyze/__init__.py +51 -0
  4. polyphys/analyze/clusters.py +1738 -0
  5. polyphys/analyze/correlations.py +995 -0
  6. polyphys/analyze/density.py +1138 -0
  7. polyphys/analyze/measurer.py +1338 -0
  8. polyphys/analyze/pipeline.py +1868 -0
  9. polyphys/analyze/stats.py +810 -0
  10. polyphys/analyze/sumrule.py +1159 -0
  11. polyphys/manage/__init__.py +11 -0
  12. polyphys/manage/organizer.py +2188 -0
  13. polyphys/manage/parser.py +1264 -0
  14. polyphys/manage/types.py +171 -0
  15. polyphys/manage/utils.py +777 -0
  16. polyphys/probe/__init__.py +25 -0
  17. polyphys/probe/binders.py +711 -0
  18. polyphys/probe/chain_size.py +583 -0
  19. polyphys/probe/core.py +437 -0
  20. polyphys/probe/histograms.py +863 -0
  21. polyphys/probe/pairs.py +555 -0
  22. polyphys/simulate/__init__.py +48 -0
  23. polyphys/simulate/build.py +1231 -0
  24. polyphys/simulate/core.py +237 -0
  25. polyphys/simulate/data.py +977 -0
  26. polyphys/simulate/dump.py +752 -0
  27. polyphys/simulate/log.py +788 -0
  28. polyphys/spec.py +1359 -0
  29. polyphys/tests/analyze/test_clusters.py +1058 -0
  30. polyphys/tests/analyze/test_correlations.py +708 -0
  31. polyphys/tests/analyze/test_density.py +813 -0
  32. polyphys/tests/analyze/test_measurer.py +785 -0
  33. polyphys/tests/analyze/test_pipeline.py +1881 -0
  34. polyphys/tests/analyze/test_stats.py +732 -0
  35. polyphys/tests/analyze/test_sumrule.py +917 -0
  36. polyphys/tests/conftest.py +753 -0
  37. polyphys/tests/data/lammps_build/make_starts.py +70 -0
  38. polyphys/tests/manage/test_organizer.py +1495 -0
  39. polyphys/tests/manage/test_parser.py +1222 -0
  40. polyphys/tests/manage/test_utils.py +498 -0
  41. polyphys/tests/probe/conftest.py +407 -0
  42. polyphys/tests/probe/test_binders.py +686 -0
  43. polyphys/tests/probe/test_chain_size.py +557 -0
  44. polyphys/tests/probe/test_core.py +391 -0
  45. polyphys/tests/probe/test_histograms.py +779 -0
  46. polyphys/tests/probe/test_pairs.py +475 -0
  47. polyphys/tests/simulate/test_build.py +1042 -0
  48. polyphys/tests/simulate/test_core.py +106 -0
  49. polyphys/tests/simulate/test_data.py +858 -0
  50. polyphys/tests/simulate/test_dump.py +487 -0
  51. polyphys/tests/simulate/test_log.py +667 -0
  52. polyphys/tests/strategies.py +120 -0
  53. polyphys/tests/synthetic.py +410 -0
  54. polyphys/tests/test_conftest.py +414 -0
  55. polyphys/tests/test_docs.py +82 -0
  56. polyphys/tests/test_packaging.py +97 -0
  57. polyphys/tests/test_spec.py +772 -0
  58. polyphys/tests/viz/conftest.py +32 -0
  59. polyphys/tests/viz/test_colors.py +113 -0
  60. polyphys/tests/viz/test_helpers.py +292 -0
  61. polyphys/viz/__init__.py +15 -0
  62. polyphys/viz/colors.py +204 -0
  63. polyphys/viz/helpers.py +549 -0
  64. polyphys-1.0.0.dist-info/METADATA +339 -0
  65. polyphys-1.0.0.dist-info/RECORD +69 -0
  66. polyphys-1.0.0.dist-info/WHEEL +5 -0
  67. polyphys-1.0.0.dist-info/licenses/AUTHORS.rst +13 -0
  68. polyphys-1.0.0.dist-info/licenses/LICENSE +22 -0
  69. polyphys-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1868 @@
1
+ """
2
+ ==========================================================
3
+ :mod:`polyphys.analyze.pipeline`
4
+ ==========================================================
5
+
6
+ The :mod:`polyphys.analyze.pipeline` module reduces the probe artifacts of one
7
+ space down the lineage, ``segment`` to ``whole`` to ``ensemble`` to
8
+ ``ensemble average``, for every property that a study's analysis table
9
+ declares. It holds no physics of its own: it selects the files of each
10
+ property, calls :mod:`polyphys.manage.organizer` to build the wholes, the
11
+ ensembles, and the ensemble averages, and lets a *post-hook* derive further
12
+ artifacts from the wholes, such as a time average, an autocorrelation
13
+ function, a density profile, or the pair-separation histograms of the foci.
14
+
15
+ A study describes its analysis as data: one
16
+ :class:`~polyphys.spec.AnalysisTable` per particle group, listing a
17
+ :class:`~polyphys.spec.PropertySpec` per probe artifact; both records are in
18
+ :mod:`polyphys.spec`. The tables of the maintainer's studies live beside each
19
+ study's :class:`~polyphys.spec.ProjectSpec` in the repository's
20
+ ``projects/`` folder, so a data correction or a special case is code in the
21
+ study's folder, never an option of this module.
22
+
23
+ Notation
24
+ ========
25
+
26
+ *property*
27
+ The name of one probe artifact without its lineage name and group, such
28
+ as ``gyrTMon`` or ``rHistCrd``: a *name* (``gyrT``, ``rHist``) followed
29
+ by a *species* (``Mon``, ``Crd``). Its files end in ``-<property>.npy``
30
+ (a whole) or ``.<property>.npy`` (a segment).
31
+ *relation*
32
+ How the segments of a whole combine, as
33
+ :func:`~polyphys.manage.organizer.whole_from_segments` defines it:
34
+ ``'tseries'`` concatenates along the frames, ``'histogram'`` sums, and
35
+ ``'bin_edge'`` keeps one copy of identical edges.
36
+ *whole type*
37
+ The shape of one whole, as :func:`~polyphys.manage.organizer.ensemble`
38
+ defines it: ``'vector'`` (a 1-D array), ``'matrix'`` (an N-D array,
39
+ stacked along a new first axis), ``'bin_edge'`` (edges shared by the
40
+ wholes), or ``'dataframe'`` (a table whose columns are elements, such as
41
+ the pairs of foci). The ensemble average of a vector is a DataFrame of
42
+ ``-mean``, ``-var``, and ``-sem`` columns; of a matrix, one array per
43
+ statistic; of edges, the edges; of a DataFrame, a DataFrame with those
44
+ three statistics per column.
45
+ *post-hook*
46
+ A callable ``(wholes, context) -> tuple[Derived, ...]`` applied to a
47
+ property's wholes before the ensembles are built. It returns every
48
+ artifact to reduce, so a hook can pass the wholes through under the
49
+ property's own name, add derived artifacts, or replace the wholes with a
50
+ reduction of them.
51
+ *lineage of a table*
52
+ ``'segment'`` when the probe phase wrote the group's artifacts in
53
+ segments, ``'whole'`` when it wrote one file per whole. A property of
54
+ the study and the group, not of this module.
55
+ :math:`r_k`
56
+ The autocorrelation at lag :math:`k` frames, from
57
+ :func:`~polyphys.analyze.correlations.acf`.
58
+ :math:`\\bar P(R)`
59
+ The probability density of the separation vector of a pair of foci, in
60
+ one over the length unit cubed, from
61
+ :func:`~polyphys.analyze.clusters.pair_separation_density`.
62
+
63
+ Contents
64
+ ========
65
+
66
+ Records:
67
+ :class:`Derived` is one artifact a hook returns, and :class:`HookContext`
68
+ what a hook may read. The table records, :class:`~polyphys.spec.PropertySpec`
69
+ and :class:`~polyphys.spec.AnalysisTable`, and their builders are in
70
+ :mod:`polyphys.spec`.
71
+
72
+ Hooks:
73
+ :class:`TimeAverage` averages a time series over its frames;
74
+ :class:`Autocorrelation` adds the autocorrelation function of a series;
75
+ :class:`LocalDensities` adds the number density and volume fraction profiles
76
+ of a histogram through an injected :class:`DensityKernel`;
77
+ :class:`FociPairDistances` turns the distance-matrix time series of the foci
78
+ into their pair histograms, separation densities, and separation time series.
79
+ :func:`half_contour` is the separation bound of a ring at full bond extension.
80
+
81
+ Drivers:
82
+ :func:`reduce_space` runs every property of a table over one space;
83
+ :func:`space_tseries_from_spec` and :func:`space_hists_from_spec` stack the
84
+ ensemble averages of one property into the ``space`` table with the spec's
85
+ columns and rounding step;
86
+ :func:`reduce_stamps` combines the stamps of a space;
87
+ :func:`equilibrium_wholes` reduces each whole's time series to its
88
+ equilibrium value with a blocked error, beside the whole's stamps.
89
+
90
+ Conventions
91
+ ===========
92
+
93
+ - A property whose files are not found raises ``FileNotFoundError`` before
94
+ any directory is made or any property reduced; nothing is skipped in
95
+ silence.
96
+ - No output overwrites another: :class:`~polyphys.spec.AnalysisTable`
97
+ rejects a repeated name among the outputs its hooks declare, and
98
+ :func:`reduce_space` raises ``ValueError`` when a hook that declares none
99
+ returns a name already produced.
100
+ - The wholes of one ensemble must hold identical bin edges, or ``ValueError``
101
+ names the ensemble and the two wholes; the organizer takes the first
102
+ whole's edges without comparing them.
103
+ - Outputs go to ``<root>/analysis/<prefix>-<group>-<stage>`` for the stages
104
+ ``wholeSim`` (the ``segment`` lineage only), ``ens``, and ``ensAvg``, as
105
+ :func:`~polyphys.manage.organizer.make_database` builds them, with the
106
+ names :func:`~polyphys.manage.organizer.save_artifact` gives.
107
+ - The ensemble of ``'dataframe'`` wholes is their long form, every whole's
108
+ rows with a ``whole`` and a ``row`` column, so no whole is lost before the
109
+ ensemble average; the average is taken row by row across the wholes.
110
+ - The error of a whole's time average is the blocked standard error of
111
+ :func:`~polyphys.analyze.stats.sem_blocked`, since the frames of one
112
+ run are correlated; across the wholes of an ensemble, independent
113
+ runs, the organizer's plain standard error stays.
114
+ - A density profile's ensemble average also carries ``<property>-norm-sem``,
115
+ the standard error over the wholes of each whole's profile divided by its
116
+ own value in the bin that the sum rule's default normalizer names
117
+ (:data:`~polyphys.analyze.sumrule.PROFILE_NORMALIZERS`): the SEM across
118
+ profiles normalized separately for each whole. Normalization is
119
+ nonlinear, so it is not generally an uncertainty estimate for the
120
+ normalized ensemble mean, and neither is the ensemble average's
121
+ ``-sem``. See :class:`Derived`.
122
+ - Progress is logged at INFO level; no warning is raised for a renamed
123
+ artifact, because every output name is declared.
124
+ - Complexities are stated in ``n_files`` (the files of one property),
125
+ ``n_frames``, ``n_bins``, ``n_wholes``, and ``n_pairs``.
126
+
127
+ Dependencies
128
+ ============
129
+
130
+ - Python standard library: `dataclasses`, `glob`, `logging`, `os`, `re`.
131
+ - `numpy`, `pandas`.
132
+ - :mod:`polyphys.analyze.clusters`: the foci pair kernels.
133
+ - :mod:`polyphys.analyze.correlations`:
134
+ :func:`~polyphys.analyze.correlations.acf`.
135
+ - :mod:`polyphys.analyze.stats`:
136
+ :func:`~polyphys.analyze.stats.sem_blocked`.
137
+ - :mod:`polyphys.manage.organizer`: the lineage reducers.
138
+ - :mod:`polyphys.spec`: :class:`~polyphys.spec.ProjectSpec`, the table
139
+ records, and :data:`~polyphys.spec.PerWhole`.
140
+
141
+ References
142
+ ==========
143
+
144
+ - K. Kremer and G. S. Grest, "Dynamics of entangled linear polymer melts: A
145
+ molecular-dynamics simulation", J. Chem. Phys. 92, 5057 (1990), for the
146
+ FENE bond and its maximum extension :math:`R_0 = 1.5\\,\\sigma`.
147
+ https://doi.org/10.1063/1.458541
148
+
149
+ Examples
150
+ ========
151
+ A table that reads as data: the gyration radius time series with its
152
+ autocorrelation, the radial histogram of the crowders with their density
153
+ profiles, and the planar histograms with their edges:
154
+
155
+ >>> from polyphys.analyze.pipeline import Autocorrelation, LocalDensities
156
+ >>> from polyphys.spec import (
157
+ ... AnalysisTable, edge, histogram, histogram_2d, tseries)
158
+ >>> table = AnalysisTable(
159
+ ... group='all', lineage='segment', stamps=False, properties=(
160
+ ... tseries('gyrT', 'Mon', post_hook=Autocorrelation(nlags=100)),
161
+ ... histogram('r', 'Crd', post_hook=LocalDensities()),
162
+ ... histogram_2d('xy', 'Crd'),
163
+ ... edge('x'),
164
+ ... ))
165
+ >>> [prop.prop for prop in table.properties]
166
+ ['gyrTMon', 'rHistCrd', 'xyHistCrd', 'xEdge']
167
+
168
+ .. versionadded:: 0.6.0
169
+
170
+ .. versionchanged:: 0.9.0
171
+ :class:`~polyphys.spec.AnalysisTable`,
172
+ :class:`~polyphys.spec.PropertySpec`, :data:`~polyphys.spec.PostHook`,
173
+ and the builders :func:`~polyphys.spec.tseries`,
174
+ :func:`~polyphys.spec.histogram`, :func:`~polyphys.spec.histogram_2d`,
175
+ :func:`~polyphys.spec.edge`, and :func:`~polyphys.spec.matrix_tseries`
176
+ moved to :mod:`polyphys.spec`; importing them from here warned with
177
+ :exc:`DeprecationWarning`.
178
+
179
+ .. versionchanged:: 1.0.0
180
+ Those names are no longer served from here; import them from
181
+ :mod:`polyphys.spec`.
182
+ """
183
+ import glob
184
+ import logging
185
+ import numbers
186
+ import os
187
+ import re
188
+ import warnings
189
+ from collections.abc import Callable, Mapping, Sequence
190
+ from dataclasses import dataclass, field
191
+ from typing import Literal, Protocol, TypeGuard, cast
192
+
193
+ import numpy as np
194
+ import numpy.typing as npt
195
+ import pandas as pd
196
+
197
+ from polyphys.analyze import clusters, measurer, stats
198
+ from polyphys.analyze.correlations import acf
199
+ from polyphys.analyze.stats import PlateauMethod
200
+ from polyphys.analyze.sumrule import PROFILE_NORMALIZERS
201
+ from polyphys import spec as _spec
202
+ from polyphys.manage import organizer
203
+ from polyphys.manage.parser import ParserBase
204
+ from polyphys.manage.types import (
205
+ DirectionT,
206
+ GroupT,
207
+ Pathish,
208
+ PrimitiveLineageT,
209
+ PropertyT,
210
+ WholeRelationT,
211
+ )
212
+ from polyphys.spec import PerWhole, ProjectSpec, _check_resolved
213
+
214
+ __all__ = [
215
+ 'Autocorrelation',
216
+ 'DensityKernel',
217
+ 'Derived',
218
+ 'FociPairDistances',
219
+ 'HookContext',
220
+ 'LocalDensities',
221
+ 'TimeAverage',
222
+ 'equilibrium_wholes',
223
+ 'half_contour',
224
+ 'reduce_space',
225
+ 'reduce_stamps',
226
+ 'space_hists_from_spec',
227
+ 'space_tseries_from_spec',
228
+ ]
229
+
230
+ logger = logging.getLogger(__name__)
231
+
232
+
233
+ #: The shape of one whole; see "whole type" in the module notation.
234
+ WholeTypeT = Literal['vector', 'matrix', 'bin_edge', 'dataframe']
235
+
236
+ #: The organizer's ensemble-average type for each array whole type.
237
+ _ENS_TYPES: Mapping[str, Literal['dataframe', 'ndarray', 'bin_edge']] = {
238
+ 'vector': 'dataframe',
239
+ 'matrix': 'ndarray',
240
+ 'bin_edge': 'bin_edge',
241
+ }
242
+
243
+ #: The reference bin of a per-whole normalization; see :class:`Derived`.
244
+ ProfileNormalizerT = Literal['first', 'last', 'max']
245
+
246
+ #: The wholes of one property as loaded from its files, by whole name.
247
+ ArrayWholes = dict[str, np.ndarray]
248
+ #: The wholes a hook may return, by whole name.
249
+ Wholes = dict[str, np.ndarray] | dict[str, pd.DataFrame]
250
+ #: Bin edges by whole name.
251
+ Edges = Mapping[str, np.ndarray]
252
+
253
+ FloatArray = npt.NDArray[np.float64]
254
+
255
+
256
+ class DensityKernel(Protocol):
257
+ """
258
+ The volume-fraction kernel that :class:`LocalDensities` calls.
259
+
260
+ A kernel turns the histogram of one species in one whole into its local
261
+ number density and local volume fraction, bin by bin, from the
262
+ geometry the whole's parser instance describes. The study's driver
263
+ supplies one, so this module imports no density model.
264
+
265
+ .. versionadded:: 0.6.0
266
+ """
267
+
268
+ def __call__(
269
+ self,
270
+ counts: np.ndarray,
271
+ edges: np.ndarray,
272
+ info: ParserBase,
273
+ direction: str,
274
+ diameter: float,
275
+ ) -> tuple[npt.ArrayLike, npt.ArrayLike]:
276
+ """
277
+ Parameters
278
+ ----------
279
+ counts : numpy.ndarray
280
+ The histogram of one whole, shape ``(n_bins,)``: the number of
281
+ particle centers in each bin, summed over the frames.
282
+ edges : numpy.ndarray
283
+ Its bin edges, shape ``(n_bins + 1,)``, in the length unit.
284
+ info : ParserBase
285
+ The whole's parser instance, which holds the box sizes. It holds
286
+ no number of frames, so a kernel that divides `counts` by the
287
+ frames gets them from the study's driver.
288
+ direction : str
289
+ The direction of the histogram, such as ``'r'`` or ``'z'``.
290
+ diameter : float
291
+ The diameter of the species, in the length unit.
292
+
293
+ Returns
294
+ -------
295
+ tuple of array_like
296
+ The number density and the volume fraction, each of shape
297
+ ``(n_bins,)``.
298
+ """
299
+ ...
300
+
301
+
302
+ # ---------------------------------------------------------------------------
303
+ # Records
304
+ # ---------------------------------------------------------------------------
305
+
306
+
307
+ @dataclass(frozen=True)
308
+ class Derived:
309
+ """
310
+ One artifact a post-hook returns, ready for the ensemble stages.
311
+
312
+ Attributes
313
+ ----------
314
+ name : str
315
+ The output property, which names the files; for example
316
+ ``'rRhoCrd'`` or ``'gyrTMon-acf'``.
317
+ wholes : dict of str to numpy.ndarray or pandas.DataFrame
318
+ The artifact of each whole, by whole name.
319
+ whole_type : {'vector', 'matrix', 'bin_edge', 'dataframe'}
320
+ The shape of each whole; see the module notation.
321
+ edges : mapping of str to numpy.ndarray, optional
322
+ The bin edges of each whole, for a ``'vector'`` histogram, so its
323
+ ensembles carry a ``bin_center`` column.
324
+ normalizer : {'first', 'last', 'max'}, optional
325
+ For a ``'vector'`` profile, the reference bin of a per-whole
326
+ normalization: the first bin, the last bin, or the bin where the
327
+ ensemble average is largest. Each whole is divided by its own value
328
+ in that bin, and the standard error over the wholes of these
329
+ separately normalized profiles is written beside the ensemble
330
+ average as ``<name>-norm-sem``; it is not generally an uncertainty
331
+ estimate for the normalized ensemble mean. The ensemble average
332
+ itself is not normalized;
333
+ :func:`polyphys.analyze.sumrule.space_profiles` does that and
334
+ carries the column through.
335
+
336
+ Raises
337
+ ------
338
+ ValueError
339
+ If `whole_type` is unknown, if `edges` or `normalizer` is given for
340
+ a whole type other than ``'vector'``, or if `normalizer` is not one
341
+ of its three values.
342
+
343
+ Notes
344
+ -----
345
+ The wholes of an ensemble are independent runs, so their spread is a
346
+ valid error of any function of the profile, the normalized one
347
+ included, with no covariance between a bin and its normalizer to track.
348
+ The ensemble picks the bin so that every whole is normalized at the
349
+ same place: a per-whole ``'max'`` would bias each ratio towards one
350
+ before the spread is taken. The error is zero in the reference bin by
351
+ construction. The scale of a profile, such as the ``1 / a`` or ``a^2``
352
+ of the sum rule, cancels in the ratio, so the column holds for the
353
+ scaled profile too.
354
+
355
+ Examples
356
+ --------
357
+ >>> import numpy as np
358
+ >>> out = Derived('gyrMon', {'w1': np.array([1.0])}, 'vector')
359
+ >>> out.name, out.whole_type
360
+ ('gyrMon', 'vector')
361
+
362
+ .. versionadded:: 0.6.0
363
+ .. versionchanged:: 0.7.0
364
+ Added `normalizer`.
365
+ """
366
+
367
+ name: str
368
+ wholes: Wholes
369
+ whole_type: WholeTypeT
370
+ edges: Edges | None = None
371
+ normalizer: ProfileNormalizerT | None = None
372
+
373
+ def __post_init__(self) -> None:
374
+ _check_whole_type(self.whole_type)
375
+ if self.edges is not None and self.whole_type != 'vector':
376
+ raise ValueError(
377
+ f"'{self.name}': edges are only valid with whole_type "
378
+ f"'vector', not '{self.whole_type}'."
379
+ )
380
+ if self.normalizer is not None:
381
+ if self.whole_type != 'vector':
382
+ raise ValueError(
383
+ f"'{self.name}': a normalizer is only valid with "
384
+ f"whole_type 'vector', not '{self.whole_type}'."
385
+ )
386
+ if self.normalizer not in ('first', 'last', 'max'):
387
+ raise ValueError(
388
+ f"'{self.name}': normalizer must be 'first', 'last', or "
389
+ f"'max', not {self.normalizer!r}."
390
+ )
391
+
392
+
393
+ @dataclass(frozen=True)
394
+ class HookContext:
395
+ """
396
+ What a post-hook may read besides the wholes.
397
+
398
+ .. versionadded:: 0.6.0
399
+
400
+ Attributes
401
+ ----------
402
+ prop : ~polyphys.spec.PropertySpec
403
+ The property the wholes belong to.
404
+ spec : ~polyphys.spec.ProjectSpec, optional
405
+ The study's spec: its parser, topology, and species sizes.
406
+ group : str
407
+ The particle group.
408
+ infos : mapping of str to ~polyphys.manage.parser.ParserBase
409
+ The parser instance of each whole, by whole name, which holds the
410
+ whole's attributes such as sizes and counts.
411
+ edges : mapping of str to numpy.ndarray, optional
412
+ The bin edges of each whole, for a property with an edge file.
413
+ density : DensityKernel, optional
414
+ The injected volume-fraction kernel.
415
+ """
416
+
417
+ prop: _spec.PropertySpec
418
+ spec: ProjectSpec | None = None
419
+ group: GroupT = 'bug'
420
+ infos: Mapping[str, ParserBase] = field(default_factory=dict)
421
+ edges: Edges | None = None
422
+ density: DensityKernel | None = None
423
+
424
+
425
+ # ---------------------------------------------------------------------------
426
+ # Hooks
427
+ # ---------------------------------------------------------------------------
428
+
429
+
430
+ @dataclass(frozen=True)
431
+ class TimeAverage:
432
+ """
433
+ Average a time series over its frames; a post-hook.
434
+
435
+ Attributes
436
+ ----------
437
+ output : str
438
+ The name of the average, without the species, which is appended:
439
+ ``TimeAverage('clustersHist')`` on ``'clustersHistTFoci'`` writes
440
+ ``'clustersHistFoci'``.
441
+ drop_first_column : bool, default False
442
+ Drop column 0 of each frame before averaging. The size histograms of
443
+ :func:`~polyphys.analyze.clusters.count_clusters` carry the count of
444
+ empty clusters, always 0, in column 0.
445
+
446
+ Notes
447
+ -----
448
+ The frames are axis 0; the average has the shape of one frame, so a
449
+ ``(n_frames, n_sizes)`` histogram time series becomes an ``n_sizes``
450
+ vector, reduced as a ``'vector'`` whole. Time and space are
451
+ :math:`O(n_\\mathrm{frames} n_\\mathrm{sizes})` per whole.
452
+
453
+ Examples
454
+ --------
455
+ >>> import numpy as np
456
+ >>> from polyphys.spec import matrix_tseries
457
+ >>> hook = TimeAverage('clustersHist', drop_first_column=True)
458
+ >>> context = HookContext(matrix_tseries('clustersHistT', 'Foci'))
459
+ >>> wholes = {'w1': np.array([[0, 1, 2], [0, 3, 4], [0, 5, 6]])}
460
+ >>> (out,) = hook(wholes, context)
461
+ >>> out.name, out.wholes['w1']
462
+ ('clustersHistFoci', array([3., 4.]))
463
+
464
+ .. versionadded:: 0.6.0
465
+ """
466
+
467
+ output: str
468
+ drop_first_column: bool = False
469
+
470
+ def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
471
+ """
472
+ The name of the average: `output` then the species of `prop`.
473
+
474
+ Examples
475
+ --------
476
+ >>> from polyphys.spec import matrix_tseries
477
+ >>> TimeAverage('clustersHist').outputs(
478
+ ... matrix_tseries('clustersHistT', 'Foci'))
479
+ ('clustersHistFoci',)
480
+
481
+ .. versionadded:: 0.9.0
482
+ """
483
+ return (self.output + prop.species,)
484
+
485
+ def __call__(
486
+ self, wholes: ArrayWholes, context: HookContext
487
+ ) -> tuple[Derived, ...]:
488
+ averaged: dict[str, np.ndarray] = {}
489
+ for whole_name, whole in wholes.items():
490
+ frames = np.asarray(whole, dtype=np.float64)
491
+ if self.drop_first_column:
492
+ frames = frames[:, 1:]
493
+ averaged[whole_name] = frames.mean(axis=0)
494
+ return (Derived(self.output + context.prop.species, averaged,
495
+ 'vector'),)
496
+
497
+
498
+ @dataclass(frozen=True)
499
+ class Autocorrelation:
500
+ """
501
+ Pass a time series through and add its autocorrelation; a post-hook.
502
+
503
+ Attributes
504
+ ----------
505
+ nlags : int
506
+ The largest lag, in frames; see
507
+ :func:`~polyphys.analyze.correlations.acf`.
508
+
509
+ Notes
510
+ -----
511
+ Returns the series under its own name and :math:`r_k` for
512
+ :math:`k = 0, \\dots,` `nlags` under ``<property>-acf``, both
513
+ ``'vector'`` wholes. No confidence band is stored: Bartlett's band is a
514
+ function of the stored :math:`r_k`, the number of frames, and the
515
+ level, so :func:`~polyphys.analyze.correlations.bartlett_half_width`
516
+ rebuilds it at any level when it is drawn. Time is
517
+ :math:`O(n_\\mathrm{frames} \\log n_\\mathrm{frames})` per whole.
518
+
519
+ Examples
520
+ --------
521
+ The alternating series has :math:`r_1 = -7/8` and :math:`r_2 = 6/8`:
522
+
523
+ >>> import numpy as np
524
+ >>> from polyphys.spec import tseries
525
+ >>> hook = Autocorrelation(nlags=2)
526
+ >>> context = HookContext(tseries('gyrT', 'Mon'))
527
+ >>> series, acf_out = hook({'w1': np.tile([1.0, -1.0], 4)}, context)
528
+ >>> series.name, acf_out.name
529
+ ('gyrTMon', 'gyrTMon-acf')
530
+ >>> np.round(acf_out.wholes['w1'], 6)
531
+ array([ 1. , -0.875, 0.75 ])
532
+
533
+ .. versionadded:: 0.6.0
534
+ """
535
+
536
+ nlags: int
537
+
538
+ def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
539
+ """
540
+ The series, passed through, and ``<property>-acf``.
541
+
542
+ Examples
543
+ --------
544
+ >>> from polyphys.spec import tseries
545
+ >>> Autocorrelation(nlags=2).outputs(tseries('gyrT', 'Mon'))
546
+ ('gyrTMon', 'gyrTMon-acf')
547
+
548
+ .. versionadded:: 0.9.0
549
+ """
550
+ return (prop.prop, f'{prop.prop}-acf')
551
+
552
+ def __call__(
553
+ self, wholes: ArrayWholes, context: HookContext
554
+ ) -> tuple[Derived, ...]:
555
+ prop = context.prop.prop
556
+ acfs: dict[str, np.ndarray] = {
557
+ whole_name: acf(whole, self.nlags, alpha=None).values
558
+ for whole_name, whole in wholes.items()
559
+ }
560
+ return (Derived(prop, wholes, 'vector'),
561
+ Derived(f'{prop}-acf', acfs, 'vector'))
562
+
563
+
564
+ @dataclass(frozen=True)
565
+ class LocalDensities:
566
+ """
567
+ Pass a histogram through and add its density profiles; a post-hook.
568
+
569
+ On ``<direction>Hist<species>`` it adds ``<direction>Rho<species>``, the
570
+ local number density, and ``<direction>Phi<species>``, the local volume
571
+ fraction, from the :class:`DensityKernel` given to :func:`reduce_space`,
572
+ with the species' diameter read from the attribute the study's
573
+ :attr:`~polyphys.spec.ProjectSpec.species_sizes` names.
574
+
575
+ Raises
576
+ ------
577
+ ValueError
578
+ When called: if the property is not a :func:`~polyphys.spec.histogram`,
579
+ if the context has no kernel, no spec, or no edges, or if the species
580
+ is not in the spec's `species_sizes`.
581
+
582
+ Notes
583
+ -----
584
+ The three outputs share the histogram's edges, so their ensembles carry
585
+ the same ``bin_center`` column. The two profiles get the normalizer of
586
+ :data:`~polyphys.analyze.sumrule.PROFILE_NORMALIZERS` for the parser's
587
+ geometry and the histogram's direction, when the pair has one, so their
588
+ ensemble averages carry ``<property>-norm-sem``, the SEM across
589
+ profiles normalized separately for each whole (see :class:`Derived`).
590
+ Time is one kernel call per whole.
591
+
592
+ Examples
593
+ --------
594
+ With a stub kernel that returns the counts and a constant, on one whole
595
+ of the ``ChainCyl`` study:
596
+
597
+ >>> import numpy as np
598
+ >>> from polyphys.tests.synthetic import ChainCyl
599
+ >>> from polyphys.spec import ProjectSpec, histogram
600
+ >>> spec = ProjectSpec(
601
+ ... parser=ChainCyl, group='bug', divisor=0.025,
602
+ ... crowder_attr='phi_bulk_c',
603
+ ... physical_attrs=('space',), species_sizes={'Crd': 'dcrowd'},
604
+ ... tseries_properties=())
605
+ >>> whole = ('N20r5.5lz25.0ac1.0nc100'
606
+ ... 'dt0.005bdump1000adump5000ens1')
607
+ >>> def kernel(counts, edges, info, direction, size):
608
+ ... return counts, np.full(counts.shape, size)
609
+ >>> context = HookContext(
610
+ ... histogram('r', 'Crd'), spec=spec, group='all',
611
+ ... infos={whole: ChainCyl(whole, 'whole', 'all')},
612
+ ... edges={whole: np.array([0.0, 1.0, 2.0])}, density=kernel)
613
+ >>> outputs = LocalDensities()({whole: np.array([3.0, 5.0])}, context)
614
+ >>> [out.name for out in outputs]
615
+ ['rHistCrd', 'rRhoCrd', 'rPhiCrd']
616
+ >>> outputs[2].wholes[whole]
617
+ array([1., 1.])
618
+ >>> outputs[2].normalizer
619
+ 'first'
620
+
621
+ .. versionadded:: 0.6.0
622
+ .. versionchanged:: 0.7.0
623
+ The profiles carry the sum rule's default normalizer.
624
+ """
625
+
626
+ @staticmethod
627
+ def _direction(prop: _spec.PropertySpec) -> str:
628
+ """The direction of a histogram property, which must be one."""
629
+ if not prop.name.endswith('Hist') or not prop.has_edges:
630
+ raise ValueError(
631
+ f"LocalDensities needs a histogram(...) property, not "
632
+ f"'{prop.prop}'."
633
+ )
634
+ return prop.name[: -len('Hist')]
635
+
636
+ def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
637
+ """
638
+ The histogram, passed through, and its ``<direction>Rho`` and
639
+ ``<direction>Phi`` profiles.
640
+
641
+ Raises
642
+ ------
643
+ ValueError
644
+ If `prop` is not a :func:`~polyphys.spec.histogram` property.
645
+
646
+ Examples
647
+ --------
648
+ >>> from polyphys.spec import histogram
649
+ >>> LocalDensities().outputs(histogram('z', 'Crd'))
650
+ ('zHistCrd', 'zRhoCrd', 'zPhiCrd')
651
+
652
+ .. versionadded:: 0.9.0
653
+ """
654
+ direction = self._direction(prop)
655
+ return (prop.prop, f'{direction}Rho{prop.species}',
656
+ f'{direction}Phi{prop.species}')
657
+
658
+ def __call__(
659
+ self, wholes: ArrayWholes, context: HookContext
660
+ ) -> tuple[Derived, ...]:
661
+ prop = context.prop
662
+ direction = self._direction(prop)
663
+ if context.density is None:
664
+ raise ValueError(
665
+ f"'{prop.prop}': LocalDensities needs the density kernel "
666
+ "passed to reduce_space(..., density=...)."
667
+ )
668
+ if context.spec is None or context.edges is None:
669
+ raise ValueError(
670
+ f"'{prop.prop}': LocalDensities needs the study's spec and "
671
+ "the histogram's edges in its context."
672
+ )
673
+ size_attr = context.spec.species_sizes.get(prop.species)
674
+ if size_attr is None:
675
+ raise ValueError(
676
+ f"'{prop.prop}': species '{prop.species}' has no size in "
677
+ f"{context.spec.name}.species_sizes "
678
+ f"{dict(context.spec.species_sizes)}."
679
+ )
680
+ # The per-whole spread is defined for a bin reference only, which is
681
+ # what every default is; a numeric or callable one would be skipped.
682
+ default = PROFILE_NORMALIZERS.get(
683
+ (context.spec.parser._geometry, cast(DirectionT, direction)))
684
+ normalizer = default if isinstance(default, str) else None
685
+ rhos: dict[str, np.ndarray] = {}
686
+ phis: dict[str, np.ndarray] = {}
687
+ for whole_name, counts in wholes.items():
688
+ info = context.infos[whole_name]
689
+ rho, phi = context.density(
690
+ np.asarray(counts), np.asarray(context.edges[whole_name]),
691
+ info, direction, float(getattr(info, size_attr)),
692
+ )
693
+ rhos[whole_name] = np.asarray(rho, dtype=np.float64)
694
+ phis[whole_name] = np.asarray(phi, dtype=np.float64)
695
+ return (
696
+ Derived(prop.prop, wholes, 'vector', context.edges),
697
+ Derived(f'{direction}Rho{prop.species}', rhos, 'vector',
698
+ context.edges, normalizer),
699
+ Derived(f'{direction}Phi{prop.species}', phis, 'vector',
700
+ context.edges, normalizer),
701
+ )
702
+
703
+
704
+ def _value_for(
705
+ value: PerWhole, info: ParserBase | None, what: str, whole: str
706
+ ) -> object:
707
+ """
708
+ Return `value`, or what it gives on `info` when it is a function.
709
+
710
+ `what` names the quantity and `whole` the whole in an error.
711
+ """
712
+ if not callable(value):
713
+ return value
714
+ if info is None:
715
+ raise ValueError(
716
+ f"{what} is a function of the whole, but the context has no "
717
+ f"parser instance for whole '{whole}'."
718
+ )
719
+ return value(info)
720
+
721
+
722
+ def _resolve(
723
+ value: PerWhole, info: ParserBase | None, what: str, whole: str
724
+ ) -> float:
725
+ """
726
+ Return `value`, calling it on `info` when it is a function.
727
+
728
+ `what` names the quantity and `whole` the whole in an error. The
729
+ constant or what the callable gives must be a finite positive number,
730
+ the check of :meth:`~polyphys.spec.ProbeSpec.resolve`.
731
+ """
732
+ return _check_resolved(_value_for(value, info, what, whole),
733
+ f"{what} of whole '{whole}'")
734
+
735
+
736
+ def _is_positive_integer(value: object) -> TypeGuard[numbers.Integral]:
737
+ """Whether `value` is a positive Python or NumPy integer, never a bool
738
+ and never a float, even one equal to a whole number."""
739
+ return (isinstance(value, numbers.Integral)
740
+ and not isinstance(value, bool) and int(value) > 0)
741
+
742
+
743
+ def _resolve_count(
744
+ value: PerWhole, info: ParserBase | None, what: str, whole: str
745
+ ) -> int:
746
+ """
747
+ Return `value`, calling it on `info` when it is a function, as an int.
748
+
749
+ `what` names the quantity and `whole` the whole in an error. The
750
+ constant or what the callable gives must be a positive integer
751
+ (``_is_positive_integer``); it is never truncated.
752
+ """
753
+ result = _value_for(value, info, what, whole)
754
+ if _is_positive_integer(result):
755
+ return int(result)
756
+ raise ValueError(
757
+ f"{what} of whole '{whole}' must resolve to a positive integer, not "
758
+ f"{result!r}.")
759
+
760
+
761
+ def half_contour(n_monomers: int, bond_max: float) -> float:
762
+ """
763
+ The largest separation two monomers of a ring can have.
764
+
765
+ Parameters
766
+ ----------
767
+ n_monomers : int
768
+ The number of monomers on the ring, at least 2.
769
+ bond_max : float
770
+ The largest length a bond can have, in the length unit; for the FENE
771
+ bond, :math:`R_0 = 1.5\\,\\sigma_{ij}` with :math:`\\sigma_{ij}` the
772
+ collision diameter of the bonded pair (Kremer and Grest, 1990).
773
+
774
+ Returns
775
+ -------
776
+ float
777
+ ``(n_monomers // 2) * bond_max``: the two monomers are at most
778
+ ``n_monomers // 2`` bonds apart along the ring, and a path of bonds
779
+ is longest when straight and fully extended.
780
+
781
+ Raises
782
+ ------
783
+ ValueError
784
+ If `n_monomers` is below 2 or `bond_max` is not positive.
785
+
786
+ Notes
787
+ -----
788
+ A bound, not a typical value: it lets a pair-distance histogram cover
789
+ every separation the chain allows, so that its counts conserve the
790
+ number of frames and a separation above it marks a broken bond or an
791
+ unwrapping error. Pass the largest bond's :math:`R_0` when the bonds
792
+ differ; the bound is then loose by the ratio of the largest to the
793
+ typical bond, which costs bins, not correctness.
794
+
795
+ Examples
796
+ --------
797
+ A ring of 10 beads with bonds that stretch to 1.5:
798
+
799
+ >>> half_contour(10, 1.5)
800
+ 7.5
801
+
802
+ .. versionadded:: 0.6.0
803
+ """
804
+ if n_monomers < 2:
805
+ raise ValueError(f'n_monomers must be at least 2; got {n_monomers}.')
806
+ if not bond_max > 0:
807
+ raise ValueError(f'bond_max must be positive; got {bond_max}.')
808
+ return (n_monomers // 2) * bond_max
809
+
810
+
811
+ @dataclass(frozen=True)
812
+ class FociPairDistances:
813
+ """
814
+ Reduce the distance-matrix time series of the foci; a post-hook.
815
+
816
+ On ``distMatT<species>``, whose whole is the ``(n_frames, n_foci,
817
+ n_foci)`` matrix of center-to-center distances between the foci with the
818
+ foci's backbone indices on the diagonal of frame 0, it writes three
819
+ ``'dataframe'`` wholes with one column per pair, tagged
820
+ ``loc<i>loc<j>genDist<d>`` with ``i < j`` the backbone indices and ``d``
821
+ their genomic distance in bonds:
822
+
823
+ - ``pairDistHist<species>``: the histogram of each pair's separation on
824
+ one grid shared by every pair and whole, with a ``bin_center`` column;
825
+ - ``pairDistRdf<species>``: :math:`\\bar P(R)`, the probability density
826
+ of each pair's separation vector on that grid, with ``bin_center``;
827
+ - ``pairDistT<species>``: each pair's separation over the frames.
828
+
829
+ Attributes
830
+ ----------
831
+ bin_size : float or callable
832
+ The bin width :math:`\\Delta r`, in the length unit, or a function of
833
+ the whole's parser instance that gives it.
834
+ r_max : float or callable
835
+ The upper end of the grid, or a function of the parser instance that
836
+ gives it, such as :func:`half_contour` of the ring; the grid runs to
837
+ the first multiple of `bin_size` at or above it.
838
+ n_monomers : int or callable
839
+ The number of monomers on the backbone, or a function of the parser
840
+ instance that gives it.
841
+
842
+ Raises
843
+ ------
844
+ ValueError
845
+ When built: if a constant `bin_size` or `r_max` is not a finite
846
+ positive number, or a constant `n_monomers` is not a positive
847
+ integer; a Python or NumPy number, never a bool, and for
848
+ `n_monomers` never a float, even ``10.0``. When called: if the
849
+ context has no spec (the topology comes from it); if a whole is not
850
+ a ``(n_frames, n_foci, n_foci)`` array; if a callable quantity gives
851
+ a value that breaks the same rule, which names the quantity and the
852
+ whole; or if a separation lies above the grid, which names the whole
853
+ and the separation, since a dropped frame would break the
854
+ normalization.
855
+
856
+ Notes
857
+ -----
858
+ Every pair has one separation per frame, so each histogram column sums
859
+ to ``n_frames`` and :math:`\\sum_k \\bar P_k \\Delta V_k = 1` exactly.
860
+ The research code set the grid from each whole's own largest separation,
861
+ so wholes of one ensemble had different edges; a bound from the system
862
+ gives one grid. Time is :math:`O(n_\\mathrm{pairs} n_\\mathrm{frames}
863
+ \\log n_\\mathrm{bins})` and space :math:`O(n_\\mathrm{pairs}
864
+ (n_\\mathrm{frames} + n_\\mathrm{bins}))` per whole.
865
+
866
+ Examples
867
+ --------
868
+ Three foci at backbone indices 0, 4, and 8 of a ring of 10 monomers,
869
+ over two frames, on a grid of unit bins to 3:
870
+
871
+ >>> import numpy as np
872
+ >>> from polyphys.tests.synthetic import RingCub
873
+ >>> from polyphys.spec import ProjectSpec, matrix_tseries
874
+ >>> spec = ProjectSpec(
875
+ ... parser=RingCub, group='bug', divisor=0.025,
876
+ ... crowder_attr='phi_bulk_c',
877
+ ... physical_attrs=('space',), species_sizes={},
878
+ ... tseries_properties=())
879
+ >>> frame0 = np.array([[0.0, 1.5, 2.5], [1.5, 4.0, 0.5], [2.5, 0.5, 8.0]])
880
+ >>> frame1 = np.array([[0.0, 1.5, 2.9], [1.5, 0.0, 2.5], [2.9, 2.5, 0.0]])
881
+ >>> hook = FociPairDistances(bin_size=1.0, r_max=3.0, n_monomers=10)
882
+ >>> context = HookContext(matrix_tseries('distMatT', 'Foci'), spec=spec)
883
+ >>> hist, density, series = hook({'w1': np.stack([frame0, frame1])},
884
+ ... context)
885
+ >>> list(hist.wholes['w1'].columns)
886
+ ['loc0loc4genDist4', 'loc0loc8genDist2', 'loc4loc8genDist4', 'bin_center']
887
+ >>> hist.wholes['w1']['loc0loc8genDist2'].to_list()
888
+ [0, 0, 2]
889
+ >>> series.wholes['w1']['loc4loc8genDist4'].to_list()
890
+ [0.5, 2.5]
891
+
892
+ A constant that breaks the rule raises when the hook is built:
893
+
894
+ >>> FociPairDistances(bin_size=1.0, r_max=3.0, n_monomers=10.5)
895
+ Traceback (most recent call last):
896
+ ...
897
+ ValueError: FociPairDistances: 'n_monomers' must be a positive integer ...
898
+
899
+ .. versionadded:: 0.6.0
900
+
901
+ .. versionchanged:: 1.0.0
902
+ The constants are checked when the hook is built, and `n_monomers`,
903
+ a constant or what a callable gives, must be a positive integer,
904
+ where a float such as 10.5 was truncated to 10.
905
+ """
906
+
907
+ bin_size: PerWhole
908
+ r_max: PerWhole
909
+ n_monomers: int | Callable[[ParserBase], int]
910
+
911
+ def __post_init__(self) -> None:
912
+ for name in ('bin_size', 'r_max'):
913
+ value = getattr(self, name)
914
+ if not _spec._is_per_whole(value):
915
+ raise ValueError(
916
+ f"FociPairDistances: '{name}' must be a finite positive "
917
+ f"number or a callable, not {value!r}.")
918
+ if not (callable(self.n_monomers)
919
+ or _is_positive_integer(self.n_monomers)):
920
+ raise ValueError(
921
+ "FociPairDistances: 'n_monomers' must be a positive integer "
922
+ f"or a callable, not {self.n_monomers!r}.")
923
+
924
+ def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
925
+ """
926
+ The pair histograms, separation densities, and separation time
927
+ series of the species of `prop`.
928
+
929
+ Examples
930
+ --------
931
+ >>> from polyphys.spec import matrix_tseries
932
+ >>> hook = FociPairDistances(bin_size=1.0, r_max=3.0, n_monomers=10)
933
+ >>> hook.outputs(matrix_tseries('distMatT', 'Foci'))
934
+ ('pairDistHistFoci', 'pairDistRdfFoci', 'pairDistTFoci')
935
+
936
+ .. versionadded:: 0.9.0
937
+ """
938
+ species = prop.species
939
+ return (f'pairDistHist{species}', f'pairDistRdf{species}',
940
+ f'pairDistT{species}')
941
+
942
+ def __call__(
943
+ self, wholes: ArrayWholes, context: HookContext
944
+ ) -> tuple[Derived, ...]:
945
+ prop = context.prop
946
+ if context.spec is None:
947
+ raise ValueError(
948
+ f"'{prop.prop}': FociPairDistances needs the study's spec in "
949
+ "its context, for the topology."
950
+ )
951
+ # foci_pair_info checks the topology itself.
952
+ topology = cast(clusters.Topology, context.spec.topology)
953
+ hists: dict[str, pd.DataFrame] = {}
954
+ densities: dict[str, pd.DataFrame] = {}
955
+ series: dict[str, pd.DataFrame] = {}
956
+ for whole_name, whole in wholes.items():
957
+ matrix = np.asarray(whole, dtype=np.float64)
958
+ if matrix.ndim != 3 or matrix.shape[1] != matrix.shape[2]:
959
+ raise ValueError(
960
+ f"'{prop.prop}', whole '{whole_name}': expected a "
961
+ "(n_frames, n_foci, n_foci) distance matrix time series,"
962
+ f" got shape {matrix.shape}."
963
+ )
964
+ info = context.infos.get(whole_name)
965
+ n_foci = matrix.shape[1]
966
+ genomic_pos = np.rint(np.diagonal(matrix[0])).astype(np.int64)
967
+ n_monomers = _resolve_count(self.n_monomers, info, 'n_monomers',
968
+ whole_name)
969
+ pairs = clusters.foci_pair_info(genomic_pos, n_monomers, topology)
970
+ tags = [f'loc{i}loc{j}genDist{d}' for i, j, d in pairs]
971
+ first, second = np.triu_indices(n_foci, 1)
972
+ distances = matrix[:, first, second].T
973
+ bin_size = _resolve(self.bin_size, info, 'bin_size', whole_name)
974
+ r_max = _resolve(self.r_max, info, 'r_max', whole_name)
975
+ # The grid pair_distance_histograms builds, so the check below
976
+ # tolerates the same round-off.
977
+ upper = measurer.create_bin_edge_and_hist(
978
+ bin_size, 0.0, r_max)[0][-1]
979
+ largest = float(distances.max())
980
+ if largest > upper:
981
+ raise ValueError(
982
+ f"'{prop.prop}', whole '{whole_name}': a pair separation "
983
+ f"of {largest:.6g} lies above r_max {upper:.6g}; the "
984
+ "chain cannot reach it, so check the whole."
985
+ )
986
+ counts, edges = clusters.pair_distance_histograms(
987
+ distances, bin_size, r_max)
988
+ counts = counts.reshape(len(tags), -1)
989
+ density = clusters.pair_separation_density(counts, edges)
990
+ centers = 0.5 * (edges[:-1] + edges[1:])
991
+ hists[whole_name] = pd.DataFrame(
992
+ dict(zip(tags, counts)) | {'bin_center': centers})
993
+ densities[whole_name] = pd.DataFrame(
994
+ dict(zip(tags, density)) | {'bin_center': centers})
995
+ series[whole_name] = pd.DataFrame(dict(zip(tags, distances)))
996
+ species = prop.species
997
+ return (
998
+ Derived(f'pairDistHist{species}', hists, 'dataframe'),
999
+ Derived(f'pairDistRdf{species}', densities, 'dataframe'),
1000
+ Derived(f'pairDistT{species}', series, 'dataframe'),
1001
+ )
1002
+
1003
+
1004
+ # ---------------------------------------------------------------------------
1005
+ # Drivers
1006
+ # ---------------------------------------------------------------------------
1007
+
1008
+
1009
+ SaveDirs = tuple[Pathish | None, Pathish | None, Pathish | None]
1010
+
1011
+
1012
+ def reduce_stamps(
1013
+ stamp_files: Sequence[tuple[str]],
1014
+ lineage: PrimitiveLineageT,
1015
+ save_to: SaveDirs = (None, None, None),
1016
+ ) -> tuple[pd.DataFrame, pd.DataFrame]:
1017
+ """
1018
+ Combine the stamps of one space into whole and ensemble stamps.
1019
+
1020
+ Parameters
1021
+ ----------
1022
+ stamp_files : sequence of tuple of str
1023
+ The stamp CSV files, each path in a 1-tuple, as
1024
+ :func:`~polyphys.manage.utils.sort_filenames` returns them.
1025
+ lineage : {'segment', 'whole'}
1026
+ The lineage of the files.
1027
+ save_to : tuple of three paths or None, default (None, None, None)
1028
+ The ``wholeSim``, ``ens``, and ``ensAvg`` directories; ``None`` skips
1029
+ a stage's file. For ``'segment'`` stamps the combined segment
1030
+ stamps go to the first, the whole stamps to the second, and the
1031
+ ensemble stamps to the third; for ``'whole'`` stamps the whole
1032
+ stamps go to the second and the ensemble stamps to the third.
1033
+
1034
+ Returns
1035
+ -------
1036
+ whole_stamps : pandas.DataFrame
1037
+ One row per whole.
1038
+ ensemble_stamps : pandas.DataFrame
1039
+ One row per ensemble.
1040
+
1041
+ Raises
1042
+ ------
1043
+ ValueError
1044
+ As :func:`~polyphys.manage.organizer.children_stamps` and
1045
+ :func:`~polyphys.manage.organizer.parents_stamps`.
1046
+
1047
+ Notes
1048
+ -----
1049
+ Segment stamps combine with
1050
+ :func:`~polyphys.manage.organizer.children_stamps`, then roll up once
1051
+ with :func:`~polyphys.manage.organizer.parents_stamps` into whole
1052
+ stamps; whole stamps combine directly. Either way the whole stamps roll
1053
+ up once more into ensemble stamps. Time and space are
1054
+ :math:`O(n_\\mathrm{files}\\, n_\\mathrm{columns})`.
1055
+
1056
+ Examples
1057
+ --------
1058
+ .. code-block:: python
1059
+
1060
+ whole_stamps, ensemble_stamps = reduce_stamps(
1061
+ [(path,) for path in sorted(stamp_paths)], 'whole')
1062
+
1063
+ .. versionadded:: 0.6.0
1064
+ """
1065
+ whole_dir, ens_dir, ens_avg_dir = save_to
1066
+ if lineage == 'segment':
1067
+ segment_stamps = organizer.children_stamps(
1068
+ list(stamp_files), 'segment', save_to=whole_dir)
1069
+ whole_stamps = organizer.parents_stamps(
1070
+ segment_stamps, 'segment', save_to=ens_dir)
1071
+ else:
1072
+ whole_stamps = organizer.children_stamps(
1073
+ list(stamp_files), 'whole', save_to=ens_dir)
1074
+ ensemble_stamps = organizer.parents_stamps(
1075
+ whole_stamps, 'whole', save_to=ens_avg_dir)
1076
+ return whole_stamps, ensemble_stamps
1077
+
1078
+
1079
+ def reduce_space(
1080
+ input_database: Pathish,
1081
+ spec: ProjectSpec,
1082
+ table: _spec.AnalysisTable,
1083
+ *,
1084
+ hierarchy: str | None = None,
1085
+ density: DensityKernel | None = None,
1086
+ save: bool = True,
1087
+ ) -> dict[str, dict[str, np.ndarray | pd.DataFrame]]:
1088
+ """
1089
+ Reduce every property of a table over one space of probe artifacts.
1090
+
1091
+ Parameters
1092
+ ----------
1093
+ input_database : path-like
1094
+ The probe directory of the space,
1095
+ ``<root>/probes/<prefix>-<group>-probe``; it may hold a glob, as the
1096
+ research code's drivers pass it.
1097
+ spec : ProjectSpec
1098
+ The study's spec: its parser, topology, and species sizes.
1099
+ table : ~polyphys.spec.AnalysisTable
1100
+ The group's analysis table.
1101
+ hierarchy : str, optional
1102
+ The glob, relative to `input_database`, that lists the artifact
1103
+ files; the default is ``<spec.hierarchy>/<spec.hierarchy>``, the
1104
+ research code's ``dir/file`` layout.
1105
+ density : DensityKernel, optional
1106
+ The volume-fraction kernel for :class:`LocalDensities`.
1107
+ save : bool, default True
1108
+ Write the stages' files under ``<root>/analysis/``; ``False``
1109
+ returns the ensemble averages without writing anything.
1110
+
1111
+ Returns
1112
+ -------
1113
+ dict of str to dict
1114
+ The ensemble averages by output property, then by ensemble name:
1115
+ a DataFrame for a ``'vector'`` or ``'dataframe'`` output, a dict of
1116
+ arrays for a ``'matrix'``, and the edges for a ``'bin_edge'``.
1117
+
1118
+ Raises
1119
+ ------
1120
+ FileNotFoundError
1121
+ If the glob matches no file, or a property or its edge property
1122
+ matches no file; checked for every property before any directory is
1123
+ made.
1124
+ ValueError
1125
+ If the wholes of one ensemble hold different bin edges; if a
1126
+ histogram's edge files do not cover its wholes; if a hook returns
1127
+ an output name the table already produced; as the organizer and the
1128
+ hooks raise.
1129
+
1130
+ Notes
1131
+ -----
1132
+ For each property in turn: its files are selected by their last name
1133
+ token, wholes are built (for the ``'segment'`` lineage with
1134
+ :func:`~polyphys.manage.organizer.whole_from_segments`, written to
1135
+ ``wholeSim``; else with
1136
+ :func:`~polyphys.manage.organizer.whole_from_file`), the edges are
1137
+ loaded and checked, the post-hook is applied, and every returned
1138
+ artifact goes through :func:`~polyphys.manage.organizer.ensemble` and
1139
+ :func:`~polyphys.manage.organizer.ensemble_avg`, or their long-form
1140
+ counterparts for ``'dataframe'`` wholes. With `table.stamps`, the
1141
+ stamps are combined first by :func:`reduce_stamps`. One property's
1142
+ wholes are in memory at a time, so peak memory is the largest property's
1143
+ ``n_wholes`` wholes plus its ensembles; time is
1144
+ :math:`O(n_\\mathrm{files}\\, n_\\mathrm{frames})` per property plus the
1145
+ hook's.
1146
+
1147
+ Examples
1148
+ --------
1149
+ Two replicates of one state point of the ``ChainCyl`` study, written
1150
+ as wholes and reduced without saving:
1151
+
1152
+ >>> import tempfile
1153
+ >>> import numpy as np
1154
+ >>> from polyphys.tests.synthetic import ChainCyl
1155
+ >>> from polyphys.spec import AnalysisTable, ProjectSpec, tseries
1156
+ >>> spec = ProjectSpec(
1157
+ ... parser=ChainCyl, group='bug', divisor=0.025,
1158
+ ... crowder_attr='phi_bulk_c',
1159
+ ... physical_attrs=('space',), species_sizes={'Mon': 'dmon'},
1160
+ ... tseries_properties=('gyrTMon',))
1161
+ >>> stem = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
1162
+ >>> root = tempfile.mkdtemp()
1163
+ >>> np.save(f'{root}/{stem}ens1-bug-gyrTMon.npy', np.array([1.0, 2.0]))
1164
+ >>> np.save(f'{root}/{stem}ens2-bug-gyrTMon.npy', np.array([3.0, 6.0]))
1165
+ >>> table = AnalysisTable('bug', 'whole', False, (tseries('gyrT', 'Mon'),))
1166
+ >>> averages = reduce_space(root, spec, table, hierarchy='*', save=False)
1167
+ >>> averages['gyrTMon'][stem]
1168
+ gyrTMon-mean gyrTMon-var gyrTMon-sem
1169
+ 0 2.0 2.0 1.0
1170
+ 1 4.0 8.0 2.0
1171
+
1172
+ .. versionadded:: 0.6.0
1173
+ """
1174
+ parser = spec.parser
1175
+ group = table.group
1176
+ if hierarchy is None:
1177
+ hierarchy = f'{spec.hierarchy}/{spec.hierarchy}'
1178
+ pattern = os.path.join(os.fspath(input_database), hierarchy)
1179
+ observations = sorted(glob.glob(pattern))
1180
+ if not observations:
1181
+ raise FileNotFoundError(f"No file matches '{pattern}'.")
1182
+ logger.info("Reducing %d files of group '%s' under '%s'.",
1183
+ len(observations), group, pattern)
1184
+
1185
+ # Select every property's files first, so that a property without
1186
+ # files, such as a typo, raises before any directory is made.
1187
+ for prop in table.properties:
1188
+ _select(observations, prop.prop)
1189
+ if prop.edge_prop is not None:
1190
+ _select(observations, prop.edge_prop)
1191
+
1192
+ whole_dir: str | None = None
1193
+ ens_dir: str | None = None
1194
+ ens_avg_dir: str | None = None
1195
+ if save:
1196
+ if table.lineage == 'segment':
1197
+ whole_dir = organizer.make_database(
1198
+ input_database, 'analysis', 'wholeSim', group)
1199
+ ens_dir = organizer.make_database(
1200
+ input_database, 'analysis', 'ens', group)
1201
+ ens_avg_dir = organizer.make_database(
1202
+ input_database, 'analysis', 'ensAvg', group)
1203
+
1204
+ if table.stamps:
1205
+ stamp_files = _select(observations, 'stamps', ext='csv')
1206
+ reduce_stamps(stamp_files, table.lineage,
1207
+ save_to=(whole_dir, ens_dir, ens_avg_dir))
1208
+
1209
+ results: dict[str, dict[str, np.ndarray | pd.DataFrame]] = {}
1210
+ for prop in table.properties:
1211
+ logger.info("Reducing property '%s'.", prop.prop)
1212
+ wholes = _load_wholes(prop.prop, prop.relation, observations, parser,
1213
+ group, table.lineage, whole_dir)
1214
+ edges: dict[str, np.ndarray] | None = None
1215
+ if prop.edge_prop is not None:
1216
+ edges = _load_wholes(prop.edge_prop, 'bin_edge', observations,
1217
+ parser, group, table.lineage, whole_dir)
1218
+ _check_edges(prop, wholes, edges, parser, group)
1219
+ infos = {name: parser(name, 'whole', group) for name in wholes}
1220
+ context = HookContext(prop, spec=spec, group=group, infos=infos,
1221
+ edges=edges, density=density)
1222
+ if prop.post_hook is None:
1223
+ outputs: tuple[Derived, ...] = (
1224
+ Derived(prop.prop, wholes, prop.whole_type, edges),)
1225
+ else:
1226
+ outputs = prop.post_hook(wholes, context)
1227
+ for output in outputs:
1228
+ if output.name in results:
1229
+ raise ValueError(
1230
+ f"'{prop.prop}': the output '{output.name}' is named "
1231
+ f"more than once in the table; it would overwrite an "
1232
+ f"earlier one.")
1233
+ if whole_dir is not None and output.name != prop.prop:
1234
+ for whole_name, data in output.wholes.items():
1235
+ organizer.save_artifact(
1236
+ organizer.create_fullname(whole_name, group,
1237
+ output.name),
1238
+ data, whole_dir)
1239
+ results[output.name] = _reduce_output(
1240
+ output, parser, group, ens_dir, ens_avg_dir)
1241
+ return results
1242
+
1243
+
1244
+ def space_tseries_from_spec(
1245
+ input_database: Pathish,
1246
+ spec: ProjectSpec,
1247
+ prop: PropertyT,
1248
+ *,
1249
+ group: GroupT | None = None,
1250
+ round_to: int = 3,
1251
+ save: bool = False,
1252
+ ) -> pd.DataFrame:
1253
+ """
1254
+ Stack the ensemble-averaged time series of one property over a space,
1255
+ with the columns and the rounding step the study's spec declares.
1256
+
1257
+ The spec supplies every study choice of
1258
+ :func:`~polyphys.manage.organizer.space_tseries`: the parser, the
1259
+ ``ensemble_long`` glob prefix (`spec.hierarchy`), the attributes that
1260
+ become columns (`spec.physical_attrs`), the rounding step
1261
+ :math:`\\Delta` of the bulk volume fraction of crowders
1262
+ :math:`\\phi_c` (`spec.divisor`), and the parser attribute that holds
1263
+ :math:`\\phi_c` (`spec.crowder_attr`). No study default is taken from
1264
+ the library.
1265
+
1266
+ Parameters
1267
+ ----------
1268
+ input_database : path-like
1269
+ The ``ensAvg`` directory of the space, which holds the files
1270
+ ``<ensemble_long>-<group>-<prop>-ensAvg.csv``.
1271
+ spec : ~polyphys.spec.ProjectSpec
1272
+ The study's spec.
1273
+ prop : str
1274
+ The property, such as ``'gyrTMon'``.
1275
+ group : str, optional
1276
+ The particle group of the files; the default is the polymer group,
1277
+ `spec.group`.
1278
+ round_to : int, default 3
1279
+ The number of decimal places of the rounded :math:`\\phi_c`.
1280
+ save : bool, default False
1281
+ Also write the table to ``<space>-<group>-<prop>-space.csv`` in the
1282
+ ``analysis`` phase's ``space`` directory next to `input_database`.
1283
+
1284
+ Returns
1285
+ -------
1286
+ pandas.DataFrame
1287
+ The table of
1288
+ :func:`~polyphys.manage.organizer.space_tseries`: each file's
1289
+ columns, ``t_index``, ``time``, one column per physical attribute,
1290
+ a `spec.crowder_attr` column, and ``phi_bulk_c_round``.
1291
+
1292
+ Notes
1293
+ -----
1294
+ The cost is that of
1295
+ :func:`~polyphys.manage.organizer.space_tseries`:
1296
+ :math:`O(n_\\mathrm{rows}\\, n_\\mathrm{cols})` time and memory for the
1297
+ rows and columns of the result.
1298
+
1299
+ Examples
1300
+ --------
1301
+ The spec's step of 0.1 rounds :math:`\\phi_c \\approx 0.0123` of this
1302
+ state point to 0.0:
1303
+
1304
+ >>> import tempfile
1305
+ >>> import pandas as pd
1306
+ >>> from pathlib import Path
1307
+ >>> from polyphys.tests.synthetic import ChainCyl
1308
+ >>> from polyphys.spec import ProjectSpec
1309
+ >>> spec = ProjectSpec(
1310
+ ... parser=ChainCyl, group='bug', divisor=0.1,
1311
+ ... crowder_attr='phi_bulk_c', physical_attrs=('nmon',),
1312
+ ... species_sizes={'Mon': 'dmon'}, tseries_properties=('gyrTMon',))
1313
+ >>> database = Path(tempfile.mkdtemp()) / 'analysis' / 'N20-bug-ensAvg'
1314
+ >>> database.mkdir(parents=True)
1315
+ >>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
1316
+ >>> pd.DataFrame({'gyrTMon-mean': [1.0, 2.0]}).to_csv(
1317
+ ... database / f'{state}-bug-gyrTMon-ensAvg.csv', index=False)
1318
+ >>> space = space_tseries_from_spec(database, spec, 'gyrTMon')
1319
+ >>> space[['time', 'gyrTMon-mean', 'nmon', 'phi_bulk_c_round']]
1320
+ time gyrTMon-mean nmon phi_bulk_c_round
1321
+ 0 0.0 1.0 20 0.0
1322
+ 1 5.0 2.0 20 0.0
1323
+
1324
+ .. versionadded:: 0.9.0
1325
+ """
1326
+ return organizer.space_tseries(
1327
+ os.path.join(os.fspath(input_database), ''), prop, spec.parser,
1328
+ spec.hierarchy, list(spec.physical_attrs),
1329
+ spec.group if group is None else group,
1330
+ divisor=spec.divisor, round_to=round_to, is_save=save,
1331
+ crowder_attr=spec.crowder_attr)
1332
+
1333
+
1334
+ def space_hists_from_spec(
1335
+ input_database: Pathish,
1336
+ spec: ProjectSpec,
1337
+ prop: PropertyT,
1338
+ *,
1339
+ group: GroupT | None = None,
1340
+ bin_center: np.ndarray | None = None,
1341
+ normalize: bool = False,
1342
+ round_to: int = 3,
1343
+ save: bool = False,
1344
+ ) -> pd.DataFrame:
1345
+ """
1346
+ Stack the ensemble-averaged histograms of one property over a space,
1347
+ with the columns and the rounding step the study's spec declares.
1348
+
1349
+ The spec supplies every study choice of
1350
+ :func:`~polyphys.manage.organizer.space_hists`, as
1351
+ :func:`space_tseries_from_spec` describes; no study default is taken
1352
+ from the library.
1353
+
1354
+ Parameters
1355
+ ----------
1356
+ input_database : path-like
1357
+ The ``ensAvg`` directory of the space, which holds the files
1358
+ ``<ensemble_long>-<group>-<prop>-ensAvg.csv``.
1359
+ spec : ~polyphys.spec.ProjectSpec
1360
+ The study's spec.
1361
+ prop : str
1362
+ The property, such as ``'rHistCrd'``.
1363
+ group : str, optional
1364
+ The particle group of the files; the default is the polymer group,
1365
+ `spec.group`. A histogram of the crowders is in the ``'all'``
1366
+ group.
1367
+ bin_center : numpy.ndarray, optional
1368
+ The bin centers, shape ``(n_bins,)``; otherwise each file must hold
1369
+ a ``bin_center`` column.
1370
+ normalize : bool, default False
1371
+ Add ``<prop>-norm``, each file's ``<prop>-mean`` divided by its sum.
1372
+ round_to : int, default 3
1373
+ The number of decimal places of the rounded :math:`\\phi_c`.
1374
+ save : bool, default False
1375
+ Also write the table to ``<space>-<group>-<prop>-space.csv`` in the
1376
+ ``analysis`` phase's ``space`` directory next to `input_database`.
1377
+
1378
+ Returns
1379
+ -------
1380
+ pandas.DataFrame
1381
+ The table of :func:`~polyphys.manage.organizer.space_hists`: each
1382
+ file's columns, one column per physical attribute, a
1383
+ `spec.crowder_attr` column, and ``phi_bulk_c_round``.
1384
+
1385
+ Warns
1386
+ -----
1387
+ UserWarning
1388
+ If `normalize` is true and a histogram sums to zero, as
1389
+ :func:`~polyphys.manage.organizer.space_hists` warns.
1390
+
1391
+ Notes
1392
+ -----
1393
+ The cost is that of :func:`~polyphys.manage.organizer.space_hists`:
1394
+ :math:`O(n_\\mathrm{rows}\\, n_\\mathrm{cols})` time and memory.
1395
+
1396
+ Examples
1397
+ --------
1398
+ >>> import tempfile
1399
+ >>> import numpy as np
1400
+ >>> import pandas as pd
1401
+ >>> from pathlib import Path
1402
+ >>> from polyphys.tests.synthetic import ChainCyl
1403
+ >>> from polyphys.spec import ProjectSpec
1404
+ >>> spec = ProjectSpec(
1405
+ ... parser=ChainCyl, group='bug', divisor=0.1,
1406
+ ... crowder_attr='phi_bulk_c', physical_attrs=('nmon',),
1407
+ ... species_sizes={'Mon': 'dmon'}, tseries_properties=())
1408
+ >>> database = Path(tempfile.mkdtemp()) / 'analysis' / 'N20-all-ensAvg'
1409
+ >>> database.mkdir(parents=True)
1410
+ >>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
1411
+ >>> pd.DataFrame({'rHistMon-mean': [1.0, 3.0]}).to_csv(
1412
+ ... database / f'{state}-all-rHistMon-ensAvg.csv', index=False)
1413
+ >>> space = space_hists_from_spec(
1414
+ ... database, spec, 'rHistMon', group='all',
1415
+ ... bin_center=np.array([0.5, 1.5]), normalize=True)
1416
+ >>> space[['bin_center', 'rHistMon-norm', 'phi_bulk_c_round']]
1417
+ bin_center rHistMon-norm phi_bulk_c_round
1418
+ 0 0.5 0.25 0.0
1419
+ 1 1.5 0.75 0.0
1420
+
1421
+ .. versionadded:: 0.9.0
1422
+ """
1423
+ return organizer.space_hists(
1424
+ os.path.join(os.fspath(input_database), ''), prop, spec.parser,
1425
+ spec.hierarchy, list(spec.physical_attrs),
1426
+ spec.group if group is None else group,
1427
+ bin_center=bin_center, normalize=normalize, divisor=spec.divisor,
1428
+ round_to=round_to, is_save=save, crowder_attr=spec.crowder_attr)
1429
+
1430
+
1431
+ def equilibrium_wholes(
1432
+ ens_database: Pathish,
1433
+ properties: Mapping[PropertyT, str],
1434
+ whole_stamps: pd.DataFrame,
1435
+ *,
1436
+ method: PlateauMethod = 'lee',
1437
+ topology: str | None = None,
1438
+ ) -> pd.DataFrame:
1439
+ """
1440
+ The equilibrium value of each whole's time series, with its error,
1441
+ beside the whole's stamps.
1442
+
1443
+ For each property, every ensemble file of the ``ens`` stage is read
1444
+ once, and each of its columns, one whole's time series, is reduced to
1445
+ its time average, its variance, and the blocked standard error of the
1446
+ average. The results of all the properties are merged with
1447
+ `whole_stamps` on ``whole``, one row per whole: the table the
1448
+ research code's ``equilibrium_wholes`` built for the ``allInOne``
1449
+ phase.
1450
+
1451
+ Parameters
1452
+ ----------
1453
+ ens_database : path-like
1454
+ The ``ens`` directory of one space and group,
1455
+ ``<root>/analysis/<prefix>-<group>-ens``, whose
1456
+ ``<ensemble_long>-<group>-<property>.csv`` files
1457
+ :func:`reduce_space` wrote.
1458
+ properties : mapping of str to str
1459
+ The time-series property of each ensemble file, mapped to its
1460
+ output name: ``{'gyrTMon': 'gyrMon'}`` reads the ``-gyrTMon.csv``
1461
+ files and writes ``gyrMon-mean``, ``gyrMon-var``, and
1462
+ ``gyrMon-sem``.
1463
+ whole_stamps : pandas.DataFrame
1464
+ The stamps of the space's wholes, one row per whole, with a
1465
+ ``whole`` column: the first result of :func:`reduce_stamps`.
1466
+ method : {'lee', 'sem_err'}, default 'lee'
1467
+ The plateau rule of :func:`~polyphys.analyze.stats.sem_blocked`;
1468
+ ``'lee'`` is the published rule.
1469
+ topology : str, optional
1470
+ Appended to each whole's name as ``.<topology>`` before the merge,
1471
+ for stamps that the probe phase keyed by the whole's file name,
1472
+ such as ``...ens1.ring``; the ensemble files name a whole without
1473
+ it.
1474
+
1475
+ Returns
1476
+ -------
1477
+ pandas.DataFrame
1478
+ `whole_stamps` followed by, for each property in the order of
1479
+ `properties`, the columns ``<output>-mean``, ``<output>-var``, and
1480
+ ``<output>-sem``, with the rows in the order of `whole_stamps`.
1481
+
1482
+ Raises
1483
+ ------
1484
+ FileNotFoundError
1485
+ If `ens_database` holds no CSV file, or a property matches none.
1486
+ ValueError
1487
+ If `whole_stamps` has no ``whole`` column or lists a whole twice;
1488
+ if two properties share an output name; if the wholes of a
1489
+ property's files differ from those of the stamps, which names the
1490
+ wholes on each side; or as
1491
+ :func:`~polyphys.analyze.stats.block_average` raises for a series
1492
+ that is too short or holds NaN.
1493
+
1494
+ Warns
1495
+ -----
1496
+ RuntimeWarning
1497
+ As :func:`~polyphys.analyze.stats.select_plateau`: when a whole's
1498
+ series is too short for its correlation time, its ``-sem`` is NaN
1499
+ or unreliable.
1500
+
1501
+ Notes
1502
+ -----
1503
+ ``-mean`` is the time average over the frames, ``-var`` the sample
1504
+ variance of the frames (``ddof = 1``), and ``-sem`` the standard error
1505
+ of the time average by block averaging, which accounts for the
1506
+ correlation between consecutive frames of one run (Flyvbjerg and
1507
+ Petersen, 1989; the plateau rule of Lee et al., 2011; both in
1508
+ :mod:`polyphys.analyze.stats`). A constant series has a ``-sem`` of 0.
1509
+ Blocking applies within a whole only: the wholes of an ensemble are
1510
+ independent runs, so :func:`~polyphys.manage.organizer.ensemble_avg`
1511
+ keeps the plain standard error across them. The research code took
1512
+ ``-sem`` as the per-frame standard error, which the blocked one
1513
+ exceeds by about :math:`\\sqrt{s}` for a statistical inefficiency
1514
+ :math:`s`, and its ``-var`` as the population variance.
1515
+
1516
+ With :math:`n_\\mathrm{frames}` the frames of the longest series and
1517
+ :math:`n_\\mathrm{wholes}` the wholes of the space, time is
1518
+ :math:`O(n_\\mathrm{wholes}\\, n_\\mathrm{frames})` per property, and
1519
+ space is one ensemble file at a time plus a ``float64`` copy of it.
1520
+
1521
+ Examples
1522
+ --------
1523
+ Two wholes of one ensemble whose series alternate between two values,
1524
+ so their time averages are exact and the blocked error vanishes at a
1525
+ block size of two frames:
1526
+
1527
+ >>> import tempfile
1528
+ >>> import numpy as np
1529
+ >>> import pandas as pd
1530
+ >>> stem = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
1531
+ >>> wholes = [f'{stem}ens1', f'{stem}ens2']
1532
+ >>> root = tempfile.mkdtemp()
1533
+ >>> pd.DataFrame({wholes[0]: np.tile([0.0, 2.0], 2048),
1534
+ ... wholes[1]: np.tile([1.0, 3.0], 2048)}
1535
+ ... ).to_csv(f'{root}/{stem}-bug-gyrTMon.csv', index=False)
1536
+ >>> stamps = pd.DataFrame({'whole': wholes, 'n_frames': [4096, 4096]})
1537
+ >>> table = equilibrium_wholes(root, {'gyrTMon': 'gyrMon'}, stamps)
1538
+ >>> table.drop(columns='whole').round(3)
1539
+ n_frames gyrMon-mean gyrMon-var gyrMon-sem
1540
+ 0 4096 1.0 1.0 0.0
1541
+ 1 4096 2.0 1.0 0.0
1542
+
1543
+ .. versionadded:: 0.6.0
1544
+ """
1545
+ if 'whole' not in whole_stamps.columns:
1546
+ raise ValueError("whole_stamps needs a 'whole' column.")
1547
+ expected = whole_stamps['whole'].astype(str).to_list()
1548
+ repeated_wholes = sorted({w for w in expected if expected.count(w) > 1})
1549
+ if repeated_wholes:
1550
+ raise ValueError(
1551
+ f"whole_stamps lists wholes more than once: {repeated_wholes}."
1552
+ )
1553
+ outputs = list(properties.values())
1554
+ repeated = sorted({out for out in outputs if outputs.count(out) > 1})
1555
+ if repeated:
1556
+ raise ValueError(f"Output names given more than once: {repeated}.")
1557
+ pattern = os.path.join(os.fspath(ens_database), '*.csv')
1558
+ observations = sorted(glob.glob(pattern))
1559
+ if not observations:
1560
+ raise FileNotFoundError(f"No file matches '{pattern}'.")
1561
+ suffix = '' if topology is None else f'.{topology}'
1562
+
1563
+ frames: list[pd.DataFrame] = []
1564
+ for prop, output in properties.items():
1565
+ logger.info("Reducing the time series of '%s' to '%s'.", prop,
1566
+ output)
1567
+ parts: list[pd.DataFrame] = []
1568
+ for (path,) in _select(observations, prop, ext='csv'):
1569
+ ensemble = pd.read_csv(path)
1570
+ values = ensemble.to_numpy(dtype=np.float64)
1571
+ parts.append(pd.DataFrame({
1572
+ f'{output}-mean': values.mean(axis=0),
1573
+ f'{output}-var': values.var(axis=0, ddof=1),
1574
+ f'{output}-sem': _blocked_errors(values, method),
1575
+ }, index=[f'{name}{suffix}' for name in ensemble.columns]))
1576
+ measured = pd.concat(parts)
1577
+ _check_wholes(prop, list(measured.index), expected)
1578
+ frames.append(measured.loc[expected])
1579
+ combined = pd.concat(frames, axis=1)
1580
+ combined.index = whole_stamps.index
1581
+ return pd.concat([whole_stamps, combined], axis=1)
1582
+
1583
+
1584
+ # ---------------------------------------------------------------------------
1585
+ # Private helpers
1586
+ # ---------------------------------------------------------------------------
1587
+
1588
+
1589
+ def _check_whole_type(whole_type: str) -> None:
1590
+ """Raise ``ValueError`` unless `whole_type` is a known whole type."""
1591
+ if whole_type not in ('vector', 'matrix', 'bin_edge', 'dataframe'):
1592
+ raise ValueError(
1593
+ "whole_type must be 'vector', 'matrix', 'bin_edge', or "
1594
+ f"'dataframe', not '{whole_type}'."
1595
+ )
1596
+
1597
+
1598
+ def _natural_key(path: str) -> list[str | int]:
1599
+ """
1600
+ The sort key that orders a file name's digit runs as numbers, so
1601
+ ``j2`` comes before ``j10``.
1602
+ """
1603
+ return [int(token) if token.isdigit() else token
1604
+ for token in re.split(r'(\d+)', os.path.basename(path))]
1605
+
1606
+
1607
+ def _select(
1608
+ observations: Sequence[str], prop: str, ext: str = 'npy'
1609
+ ) -> list[tuple[str]]:
1610
+ """
1611
+ The files of property `prop` among `observations`, as 1-tuples in
1612
+ natural order of their names.
1613
+
1614
+ A file belongs to `prop` when its name ends in ``-<prop>.<ext>`` (a
1615
+ whole) or ``.<prop>.<ext>`` (a segment), so ``gyrTMon`` never matches
1616
+ ``xgyrTMon``. Raises ``FileNotFoundError`` when nothing matches.
1617
+ :func:`~polyphys.manage.utils.sort_filenames` is not used because it
1618
+ reads a name's stem up to its first dot, which a keyword value such as
1619
+ ``dt0.005`` holds.
1620
+ """
1621
+ suffixes = (f'-{prop}.{ext}', f'.{prop}.{ext}')
1622
+ matches = [path for path in observations if path.endswith(suffixes)]
1623
+ if not matches:
1624
+ raise FileNotFoundError(
1625
+ f"No file of property '{prop}': none of the {len(observations)}"
1626
+ f" files ends in '{suffixes[0]}' or '{suffixes[1]}'."
1627
+ )
1628
+ return [(path,) for path in sorted(matches, key=_natural_key)]
1629
+
1630
+
1631
+ def _load_wholes(
1632
+ prop: str,
1633
+ relation: WholeRelationT,
1634
+ observations: Sequence[str],
1635
+ parser: type[ParserBase],
1636
+ group: GroupT,
1637
+ lineage: PrimitiveLineageT,
1638
+ whole_dir: str | None,
1639
+ ) -> dict[str, np.ndarray]:
1640
+ """Build the wholes of `prop` from its segment or whole files."""
1641
+ paths = _select(observations, prop)
1642
+ if lineage == 'segment':
1643
+ return organizer.whole_from_segments(
1644
+ prop, paths, parser, group, relation, save_to=whole_dir)
1645
+ return organizer.whole_from_file(paths, parser, group)
1646
+
1647
+
1648
+ def _check_edges(
1649
+ prop: _spec.PropertySpec,
1650
+ wholes: Mapping[str, np.ndarray],
1651
+ edges: Mapping[str, np.ndarray],
1652
+ parser: type[ParserBase],
1653
+ group: GroupT,
1654
+ ) -> None:
1655
+ """
1656
+ Raise ``ValueError`` unless every whole has edges and the wholes of each
1657
+ ensemble hold identical edges.
1658
+ """
1659
+ missing = sorted(set(wholes) - set(edges))
1660
+ if missing:
1661
+ raise ValueError(
1662
+ f"'{prop.prop}': no '{prop.edge_prop}' file for the wholes "
1663
+ f"{missing}."
1664
+ )
1665
+ first_of: dict[str, str] = {}
1666
+ for whole_name in wholes:
1667
+ ens_name = str(getattr(parser(whole_name, 'whole', group),
1668
+ 'ensemble_long'))
1669
+ reference = first_of.setdefault(ens_name, whole_name)
1670
+ if not np.array_equal(edges[reference], edges[whole_name]):
1671
+ raise ValueError(
1672
+ f"'{prop.prop}': the wholes '{reference}' and '{whole_name}'"
1673
+ f" of ensemble '{ens_name}' hold different bin edges."
1674
+ )
1675
+
1676
+
1677
+ def _reduce_output(
1678
+ output: Derived,
1679
+ parser: type[ParserBase],
1680
+ group: GroupT,
1681
+ ens_dir: str | None,
1682
+ ens_avg_dir: str | None,
1683
+ ) -> dict[str, np.ndarray | pd.DataFrame]:
1684
+ """Build and save the ensembles and ensemble averages of one output."""
1685
+ if output.whole_type == 'dataframe':
1686
+ frames = {name: data for name, data in output.wholes.items()
1687
+ if isinstance(data, pd.DataFrame)}
1688
+ if len(frames) != len(output.wholes):
1689
+ raise ValueError(
1690
+ f"'{output.name}': every 'dataframe' whole must be a "
1691
+ "pandas.DataFrame."
1692
+ )
1693
+ long_ensembles = _frame_ensembles(output.name, frames, parser, group,
1694
+ ens_dir)
1695
+ return {
1696
+ ens_name: _frame_ensemble_avg(output.name, long_frame,
1697
+ ens_name, group, ens_avg_dir)
1698
+ for ens_name, long_frame in long_ensembles.items()
1699
+ }
1700
+ ensembles = organizer.ensemble(
1701
+ output.name, output.wholes, parser, group, output.whole_type,
1702
+ whole_edges=(dict(output.edges) if output.edges is not None
1703
+ else None),
1704
+ save_to=ens_dir,
1705
+ )
1706
+ if output.normalizer is None:
1707
+ averages = organizer.ensemble_avg(
1708
+ output.name, ensembles, group, _ENS_TYPES[output.whole_type],
1709
+ save_to=ens_avg_dir,
1710
+ )
1711
+ return dict(averages)
1712
+ # The normalized spread needs the wholes and the average together, so
1713
+ # the column is added before the average is saved.
1714
+ averages = organizer.ensemble_avg(
1715
+ output.name, ensembles, group, _ENS_TYPES[output.whole_type])
1716
+ normalized: dict[str, np.ndarray | pd.DataFrame] = {}
1717
+ for ens_name, average in averages.items():
1718
+ ensemble = cast(pd.DataFrame, ensembles[ens_name])
1719
+ frame = cast(pd.DataFrame, average)
1720
+ frame[f'{output.name}-norm-sem'] = _normalized_sem(
1721
+ output.name, ensemble, frame, output.normalizer, ens_name)
1722
+ normalized[ens_name] = frame
1723
+ if ens_avg_dir is not None:
1724
+ organizer.save_artifact(
1725
+ organizer.create_fullname(ens_name, group,
1726
+ f'{output.name}-ensAvg'),
1727
+ frame, ens_avg_dir)
1728
+ return normalized
1729
+
1730
+
1731
+ def _normalized_sem(
1732
+ prop: str,
1733
+ ensemble: pd.DataFrame,
1734
+ average: pd.DataFrame,
1735
+ normalizer: ProfileNormalizerT,
1736
+ ens_name: str,
1737
+ ) -> np.ndarray:
1738
+ """
1739
+ The standard error over the wholes of `ensemble` (one column per whole,
1740
+ after ``bin_center``) of each whole divided by its own value in the
1741
+ reference bin: bin 0 for ``'first'``, the last bin for ``'last'``, and
1742
+ the bin where ``<prop>-mean`` of `average` is largest for ``'max'``.
1743
+ A whole whose reference value is zero warns, naming the ensemble, and
1744
+ gives NaN in every bin. Time and space are
1745
+ :math:`O(n_\\mathrm{bins} n_\\mathrm{wholes})`.
1746
+ """
1747
+ wholes = ensemble.drop(columns='bin_center', errors='ignore')
1748
+ values = wholes.to_numpy(dtype=np.float64)
1749
+ if normalizer == 'first':
1750
+ index = 0
1751
+ elif normalizer == 'last':
1752
+ index = len(values) - 1
1753
+ else:
1754
+ index = int(np.argmax(average[f'{prop}-mean'].to_numpy()))
1755
+ references = values[index]
1756
+ if np.any(references == 0):
1757
+ warnings.warn(
1758
+ f"'{prop}': a whole of ensemble '{ens_name}' is zero in its "
1759
+ f"reference bin {index}; its '{prop}-norm-sem' is NaN.",
1760
+ UserWarning,
1761
+ )
1762
+ return np.full(len(values), np.nan)
1763
+ normalized = pd.DataFrame(values / references)
1764
+ return normalized.sem(axis=1, ddof=1).to_numpy()
1765
+
1766
+
1767
+ def _frame_ensembles(
1768
+ prop: str,
1769
+ wholes: Mapping[str, pd.DataFrame],
1770
+ parser: type[ParserBase],
1771
+ group: GroupT,
1772
+ save_to: str | None,
1773
+ ) -> dict[str, pd.DataFrame]:
1774
+ """
1775
+ The long-form ensembles of ``'dataframe'`` wholes: every whole's rows,
1776
+ with a ``whole`` and a ``row`` column, saved as
1777
+ ``<ensemble_long>-<group>-<prop>.csv``. Raises ``ValueError`` if the
1778
+ wholes of an ensemble have unequal numbers of rows, as
1779
+ :func:`~polyphys.manage.organizer.ens_from_df` does, before anything is
1780
+ saved; ``_frame_ensemble_avg`` would otherwise take its rows from
1781
+ the first whole.
1782
+ """
1783
+ members: dict[str, dict[str, pd.DataFrame]] = {}
1784
+ for whole_name, frame in wholes.items():
1785
+ ens_name = str(getattr(parser(whole_name, 'whole', group),
1786
+ 'ensemble_long'))
1787
+ members.setdefault(ens_name, {})[whole_name] = frame
1788
+ ensembles: dict[str, pd.DataFrame] = {}
1789
+ for ens_name, frames in members.items():
1790
+ organizer._assert_equal_lengths(
1791
+ ens_name, {name: len(frame) for name, frame in frames.items()})
1792
+ long_frame = pd.concat(
1793
+ list(frames.values()), keys=list(frames),
1794
+ names=['whole', 'row'],
1795
+ ).reset_index()
1796
+ ensembles[ens_name] = long_frame
1797
+ if save_to is not None:
1798
+ organizer.save_artifact(
1799
+ organizer.create_fullname(ens_name, group, prop),
1800
+ long_frame, save_to)
1801
+ return ensembles
1802
+
1803
+
1804
+ def _frame_ensemble_avg(
1805
+ prop: str,
1806
+ long_frame: pd.DataFrame,
1807
+ ens_name: str,
1808
+ group: GroupT,
1809
+ save_to: str | None,
1810
+ exclude: Sequence[str] = ('bin_center',),
1811
+ ) -> pd.DataFrame:
1812
+ """
1813
+ The row-by-row mean, variance, and standard error across the wholes of
1814
+ one long-form ensemble, as ``<column>-mean``, ``<column>-var``, and
1815
+ ``<column>-sem`` per column, after the `exclude` columns of the first
1816
+ whole; saved as ``<ensemble_long>-<group>-<prop>-ensAvg.csv``.
1817
+ """
1818
+ kept = [col for col in exclude if col in long_frame.columns]
1819
+ first_whole = long_frame['whole'].iloc[0]
1820
+ first = long_frame.loc[long_frame['whole'] == first_whole]
1821
+ average: pd.DataFrame = first.set_index('row').loc[:, kept]
1822
+ grouped = long_frame.drop(columns=['whole', *kept]).groupby('row')
1823
+ means = grouped.mean()
1824
+ variances = grouped.var(ddof=1)
1825
+ errors = grouped.sem(ddof=1)
1826
+ for column in means.columns:
1827
+ average[f'{column}-mean'] = means[column]
1828
+ average[f'{column}-var'] = variances[column]
1829
+ average[f'{column}-sem'] = errors[column]
1830
+ result: pd.DataFrame = average.reset_index(drop=True)
1831
+ if save_to is not None:
1832
+ organizer.save_artifact(
1833
+ organizer.create_fullname(ens_name, group, f'{prop}-ensAvg'),
1834
+ result, save_to)
1835
+ return result
1836
+
1837
+
1838
+ def _blocked_errors(values: np.ndarray, method: PlateauMethod) -> np.ndarray:
1839
+ """
1840
+ The blocked standard error of each column of `values`, shape
1841
+ ``(n_frames, n_series)``, by :func:`~polyphys.analyze.stats.sem_blocked`;
1842
+ 0 for a constant column, which
1843
+ :func:`~polyphys.analyze.stats.block_average` rejects. A column
1844
+ with NaN is passed on, so that it raises there.
1845
+ """
1846
+ errors = np.zeros(values.shape[1])
1847
+ varying = ~(values.var(axis=0) == 0.0)
1848
+ if varying.any():
1849
+ errors[varying] = np.asarray(
1850
+ stats.sem_blocked(values[:, varying], method=method))
1851
+ return errors
1852
+
1853
+
1854
+ def _check_wholes(
1855
+ prop: str, found: Sequence[str], expected: Sequence[str]
1856
+ ) -> None:
1857
+ """
1858
+ Raise ``ValueError`` unless the wholes `found` in a property's ensemble
1859
+ files are exactly the `expected` wholes of the stamps.
1860
+ """
1861
+ missing = sorted(set(expected) - set(found))
1862
+ unknown = sorted(set(found) - set(expected))
1863
+ if missing or unknown:
1864
+ raise ValueError(
1865
+ f"'{prop}': the wholes of its ensemble files and of whole_stamps "
1866
+ f"differ. In the stamps only: {missing}. In the files only: "
1867
+ f"{unknown}. If the names differ by a suffix, pass topology."
1868
+ )