polyphys 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyphys/__init__.py +7 -0
- polyphys/__version__.py +22 -0
- polyphys/analyze/__init__.py +51 -0
- polyphys/analyze/clusters.py +1738 -0
- polyphys/analyze/correlations.py +995 -0
- polyphys/analyze/density.py +1138 -0
- polyphys/analyze/measurer.py +1338 -0
- polyphys/analyze/pipeline.py +1868 -0
- polyphys/analyze/stats.py +810 -0
- polyphys/analyze/sumrule.py +1159 -0
- polyphys/manage/__init__.py +11 -0
- polyphys/manage/organizer.py +2188 -0
- polyphys/manage/parser.py +1264 -0
- polyphys/manage/types.py +171 -0
- polyphys/manage/utils.py +777 -0
- polyphys/probe/__init__.py +25 -0
- polyphys/probe/binders.py +711 -0
- polyphys/probe/chain_size.py +583 -0
- polyphys/probe/core.py +437 -0
- polyphys/probe/histograms.py +863 -0
- polyphys/probe/pairs.py +555 -0
- polyphys/simulate/__init__.py +48 -0
- polyphys/simulate/build.py +1231 -0
- polyphys/simulate/core.py +237 -0
- polyphys/simulate/data.py +977 -0
- polyphys/simulate/dump.py +752 -0
- polyphys/simulate/log.py +788 -0
- polyphys/spec.py +1359 -0
- polyphys/tests/analyze/test_clusters.py +1058 -0
- polyphys/tests/analyze/test_correlations.py +708 -0
- polyphys/tests/analyze/test_density.py +813 -0
- polyphys/tests/analyze/test_measurer.py +785 -0
- polyphys/tests/analyze/test_pipeline.py +1881 -0
- polyphys/tests/analyze/test_stats.py +732 -0
- polyphys/tests/analyze/test_sumrule.py +917 -0
- polyphys/tests/conftest.py +753 -0
- polyphys/tests/data/lammps_build/make_starts.py +70 -0
- polyphys/tests/manage/test_organizer.py +1495 -0
- polyphys/tests/manage/test_parser.py +1222 -0
- polyphys/tests/manage/test_utils.py +498 -0
- polyphys/tests/probe/conftest.py +407 -0
- polyphys/tests/probe/test_binders.py +686 -0
- polyphys/tests/probe/test_chain_size.py +557 -0
- polyphys/tests/probe/test_core.py +391 -0
- polyphys/tests/probe/test_histograms.py +779 -0
- polyphys/tests/probe/test_pairs.py +475 -0
- polyphys/tests/simulate/test_build.py +1042 -0
- polyphys/tests/simulate/test_core.py +106 -0
- polyphys/tests/simulate/test_data.py +858 -0
- polyphys/tests/simulate/test_dump.py +487 -0
- polyphys/tests/simulate/test_log.py +667 -0
- polyphys/tests/strategies.py +120 -0
- polyphys/tests/synthetic.py +410 -0
- polyphys/tests/test_conftest.py +414 -0
- polyphys/tests/test_docs.py +82 -0
- polyphys/tests/test_packaging.py +97 -0
- polyphys/tests/test_spec.py +772 -0
- polyphys/tests/viz/conftest.py +32 -0
- polyphys/tests/viz/test_colors.py +113 -0
- polyphys/tests/viz/test_helpers.py +292 -0
- polyphys/viz/__init__.py +15 -0
- polyphys/viz/colors.py +204 -0
- polyphys/viz/helpers.py +549 -0
- polyphys-1.0.0.dist-info/METADATA +339 -0
- polyphys-1.0.0.dist-info/RECORD +69 -0
- polyphys-1.0.0.dist-info/WHEEL +5 -0
- polyphys-1.0.0.dist-info/licenses/AUTHORS.rst +13 -0
- polyphys-1.0.0.dist-info/licenses/LICENSE +22 -0
- polyphys-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1868 @@
|
|
|
1
|
+
"""
|
|
2
|
+
==========================================================
|
|
3
|
+
:mod:`polyphys.analyze.pipeline`
|
|
4
|
+
==========================================================
|
|
5
|
+
|
|
6
|
+
The :mod:`polyphys.analyze.pipeline` module reduces the probe artifacts of one
|
|
7
|
+
space down the lineage, ``segment`` to ``whole`` to ``ensemble`` to
|
|
8
|
+
``ensemble average``, for every property that a study's analysis table
|
|
9
|
+
declares. It holds no physics of its own: it selects the files of each
|
|
10
|
+
property, calls :mod:`polyphys.manage.organizer` to build the wholes, the
|
|
11
|
+
ensembles, and the ensemble averages, and lets a *post-hook* derive further
|
|
12
|
+
artifacts from the wholes, such as a time average, an autocorrelation
|
|
13
|
+
function, a density profile, or the pair-separation histograms of the foci.
|
|
14
|
+
|
|
15
|
+
A study describes its analysis as data: one
|
|
16
|
+
:class:`~polyphys.spec.AnalysisTable` per particle group, listing a
|
|
17
|
+
:class:`~polyphys.spec.PropertySpec` per probe artifact; both records are in
|
|
18
|
+
:mod:`polyphys.spec`. The tables of the maintainer's studies live beside each
|
|
19
|
+
study's :class:`~polyphys.spec.ProjectSpec` in the repository's
|
|
20
|
+
``projects/`` folder, so a data correction or a special case is code in the
|
|
21
|
+
study's folder, never an option of this module.
|
|
22
|
+
|
|
23
|
+
Notation
|
|
24
|
+
========
|
|
25
|
+
|
|
26
|
+
*property*
|
|
27
|
+
The name of one probe artifact without its lineage name and group, such
|
|
28
|
+
as ``gyrTMon`` or ``rHistCrd``: a *name* (``gyrT``, ``rHist``) followed
|
|
29
|
+
by a *species* (``Mon``, ``Crd``). Its files end in ``-<property>.npy``
|
|
30
|
+
(a whole) or ``.<property>.npy`` (a segment).
|
|
31
|
+
*relation*
|
|
32
|
+
How the segments of a whole combine, as
|
|
33
|
+
:func:`~polyphys.manage.organizer.whole_from_segments` defines it:
|
|
34
|
+
``'tseries'`` concatenates along the frames, ``'histogram'`` sums, and
|
|
35
|
+
``'bin_edge'`` keeps one copy of identical edges.
|
|
36
|
+
*whole type*
|
|
37
|
+
The shape of one whole, as :func:`~polyphys.manage.organizer.ensemble`
|
|
38
|
+
defines it: ``'vector'`` (a 1-D array), ``'matrix'`` (an N-D array,
|
|
39
|
+
stacked along a new first axis), ``'bin_edge'`` (edges shared by the
|
|
40
|
+
wholes), or ``'dataframe'`` (a table whose columns are elements, such as
|
|
41
|
+
the pairs of foci). The ensemble average of a vector is a DataFrame of
|
|
42
|
+
``-mean``, ``-var``, and ``-sem`` columns; of a matrix, one array per
|
|
43
|
+
statistic; of edges, the edges; of a DataFrame, a DataFrame with those
|
|
44
|
+
three statistics per column.
|
|
45
|
+
*post-hook*
|
|
46
|
+
A callable ``(wholes, context) -> tuple[Derived, ...]`` applied to a
|
|
47
|
+
property's wholes before the ensembles are built. It returns every
|
|
48
|
+
artifact to reduce, so a hook can pass the wholes through under the
|
|
49
|
+
property's own name, add derived artifacts, or replace the wholes with a
|
|
50
|
+
reduction of them.
|
|
51
|
+
*lineage of a table*
|
|
52
|
+
``'segment'`` when the probe phase wrote the group's artifacts in
|
|
53
|
+
segments, ``'whole'`` when it wrote one file per whole. A property of
|
|
54
|
+
the study and the group, not of this module.
|
|
55
|
+
:math:`r_k`
|
|
56
|
+
The autocorrelation at lag :math:`k` frames, from
|
|
57
|
+
:func:`~polyphys.analyze.correlations.acf`.
|
|
58
|
+
:math:`\\bar P(R)`
|
|
59
|
+
The probability density of the separation vector of a pair of foci, in
|
|
60
|
+
one over the length unit cubed, from
|
|
61
|
+
:func:`~polyphys.analyze.clusters.pair_separation_density`.
|
|
62
|
+
|
|
63
|
+
Contents
|
|
64
|
+
========
|
|
65
|
+
|
|
66
|
+
Records:
|
|
67
|
+
:class:`Derived` is one artifact a hook returns, and :class:`HookContext`
|
|
68
|
+
what a hook may read. The table records, :class:`~polyphys.spec.PropertySpec`
|
|
69
|
+
and :class:`~polyphys.spec.AnalysisTable`, and their builders are in
|
|
70
|
+
:mod:`polyphys.spec`.
|
|
71
|
+
|
|
72
|
+
Hooks:
|
|
73
|
+
:class:`TimeAverage` averages a time series over its frames;
|
|
74
|
+
:class:`Autocorrelation` adds the autocorrelation function of a series;
|
|
75
|
+
:class:`LocalDensities` adds the number density and volume fraction profiles
|
|
76
|
+
of a histogram through an injected :class:`DensityKernel`;
|
|
77
|
+
:class:`FociPairDistances` turns the distance-matrix time series of the foci
|
|
78
|
+
into their pair histograms, separation densities, and separation time series.
|
|
79
|
+
:func:`half_contour` is the separation bound of a ring at full bond extension.
|
|
80
|
+
|
|
81
|
+
Drivers:
|
|
82
|
+
:func:`reduce_space` runs every property of a table over one space;
|
|
83
|
+
:func:`space_tseries_from_spec` and :func:`space_hists_from_spec` stack the
|
|
84
|
+
ensemble averages of one property into the ``space`` table with the spec's
|
|
85
|
+
columns and rounding step;
|
|
86
|
+
:func:`reduce_stamps` combines the stamps of a space;
|
|
87
|
+
:func:`equilibrium_wholes` reduces each whole's time series to its
|
|
88
|
+
equilibrium value with a blocked error, beside the whole's stamps.
|
|
89
|
+
|
|
90
|
+
Conventions
|
|
91
|
+
===========
|
|
92
|
+
|
|
93
|
+
- A property whose files are not found raises ``FileNotFoundError`` before
|
|
94
|
+
any directory is made or any property reduced; nothing is skipped in
|
|
95
|
+
silence.
|
|
96
|
+
- No output overwrites another: :class:`~polyphys.spec.AnalysisTable`
|
|
97
|
+
rejects a repeated name among the outputs its hooks declare, and
|
|
98
|
+
:func:`reduce_space` raises ``ValueError`` when a hook that declares none
|
|
99
|
+
returns a name already produced.
|
|
100
|
+
- The wholes of one ensemble must hold identical bin edges, or ``ValueError``
|
|
101
|
+
names the ensemble and the two wholes; the organizer takes the first
|
|
102
|
+
whole's edges without comparing them.
|
|
103
|
+
- Outputs go to ``<root>/analysis/<prefix>-<group>-<stage>`` for the stages
|
|
104
|
+
``wholeSim`` (the ``segment`` lineage only), ``ens``, and ``ensAvg``, as
|
|
105
|
+
:func:`~polyphys.manage.organizer.make_database` builds them, with the
|
|
106
|
+
names :func:`~polyphys.manage.organizer.save_artifact` gives.
|
|
107
|
+
- The ensemble of ``'dataframe'`` wholes is their long form, every whole's
|
|
108
|
+
rows with a ``whole`` and a ``row`` column, so no whole is lost before the
|
|
109
|
+
ensemble average; the average is taken row by row across the wholes.
|
|
110
|
+
- The error of a whole's time average is the blocked standard error of
|
|
111
|
+
:func:`~polyphys.analyze.stats.sem_blocked`, since the frames of one
|
|
112
|
+
run are correlated; across the wholes of an ensemble, independent
|
|
113
|
+
runs, the organizer's plain standard error stays.
|
|
114
|
+
- A density profile's ensemble average also carries ``<property>-norm-sem``,
|
|
115
|
+
the standard error over the wholes of each whole's profile divided by its
|
|
116
|
+
own value in the bin that the sum rule's default normalizer names
|
|
117
|
+
(:data:`~polyphys.analyze.sumrule.PROFILE_NORMALIZERS`): the SEM across
|
|
118
|
+
profiles normalized separately for each whole. Normalization is
|
|
119
|
+
nonlinear, so it is not generally an uncertainty estimate for the
|
|
120
|
+
normalized ensemble mean, and neither is the ensemble average's
|
|
121
|
+
``-sem``. See :class:`Derived`.
|
|
122
|
+
- Progress is logged at INFO level; no warning is raised for a renamed
|
|
123
|
+
artifact, because every output name is declared.
|
|
124
|
+
- Complexities are stated in ``n_files`` (the files of one property),
|
|
125
|
+
``n_frames``, ``n_bins``, ``n_wholes``, and ``n_pairs``.
|
|
126
|
+
|
|
127
|
+
Dependencies
|
|
128
|
+
============
|
|
129
|
+
|
|
130
|
+
- Python standard library: `dataclasses`, `glob`, `logging`, `os`, `re`.
|
|
131
|
+
- `numpy`, `pandas`.
|
|
132
|
+
- :mod:`polyphys.analyze.clusters`: the foci pair kernels.
|
|
133
|
+
- :mod:`polyphys.analyze.correlations`:
|
|
134
|
+
:func:`~polyphys.analyze.correlations.acf`.
|
|
135
|
+
- :mod:`polyphys.analyze.stats`:
|
|
136
|
+
:func:`~polyphys.analyze.stats.sem_blocked`.
|
|
137
|
+
- :mod:`polyphys.manage.organizer`: the lineage reducers.
|
|
138
|
+
- :mod:`polyphys.spec`: :class:`~polyphys.spec.ProjectSpec`, the table
|
|
139
|
+
records, and :data:`~polyphys.spec.PerWhole`.
|
|
140
|
+
|
|
141
|
+
References
|
|
142
|
+
==========
|
|
143
|
+
|
|
144
|
+
- K. Kremer and G. S. Grest, "Dynamics of entangled linear polymer melts: A
|
|
145
|
+
molecular-dynamics simulation", J. Chem. Phys. 92, 5057 (1990), for the
|
|
146
|
+
FENE bond and its maximum extension :math:`R_0 = 1.5\\,\\sigma`.
|
|
147
|
+
https://doi.org/10.1063/1.458541
|
|
148
|
+
|
|
149
|
+
Examples
|
|
150
|
+
========
|
|
151
|
+
A table that reads as data: the gyration radius time series with its
|
|
152
|
+
autocorrelation, the radial histogram of the crowders with their density
|
|
153
|
+
profiles, and the planar histograms with their edges:
|
|
154
|
+
|
|
155
|
+
>>> from polyphys.analyze.pipeline import Autocorrelation, LocalDensities
|
|
156
|
+
>>> from polyphys.spec import (
|
|
157
|
+
... AnalysisTable, edge, histogram, histogram_2d, tseries)
|
|
158
|
+
>>> table = AnalysisTable(
|
|
159
|
+
... group='all', lineage='segment', stamps=False, properties=(
|
|
160
|
+
... tseries('gyrT', 'Mon', post_hook=Autocorrelation(nlags=100)),
|
|
161
|
+
... histogram('r', 'Crd', post_hook=LocalDensities()),
|
|
162
|
+
... histogram_2d('xy', 'Crd'),
|
|
163
|
+
... edge('x'),
|
|
164
|
+
... ))
|
|
165
|
+
>>> [prop.prop for prop in table.properties]
|
|
166
|
+
['gyrTMon', 'rHistCrd', 'xyHistCrd', 'xEdge']
|
|
167
|
+
|
|
168
|
+
.. versionadded:: 0.6.0
|
|
169
|
+
|
|
170
|
+
.. versionchanged:: 0.9.0
|
|
171
|
+
:class:`~polyphys.spec.AnalysisTable`,
|
|
172
|
+
:class:`~polyphys.spec.PropertySpec`, :data:`~polyphys.spec.PostHook`,
|
|
173
|
+
and the builders :func:`~polyphys.spec.tseries`,
|
|
174
|
+
:func:`~polyphys.spec.histogram`, :func:`~polyphys.spec.histogram_2d`,
|
|
175
|
+
:func:`~polyphys.spec.edge`, and :func:`~polyphys.spec.matrix_tseries`
|
|
176
|
+
moved to :mod:`polyphys.spec`; importing them from here warned with
|
|
177
|
+
:exc:`DeprecationWarning`.
|
|
178
|
+
|
|
179
|
+
.. versionchanged:: 1.0.0
|
|
180
|
+
Those names are no longer served from here; import them from
|
|
181
|
+
:mod:`polyphys.spec`.
|
|
182
|
+
"""
|
|
183
|
+
import glob
|
|
184
|
+
import logging
|
|
185
|
+
import numbers
|
|
186
|
+
import os
|
|
187
|
+
import re
|
|
188
|
+
import warnings
|
|
189
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
190
|
+
from dataclasses import dataclass, field
|
|
191
|
+
from typing import Literal, Protocol, TypeGuard, cast
|
|
192
|
+
|
|
193
|
+
import numpy as np
|
|
194
|
+
import numpy.typing as npt
|
|
195
|
+
import pandas as pd
|
|
196
|
+
|
|
197
|
+
from polyphys.analyze import clusters, measurer, stats
|
|
198
|
+
from polyphys.analyze.correlations import acf
|
|
199
|
+
from polyphys.analyze.stats import PlateauMethod
|
|
200
|
+
from polyphys.analyze.sumrule import PROFILE_NORMALIZERS
|
|
201
|
+
from polyphys import spec as _spec
|
|
202
|
+
from polyphys.manage import organizer
|
|
203
|
+
from polyphys.manage.parser import ParserBase
|
|
204
|
+
from polyphys.manage.types import (
|
|
205
|
+
DirectionT,
|
|
206
|
+
GroupT,
|
|
207
|
+
Pathish,
|
|
208
|
+
PrimitiveLineageT,
|
|
209
|
+
PropertyT,
|
|
210
|
+
WholeRelationT,
|
|
211
|
+
)
|
|
212
|
+
from polyphys.spec import PerWhole, ProjectSpec, _check_resolved
|
|
213
|
+
|
|
214
|
+
__all__ = [
|
|
215
|
+
'Autocorrelation',
|
|
216
|
+
'DensityKernel',
|
|
217
|
+
'Derived',
|
|
218
|
+
'FociPairDistances',
|
|
219
|
+
'HookContext',
|
|
220
|
+
'LocalDensities',
|
|
221
|
+
'TimeAverage',
|
|
222
|
+
'equilibrium_wholes',
|
|
223
|
+
'half_contour',
|
|
224
|
+
'reduce_space',
|
|
225
|
+
'reduce_stamps',
|
|
226
|
+
'space_hists_from_spec',
|
|
227
|
+
'space_tseries_from_spec',
|
|
228
|
+
]
|
|
229
|
+
|
|
230
|
+
logger = logging.getLogger(__name__)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
#: The shape of one whole; see "whole type" in the module notation.
|
|
234
|
+
WholeTypeT = Literal['vector', 'matrix', 'bin_edge', 'dataframe']
|
|
235
|
+
|
|
236
|
+
#: The organizer's ensemble-average type for each array whole type.
|
|
237
|
+
_ENS_TYPES: Mapping[str, Literal['dataframe', 'ndarray', 'bin_edge']] = {
|
|
238
|
+
'vector': 'dataframe',
|
|
239
|
+
'matrix': 'ndarray',
|
|
240
|
+
'bin_edge': 'bin_edge',
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
#: The reference bin of a per-whole normalization; see :class:`Derived`.
|
|
244
|
+
ProfileNormalizerT = Literal['first', 'last', 'max']
|
|
245
|
+
|
|
246
|
+
#: The wholes of one property as loaded from its files, by whole name.
|
|
247
|
+
ArrayWholes = dict[str, np.ndarray]
|
|
248
|
+
#: The wholes a hook may return, by whole name.
|
|
249
|
+
Wholes = dict[str, np.ndarray] | dict[str, pd.DataFrame]
|
|
250
|
+
#: Bin edges by whole name.
|
|
251
|
+
Edges = Mapping[str, np.ndarray]
|
|
252
|
+
|
|
253
|
+
FloatArray = npt.NDArray[np.float64]
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
class DensityKernel(Protocol):
|
|
257
|
+
"""
|
|
258
|
+
The volume-fraction kernel that :class:`LocalDensities` calls.
|
|
259
|
+
|
|
260
|
+
A kernel turns the histogram of one species in one whole into its local
|
|
261
|
+
number density and local volume fraction, bin by bin, from the
|
|
262
|
+
geometry the whole's parser instance describes. The study's driver
|
|
263
|
+
supplies one, so this module imports no density model.
|
|
264
|
+
|
|
265
|
+
.. versionadded:: 0.6.0
|
|
266
|
+
"""
|
|
267
|
+
|
|
268
|
+
def __call__(
|
|
269
|
+
self,
|
|
270
|
+
counts: np.ndarray,
|
|
271
|
+
edges: np.ndarray,
|
|
272
|
+
info: ParserBase,
|
|
273
|
+
direction: str,
|
|
274
|
+
diameter: float,
|
|
275
|
+
) -> tuple[npt.ArrayLike, npt.ArrayLike]:
|
|
276
|
+
"""
|
|
277
|
+
Parameters
|
|
278
|
+
----------
|
|
279
|
+
counts : numpy.ndarray
|
|
280
|
+
The histogram of one whole, shape ``(n_bins,)``: the number of
|
|
281
|
+
particle centers in each bin, summed over the frames.
|
|
282
|
+
edges : numpy.ndarray
|
|
283
|
+
Its bin edges, shape ``(n_bins + 1,)``, in the length unit.
|
|
284
|
+
info : ParserBase
|
|
285
|
+
The whole's parser instance, which holds the box sizes. It holds
|
|
286
|
+
no number of frames, so a kernel that divides `counts` by the
|
|
287
|
+
frames gets them from the study's driver.
|
|
288
|
+
direction : str
|
|
289
|
+
The direction of the histogram, such as ``'r'`` or ``'z'``.
|
|
290
|
+
diameter : float
|
|
291
|
+
The diameter of the species, in the length unit.
|
|
292
|
+
|
|
293
|
+
Returns
|
|
294
|
+
-------
|
|
295
|
+
tuple of array_like
|
|
296
|
+
The number density and the volume fraction, each of shape
|
|
297
|
+
``(n_bins,)``.
|
|
298
|
+
"""
|
|
299
|
+
...
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# ---------------------------------------------------------------------------
|
|
303
|
+
# Records
|
|
304
|
+
# ---------------------------------------------------------------------------
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
@dataclass(frozen=True)
|
|
308
|
+
class Derived:
|
|
309
|
+
"""
|
|
310
|
+
One artifact a post-hook returns, ready for the ensemble stages.
|
|
311
|
+
|
|
312
|
+
Attributes
|
|
313
|
+
----------
|
|
314
|
+
name : str
|
|
315
|
+
The output property, which names the files; for example
|
|
316
|
+
``'rRhoCrd'`` or ``'gyrTMon-acf'``.
|
|
317
|
+
wholes : dict of str to numpy.ndarray or pandas.DataFrame
|
|
318
|
+
The artifact of each whole, by whole name.
|
|
319
|
+
whole_type : {'vector', 'matrix', 'bin_edge', 'dataframe'}
|
|
320
|
+
The shape of each whole; see the module notation.
|
|
321
|
+
edges : mapping of str to numpy.ndarray, optional
|
|
322
|
+
The bin edges of each whole, for a ``'vector'`` histogram, so its
|
|
323
|
+
ensembles carry a ``bin_center`` column.
|
|
324
|
+
normalizer : {'first', 'last', 'max'}, optional
|
|
325
|
+
For a ``'vector'`` profile, the reference bin of a per-whole
|
|
326
|
+
normalization: the first bin, the last bin, or the bin where the
|
|
327
|
+
ensemble average is largest. Each whole is divided by its own value
|
|
328
|
+
in that bin, and the standard error over the wholes of these
|
|
329
|
+
separately normalized profiles is written beside the ensemble
|
|
330
|
+
average as ``<name>-norm-sem``; it is not generally an uncertainty
|
|
331
|
+
estimate for the normalized ensemble mean. The ensemble average
|
|
332
|
+
itself is not normalized;
|
|
333
|
+
:func:`polyphys.analyze.sumrule.space_profiles` does that and
|
|
334
|
+
carries the column through.
|
|
335
|
+
|
|
336
|
+
Raises
|
|
337
|
+
------
|
|
338
|
+
ValueError
|
|
339
|
+
If `whole_type` is unknown, if `edges` or `normalizer` is given for
|
|
340
|
+
a whole type other than ``'vector'``, or if `normalizer` is not one
|
|
341
|
+
of its three values.
|
|
342
|
+
|
|
343
|
+
Notes
|
|
344
|
+
-----
|
|
345
|
+
The wholes of an ensemble are independent runs, so their spread is a
|
|
346
|
+
valid error of any function of the profile, the normalized one
|
|
347
|
+
included, with no covariance between a bin and its normalizer to track.
|
|
348
|
+
The ensemble picks the bin so that every whole is normalized at the
|
|
349
|
+
same place: a per-whole ``'max'`` would bias each ratio towards one
|
|
350
|
+
before the spread is taken. The error is zero in the reference bin by
|
|
351
|
+
construction. The scale of a profile, such as the ``1 / a`` or ``a^2``
|
|
352
|
+
of the sum rule, cancels in the ratio, so the column holds for the
|
|
353
|
+
scaled profile too.
|
|
354
|
+
|
|
355
|
+
Examples
|
|
356
|
+
--------
|
|
357
|
+
>>> import numpy as np
|
|
358
|
+
>>> out = Derived('gyrMon', {'w1': np.array([1.0])}, 'vector')
|
|
359
|
+
>>> out.name, out.whole_type
|
|
360
|
+
('gyrMon', 'vector')
|
|
361
|
+
|
|
362
|
+
.. versionadded:: 0.6.0
|
|
363
|
+
.. versionchanged:: 0.7.0
|
|
364
|
+
Added `normalizer`.
|
|
365
|
+
"""
|
|
366
|
+
|
|
367
|
+
name: str
|
|
368
|
+
wholes: Wholes
|
|
369
|
+
whole_type: WholeTypeT
|
|
370
|
+
edges: Edges | None = None
|
|
371
|
+
normalizer: ProfileNormalizerT | None = None
|
|
372
|
+
|
|
373
|
+
def __post_init__(self) -> None:
|
|
374
|
+
_check_whole_type(self.whole_type)
|
|
375
|
+
if self.edges is not None and self.whole_type != 'vector':
|
|
376
|
+
raise ValueError(
|
|
377
|
+
f"'{self.name}': edges are only valid with whole_type "
|
|
378
|
+
f"'vector', not '{self.whole_type}'."
|
|
379
|
+
)
|
|
380
|
+
if self.normalizer is not None:
|
|
381
|
+
if self.whole_type != 'vector':
|
|
382
|
+
raise ValueError(
|
|
383
|
+
f"'{self.name}': a normalizer is only valid with "
|
|
384
|
+
f"whole_type 'vector', not '{self.whole_type}'."
|
|
385
|
+
)
|
|
386
|
+
if self.normalizer not in ('first', 'last', 'max'):
|
|
387
|
+
raise ValueError(
|
|
388
|
+
f"'{self.name}': normalizer must be 'first', 'last', or "
|
|
389
|
+
f"'max', not {self.normalizer!r}."
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@dataclass(frozen=True)
|
|
394
|
+
class HookContext:
|
|
395
|
+
"""
|
|
396
|
+
What a post-hook may read besides the wholes.
|
|
397
|
+
|
|
398
|
+
.. versionadded:: 0.6.0
|
|
399
|
+
|
|
400
|
+
Attributes
|
|
401
|
+
----------
|
|
402
|
+
prop : ~polyphys.spec.PropertySpec
|
|
403
|
+
The property the wholes belong to.
|
|
404
|
+
spec : ~polyphys.spec.ProjectSpec, optional
|
|
405
|
+
The study's spec: its parser, topology, and species sizes.
|
|
406
|
+
group : str
|
|
407
|
+
The particle group.
|
|
408
|
+
infos : mapping of str to ~polyphys.manage.parser.ParserBase
|
|
409
|
+
The parser instance of each whole, by whole name, which holds the
|
|
410
|
+
whole's attributes such as sizes and counts.
|
|
411
|
+
edges : mapping of str to numpy.ndarray, optional
|
|
412
|
+
The bin edges of each whole, for a property with an edge file.
|
|
413
|
+
density : DensityKernel, optional
|
|
414
|
+
The injected volume-fraction kernel.
|
|
415
|
+
"""
|
|
416
|
+
|
|
417
|
+
prop: _spec.PropertySpec
|
|
418
|
+
spec: ProjectSpec | None = None
|
|
419
|
+
group: GroupT = 'bug'
|
|
420
|
+
infos: Mapping[str, ParserBase] = field(default_factory=dict)
|
|
421
|
+
edges: Edges | None = None
|
|
422
|
+
density: DensityKernel | None = None
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
# ---------------------------------------------------------------------------
|
|
426
|
+
# Hooks
|
|
427
|
+
# ---------------------------------------------------------------------------
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
@dataclass(frozen=True)
|
|
431
|
+
class TimeAverage:
|
|
432
|
+
"""
|
|
433
|
+
Average a time series over its frames; a post-hook.
|
|
434
|
+
|
|
435
|
+
Attributes
|
|
436
|
+
----------
|
|
437
|
+
output : str
|
|
438
|
+
The name of the average, without the species, which is appended:
|
|
439
|
+
``TimeAverage('clustersHist')`` on ``'clustersHistTFoci'`` writes
|
|
440
|
+
``'clustersHistFoci'``.
|
|
441
|
+
drop_first_column : bool, default False
|
|
442
|
+
Drop column 0 of each frame before averaging. The size histograms of
|
|
443
|
+
:func:`~polyphys.analyze.clusters.count_clusters` carry the count of
|
|
444
|
+
empty clusters, always 0, in column 0.
|
|
445
|
+
|
|
446
|
+
Notes
|
|
447
|
+
-----
|
|
448
|
+
The frames are axis 0; the average has the shape of one frame, so a
|
|
449
|
+
``(n_frames, n_sizes)`` histogram time series becomes an ``n_sizes``
|
|
450
|
+
vector, reduced as a ``'vector'`` whole. Time and space are
|
|
451
|
+
:math:`O(n_\\mathrm{frames} n_\\mathrm{sizes})` per whole.
|
|
452
|
+
|
|
453
|
+
Examples
|
|
454
|
+
--------
|
|
455
|
+
>>> import numpy as np
|
|
456
|
+
>>> from polyphys.spec import matrix_tseries
|
|
457
|
+
>>> hook = TimeAverage('clustersHist', drop_first_column=True)
|
|
458
|
+
>>> context = HookContext(matrix_tseries('clustersHistT', 'Foci'))
|
|
459
|
+
>>> wholes = {'w1': np.array([[0, 1, 2], [0, 3, 4], [0, 5, 6]])}
|
|
460
|
+
>>> (out,) = hook(wholes, context)
|
|
461
|
+
>>> out.name, out.wholes['w1']
|
|
462
|
+
('clustersHistFoci', array([3., 4.]))
|
|
463
|
+
|
|
464
|
+
.. versionadded:: 0.6.0
|
|
465
|
+
"""
|
|
466
|
+
|
|
467
|
+
output: str
|
|
468
|
+
drop_first_column: bool = False
|
|
469
|
+
|
|
470
|
+
def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
|
|
471
|
+
"""
|
|
472
|
+
The name of the average: `output` then the species of `prop`.
|
|
473
|
+
|
|
474
|
+
Examples
|
|
475
|
+
--------
|
|
476
|
+
>>> from polyphys.spec import matrix_tseries
|
|
477
|
+
>>> TimeAverage('clustersHist').outputs(
|
|
478
|
+
... matrix_tseries('clustersHistT', 'Foci'))
|
|
479
|
+
('clustersHistFoci',)
|
|
480
|
+
|
|
481
|
+
.. versionadded:: 0.9.0
|
|
482
|
+
"""
|
|
483
|
+
return (self.output + prop.species,)
|
|
484
|
+
|
|
485
|
+
def __call__(
|
|
486
|
+
self, wholes: ArrayWholes, context: HookContext
|
|
487
|
+
) -> tuple[Derived, ...]:
|
|
488
|
+
averaged: dict[str, np.ndarray] = {}
|
|
489
|
+
for whole_name, whole in wholes.items():
|
|
490
|
+
frames = np.asarray(whole, dtype=np.float64)
|
|
491
|
+
if self.drop_first_column:
|
|
492
|
+
frames = frames[:, 1:]
|
|
493
|
+
averaged[whole_name] = frames.mean(axis=0)
|
|
494
|
+
return (Derived(self.output + context.prop.species, averaged,
|
|
495
|
+
'vector'),)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
@dataclass(frozen=True)
|
|
499
|
+
class Autocorrelation:
|
|
500
|
+
"""
|
|
501
|
+
Pass a time series through and add its autocorrelation; a post-hook.
|
|
502
|
+
|
|
503
|
+
Attributes
|
|
504
|
+
----------
|
|
505
|
+
nlags : int
|
|
506
|
+
The largest lag, in frames; see
|
|
507
|
+
:func:`~polyphys.analyze.correlations.acf`.
|
|
508
|
+
|
|
509
|
+
Notes
|
|
510
|
+
-----
|
|
511
|
+
Returns the series under its own name and :math:`r_k` for
|
|
512
|
+
:math:`k = 0, \\dots,` `nlags` under ``<property>-acf``, both
|
|
513
|
+
``'vector'`` wholes. No confidence band is stored: Bartlett's band is a
|
|
514
|
+
function of the stored :math:`r_k`, the number of frames, and the
|
|
515
|
+
level, so :func:`~polyphys.analyze.correlations.bartlett_half_width`
|
|
516
|
+
rebuilds it at any level when it is drawn. Time is
|
|
517
|
+
:math:`O(n_\\mathrm{frames} \\log n_\\mathrm{frames})` per whole.
|
|
518
|
+
|
|
519
|
+
Examples
|
|
520
|
+
--------
|
|
521
|
+
The alternating series has :math:`r_1 = -7/8` and :math:`r_2 = 6/8`:
|
|
522
|
+
|
|
523
|
+
>>> import numpy as np
|
|
524
|
+
>>> from polyphys.spec import tseries
|
|
525
|
+
>>> hook = Autocorrelation(nlags=2)
|
|
526
|
+
>>> context = HookContext(tseries('gyrT', 'Mon'))
|
|
527
|
+
>>> series, acf_out = hook({'w1': np.tile([1.0, -1.0], 4)}, context)
|
|
528
|
+
>>> series.name, acf_out.name
|
|
529
|
+
('gyrTMon', 'gyrTMon-acf')
|
|
530
|
+
>>> np.round(acf_out.wholes['w1'], 6)
|
|
531
|
+
array([ 1. , -0.875, 0.75 ])
|
|
532
|
+
|
|
533
|
+
.. versionadded:: 0.6.0
|
|
534
|
+
"""
|
|
535
|
+
|
|
536
|
+
nlags: int
|
|
537
|
+
|
|
538
|
+
def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
|
|
539
|
+
"""
|
|
540
|
+
The series, passed through, and ``<property>-acf``.
|
|
541
|
+
|
|
542
|
+
Examples
|
|
543
|
+
--------
|
|
544
|
+
>>> from polyphys.spec import tseries
|
|
545
|
+
>>> Autocorrelation(nlags=2).outputs(tseries('gyrT', 'Mon'))
|
|
546
|
+
('gyrTMon', 'gyrTMon-acf')
|
|
547
|
+
|
|
548
|
+
.. versionadded:: 0.9.0
|
|
549
|
+
"""
|
|
550
|
+
return (prop.prop, f'{prop.prop}-acf')
|
|
551
|
+
|
|
552
|
+
def __call__(
|
|
553
|
+
self, wholes: ArrayWholes, context: HookContext
|
|
554
|
+
) -> tuple[Derived, ...]:
|
|
555
|
+
prop = context.prop.prop
|
|
556
|
+
acfs: dict[str, np.ndarray] = {
|
|
557
|
+
whole_name: acf(whole, self.nlags, alpha=None).values
|
|
558
|
+
for whole_name, whole in wholes.items()
|
|
559
|
+
}
|
|
560
|
+
return (Derived(prop, wholes, 'vector'),
|
|
561
|
+
Derived(f'{prop}-acf', acfs, 'vector'))
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
@dataclass(frozen=True)
|
|
565
|
+
class LocalDensities:
|
|
566
|
+
"""
|
|
567
|
+
Pass a histogram through and add its density profiles; a post-hook.
|
|
568
|
+
|
|
569
|
+
On ``<direction>Hist<species>`` it adds ``<direction>Rho<species>``, the
|
|
570
|
+
local number density, and ``<direction>Phi<species>``, the local volume
|
|
571
|
+
fraction, from the :class:`DensityKernel` given to :func:`reduce_space`,
|
|
572
|
+
with the species' diameter read from the attribute the study's
|
|
573
|
+
:attr:`~polyphys.spec.ProjectSpec.species_sizes` names.
|
|
574
|
+
|
|
575
|
+
Raises
|
|
576
|
+
------
|
|
577
|
+
ValueError
|
|
578
|
+
When called: if the property is not a :func:`~polyphys.spec.histogram`,
|
|
579
|
+
if the context has no kernel, no spec, or no edges, or if the species
|
|
580
|
+
is not in the spec's `species_sizes`.
|
|
581
|
+
|
|
582
|
+
Notes
|
|
583
|
+
-----
|
|
584
|
+
The three outputs share the histogram's edges, so their ensembles carry
|
|
585
|
+
the same ``bin_center`` column. The two profiles get the normalizer of
|
|
586
|
+
:data:`~polyphys.analyze.sumrule.PROFILE_NORMALIZERS` for the parser's
|
|
587
|
+
geometry and the histogram's direction, when the pair has one, so their
|
|
588
|
+
ensemble averages carry ``<property>-norm-sem``, the SEM across
|
|
589
|
+
profiles normalized separately for each whole (see :class:`Derived`).
|
|
590
|
+
Time is one kernel call per whole.
|
|
591
|
+
|
|
592
|
+
Examples
|
|
593
|
+
--------
|
|
594
|
+
With a stub kernel that returns the counts and a constant, on one whole
|
|
595
|
+
of the ``ChainCyl`` study:
|
|
596
|
+
|
|
597
|
+
>>> import numpy as np
|
|
598
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
599
|
+
>>> from polyphys.spec import ProjectSpec, histogram
|
|
600
|
+
>>> spec = ProjectSpec(
|
|
601
|
+
... parser=ChainCyl, group='bug', divisor=0.025,
|
|
602
|
+
... crowder_attr='phi_bulk_c',
|
|
603
|
+
... physical_attrs=('space',), species_sizes={'Crd': 'dcrowd'},
|
|
604
|
+
... tseries_properties=())
|
|
605
|
+
>>> whole = ('N20r5.5lz25.0ac1.0nc100'
|
|
606
|
+
... 'dt0.005bdump1000adump5000ens1')
|
|
607
|
+
>>> def kernel(counts, edges, info, direction, size):
|
|
608
|
+
... return counts, np.full(counts.shape, size)
|
|
609
|
+
>>> context = HookContext(
|
|
610
|
+
... histogram('r', 'Crd'), spec=spec, group='all',
|
|
611
|
+
... infos={whole: ChainCyl(whole, 'whole', 'all')},
|
|
612
|
+
... edges={whole: np.array([0.0, 1.0, 2.0])}, density=kernel)
|
|
613
|
+
>>> outputs = LocalDensities()({whole: np.array([3.0, 5.0])}, context)
|
|
614
|
+
>>> [out.name for out in outputs]
|
|
615
|
+
['rHistCrd', 'rRhoCrd', 'rPhiCrd']
|
|
616
|
+
>>> outputs[2].wholes[whole]
|
|
617
|
+
array([1., 1.])
|
|
618
|
+
>>> outputs[2].normalizer
|
|
619
|
+
'first'
|
|
620
|
+
|
|
621
|
+
.. versionadded:: 0.6.0
|
|
622
|
+
.. versionchanged:: 0.7.0
|
|
623
|
+
The profiles carry the sum rule's default normalizer.
|
|
624
|
+
"""
|
|
625
|
+
|
|
626
|
+
@staticmethod
|
|
627
|
+
def _direction(prop: _spec.PropertySpec) -> str:
|
|
628
|
+
"""The direction of a histogram property, which must be one."""
|
|
629
|
+
if not prop.name.endswith('Hist') or not prop.has_edges:
|
|
630
|
+
raise ValueError(
|
|
631
|
+
f"LocalDensities needs a histogram(...) property, not "
|
|
632
|
+
f"'{prop.prop}'."
|
|
633
|
+
)
|
|
634
|
+
return prop.name[: -len('Hist')]
|
|
635
|
+
|
|
636
|
+
def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
|
|
637
|
+
"""
|
|
638
|
+
The histogram, passed through, and its ``<direction>Rho`` and
|
|
639
|
+
``<direction>Phi`` profiles.
|
|
640
|
+
|
|
641
|
+
Raises
|
|
642
|
+
------
|
|
643
|
+
ValueError
|
|
644
|
+
If `prop` is not a :func:`~polyphys.spec.histogram` property.
|
|
645
|
+
|
|
646
|
+
Examples
|
|
647
|
+
--------
|
|
648
|
+
>>> from polyphys.spec import histogram
|
|
649
|
+
>>> LocalDensities().outputs(histogram('z', 'Crd'))
|
|
650
|
+
('zHistCrd', 'zRhoCrd', 'zPhiCrd')
|
|
651
|
+
|
|
652
|
+
.. versionadded:: 0.9.0
|
|
653
|
+
"""
|
|
654
|
+
direction = self._direction(prop)
|
|
655
|
+
return (prop.prop, f'{direction}Rho{prop.species}',
|
|
656
|
+
f'{direction}Phi{prop.species}')
|
|
657
|
+
|
|
658
|
+
def __call__(
|
|
659
|
+
self, wholes: ArrayWholes, context: HookContext
|
|
660
|
+
) -> tuple[Derived, ...]:
|
|
661
|
+
prop = context.prop
|
|
662
|
+
direction = self._direction(prop)
|
|
663
|
+
if context.density is None:
|
|
664
|
+
raise ValueError(
|
|
665
|
+
f"'{prop.prop}': LocalDensities needs the density kernel "
|
|
666
|
+
"passed to reduce_space(..., density=...)."
|
|
667
|
+
)
|
|
668
|
+
if context.spec is None or context.edges is None:
|
|
669
|
+
raise ValueError(
|
|
670
|
+
f"'{prop.prop}': LocalDensities needs the study's spec and "
|
|
671
|
+
"the histogram's edges in its context."
|
|
672
|
+
)
|
|
673
|
+
size_attr = context.spec.species_sizes.get(prop.species)
|
|
674
|
+
if size_attr is None:
|
|
675
|
+
raise ValueError(
|
|
676
|
+
f"'{prop.prop}': species '{prop.species}' has no size in "
|
|
677
|
+
f"{context.spec.name}.species_sizes "
|
|
678
|
+
f"{dict(context.spec.species_sizes)}."
|
|
679
|
+
)
|
|
680
|
+
# The per-whole spread is defined for a bin reference only, which is
|
|
681
|
+
# what every default is; a numeric or callable one would be skipped.
|
|
682
|
+
default = PROFILE_NORMALIZERS.get(
|
|
683
|
+
(context.spec.parser._geometry, cast(DirectionT, direction)))
|
|
684
|
+
normalizer = default if isinstance(default, str) else None
|
|
685
|
+
rhos: dict[str, np.ndarray] = {}
|
|
686
|
+
phis: dict[str, np.ndarray] = {}
|
|
687
|
+
for whole_name, counts in wholes.items():
|
|
688
|
+
info = context.infos[whole_name]
|
|
689
|
+
rho, phi = context.density(
|
|
690
|
+
np.asarray(counts), np.asarray(context.edges[whole_name]),
|
|
691
|
+
info, direction, float(getattr(info, size_attr)),
|
|
692
|
+
)
|
|
693
|
+
rhos[whole_name] = np.asarray(rho, dtype=np.float64)
|
|
694
|
+
phis[whole_name] = np.asarray(phi, dtype=np.float64)
|
|
695
|
+
return (
|
|
696
|
+
Derived(prop.prop, wholes, 'vector', context.edges),
|
|
697
|
+
Derived(f'{direction}Rho{prop.species}', rhos, 'vector',
|
|
698
|
+
context.edges, normalizer),
|
|
699
|
+
Derived(f'{direction}Phi{prop.species}', phis, 'vector',
|
|
700
|
+
context.edges, normalizer),
|
|
701
|
+
)
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def _value_for(
|
|
705
|
+
value: PerWhole, info: ParserBase | None, what: str, whole: str
|
|
706
|
+
) -> object:
|
|
707
|
+
"""
|
|
708
|
+
Return `value`, or what it gives on `info` when it is a function.
|
|
709
|
+
|
|
710
|
+
`what` names the quantity and `whole` the whole in an error.
|
|
711
|
+
"""
|
|
712
|
+
if not callable(value):
|
|
713
|
+
return value
|
|
714
|
+
if info is None:
|
|
715
|
+
raise ValueError(
|
|
716
|
+
f"{what} is a function of the whole, but the context has no "
|
|
717
|
+
f"parser instance for whole '{whole}'."
|
|
718
|
+
)
|
|
719
|
+
return value(info)
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def _resolve(
|
|
723
|
+
value: PerWhole, info: ParserBase | None, what: str, whole: str
|
|
724
|
+
) -> float:
|
|
725
|
+
"""
|
|
726
|
+
Return `value`, calling it on `info` when it is a function.
|
|
727
|
+
|
|
728
|
+
`what` names the quantity and `whole` the whole in an error. The
|
|
729
|
+
constant or what the callable gives must be a finite positive number,
|
|
730
|
+
the check of :meth:`~polyphys.spec.ProbeSpec.resolve`.
|
|
731
|
+
"""
|
|
732
|
+
return _check_resolved(_value_for(value, info, what, whole),
|
|
733
|
+
f"{what} of whole '{whole}'")
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _is_positive_integer(value: object) -> TypeGuard[numbers.Integral]:
|
|
737
|
+
"""Whether `value` is a positive Python or NumPy integer, never a bool
|
|
738
|
+
and never a float, even one equal to a whole number."""
|
|
739
|
+
return (isinstance(value, numbers.Integral)
|
|
740
|
+
and not isinstance(value, bool) and int(value) > 0)
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _resolve_count(
|
|
744
|
+
value: PerWhole, info: ParserBase | None, what: str, whole: str
|
|
745
|
+
) -> int:
|
|
746
|
+
"""
|
|
747
|
+
Return `value`, calling it on `info` when it is a function, as an int.
|
|
748
|
+
|
|
749
|
+
`what` names the quantity and `whole` the whole in an error. The
|
|
750
|
+
constant or what the callable gives must be a positive integer
|
|
751
|
+
(``_is_positive_integer``); it is never truncated.
|
|
752
|
+
"""
|
|
753
|
+
result = _value_for(value, info, what, whole)
|
|
754
|
+
if _is_positive_integer(result):
|
|
755
|
+
return int(result)
|
|
756
|
+
raise ValueError(
|
|
757
|
+
f"{what} of whole '{whole}' must resolve to a positive integer, not "
|
|
758
|
+
f"{result!r}.")
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def half_contour(n_monomers: int, bond_max: float) -> float:
|
|
762
|
+
"""
|
|
763
|
+
The largest separation two monomers of a ring can have.
|
|
764
|
+
|
|
765
|
+
Parameters
|
|
766
|
+
----------
|
|
767
|
+
n_monomers : int
|
|
768
|
+
The number of monomers on the ring, at least 2.
|
|
769
|
+
bond_max : float
|
|
770
|
+
The largest length a bond can have, in the length unit; for the FENE
|
|
771
|
+
bond, :math:`R_0 = 1.5\\,\\sigma_{ij}` with :math:`\\sigma_{ij}` the
|
|
772
|
+
collision diameter of the bonded pair (Kremer and Grest, 1990).
|
|
773
|
+
|
|
774
|
+
Returns
|
|
775
|
+
-------
|
|
776
|
+
float
|
|
777
|
+
``(n_monomers // 2) * bond_max``: the two monomers are at most
|
|
778
|
+
``n_monomers // 2`` bonds apart along the ring, and a path of bonds
|
|
779
|
+
is longest when straight and fully extended.
|
|
780
|
+
|
|
781
|
+
Raises
|
|
782
|
+
------
|
|
783
|
+
ValueError
|
|
784
|
+
If `n_monomers` is below 2 or `bond_max` is not positive.
|
|
785
|
+
|
|
786
|
+
Notes
|
|
787
|
+
-----
|
|
788
|
+
A bound, not a typical value: it lets a pair-distance histogram cover
|
|
789
|
+
every separation the chain allows, so that its counts conserve the
|
|
790
|
+
number of frames and a separation above it marks a broken bond or an
|
|
791
|
+
unwrapping error. Pass the largest bond's :math:`R_0` when the bonds
|
|
792
|
+
differ; the bound is then loose by the ratio of the largest to the
|
|
793
|
+
typical bond, which costs bins, not correctness.
|
|
794
|
+
|
|
795
|
+
Examples
|
|
796
|
+
--------
|
|
797
|
+
A ring of 10 beads with bonds that stretch to 1.5:
|
|
798
|
+
|
|
799
|
+
>>> half_contour(10, 1.5)
|
|
800
|
+
7.5
|
|
801
|
+
|
|
802
|
+
.. versionadded:: 0.6.0
|
|
803
|
+
"""
|
|
804
|
+
if n_monomers < 2:
|
|
805
|
+
raise ValueError(f'n_monomers must be at least 2; got {n_monomers}.')
|
|
806
|
+
if not bond_max > 0:
|
|
807
|
+
raise ValueError(f'bond_max must be positive; got {bond_max}.')
|
|
808
|
+
return (n_monomers // 2) * bond_max
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
@dataclass(frozen=True)
|
|
812
|
+
class FociPairDistances:
|
|
813
|
+
"""
|
|
814
|
+
Reduce the distance-matrix time series of the foci; a post-hook.
|
|
815
|
+
|
|
816
|
+
On ``distMatT<species>``, whose whole is the ``(n_frames, n_foci,
|
|
817
|
+
n_foci)`` matrix of center-to-center distances between the foci with the
|
|
818
|
+
foci's backbone indices on the diagonal of frame 0, it writes three
|
|
819
|
+
``'dataframe'`` wholes with one column per pair, tagged
|
|
820
|
+
``loc<i>loc<j>genDist<d>`` with ``i < j`` the backbone indices and ``d``
|
|
821
|
+
their genomic distance in bonds:
|
|
822
|
+
|
|
823
|
+
- ``pairDistHist<species>``: the histogram of each pair's separation on
|
|
824
|
+
one grid shared by every pair and whole, with a ``bin_center`` column;
|
|
825
|
+
- ``pairDistRdf<species>``: :math:`\\bar P(R)`, the probability density
|
|
826
|
+
of each pair's separation vector on that grid, with ``bin_center``;
|
|
827
|
+
- ``pairDistT<species>``: each pair's separation over the frames.
|
|
828
|
+
|
|
829
|
+
Attributes
|
|
830
|
+
----------
|
|
831
|
+
bin_size : float or callable
|
|
832
|
+
The bin width :math:`\\Delta r`, in the length unit, or a function of
|
|
833
|
+
the whole's parser instance that gives it.
|
|
834
|
+
r_max : float or callable
|
|
835
|
+
The upper end of the grid, or a function of the parser instance that
|
|
836
|
+
gives it, such as :func:`half_contour` of the ring; the grid runs to
|
|
837
|
+
the first multiple of `bin_size` at or above it.
|
|
838
|
+
n_monomers : int or callable
|
|
839
|
+
The number of monomers on the backbone, or a function of the parser
|
|
840
|
+
instance that gives it.
|
|
841
|
+
|
|
842
|
+
Raises
|
|
843
|
+
------
|
|
844
|
+
ValueError
|
|
845
|
+
When built: if a constant `bin_size` or `r_max` is not a finite
|
|
846
|
+
positive number, or a constant `n_monomers` is not a positive
|
|
847
|
+
integer; a Python or NumPy number, never a bool, and for
|
|
848
|
+
`n_monomers` never a float, even ``10.0``. When called: if the
|
|
849
|
+
context has no spec (the topology comes from it); if a whole is not
|
|
850
|
+
a ``(n_frames, n_foci, n_foci)`` array; if a callable quantity gives
|
|
851
|
+
a value that breaks the same rule, which names the quantity and the
|
|
852
|
+
whole; or if a separation lies above the grid, which names the whole
|
|
853
|
+
and the separation, since a dropped frame would break the
|
|
854
|
+
normalization.
|
|
855
|
+
|
|
856
|
+
Notes
|
|
857
|
+
-----
|
|
858
|
+
Every pair has one separation per frame, so each histogram column sums
|
|
859
|
+
to ``n_frames`` and :math:`\\sum_k \\bar P_k \\Delta V_k = 1` exactly.
|
|
860
|
+
The research code set the grid from each whole's own largest separation,
|
|
861
|
+
so wholes of one ensemble had different edges; a bound from the system
|
|
862
|
+
gives one grid. Time is :math:`O(n_\\mathrm{pairs} n_\\mathrm{frames}
|
|
863
|
+
\\log n_\\mathrm{bins})` and space :math:`O(n_\\mathrm{pairs}
|
|
864
|
+
(n_\\mathrm{frames} + n_\\mathrm{bins}))` per whole.
|
|
865
|
+
|
|
866
|
+
Examples
|
|
867
|
+
--------
|
|
868
|
+
Three foci at backbone indices 0, 4, and 8 of a ring of 10 monomers,
|
|
869
|
+
over two frames, on a grid of unit bins to 3:
|
|
870
|
+
|
|
871
|
+
>>> import numpy as np
|
|
872
|
+
>>> from polyphys.tests.synthetic import RingCub
|
|
873
|
+
>>> from polyphys.spec import ProjectSpec, matrix_tseries
|
|
874
|
+
>>> spec = ProjectSpec(
|
|
875
|
+
... parser=RingCub, group='bug', divisor=0.025,
|
|
876
|
+
... crowder_attr='phi_bulk_c',
|
|
877
|
+
... physical_attrs=('space',), species_sizes={},
|
|
878
|
+
... tseries_properties=())
|
|
879
|
+
>>> frame0 = np.array([[0.0, 1.5, 2.5], [1.5, 4.0, 0.5], [2.5, 0.5, 8.0]])
|
|
880
|
+
>>> frame1 = np.array([[0.0, 1.5, 2.9], [1.5, 0.0, 2.5], [2.9, 2.5, 0.0]])
|
|
881
|
+
>>> hook = FociPairDistances(bin_size=1.0, r_max=3.0, n_monomers=10)
|
|
882
|
+
>>> context = HookContext(matrix_tseries('distMatT', 'Foci'), spec=spec)
|
|
883
|
+
>>> hist, density, series = hook({'w1': np.stack([frame0, frame1])},
|
|
884
|
+
... context)
|
|
885
|
+
>>> list(hist.wholes['w1'].columns)
|
|
886
|
+
['loc0loc4genDist4', 'loc0loc8genDist2', 'loc4loc8genDist4', 'bin_center']
|
|
887
|
+
>>> hist.wholes['w1']['loc0loc8genDist2'].to_list()
|
|
888
|
+
[0, 0, 2]
|
|
889
|
+
>>> series.wholes['w1']['loc4loc8genDist4'].to_list()
|
|
890
|
+
[0.5, 2.5]
|
|
891
|
+
|
|
892
|
+
A constant that breaks the rule raises when the hook is built:
|
|
893
|
+
|
|
894
|
+
>>> FociPairDistances(bin_size=1.0, r_max=3.0, n_monomers=10.5)
|
|
895
|
+
Traceback (most recent call last):
|
|
896
|
+
...
|
|
897
|
+
ValueError: FociPairDistances: 'n_monomers' must be a positive integer ...
|
|
898
|
+
|
|
899
|
+
.. versionadded:: 0.6.0
|
|
900
|
+
|
|
901
|
+
.. versionchanged:: 1.0.0
|
|
902
|
+
The constants are checked when the hook is built, and `n_monomers`,
|
|
903
|
+
a constant or what a callable gives, must be a positive integer,
|
|
904
|
+
where a float such as 10.5 was truncated to 10.
|
|
905
|
+
"""
|
|
906
|
+
|
|
907
|
+
bin_size: PerWhole
|
|
908
|
+
r_max: PerWhole
|
|
909
|
+
n_monomers: int | Callable[[ParserBase], int]
|
|
910
|
+
|
|
911
|
+
def __post_init__(self) -> None:
|
|
912
|
+
for name in ('bin_size', 'r_max'):
|
|
913
|
+
value = getattr(self, name)
|
|
914
|
+
if not _spec._is_per_whole(value):
|
|
915
|
+
raise ValueError(
|
|
916
|
+
f"FociPairDistances: '{name}' must be a finite positive "
|
|
917
|
+
f"number or a callable, not {value!r}.")
|
|
918
|
+
if not (callable(self.n_monomers)
|
|
919
|
+
or _is_positive_integer(self.n_monomers)):
|
|
920
|
+
raise ValueError(
|
|
921
|
+
"FociPairDistances: 'n_monomers' must be a positive integer "
|
|
922
|
+
f"or a callable, not {self.n_monomers!r}.")
|
|
923
|
+
|
|
924
|
+
def outputs(self, prop: _spec.PropertySpec) -> tuple[PropertyT, ...]:
|
|
925
|
+
"""
|
|
926
|
+
The pair histograms, separation densities, and separation time
|
|
927
|
+
series of the species of `prop`.
|
|
928
|
+
|
|
929
|
+
Examples
|
|
930
|
+
--------
|
|
931
|
+
>>> from polyphys.spec import matrix_tseries
|
|
932
|
+
>>> hook = FociPairDistances(bin_size=1.0, r_max=3.0, n_monomers=10)
|
|
933
|
+
>>> hook.outputs(matrix_tseries('distMatT', 'Foci'))
|
|
934
|
+
('pairDistHistFoci', 'pairDistRdfFoci', 'pairDistTFoci')
|
|
935
|
+
|
|
936
|
+
.. versionadded:: 0.9.0
|
|
937
|
+
"""
|
|
938
|
+
species = prop.species
|
|
939
|
+
return (f'pairDistHist{species}', f'pairDistRdf{species}',
|
|
940
|
+
f'pairDistT{species}')
|
|
941
|
+
|
|
942
|
+
def __call__(
|
|
943
|
+
self, wholes: ArrayWholes, context: HookContext
|
|
944
|
+
) -> tuple[Derived, ...]:
|
|
945
|
+
prop = context.prop
|
|
946
|
+
if context.spec is None:
|
|
947
|
+
raise ValueError(
|
|
948
|
+
f"'{prop.prop}': FociPairDistances needs the study's spec in "
|
|
949
|
+
"its context, for the topology."
|
|
950
|
+
)
|
|
951
|
+
# foci_pair_info checks the topology itself.
|
|
952
|
+
topology = cast(clusters.Topology, context.spec.topology)
|
|
953
|
+
hists: dict[str, pd.DataFrame] = {}
|
|
954
|
+
densities: dict[str, pd.DataFrame] = {}
|
|
955
|
+
series: dict[str, pd.DataFrame] = {}
|
|
956
|
+
for whole_name, whole in wholes.items():
|
|
957
|
+
matrix = np.asarray(whole, dtype=np.float64)
|
|
958
|
+
if matrix.ndim != 3 or matrix.shape[1] != matrix.shape[2]:
|
|
959
|
+
raise ValueError(
|
|
960
|
+
f"'{prop.prop}', whole '{whole_name}': expected a "
|
|
961
|
+
"(n_frames, n_foci, n_foci) distance matrix time series,"
|
|
962
|
+
f" got shape {matrix.shape}."
|
|
963
|
+
)
|
|
964
|
+
info = context.infos.get(whole_name)
|
|
965
|
+
n_foci = matrix.shape[1]
|
|
966
|
+
genomic_pos = np.rint(np.diagonal(matrix[0])).astype(np.int64)
|
|
967
|
+
n_monomers = _resolve_count(self.n_monomers, info, 'n_monomers',
|
|
968
|
+
whole_name)
|
|
969
|
+
pairs = clusters.foci_pair_info(genomic_pos, n_monomers, topology)
|
|
970
|
+
tags = [f'loc{i}loc{j}genDist{d}' for i, j, d in pairs]
|
|
971
|
+
first, second = np.triu_indices(n_foci, 1)
|
|
972
|
+
distances = matrix[:, first, second].T
|
|
973
|
+
bin_size = _resolve(self.bin_size, info, 'bin_size', whole_name)
|
|
974
|
+
r_max = _resolve(self.r_max, info, 'r_max', whole_name)
|
|
975
|
+
# The grid pair_distance_histograms builds, so the check below
|
|
976
|
+
# tolerates the same round-off.
|
|
977
|
+
upper = measurer.create_bin_edge_and_hist(
|
|
978
|
+
bin_size, 0.0, r_max)[0][-1]
|
|
979
|
+
largest = float(distances.max())
|
|
980
|
+
if largest > upper:
|
|
981
|
+
raise ValueError(
|
|
982
|
+
f"'{prop.prop}', whole '{whole_name}': a pair separation "
|
|
983
|
+
f"of {largest:.6g} lies above r_max {upper:.6g}; the "
|
|
984
|
+
"chain cannot reach it, so check the whole."
|
|
985
|
+
)
|
|
986
|
+
counts, edges = clusters.pair_distance_histograms(
|
|
987
|
+
distances, bin_size, r_max)
|
|
988
|
+
counts = counts.reshape(len(tags), -1)
|
|
989
|
+
density = clusters.pair_separation_density(counts, edges)
|
|
990
|
+
centers = 0.5 * (edges[:-1] + edges[1:])
|
|
991
|
+
hists[whole_name] = pd.DataFrame(
|
|
992
|
+
dict(zip(tags, counts)) | {'bin_center': centers})
|
|
993
|
+
densities[whole_name] = pd.DataFrame(
|
|
994
|
+
dict(zip(tags, density)) | {'bin_center': centers})
|
|
995
|
+
series[whole_name] = pd.DataFrame(dict(zip(tags, distances)))
|
|
996
|
+
species = prop.species
|
|
997
|
+
return (
|
|
998
|
+
Derived(f'pairDistHist{species}', hists, 'dataframe'),
|
|
999
|
+
Derived(f'pairDistRdf{species}', densities, 'dataframe'),
|
|
1000
|
+
Derived(f'pairDistT{species}', series, 'dataframe'),
|
|
1001
|
+
)
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
# ---------------------------------------------------------------------------
|
|
1005
|
+
# Drivers
|
|
1006
|
+
# ---------------------------------------------------------------------------
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
SaveDirs = tuple[Pathish | None, Pathish | None, Pathish | None]
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def reduce_stamps(
|
|
1013
|
+
stamp_files: Sequence[tuple[str]],
|
|
1014
|
+
lineage: PrimitiveLineageT,
|
|
1015
|
+
save_to: SaveDirs = (None, None, None),
|
|
1016
|
+
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
1017
|
+
"""
|
|
1018
|
+
Combine the stamps of one space into whole and ensemble stamps.
|
|
1019
|
+
|
|
1020
|
+
Parameters
|
|
1021
|
+
----------
|
|
1022
|
+
stamp_files : sequence of tuple of str
|
|
1023
|
+
The stamp CSV files, each path in a 1-tuple, as
|
|
1024
|
+
:func:`~polyphys.manage.utils.sort_filenames` returns them.
|
|
1025
|
+
lineage : {'segment', 'whole'}
|
|
1026
|
+
The lineage of the files.
|
|
1027
|
+
save_to : tuple of three paths or None, default (None, None, None)
|
|
1028
|
+
The ``wholeSim``, ``ens``, and ``ensAvg`` directories; ``None`` skips
|
|
1029
|
+
a stage's file. For ``'segment'`` stamps the combined segment
|
|
1030
|
+
stamps go to the first, the whole stamps to the second, and the
|
|
1031
|
+
ensemble stamps to the third; for ``'whole'`` stamps the whole
|
|
1032
|
+
stamps go to the second and the ensemble stamps to the third.
|
|
1033
|
+
|
|
1034
|
+
Returns
|
|
1035
|
+
-------
|
|
1036
|
+
whole_stamps : pandas.DataFrame
|
|
1037
|
+
One row per whole.
|
|
1038
|
+
ensemble_stamps : pandas.DataFrame
|
|
1039
|
+
One row per ensemble.
|
|
1040
|
+
|
|
1041
|
+
Raises
|
|
1042
|
+
------
|
|
1043
|
+
ValueError
|
|
1044
|
+
As :func:`~polyphys.manage.organizer.children_stamps` and
|
|
1045
|
+
:func:`~polyphys.manage.organizer.parents_stamps`.
|
|
1046
|
+
|
|
1047
|
+
Notes
|
|
1048
|
+
-----
|
|
1049
|
+
Segment stamps combine with
|
|
1050
|
+
:func:`~polyphys.manage.organizer.children_stamps`, then roll up once
|
|
1051
|
+
with :func:`~polyphys.manage.organizer.parents_stamps` into whole
|
|
1052
|
+
stamps; whole stamps combine directly. Either way the whole stamps roll
|
|
1053
|
+
up once more into ensemble stamps. Time and space are
|
|
1054
|
+
:math:`O(n_\\mathrm{files}\\, n_\\mathrm{columns})`.
|
|
1055
|
+
|
|
1056
|
+
Examples
|
|
1057
|
+
--------
|
|
1058
|
+
.. code-block:: python
|
|
1059
|
+
|
|
1060
|
+
whole_stamps, ensemble_stamps = reduce_stamps(
|
|
1061
|
+
[(path,) for path in sorted(stamp_paths)], 'whole')
|
|
1062
|
+
|
|
1063
|
+
.. versionadded:: 0.6.0
|
|
1064
|
+
"""
|
|
1065
|
+
whole_dir, ens_dir, ens_avg_dir = save_to
|
|
1066
|
+
if lineage == 'segment':
|
|
1067
|
+
segment_stamps = organizer.children_stamps(
|
|
1068
|
+
list(stamp_files), 'segment', save_to=whole_dir)
|
|
1069
|
+
whole_stamps = organizer.parents_stamps(
|
|
1070
|
+
segment_stamps, 'segment', save_to=ens_dir)
|
|
1071
|
+
else:
|
|
1072
|
+
whole_stamps = organizer.children_stamps(
|
|
1073
|
+
list(stamp_files), 'whole', save_to=ens_dir)
|
|
1074
|
+
ensemble_stamps = organizer.parents_stamps(
|
|
1075
|
+
whole_stamps, 'whole', save_to=ens_avg_dir)
|
|
1076
|
+
return whole_stamps, ensemble_stamps
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
def reduce_space(
|
|
1080
|
+
input_database: Pathish,
|
|
1081
|
+
spec: ProjectSpec,
|
|
1082
|
+
table: _spec.AnalysisTable,
|
|
1083
|
+
*,
|
|
1084
|
+
hierarchy: str | None = None,
|
|
1085
|
+
density: DensityKernel | None = None,
|
|
1086
|
+
save: bool = True,
|
|
1087
|
+
) -> dict[str, dict[str, np.ndarray | pd.DataFrame]]:
|
|
1088
|
+
"""
|
|
1089
|
+
Reduce every property of a table over one space of probe artifacts.
|
|
1090
|
+
|
|
1091
|
+
Parameters
|
|
1092
|
+
----------
|
|
1093
|
+
input_database : path-like
|
|
1094
|
+
The probe directory of the space,
|
|
1095
|
+
``<root>/probes/<prefix>-<group>-probe``; it may hold a glob, as the
|
|
1096
|
+
research code's drivers pass it.
|
|
1097
|
+
spec : ProjectSpec
|
|
1098
|
+
The study's spec: its parser, topology, and species sizes.
|
|
1099
|
+
table : ~polyphys.spec.AnalysisTable
|
|
1100
|
+
The group's analysis table.
|
|
1101
|
+
hierarchy : str, optional
|
|
1102
|
+
The glob, relative to `input_database`, that lists the artifact
|
|
1103
|
+
files; the default is ``<spec.hierarchy>/<spec.hierarchy>``, the
|
|
1104
|
+
research code's ``dir/file`` layout.
|
|
1105
|
+
density : DensityKernel, optional
|
|
1106
|
+
The volume-fraction kernel for :class:`LocalDensities`.
|
|
1107
|
+
save : bool, default True
|
|
1108
|
+
Write the stages' files under ``<root>/analysis/``; ``False``
|
|
1109
|
+
returns the ensemble averages without writing anything.
|
|
1110
|
+
|
|
1111
|
+
Returns
|
|
1112
|
+
-------
|
|
1113
|
+
dict of str to dict
|
|
1114
|
+
The ensemble averages by output property, then by ensemble name:
|
|
1115
|
+
a DataFrame for a ``'vector'`` or ``'dataframe'`` output, a dict of
|
|
1116
|
+
arrays for a ``'matrix'``, and the edges for a ``'bin_edge'``.
|
|
1117
|
+
|
|
1118
|
+
Raises
|
|
1119
|
+
------
|
|
1120
|
+
FileNotFoundError
|
|
1121
|
+
If the glob matches no file, or a property or its edge property
|
|
1122
|
+
matches no file; checked for every property before any directory is
|
|
1123
|
+
made.
|
|
1124
|
+
ValueError
|
|
1125
|
+
If the wholes of one ensemble hold different bin edges; if a
|
|
1126
|
+
histogram's edge files do not cover its wholes; if a hook returns
|
|
1127
|
+
an output name the table already produced; as the organizer and the
|
|
1128
|
+
hooks raise.
|
|
1129
|
+
|
|
1130
|
+
Notes
|
|
1131
|
+
-----
|
|
1132
|
+
For each property in turn: its files are selected by their last name
|
|
1133
|
+
token, wholes are built (for the ``'segment'`` lineage with
|
|
1134
|
+
:func:`~polyphys.manage.organizer.whole_from_segments`, written to
|
|
1135
|
+
``wholeSim``; else with
|
|
1136
|
+
:func:`~polyphys.manage.organizer.whole_from_file`), the edges are
|
|
1137
|
+
loaded and checked, the post-hook is applied, and every returned
|
|
1138
|
+
artifact goes through :func:`~polyphys.manage.organizer.ensemble` and
|
|
1139
|
+
:func:`~polyphys.manage.organizer.ensemble_avg`, or their long-form
|
|
1140
|
+
counterparts for ``'dataframe'`` wholes. With `table.stamps`, the
|
|
1141
|
+
stamps are combined first by :func:`reduce_stamps`. One property's
|
|
1142
|
+
wholes are in memory at a time, so peak memory is the largest property's
|
|
1143
|
+
``n_wholes`` wholes plus its ensembles; time is
|
|
1144
|
+
:math:`O(n_\\mathrm{files}\\, n_\\mathrm{frames})` per property plus the
|
|
1145
|
+
hook's.
|
|
1146
|
+
|
|
1147
|
+
Examples
|
|
1148
|
+
--------
|
|
1149
|
+
Two replicates of one state point of the ``ChainCyl`` study, written
|
|
1150
|
+
as wholes and reduced without saving:
|
|
1151
|
+
|
|
1152
|
+
>>> import tempfile
|
|
1153
|
+
>>> import numpy as np
|
|
1154
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
1155
|
+
>>> from polyphys.spec import AnalysisTable, ProjectSpec, tseries
|
|
1156
|
+
>>> spec = ProjectSpec(
|
|
1157
|
+
... parser=ChainCyl, group='bug', divisor=0.025,
|
|
1158
|
+
... crowder_attr='phi_bulk_c',
|
|
1159
|
+
... physical_attrs=('space',), species_sizes={'Mon': 'dmon'},
|
|
1160
|
+
... tseries_properties=('gyrTMon',))
|
|
1161
|
+
>>> stem = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
|
|
1162
|
+
>>> root = tempfile.mkdtemp()
|
|
1163
|
+
>>> np.save(f'{root}/{stem}ens1-bug-gyrTMon.npy', np.array([1.0, 2.0]))
|
|
1164
|
+
>>> np.save(f'{root}/{stem}ens2-bug-gyrTMon.npy', np.array([3.0, 6.0]))
|
|
1165
|
+
>>> table = AnalysisTable('bug', 'whole', False, (tseries('gyrT', 'Mon'),))
|
|
1166
|
+
>>> averages = reduce_space(root, spec, table, hierarchy='*', save=False)
|
|
1167
|
+
>>> averages['gyrTMon'][stem]
|
|
1168
|
+
gyrTMon-mean gyrTMon-var gyrTMon-sem
|
|
1169
|
+
0 2.0 2.0 1.0
|
|
1170
|
+
1 4.0 8.0 2.0
|
|
1171
|
+
|
|
1172
|
+
.. versionadded:: 0.6.0
|
|
1173
|
+
"""
|
|
1174
|
+
parser = spec.parser
|
|
1175
|
+
group = table.group
|
|
1176
|
+
if hierarchy is None:
|
|
1177
|
+
hierarchy = f'{spec.hierarchy}/{spec.hierarchy}'
|
|
1178
|
+
pattern = os.path.join(os.fspath(input_database), hierarchy)
|
|
1179
|
+
observations = sorted(glob.glob(pattern))
|
|
1180
|
+
if not observations:
|
|
1181
|
+
raise FileNotFoundError(f"No file matches '{pattern}'.")
|
|
1182
|
+
logger.info("Reducing %d files of group '%s' under '%s'.",
|
|
1183
|
+
len(observations), group, pattern)
|
|
1184
|
+
|
|
1185
|
+
# Select every property's files first, so that a property without
|
|
1186
|
+
# files, such as a typo, raises before any directory is made.
|
|
1187
|
+
for prop in table.properties:
|
|
1188
|
+
_select(observations, prop.prop)
|
|
1189
|
+
if prop.edge_prop is not None:
|
|
1190
|
+
_select(observations, prop.edge_prop)
|
|
1191
|
+
|
|
1192
|
+
whole_dir: str | None = None
|
|
1193
|
+
ens_dir: str | None = None
|
|
1194
|
+
ens_avg_dir: str | None = None
|
|
1195
|
+
if save:
|
|
1196
|
+
if table.lineage == 'segment':
|
|
1197
|
+
whole_dir = organizer.make_database(
|
|
1198
|
+
input_database, 'analysis', 'wholeSim', group)
|
|
1199
|
+
ens_dir = organizer.make_database(
|
|
1200
|
+
input_database, 'analysis', 'ens', group)
|
|
1201
|
+
ens_avg_dir = organizer.make_database(
|
|
1202
|
+
input_database, 'analysis', 'ensAvg', group)
|
|
1203
|
+
|
|
1204
|
+
if table.stamps:
|
|
1205
|
+
stamp_files = _select(observations, 'stamps', ext='csv')
|
|
1206
|
+
reduce_stamps(stamp_files, table.lineage,
|
|
1207
|
+
save_to=(whole_dir, ens_dir, ens_avg_dir))
|
|
1208
|
+
|
|
1209
|
+
results: dict[str, dict[str, np.ndarray | pd.DataFrame]] = {}
|
|
1210
|
+
for prop in table.properties:
|
|
1211
|
+
logger.info("Reducing property '%s'.", prop.prop)
|
|
1212
|
+
wholes = _load_wholes(prop.prop, prop.relation, observations, parser,
|
|
1213
|
+
group, table.lineage, whole_dir)
|
|
1214
|
+
edges: dict[str, np.ndarray] | None = None
|
|
1215
|
+
if prop.edge_prop is not None:
|
|
1216
|
+
edges = _load_wholes(prop.edge_prop, 'bin_edge', observations,
|
|
1217
|
+
parser, group, table.lineage, whole_dir)
|
|
1218
|
+
_check_edges(prop, wholes, edges, parser, group)
|
|
1219
|
+
infos = {name: parser(name, 'whole', group) for name in wholes}
|
|
1220
|
+
context = HookContext(prop, spec=spec, group=group, infos=infos,
|
|
1221
|
+
edges=edges, density=density)
|
|
1222
|
+
if prop.post_hook is None:
|
|
1223
|
+
outputs: tuple[Derived, ...] = (
|
|
1224
|
+
Derived(prop.prop, wholes, prop.whole_type, edges),)
|
|
1225
|
+
else:
|
|
1226
|
+
outputs = prop.post_hook(wholes, context)
|
|
1227
|
+
for output in outputs:
|
|
1228
|
+
if output.name in results:
|
|
1229
|
+
raise ValueError(
|
|
1230
|
+
f"'{prop.prop}': the output '{output.name}' is named "
|
|
1231
|
+
f"more than once in the table; it would overwrite an "
|
|
1232
|
+
f"earlier one.")
|
|
1233
|
+
if whole_dir is not None and output.name != prop.prop:
|
|
1234
|
+
for whole_name, data in output.wholes.items():
|
|
1235
|
+
organizer.save_artifact(
|
|
1236
|
+
organizer.create_fullname(whole_name, group,
|
|
1237
|
+
output.name),
|
|
1238
|
+
data, whole_dir)
|
|
1239
|
+
results[output.name] = _reduce_output(
|
|
1240
|
+
output, parser, group, ens_dir, ens_avg_dir)
|
|
1241
|
+
return results
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
def space_tseries_from_spec(
|
|
1245
|
+
input_database: Pathish,
|
|
1246
|
+
spec: ProjectSpec,
|
|
1247
|
+
prop: PropertyT,
|
|
1248
|
+
*,
|
|
1249
|
+
group: GroupT | None = None,
|
|
1250
|
+
round_to: int = 3,
|
|
1251
|
+
save: bool = False,
|
|
1252
|
+
) -> pd.DataFrame:
|
|
1253
|
+
"""
|
|
1254
|
+
Stack the ensemble-averaged time series of one property over a space,
|
|
1255
|
+
with the columns and the rounding step the study's spec declares.
|
|
1256
|
+
|
|
1257
|
+
The spec supplies every study choice of
|
|
1258
|
+
:func:`~polyphys.manage.organizer.space_tseries`: the parser, the
|
|
1259
|
+
``ensemble_long`` glob prefix (`spec.hierarchy`), the attributes that
|
|
1260
|
+
become columns (`spec.physical_attrs`), the rounding step
|
|
1261
|
+
:math:`\\Delta` of the bulk volume fraction of crowders
|
|
1262
|
+
:math:`\\phi_c` (`spec.divisor`), and the parser attribute that holds
|
|
1263
|
+
:math:`\\phi_c` (`spec.crowder_attr`). No study default is taken from
|
|
1264
|
+
the library.
|
|
1265
|
+
|
|
1266
|
+
Parameters
|
|
1267
|
+
----------
|
|
1268
|
+
input_database : path-like
|
|
1269
|
+
The ``ensAvg`` directory of the space, which holds the files
|
|
1270
|
+
``<ensemble_long>-<group>-<prop>-ensAvg.csv``.
|
|
1271
|
+
spec : ~polyphys.spec.ProjectSpec
|
|
1272
|
+
The study's spec.
|
|
1273
|
+
prop : str
|
|
1274
|
+
The property, such as ``'gyrTMon'``.
|
|
1275
|
+
group : str, optional
|
|
1276
|
+
The particle group of the files; the default is the polymer group,
|
|
1277
|
+
`spec.group`.
|
|
1278
|
+
round_to : int, default 3
|
|
1279
|
+
The number of decimal places of the rounded :math:`\\phi_c`.
|
|
1280
|
+
save : bool, default False
|
|
1281
|
+
Also write the table to ``<space>-<group>-<prop>-space.csv`` in the
|
|
1282
|
+
``analysis`` phase's ``space`` directory next to `input_database`.
|
|
1283
|
+
|
|
1284
|
+
Returns
|
|
1285
|
+
-------
|
|
1286
|
+
pandas.DataFrame
|
|
1287
|
+
The table of
|
|
1288
|
+
:func:`~polyphys.manage.organizer.space_tseries`: each file's
|
|
1289
|
+
columns, ``t_index``, ``time``, one column per physical attribute,
|
|
1290
|
+
a `spec.crowder_attr` column, and ``phi_bulk_c_round``.
|
|
1291
|
+
|
|
1292
|
+
Notes
|
|
1293
|
+
-----
|
|
1294
|
+
The cost is that of
|
|
1295
|
+
:func:`~polyphys.manage.organizer.space_tseries`:
|
|
1296
|
+
:math:`O(n_\\mathrm{rows}\\, n_\\mathrm{cols})` time and memory for the
|
|
1297
|
+
rows and columns of the result.
|
|
1298
|
+
|
|
1299
|
+
Examples
|
|
1300
|
+
--------
|
|
1301
|
+
The spec's step of 0.1 rounds :math:`\\phi_c \\approx 0.0123` of this
|
|
1302
|
+
state point to 0.0:
|
|
1303
|
+
|
|
1304
|
+
>>> import tempfile
|
|
1305
|
+
>>> import pandas as pd
|
|
1306
|
+
>>> from pathlib import Path
|
|
1307
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
1308
|
+
>>> from polyphys.spec import ProjectSpec
|
|
1309
|
+
>>> spec = ProjectSpec(
|
|
1310
|
+
... parser=ChainCyl, group='bug', divisor=0.1,
|
|
1311
|
+
... crowder_attr='phi_bulk_c', physical_attrs=('nmon',),
|
|
1312
|
+
... species_sizes={'Mon': 'dmon'}, tseries_properties=('gyrTMon',))
|
|
1313
|
+
>>> database = Path(tempfile.mkdtemp()) / 'analysis' / 'N20-bug-ensAvg'
|
|
1314
|
+
>>> database.mkdir(parents=True)
|
|
1315
|
+
>>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
|
|
1316
|
+
>>> pd.DataFrame({'gyrTMon-mean': [1.0, 2.0]}).to_csv(
|
|
1317
|
+
... database / f'{state}-bug-gyrTMon-ensAvg.csv', index=False)
|
|
1318
|
+
>>> space = space_tseries_from_spec(database, spec, 'gyrTMon')
|
|
1319
|
+
>>> space[['time', 'gyrTMon-mean', 'nmon', 'phi_bulk_c_round']]
|
|
1320
|
+
time gyrTMon-mean nmon phi_bulk_c_round
|
|
1321
|
+
0 0.0 1.0 20 0.0
|
|
1322
|
+
1 5.0 2.0 20 0.0
|
|
1323
|
+
|
|
1324
|
+
.. versionadded:: 0.9.0
|
|
1325
|
+
"""
|
|
1326
|
+
return organizer.space_tseries(
|
|
1327
|
+
os.path.join(os.fspath(input_database), ''), prop, spec.parser,
|
|
1328
|
+
spec.hierarchy, list(spec.physical_attrs),
|
|
1329
|
+
spec.group if group is None else group,
|
|
1330
|
+
divisor=spec.divisor, round_to=round_to, is_save=save,
|
|
1331
|
+
crowder_attr=spec.crowder_attr)
|
|
1332
|
+
|
|
1333
|
+
|
|
1334
|
+
def space_hists_from_spec(
|
|
1335
|
+
input_database: Pathish,
|
|
1336
|
+
spec: ProjectSpec,
|
|
1337
|
+
prop: PropertyT,
|
|
1338
|
+
*,
|
|
1339
|
+
group: GroupT | None = None,
|
|
1340
|
+
bin_center: np.ndarray | None = None,
|
|
1341
|
+
normalize: bool = False,
|
|
1342
|
+
round_to: int = 3,
|
|
1343
|
+
save: bool = False,
|
|
1344
|
+
) -> pd.DataFrame:
|
|
1345
|
+
"""
|
|
1346
|
+
Stack the ensemble-averaged histograms of one property over a space,
|
|
1347
|
+
with the columns and the rounding step the study's spec declares.
|
|
1348
|
+
|
|
1349
|
+
The spec supplies every study choice of
|
|
1350
|
+
:func:`~polyphys.manage.organizer.space_hists`, as
|
|
1351
|
+
:func:`space_tseries_from_spec` describes; no study default is taken
|
|
1352
|
+
from the library.
|
|
1353
|
+
|
|
1354
|
+
Parameters
|
|
1355
|
+
----------
|
|
1356
|
+
input_database : path-like
|
|
1357
|
+
The ``ensAvg`` directory of the space, which holds the files
|
|
1358
|
+
``<ensemble_long>-<group>-<prop>-ensAvg.csv``.
|
|
1359
|
+
spec : ~polyphys.spec.ProjectSpec
|
|
1360
|
+
The study's spec.
|
|
1361
|
+
prop : str
|
|
1362
|
+
The property, such as ``'rHistCrd'``.
|
|
1363
|
+
group : str, optional
|
|
1364
|
+
The particle group of the files; the default is the polymer group,
|
|
1365
|
+
`spec.group`. A histogram of the crowders is in the ``'all'``
|
|
1366
|
+
group.
|
|
1367
|
+
bin_center : numpy.ndarray, optional
|
|
1368
|
+
The bin centers, shape ``(n_bins,)``; otherwise each file must hold
|
|
1369
|
+
a ``bin_center`` column.
|
|
1370
|
+
normalize : bool, default False
|
|
1371
|
+
Add ``<prop>-norm``, each file's ``<prop>-mean`` divided by its sum.
|
|
1372
|
+
round_to : int, default 3
|
|
1373
|
+
The number of decimal places of the rounded :math:`\\phi_c`.
|
|
1374
|
+
save : bool, default False
|
|
1375
|
+
Also write the table to ``<space>-<group>-<prop>-space.csv`` in the
|
|
1376
|
+
``analysis`` phase's ``space`` directory next to `input_database`.
|
|
1377
|
+
|
|
1378
|
+
Returns
|
|
1379
|
+
-------
|
|
1380
|
+
pandas.DataFrame
|
|
1381
|
+
The table of :func:`~polyphys.manage.organizer.space_hists`: each
|
|
1382
|
+
file's columns, one column per physical attribute, a
|
|
1383
|
+
`spec.crowder_attr` column, and ``phi_bulk_c_round``.
|
|
1384
|
+
|
|
1385
|
+
Warns
|
|
1386
|
+
-----
|
|
1387
|
+
UserWarning
|
|
1388
|
+
If `normalize` is true and a histogram sums to zero, as
|
|
1389
|
+
:func:`~polyphys.manage.organizer.space_hists` warns.
|
|
1390
|
+
|
|
1391
|
+
Notes
|
|
1392
|
+
-----
|
|
1393
|
+
The cost is that of :func:`~polyphys.manage.organizer.space_hists`:
|
|
1394
|
+
:math:`O(n_\\mathrm{rows}\\, n_\\mathrm{cols})` time and memory.
|
|
1395
|
+
|
|
1396
|
+
Examples
|
|
1397
|
+
--------
|
|
1398
|
+
>>> import tempfile
|
|
1399
|
+
>>> import numpy as np
|
|
1400
|
+
>>> import pandas as pd
|
|
1401
|
+
>>> from pathlib import Path
|
|
1402
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
1403
|
+
>>> from polyphys.spec import ProjectSpec
|
|
1404
|
+
>>> spec = ProjectSpec(
|
|
1405
|
+
... parser=ChainCyl, group='bug', divisor=0.1,
|
|
1406
|
+
... crowder_attr='phi_bulk_c', physical_attrs=('nmon',),
|
|
1407
|
+
... species_sizes={'Mon': 'dmon'}, tseries_properties=())
|
|
1408
|
+
>>> database = Path(tempfile.mkdtemp()) / 'analysis' / 'N20-all-ensAvg'
|
|
1409
|
+
>>> database.mkdir(parents=True)
|
|
1410
|
+
>>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
|
|
1411
|
+
>>> pd.DataFrame({'rHistMon-mean': [1.0, 3.0]}).to_csv(
|
|
1412
|
+
... database / f'{state}-all-rHistMon-ensAvg.csv', index=False)
|
|
1413
|
+
>>> space = space_hists_from_spec(
|
|
1414
|
+
... database, spec, 'rHistMon', group='all',
|
|
1415
|
+
... bin_center=np.array([0.5, 1.5]), normalize=True)
|
|
1416
|
+
>>> space[['bin_center', 'rHistMon-norm', 'phi_bulk_c_round']]
|
|
1417
|
+
bin_center rHistMon-norm phi_bulk_c_round
|
|
1418
|
+
0 0.5 0.25 0.0
|
|
1419
|
+
1 1.5 0.75 0.0
|
|
1420
|
+
|
|
1421
|
+
.. versionadded:: 0.9.0
|
|
1422
|
+
"""
|
|
1423
|
+
return organizer.space_hists(
|
|
1424
|
+
os.path.join(os.fspath(input_database), ''), prop, spec.parser,
|
|
1425
|
+
spec.hierarchy, list(spec.physical_attrs),
|
|
1426
|
+
spec.group if group is None else group,
|
|
1427
|
+
bin_center=bin_center, normalize=normalize, divisor=spec.divisor,
|
|
1428
|
+
round_to=round_to, is_save=save, crowder_attr=spec.crowder_attr)
|
|
1429
|
+
|
|
1430
|
+
|
|
1431
|
+
def equilibrium_wholes(
|
|
1432
|
+
ens_database: Pathish,
|
|
1433
|
+
properties: Mapping[PropertyT, str],
|
|
1434
|
+
whole_stamps: pd.DataFrame,
|
|
1435
|
+
*,
|
|
1436
|
+
method: PlateauMethod = 'lee',
|
|
1437
|
+
topology: str | None = None,
|
|
1438
|
+
) -> pd.DataFrame:
|
|
1439
|
+
"""
|
|
1440
|
+
The equilibrium value of each whole's time series, with its error,
|
|
1441
|
+
beside the whole's stamps.
|
|
1442
|
+
|
|
1443
|
+
For each property, every ensemble file of the ``ens`` stage is read
|
|
1444
|
+
once, and each of its columns, one whole's time series, is reduced to
|
|
1445
|
+
its time average, its variance, and the blocked standard error of the
|
|
1446
|
+
average. The results of all the properties are merged with
|
|
1447
|
+
`whole_stamps` on ``whole``, one row per whole: the table the
|
|
1448
|
+
research code's ``equilibrium_wholes`` built for the ``allInOne``
|
|
1449
|
+
phase.
|
|
1450
|
+
|
|
1451
|
+
Parameters
|
|
1452
|
+
----------
|
|
1453
|
+
ens_database : path-like
|
|
1454
|
+
The ``ens`` directory of one space and group,
|
|
1455
|
+
``<root>/analysis/<prefix>-<group>-ens``, whose
|
|
1456
|
+
``<ensemble_long>-<group>-<property>.csv`` files
|
|
1457
|
+
:func:`reduce_space` wrote.
|
|
1458
|
+
properties : mapping of str to str
|
|
1459
|
+
The time-series property of each ensemble file, mapped to its
|
|
1460
|
+
output name: ``{'gyrTMon': 'gyrMon'}`` reads the ``-gyrTMon.csv``
|
|
1461
|
+
files and writes ``gyrMon-mean``, ``gyrMon-var``, and
|
|
1462
|
+
``gyrMon-sem``.
|
|
1463
|
+
whole_stamps : pandas.DataFrame
|
|
1464
|
+
The stamps of the space's wholes, one row per whole, with a
|
|
1465
|
+
``whole`` column: the first result of :func:`reduce_stamps`.
|
|
1466
|
+
method : {'lee', 'sem_err'}, default 'lee'
|
|
1467
|
+
The plateau rule of :func:`~polyphys.analyze.stats.sem_blocked`;
|
|
1468
|
+
``'lee'`` is the published rule.
|
|
1469
|
+
topology : str, optional
|
|
1470
|
+
Appended to each whole's name as ``.<topology>`` before the merge,
|
|
1471
|
+
for stamps that the probe phase keyed by the whole's file name,
|
|
1472
|
+
such as ``...ens1.ring``; the ensemble files name a whole without
|
|
1473
|
+
it.
|
|
1474
|
+
|
|
1475
|
+
Returns
|
|
1476
|
+
-------
|
|
1477
|
+
pandas.DataFrame
|
|
1478
|
+
`whole_stamps` followed by, for each property in the order of
|
|
1479
|
+
`properties`, the columns ``<output>-mean``, ``<output>-var``, and
|
|
1480
|
+
``<output>-sem``, with the rows in the order of `whole_stamps`.
|
|
1481
|
+
|
|
1482
|
+
Raises
|
|
1483
|
+
------
|
|
1484
|
+
FileNotFoundError
|
|
1485
|
+
If `ens_database` holds no CSV file, or a property matches none.
|
|
1486
|
+
ValueError
|
|
1487
|
+
If `whole_stamps` has no ``whole`` column or lists a whole twice;
|
|
1488
|
+
if two properties share an output name; if the wholes of a
|
|
1489
|
+
property's files differ from those of the stamps, which names the
|
|
1490
|
+
wholes on each side; or as
|
|
1491
|
+
:func:`~polyphys.analyze.stats.block_average` raises for a series
|
|
1492
|
+
that is too short or holds NaN.
|
|
1493
|
+
|
|
1494
|
+
Warns
|
|
1495
|
+
-----
|
|
1496
|
+
RuntimeWarning
|
|
1497
|
+
As :func:`~polyphys.analyze.stats.select_plateau`: when a whole's
|
|
1498
|
+
series is too short for its correlation time, its ``-sem`` is NaN
|
|
1499
|
+
or unreliable.
|
|
1500
|
+
|
|
1501
|
+
Notes
|
|
1502
|
+
-----
|
|
1503
|
+
``-mean`` is the time average over the frames, ``-var`` the sample
|
|
1504
|
+
variance of the frames (``ddof = 1``), and ``-sem`` the standard error
|
|
1505
|
+
of the time average by block averaging, which accounts for the
|
|
1506
|
+
correlation between consecutive frames of one run (Flyvbjerg and
|
|
1507
|
+
Petersen, 1989; the plateau rule of Lee et al., 2011; both in
|
|
1508
|
+
:mod:`polyphys.analyze.stats`). A constant series has a ``-sem`` of 0.
|
|
1509
|
+
Blocking applies within a whole only: the wholes of an ensemble are
|
|
1510
|
+
independent runs, so :func:`~polyphys.manage.organizer.ensemble_avg`
|
|
1511
|
+
keeps the plain standard error across them. The research code took
|
|
1512
|
+
``-sem`` as the per-frame standard error, which the blocked one
|
|
1513
|
+
exceeds by about :math:`\\sqrt{s}` for a statistical inefficiency
|
|
1514
|
+
:math:`s`, and its ``-var`` as the population variance.
|
|
1515
|
+
|
|
1516
|
+
With :math:`n_\\mathrm{frames}` the frames of the longest series and
|
|
1517
|
+
:math:`n_\\mathrm{wholes}` the wholes of the space, time is
|
|
1518
|
+
:math:`O(n_\\mathrm{wholes}\\, n_\\mathrm{frames})` per property, and
|
|
1519
|
+
space is one ensemble file at a time plus a ``float64`` copy of it.
|
|
1520
|
+
|
|
1521
|
+
Examples
|
|
1522
|
+
--------
|
|
1523
|
+
Two wholes of one ensemble whose series alternate between two values,
|
|
1524
|
+
so their time averages are exact and the blocked error vanishes at a
|
|
1525
|
+
block size of two frames:
|
|
1526
|
+
|
|
1527
|
+
>>> import tempfile
|
|
1528
|
+
>>> import numpy as np
|
|
1529
|
+
>>> import pandas as pd
|
|
1530
|
+
>>> stem = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
|
|
1531
|
+
>>> wholes = [f'{stem}ens1', f'{stem}ens2']
|
|
1532
|
+
>>> root = tempfile.mkdtemp()
|
|
1533
|
+
>>> pd.DataFrame({wholes[0]: np.tile([0.0, 2.0], 2048),
|
|
1534
|
+
... wholes[1]: np.tile([1.0, 3.0], 2048)}
|
|
1535
|
+
... ).to_csv(f'{root}/{stem}-bug-gyrTMon.csv', index=False)
|
|
1536
|
+
>>> stamps = pd.DataFrame({'whole': wholes, 'n_frames': [4096, 4096]})
|
|
1537
|
+
>>> table = equilibrium_wholes(root, {'gyrTMon': 'gyrMon'}, stamps)
|
|
1538
|
+
>>> table.drop(columns='whole').round(3)
|
|
1539
|
+
n_frames gyrMon-mean gyrMon-var gyrMon-sem
|
|
1540
|
+
0 4096 1.0 1.0 0.0
|
|
1541
|
+
1 4096 2.0 1.0 0.0
|
|
1542
|
+
|
|
1543
|
+
.. versionadded:: 0.6.0
|
|
1544
|
+
"""
|
|
1545
|
+
if 'whole' not in whole_stamps.columns:
|
|
1546
|
+
raise ValueError("whole_stamps needs a 'whole' column.")
|
|
1547
|
+
expected = whole_stamps['whole'].astype(str).to_list()
|
|
1548
|
+
repeated_wholes = sorted({w for w in expected if expected.count(w) > 1})
|
|
1549
|
+
if repeated_wholes:
|
|
1550
|
+
raise ValueError(
|
|
1551
|
+
f"whole_stamps lists wholes more than once: {repeated_wholes}."
|
|
1552
|
+
)
|
|
1553
|
+
outputs = list(properties.values())
|
|
1554
|
+
repeated = sorted({out for out in outputs if outputs.count(out) > 1})
|
|
1555
|
+
if repeated:
|
|
1556
|
+
raise ValueError(f"Output names given more than once: {repeated}.")
|
|
1557
|
+
pattern = os.path.join(os.fspath(ens_database), '*.csv')
|
|
1558
|
+
observations = sorted(glob.glob(pattern))
|
|
1559
|
+
if not observations:
|
|
1560
|
+
raise FileNotFoundError(f"No file matches '{pattern}'.")
|
|
1561
|
+
suffix = '' if topology is None else f'.{topology}'
|
|
1562
|
+
|
|
1563
|
+
frames: list[pd.DataFrame] = []
|
|
1564
|
+
for prop, output in properties.items():
|
|
1565
|
+
logger.info("Reducing the time series of '%s' to '%s'.", prop,
|
|
1566
|
+
output)
|
|
1567
|
+
parts: list[pd.DataFrame] = []
|
|
1568
|
+
for (path,) in _select(observations, prop, ext='csv'):
|
|
1569
|
+
ensemble = pd.read_csv(path)
|
|
1570
|
+
values = ensemble.to_numpy(dtype=np.float64)
|
|
1571
|
+
parts.append(pd.DataFrame({
|
|
1572
|
+
f'{output}-mean': values.mean(axis=0),
|
|
1573
|
+
f'{output}-var': values.var(axis=0, ddof=1),
|
|
1574
|
+
f'{output}-sem': _blocked_errors(values, method),
|
|
1575
|
+
}, index=[f'{name}{suffix}' for name in ensemble.columns]))
|
|
1576
|
+
measured = pd.concat(parts)
|
|
1577
|
+
_check_wholes(prop, list(measured.index), expected)
|
|
1578
|
+
frames.append(measured.loc[expected])
|
|
1579
|
+
combined = pd.concat(frames, axis=1)
|
|
1580
|
+
combined.index = whole_stamps.index
|
|
1581
|
+
return pd.concat([whole_stamps, combined], axis=1)
|
|
1582
|
+
|
|
1583
|
+
|
|
1584
|
+
# ---------------------------------------------------------------------------
|
|
1585
|
+
# Private helpers
|
|
1586
|
+
# ---------------------------------------------------------------------------
|
|
1587
|
+
|
|
1588
|
+
|
|
1589
|
+
def _check_whole_type(whole_type: str) -> None:
|
|
1590
|
+
"""Raise ``ValueError`` unless `whole_type` is a known whole type."""
|
|
1591
|
+
if whole_type not in ('vector', 'matrix', 'bin_edge', 'dataframe'):
|
|
1592
|
+
raise ValueError(
|
|
1593
|
+
"whole_type must be 'vector', 'matrix', 'bin_edge', or "
|
|
1594
|
+
f"'dataframe', not '{whole_type}'."
|
|
1595
|
+
)
|
|
1596
|
+
|
|
1597
|
+
|
|
1598
|
+
def _natural_key(path: str) -> list[str | int]:
|
|
1599
|
+
"""
|
|
1600
|
+
The sort key that orders a file name's digit runs as numbers, so
|
|
1601
|
+
``j2`` comes before ``j10``.
|
|
1602
|
+
"""
|
|
1603
|
+
return [int(token) if token.isdigit() else token
|
|
1604
|
+
for token in re.split(r'(\d+)', os.path.basename(path))]
|
|
1605
|
+
|
|
1606
|
+
|
|
1607
|
+
def _select(
|
|
1608
|
+
observations: Sequence[str], prop: str, ext: str = 'npy'
|
|
1609
|
+
) -> list[tuple[str]]:
|
|
1610
|
+
"""
|
|
1611
|
+
The files of property `prop` among `observations`, as 1-tuples in
|
|
1612
|
+
natural order of their names.
|
|
1613
|
+
|
|
1614
|
+
A file belongs to `prop` when its name ends in ``-<prop>.<ext>`` (a
|
|
1615
|
+
whole) or ``.<prop>.<ext>`` (a segment), so ``gyrTMon`` never matches
|
|
1616
|
+
``xgyrTMon``. Raises ``FileNotFoundError`` when nothing matches.
|
|
1617
|
+
:func:`~polyphys.manage.utils.sort_filenames` is not used because it
|
|
1618
|
+
reads a name's stem up to its first dot, which a keyword value such as
|
|
1619
|
+
``dt0.005`` holds.
|
|
1620
|
+
"""
|
|
1621
|
+
suffixes = (f'-{prop}.{ext}', f'.{prop}.{ext}')
|
|
1622
|
+
matches = [path for path in observations if path.endswith(suffixes)]
|
|
1623
|
+
if not matches:
|
|
1624
|
+
raise FileNotFoundError(
|
|
1625
|
+
f"No file of property '{prop}': none of the {len(observations)}"
|
|
1626
|
+
f" files ends in '{suffixes[0]}' or '{suffixes[1]}'."
|
|
1627
|
+
)
|
|
1628
|
+
return [(path,) for path in sorted(matches, key=_natural_key)]
|
|
1629
|
+
|
|
1630
|
+
|
|
1631
|
+
def _load_wholes(
|
|
1632
|
+
prop: str,
|
|
1633
|
+
relation: WholeRelationT,
|
|
1634
|
+
observations: Sequence[str],
|
|
1635
|
+
parser: type[ParserBase],
|
|
1636
|
+
group: GroupT,
|
|
1637
|
+
lineage: PrimitiveLineageT,
|
|
1638
|
+
whole_dir: str | None,
|
|
1639
|
+
) -> dict[str, np.ndarray]:
|
|
1640
|
+
"""Build the wholes of `prop` from its segment or whole files."""
|
|
1641
|
+
paths = _select(observations, prop)
|
|
1642
|
+
if lineage == 'segment':
|
|
1643
|
+
return organizer.whole_from_segments(
|
|
1644
|
+
prop, paths, parser, group, relation, save_to=whole_dir)
|
|
1645
|
+
return organizer.whole_from_file(paths, parser, group)
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
def _check_edges(
|
|
1649
|
+
prop: _spec.PropertySpec,
|
|
1650
|
+
wholes: Mapping[str, np.ndarray],
|
|
1651
|
+
edges: Mapping[str, np.ndarray],
|
|
1652
|
+
parser: type[ParserBase],
|
|
1653
|
+
group: GroupT,
|
|
1654
|
+
) -> None:
|
|
1655
|
+
"""
|
|
1656
|
+
Raise ``ValueError`` unless every whole has edges and the wholes of each
|
|
1657
|
+
ensemble hold identical edges.
|
|
1658
|
+
"""
|
|
1659
|
+
missing = sorted(set(wholes) - set(edges))
|
|
1660
|
+
if missing:
|
|
1661
|
+
raise ValueError(
|
|
1662
|
+
f"'{prop.prop}': no '{prop.edge_prop}' file for the wholes "
|
|
1663
|
+
f"{missing}."
|
|
1664
|
+
)
|
|
1665
|
+
first_of: dict[str, str] = {}
|
|
1666
|
+
for whole_name in wholes:
|
|
1667
|
+
ens_name = str(getattr(parser(whole_name, 'whole', group),
|
|
1668
|
+
'ensemble_long'))
|
|
1669
|
+
reference = first_of.setdefault(ens_name, whole_name)
|
|
1670
|
+
if not np.array_equal(edges[reference], edges[whole_name]):
|
|
1671
|
+
raise ValueError(
|
|
1672
|
+
f"'{prop.prop}': the wholes '{reference}' and '{whole_name}'"
|
|
1673
|
+
f" of ensemble '{ens_name}' hold different bin edges."
|
|
1674
|
+
)
|
|
1675
|
+
|
|
1676
|
+
|
|
1677
|
+
def _reduce_output(
|
|
1678
|
+
output: Derived,
|
|
1679
|
+
parser: type[ParserBase],
|
|
1680
|
+
group: GroupT,
|
|
1681
|
+
ens_dir: str | None,
|
|
1682
|
+
ens_avg_dir: str | None,
|
|
1683
|
+
) -> dict[str, np.ndarray | pd.DataFrame]:
|
|
1684
|
+
"""Build and save the ensembles and ensemble averages of one output."""
|
|
1685
|
+
if output.whole_type == 'dataframe':
|
|
1686
|
+
frames = {name: data for name, data in output.wholes.items()
|
|
1687
|
+
if isinstance(data, pd.DataFrame)}
|
|
1688
|
+
if len(frames) != len(output.wholes):
|
|
1689
|
+
raise ValueError(
|
|
1690
|
+
f"'{output.name}': every 'dataframe' whole must be a "
|
|
1691
|
+
"pandas.DataFrame."
|
|
1692
|
+
)
|
|
1693
|
+
long_ensembles = _frame_ensembles(output.name, frames, parser, group,
|
|
1694
|
+
ens_dir)
|
|
1695
|
+
return {
|
|
1696
|
+
ens_name: _frame_ensemble_avg(output.name, long_frame,
|
|
1697
|
+
ens_name, group, ens_avg_dir)
|
|
1698
|
+
for ens_name, long_frame in long_ensembles.items()
|
|
1699
|
+
}
|
|
1700
|
+
ensembles = organizer.ensemble(
|
|
1701
|
+
output.name, output.wholes, parser, group, output.whole_type,
|
|
1702
|
+
whole_edges=(dict(output.edges) if output.edges is not None
|
|
1703
|
+
else None),
|
|
1704
|
+
save_to=ens_dir,
|
|
1705
|
+
)
|
|
1706
|
+
if output.normalizer is None:
|
|
1707
|
+
averages = organizer.ensemble_avg(
|
|
1708
|
+
output.name, ensembles, group, _ENS_TYPES[output.whole_type],
|
|
1709
|
+
save_to=ens_avg_dir,
|
|
1710
|
+
)
|
|
1711
|
+
return dict(averages)
|
|
1712
|
+
# The normalized spread needs the wholes and the average together, so
|
|
1713
|
+
# the column is added before the average is saved.
|
|
1714
|
+
averages = organizer.ensemble_avg(
|
|
1715
|
+
output.name, ensembles, group, _ENS_TYPES[output.whole_type])
|
|
1716
|
+
normalized: dict[str, np.ndarray | pd.DataFrame] = {}
|
|
1717
|
+
for ens_name, average in averages.items():
|
|
1718
|
+
ensemble = cast(pd.DataFrame, ensembles[ens_name])
|
|
1719
|
+
frame = cast(pd.DataFrame, average)
|
|
1720
|
+
frame[f'{output.name}-norm-sem'] = _normalized_sem(
|
|
1721
|
+
output.name, ensemble, frame, output.normalizer, ens_name)
|
|
1722
|
+
normalized[ens_name] = frame
|
|
1723
|
+
if ens_avg_dir is not None:
|
|
1724
|
+
organizer.save_artifact(
|
|
1725
|
+
organizer.create_fullname(ens_name, group,
|
|
1726
|
+
f'{output.name}-ensAvg'),
|
|
1727
|
+
frame, ens_avg_dir)
|
|
1728
|
+
return normalized
|
|
1729
|
+
|
|
1730
|
+
|
|
1731
|
+
def _normalized_sem(
|
|
1732
|
+
prop: str,
|
|
1733
|
+
ensemble: pd.DataFrame,
|
|
1734
|
+
average: pd.DataFrame,
|
|
1735
|
+
normalizer: ProfileNormalizerT,
|
|
1736
|
+
ens_name: str,
|
|
1737
|
+
) -> np.ndarray:
|
|
1738
|
+
"""
|
|
1739
|
+
The standard error over the wholes of `ensemble` (one column per whole,
|
|
1740
|
+
after ``bin_center``) of each whole divided by its own value in the
|
|
1741
|
+
reference bin: bin 0 for ``'first'``, the last bin for ``'last'``, and
|
|
1742
|
+
the bin where ``<prop>-mean`` of `average` is largest for ``'max'``.
|
|
1743
|
+
A whole whose reference value is zero warns, naming the ensemble, and
|
|
1744
|
+
gives NaN in every bin. Time and space are
|
|
1745
|
+
:math:`O(n_\\mathrm{bins} n_\\mathrm{wholes})`.
|
|
1746
|
+
"""
|
|
1747
|
+
wholes = ensemble.drop(columns='bin_center', errors='ignore')
|
|
1748
|
+
values = wholes.to_numpy(dtype=np.float64)
|
|
1749
|
+
if normalizer == 'first':
|
|
1750
|
+
index = 0
|
|
1751
|
+
elif normalizer == 'last':
|
|
1752
|
+
index = len(values) - 1
|
|
1753
|
+
else:
|
|
1754
|
+
index = int(np.argmax(average[f'{prop}-mean'].to_numpy()))
|
|
1755
|
+
references = values[index]
|
|
1756
|
+
if np.any(references == 0):
|
|
1757
|
+
warnings.warn(
|
|
1758
|
+
f"'{prop}': a whole of ensemble '{ens_name}' is zero in its "
|
|
1759
|
+
f"reference bin {index}; its '{prop}-norm-sem' is NaN.",
|
|
1760
|
+
UserWarning,
|
|
1761
|
+
)
|
|
1762
|
+
return np.full(len(values), np.nan)
|
|
1763
|
+
normalized = pd.DataFrame(values / references)
|
|
1764
|
+
return normalized.sem(axis=1, ddof=1).to_numpy()
|
|
1765
|
+
|
|
1766
|
+
|
|
1767
|
+
def _frame_ensembles(
|
|
1768
|
+
prop: str,
|
|
1769
|
+
wholes: Mapping[str, pd.DataFrame],
|
|
1770
|
+
parser: type[ParserBase],
|
|
1771
|
+
group: GroupT,
|
|
1772
|
+
save_to: str | None,
|
|
1773
|
+
) -> dict[str, pd.DataFrame]:
|
|
1774
|
+
"""
|
|
1775
|
+
The long-form ensembles of ``'dataframe'`` wholes: every whole's rows,
|
|
1776
|
+
with a ``whole`` and a ``row`` column, saved as
|
|
1777
|
+
``<ensemble_long>-<group>-<prop>.csv``. Raises ``ValueError`` if the
|
|
1778
|
+
wholes of an ensemble have unequal numbers of rows, as
|
|
1779
|
+
:func:`~polyphys.manage.organizer.ens_from_df` does, before anything is
|
|
1780
|
+
saved; ``_frame_ensemble_avg`` would otherwise take its rows from
|
|
1781
|
+
the first whole.
|
|
1782
|
+
"""
|
|
1783
|
+
members: dict[str, dict[str, pd.DataFrame]] = {}
|
|
1784
|
+
for whole_name, frame in wholes.items():
|
|
1785
|
+
ens_name = str(getattr(parser(whole_name, 'whole', group),
|
|
1786
|
+
'ensemble_long'))
|
|
1787
|
+
members.setdefault(ens_name, {})[whole_name] = frame
|
|
1788
|
+
ensembles: dict[str, pd.DataFrame] = {}
|
|
1789
|
+
for ens_name, frames in members.items():
|
|
1790
|
+
organizer._assert_equal_lengths(
|
|
1791
|
+
ens_name, {name: len(frame) for name, frame in frames.items()})
|
|
1792
|
+
long_frame = pd.concat(
|
|
1793
|
+
list(frames.values()), keys=list(frames),
|
|
1794
|
+
names=['whole', 'row'],
|
|
1795
|
+
).reset_index()
|
|
1796
|
+
ensembles[ens_name] = long_frame
|
|
1797
|
+
if save_to is not None:
|
|
1798
|
+
organizer.save_artifact(
|
|
1799
|
+
organizer.create_fullname(ens_name, group, prop),
|
|
1800
|
+
long_frame, save_to)
|
|
1801
|
+
return ensembles
|
|
1802
|
+
|
|
1803
|
+
|
|
1804
|
+
def _frame_ensemble_avg(
|
|
1805
|
+
prop: str,
|
|
1806
|
+
long_frame: pd.DataFrame,
|
|
1807
|
+
ens_name: str,
|
|
1808
|
+
group: GroupT,
|
|
1809
|
+
save_to: str | None,
|
|
1810
|
+
exclude: Sequence[str] = ('bin_center',),
|
|
1811
|
+
) -> pd.DataFrame:
|
|
1812
|
+
"""
|
|
1813
|
+
The row-by-row mean, variance, and standard error across the wholes of
|
|
1814
|
+
one long-form ensemble, as ``<column>-mean``, ``<column>-var``, and
|
|
1815
|
+
``<column>-sem`` per column, after the `exclude` columns of the first
|
|
1816
|
+
whole; saved as ``<ensemble_long>-<group>-<prop>-ensAvg.csv``.
|
|
1817
|
+
"""
|
|
1818
|
+
kept = [col for col in exclude if col in long_frame.columns]
|
|
1819
|
+
first_whole = long_frame['whole'].iloc[0]
|
|
1820
|
+
first = long_frame.loc[long_frame['whole'] == first_whole]
|
|
1821
|
+
average: pd.DataFrame = first.set_index('row').loc[:, kept]
|
|
1822
|
+
grouped = long_frame.drop(columns=['whole', *kept]).groupby('row')
|
|
1823
|
+
means = grouped.mean()
|
|
1824
|
+
variances = grouped.var(ddof=1)
|
|
1825
|
+
errors = grouped.sem(ddof=1)
|
|
1826
|
+
for column in means.columns:
|
|
1827
|
+
average[f'{column}-mean'] = means[column]
|
|
1828
|
+
average[f'{column}-var'] = variances[column]
|
|
1829
|
+
average[f'{column}-sem'] = errors[column]
|
|
1830
|
+
result: pd.DataFrame = average.reset_index(drop=True)
|
|
1831
|
+
if save_to is not None:
|
|
1832
|
+
organizer.save_artifact(
|
|
1833
|
+
organizer.create_fullname(ens_name, group, f'{prop}-ensAvg'),
|
|
1834
|
+
result, save_to)
|
|
1835
|
+
return result
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
def _blocked_errors(values: np.ndarray, method: PlateauMethod) -> np.ndarray:
|
|
1839
|
+
"""
|
|
1840
|
+
The blocked standard error of each column of `values`, shape
|
|
1841
|
+
``(n_frames, n_series)``, by :func:`~polyphys.analyze.stats.sem_blocked`;
|
|
1842
|
+
0 for a constant column, which
|
|
1843
|
+
:func:`~polyphys.analyze.stats.block_average` rejects. A column
|
|
1844
|
+
with NaN is passed on, so that it raises there.
|
|
1845
|
+
"""
|
|
1846
|
+
errors = np.zeros(values.shape[1])
|
|
1847
|
+
varying = ~(values.var(axis=0) == 0.0)
|
|
1848
|
+
if varying.any():
|
|
1849
|
+
errors[varying] = np.asarray(
|
|
1850
|
+
stats.sem_blocked(values[:, varying], method=method))
|
|
1851
|
+
return errors
|
|
1852
|
+
|
|
1853
|
+
|
|
1854
|
+
def _check_wholes(
|
|
1855
|
+
prop: str, found: Sequence[str], expected: Sequence[str]
|
|
1856
|
+
) -> None:
|
|
1857
|
+
"""
|
|
1858
|
+
Raise ``ValueError`` unless the wholes `found` in a property's ensemble
|
|
1859
|
+
files are exactly the `expected` wholes of the stamps.
|
|
1860
|
+
"""
|
|
1861
|
+
missing = sorted(set(expected) - set(found))
|
|
1862
|
+
unknown = sorted(set(found) - set(expected))
|
|
1863
|
+
if missing or unknown:
|
|
1864
|
+
raise ValueError(
|
|
1865
|
+
f"'{prop}': the wholes of its ensemble files and of whole_stamps "
|
|
1866
|
+
f"differ. In the stamps only: {missing}. In the files only: "
|
|
1867
|
+
f"{unknown}. If the names differ by a suffix, pass topology."
|
|
1868
|
+
)
|