polyphys 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyphys/__init__.py +7 -0
- polyphys/__version__.py +22 -0
- polyphys/analyze/__init__.py +51 -0
- polyphys/analyze/clusters.py +1738 -0
- polyphys/analyze/correlations.py +995 -0
- polyphys/analyze/density.py +1138 -0
- polyphys/analyze/measurer.py +1338 -0
- polyphys/analyze/pipeline.py +1868 -0
- polyphys/analyze/stats.py +810 -0
- polyphys/analyze/sumrule.py +1159 -0
- polyphys/manage/__init__.py +11 -0
- polyphys/manage/organizer.py +2188 -0
- polyphys/manage/parser.py +1264 -0
- polyphys/manage/types.py +171 -0
- polyphys/manage/utils.py +777 -0
- polyphys/probe/__init__.py +25 -0
- polyphys/probe/binders.py +711 -0
- polyphys/probe/chain_size.py +583 -0
- polyphys/probe/core.py +437 -0
- polyphys/probe/histograms.py +863 -0
- polyphys/probe/pairs.py +555 -0
- polyphys/simulate/__init__.py +48 -0
- polyphys/simulate/build.py +1231 -0
- polyphys/simulate/core.py +237 -0
- polyphys/simulate/data.py +977 -0
- polyphys/simulate/dump.py +752 -0
- polyphys/simulate/log.py +788 -0
- polyphys/spec.py +1359 -0
- polyphys/tests/analyze/test_clusters.py +1058 -0
- polyphys/tests/analyze/test_correlations.py +708 -0
- polyphys/tests/analyze/test_density.py +813 -0
- polyphys/tests/analyze/test_measurer.py +785 -0
- polyphys/tests/analyze/test_pipeline.py +1881 -0
- polyphys/tests/analyze/test_stats.py +732 -0
- polyphys/tests/analyze/test_sumrule.py +917 -0
- polyphys/tests/conftest.py +753 -0
- polyphys/tests/data/lammps_build/make_starts.py +70 -0
- polyphys/tests/manage/test_organizer.py +1495 -0
- polyphys/tests/manage/test_parser.py +1222 -0
- polyphys/tests/manage/test_utils.py +498 -0
- polyphys/tests/probe/conftest.py +407 -0
- polyphys/tests/probe/test_binders.py +686 -0
- polyphys/tests/probe/test_chain_size.py +557 -0
- polyphys/tests/probe/test_core.py +391 -0
- polyphys/tests/probe/test_histograms.py +779 -0
- polyphys/tests/probe/test_pairs.py +475 -0
- polyphys/tests/simulate/test_build.py +1042 -0
- polyphys/tests/simulate/test_core.py +106 -0
- polyphys/tests/simulate/test_data.py +858 -0
- polyphys/tests/simulate/test_dump.py +487 -0
- polyphys/tests/simulate/test_log.py +667 -0
- polyphys/tests/strategies.py +120 -0
- polyphys/tests/synthetic.py +410 -0
- polyphys/tests/test_conftest.py +414 -0
- polyphys/tests/test_docs.py +82 -0
- polyphys/tests/test_packaging.py +97 -0
- polyphys/tests/test_spec.py +772 -0
- polyphys/tests/viz/conftest.py +32 -0
- polyphys/tests/viz/test_colors.py +113 -0
- polyphys/tests/viz/test_helpers.py +292 -0
- polyphys/viz/__init__.py +15 -0
- polyphys/viz/colors.py +204 -0
- polyphys/viz/helpers.py +549 -0
- polyphys-1.0.0.dist-info/METADATA +339 -0
- polyphys-1.0.0.dist-info/RECORD +69 -0
- polyphys-1.0.0.dist-info/WHEEL +5 -0
- polyphys-1.0.0.dist-info/licenses/AUTHORS.rst +13 -0
- polyphys-1.0.0.dist-info/licenses/LICENSE +22 -0
- polyphys-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2188 @@
|
|
|
1
|
+
"""\
|
|
2
|
+
==========================================================
|
|
3
|
+
:mod:`polyphys.manage.organizer`
|
|
4
|
+
==========================================================
|
|
5
|
+
The :mod:`polyphys.manage.organizer` module provides tools for organizing and
|
|
6
|
+
aggregating simulation data based on filename patterns, file structures, and
|
|
7
|
+
processing stages. It facilitates handling segmented data, grouping experiments
|
|
8
|
+
into ensembles, and consolidating ensembles into spaces.
|
|
9
|
+
|
|
10
|
+
The module is tailored for use with LAMMPS and similar simulation tools, where
|
|
11
|
+
it automates data consolidation across hierarchical levels. It is particularly
|
|
12
|
+
useful for organizing data generated in high-throughput simulations, enabling
|
|
13
|
+
efficient post-processing, ensemble averaging, and preparation of data for
|
|
14
|
+
statistical and comparative analysis.
|
|
15
|
+
|
|
16
|
+
Terminology
|
|
17
|
+
===========
|
|
18
|
+
The following terms are used throughout the module to structure and organize
|
|
19
|
+
simulation output data:
|
|
20
|
+
|
|
21
|
+
- **Experiment**: A complete simulation run, typically using tools like LAMMPS,
|
|
22
|
+
representing a specific set of conditions.
|
|
23
|
+
|
|
24
|
+
- **Lineage**: The level of an artifact in the hierarchy
|
|
25
|
+
``segment → whole → ensemble_long → ensemble → space``, the values of
|
|
26
|
+
:py:obj:`~.LineageT`; each level aggregates the one
|
|
27
|
+
before it:
|
|
28
|
+
|
|
29
|
+
- **segment**: a chunk of one simulation run.
|
|
30
|
+
- **whole**: one run, its segments joined.
|
|
31
|
+
- **ensemble_long**: one state point, every input parameter in the name.
|
|
32
|
+
- **ensemble**: one state point, its replicates (wholes with different
|
|
33
|
+
random seeds) collapsed into a short name.
|
|
34
|
+
- **space**: the parameter sweep, the state points of one study.
|
|
35
|
+
|
|
36
|
+
- **Attributes**: Macroscopic characteristics such as box size or polymer
|
|
37
|
+
topology that remain constant within a space.
|
|
38
|
+
|
|
39
|
+
- **Properties**: Physical features measured during the simulation, like
|
|
40
|
+
density or radius of gyration, with optional associated measures (e.g.,
|
|
41
|
+
autocorrelation).
|
|
42
|
+
|
|
43
|
+
- **Species**: The particle or molecular entity type, such as 'Mon' for
|
|
44
|
+
monomers, each with specific behaviors or attributes as needed.
|
|
45
|
+
|
|
46
|
+
- **Phases**: Directories organized by processing phase: `simsAll` and
|
|
47
|
+
`simsCont` (simulation inputs), `logs`, `trjs` (trajectories), `probes`
|
|
48
|
+
(probing data), `analysis` (post-processing), `viz` (visualization-ready
|
|
49
|
+
data), `galaxy`, and `allInOne`; the values of
|
|
50
|
+
:py:obj:`~.PhaseT`.
|
|
51
|
+
|
|
52
|
+
- **Stages**: Directories inside a phase, each holding the artifacts of one
|
|
53
|
+
lineage level or one step of the reduction: `segment`, `wholeSim`, `ens`,
|
|
54
|
+
`ensAvg`, `space`, and `galaxy`, the values of
|
|
55
|
+
:py:obj:`~.StageT`. The organizer writes them in the
|
|
56
|
+
`analysis` phase.
|
|
57
|
+
|
|
58
|
+
The words `segment` and `space` are both lineage levels and stages, and
|
|
59
|
+
`galaxy` is both a phase and a stage; which one a word is follows from where
|
|
60
|
+
it appears: a lineage argument, a phase directory, or the last field of a
|
|
61
|
+
stage directory's name. `galaxy` is not a lineage level.
|
|
62
|
+
|
|
63
|
+
Naming Conventions
|
|
64
|
+
==================
|
|
65
|
+
The :doc:`/lineage` page is the reference; in short:
|
|
66
|
+
|
|
67
|
+
- A stage directory is ``<root>/<phase>/<prefix>-<group>-<stage>``, as
|
|
68
|
+
:func:`make_database` builds it.
|
|
69
|
+
- A file is ``<lineage name>-<group>-<property>``, as
|
|
70
|
+
:func:`create_fullname` builds it, with ``-ensAvg`` or ``-space`` added
|
|
71
|
+
where two stages hold files of one level (``ens`` and ``ensAvg``) or the
|
|
72
|
+
level is ``space``, and then the extension. A stamps table is
|
|
73
|
+
``<space>-<group>-stamps-<level>.csv``, with ``<level>`` the lineage of
|
|
74
|
+
its rows.
|
|
75
|
+
- Fields are joined by ``-``. A lineage name never contains ``-``, and a
|
|
76
|
+
group cannot; a property may, to join a measure (``gyrTMon-acf``), so it
|
|
77
|
+
is everything between the group and the stage suffix or the extension.
|
|
78
|
+
|
|
79
|
+
Dependencies
|
|
80
|
+
============
|
|
81
|
+
- `numpy`: For numerical operations on arrays, such as summing or concatenating
|
|
82
|
+
segments.
|
|
83
|
+
- `pandas`: For handling tabular data and DataFrames used in data aggregation.
|
|
84
|
+
|
|
85
|
+
Contents
|
|
86
|
+
========
|
|
87
|
+
|
|
88
|
+
Naming and persistence:
|
|
89
|
+
:func:`create_fullname`, :func:`save_artifact`, and :func:`make_database`.
|
|
90
|
+
|
|
91
|
+
Building wholes from their parts:
|
|
92
|
+
:func:`whole_from_segments` and :func:`whole_from_file`.
|
|
93
|
+
|
|
94
|
+
Grouping wholes into ensembles:
|
|
95
|
+
:func:`ens_from_bin_edge`, :func:`ens_from_vec`, :func:`ens_from_mat_t`,
|
|
96
|
+
:func:`ens_from_df`, and :func:`ensemble`.
|
|
97
|
+
|
|
98
|
+
Averaging over an ensemble:
|
|
99
|
+
:func:`ens_avg_from_bin_edge`, :func:`ens_avg_from_ndarray`,
|
|
100
|
+
:func:`ens_avg_from_df`, and :func:`ensemble_avg`.
|
|
101
|
+
|
|
102
|
+
Space-level collection and bookkeeping:
|
|
103
|
+
:func:`children_stamps`, :func:`parents_stamps`, :func:`space_tseries`, and
|
|
104
|
+
:func:`space_hists`.
|
|
105
|
+
|
|
106
|
+
Reading tables the research code wrote:
|
|
107
|
+
:data:`BULK_NAME_MAP` and :func:`rename_bulk_columns`.
|
|
108
|
+
|
|
109
|
+
The density sum rule on the ``ensAvg`` profiles these reducers build is in
|
|
110
|
+
:mod:`polyphys.analyze.sumrule`.
|
|
111
|
+
|
|
112
|
+
Usage ===== The :mod:`~polyphys.manage.organizer` module streamlines
|
|
113
|
+
organization and aggregation of simulation data. Users can assemble segmented
|
|
114
|
+
data, organize data into ensembles, and generate ensemble-averaged data at
|
|
115
|
+
various aggregation stages. Its multi-level organization supports comparative
|
|
116
|
+
analysis across varied conditions within spaces and galaxies.
|
|
117
|
+
|
|
118
|
+
Examples
|
|
119
|
+
========
|
|
120
|
+
Two replicates (``ens1`` and ``ens2``) of one state point, each probed in
|
|
121
|
+
two segments (``j01`` and ``j02``), reduce to one ensemble-averaged time
|
|
122
|
+
series. Segment ``s`` of replicate ``e`` holds two frames of the value
|
|
123
|
+
``10 * e + s``:
|
|
124
|
+
|
|
125
|
+
>>> import tempfile
|
|
126
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
127
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
128
|
+
>>> root = Path(tmp.name)
|
|
129
|
+
>>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
|
|
130
|
+
>>> paths = []
|
|
131
|
+
>>> for e in (1, 2):
|
|
132
|
+
... for s in (1, 2):
|
|
133
|
+
... path = root / f'{state}ens{e}.j0{s}.bug.gyrTMon.npy'
|
|
134
|
+
... np.save(path, np.full(2, 10.0 * e + s))
|
|
135
|
+
... paths.append(path)
|
|
136
|
+
|
|
137
|
+
Join each replicate's segments into a whole, collect the wholes of the
|
|
138
|
+
state point into an ensemble, and average over the replicates:
|
|
139
|
+
|
|
140
|
+
>>> wholes = whole_from_segments('gyrTMon', paths, ChainCyl, 'bug',
|
|
141
|
+
... 'tseries')
|
|
142
|
+
>>> wholes[state + 'ens1']
|
|
143
|
+
array([11., 11., 12., 12.])
|
|
144
|
+
>>> ensembles = ensemble('gyrTMon', wholes, ChainCyl, 'bug', 'vector')
|
|
145
|
+
>>> averages = ensemble_avg('gyrTMon', ensembles, 'bug', 'dataframe')
|
|
146
|
+
>>> averages[state]
|
|
147
|
+
gyrTMon-mean gyrTMon-var gyrTMon-sem
|
|
148
|
+
0 16.0 50.0 5.0
|
|
149
|
+
1 16.0 50.0 5.0
|
|
150
|
+
2 17.0 50.0 5.0
|
|
151
|
+
3 17.0 50.0 5.0
|
|
152
|
+
>>> tmp.cleanup()
|
|
153
|
+
|
|
154
|
+
Notes
|
|
155
|
+
=====
|
|
156
|
+
- Filenames must follow consistent patterns for the `organizer` module to
|
|
157
|
+
accurately parse and organize data.
|
|
158
|
+
- This module supports only cylindrical, slit, or cubic geometries.
|
|
159
|
+
- Functions in this module rely on consistent naming conventions for groups,
|
|
160
|
+
properties, species, and stages.
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
import warnings
|
|
164
|
+
import logging
|
|
165
|
+
from typing import (
|
|
166
|
+
Any, Callable, Literal, Mapping, Sequence, Iterator, TYPE_CHECKING,
|
|
167
|
+
get_args,
|
|
168
|
+
)
|
|
169
|
+
import os
|
|
170
|
+
import pathlib
|
|
171
|
+
from pathlib import Path
|
|
172
|
+
from glob import glob
|
|
173
|
+
from types import MappingProxyType
|
|
174
|
+
import numpy as np
|
|
175
|
+
import pandas as pd
|
|
176
|
+
from .types import (
|
|
177
|
+
Pathish,
|
|
178
|
+
PathGroup,
|
|
179
|
+
AnyPathIter,
|
|
180
|
+
GroupT,
|
|
181
|
+
PropertyT,
|
|
182
|
+
ParserClassT,
|
|
183
|
+
StageT,
|
|
184
|
+
PhaseT,
|
|
185
|
+
WholeRelationT,
|
|
186
|
+
PrimitiveLineageT,
|
|
187
|
+
)
|
|
188
|
+
from .utils import invalid_keyword, sort_filenames, round_to_multiple
|
|
189
|
+
|
|
190
|
+
if TYPE_CHECKING:
|
|
191
|
+
# Unused here, but sphinx_autodoc_typehints runs this block to resolve
|
|
192
|
+
# the "ParserBase" forward reference inside ParserClassT.
|
|
193
|
+
from .parser import ParserBase # noqa: F401
|
|
194
|
+
|
|
195
|
+
__all__ = [
|
|
196
|
+
'BULK_NAME_MAP',
|
|
197
|
+
'children_stamps',
|
|
198
|
+
'create_fullname',
|
|
199
|
+
'ens_avg_from_bin_edge',
|
|
200
|
+
'ens_avg_from_df',
|
|
201
|
+
'ens_avg_from_ndarray',
|
|
202
|
+
'ens_from_bin_edge',
|
|
203
|
+
'ens_from_df',
|
|
204
|
+
'ens_from_mat_t',
|
|
205
|
+
'ens_from_vec',
|
|
206
|
+
'ensemble',
|
|
207
|
+
'ensemble_avg',
|
|
208
|
+
'make_database',
|
|
209
|
+
'parents_stamps',
|
|
210
|
+
'rename_bulk_columns',
|
|
211
|
+
'save_artifact',
|
|
212
|
+
'space_hists',
|
|
213
|
+
'space_tseries',
|
|
214
|
+
'whole_from_file',
|
|
215
|
+
'whole_from_segments',
|
|
216
|
+
]
|
|
217
|
+
|
|
218
|
+
logger = logging.getLogger(__name__)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def create_fullname(name: str, group: GroupT, prop: PropertyT) -> str:
|
|
222
|
+
"""
|
|
223
|
+
Join a base name, a particle group, and a property into an artifact's
|
|
224
|
+
full name.
|
|
225
|
+
|
|
226
|
+
Parameters
|
|
227
|
+
----------
|
|
228
|
+
name : str
|
|
229
|
+
The base name.
|
|
230
|
+
group : :py:obj:`~.GroupT`
|
|
231
|
+
Particle group (e.g., 'bug' or 'nucleoid').
|
|
232
|
+
prop : :py:obj:`~.PropertyT`
|
|
233
|
+
Physical property name (e.g., 'density').
|
|
234
|
+
|
|
235
|
+
Returns
|
|
236
|
+
-------
|
|
237
|
+
str
|
|
238
|
+
A string that combines `name`, `group`, and `prop` in a
|
|
239
|
+
hyphen-separated format.
|
|
240
|
+
|
|
241
|
+
Examples
|
|
242
|
+
--------
|
|
243
|
+
>>> create_fullname('sample', 'bug', 'density')
|
|
244
|
+
'sample-bug-density'
|
|
245
|
+
"""
|
|
246
|
+
return "-".join([name, group, prop])
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def save_artifact(
|
|
250
|
+
filename: str,
|
|
251
|
+
data: np.ndarray | pd.DataFrame | dict[str, np.ndarray],
|
|
252
|
+
save_to: Pathish,
|
|
253
|
+
) -> None:
|
|
254
|
+
"""
|
|
255
|
+
Save `data` as ``<save_to>/<filename>``, a ``.npy`` or ``.csv`` file by
|
|
256
|
+
its type.
|
|
257
|
+
|
|
258
|
+
An existing file of the same name is overwritten.
|
|
259
|
+
|
|
260
|
+
Parameters
|
|
261
|
+
----------
|
|
262
|
+
filename : str
|
|
263
|
+
The output file name.
|
|
264
|
+
data : np.ndarray | pd.DataFrame | dict[str, np.ndarray]
|
|
265
|
+
Data to be saved. Accepted types:
|
|
266
|
+
|
|
267
|
+
- `np.ndarray`: Saved as ``<filename>.npy``.
|
|
268
|
+
- `pd.DataFrame`: Saved as ``<filename>.csv``, without the index.
|
|
269
|
+
- `dict` of `np.ndarray`: Each key must have the form
|
|
270
|
+
``<prop>-<measure>``, with exactly one hyphen, and each entry is
|
|
271
|
+
saved as ``<filename>-<measure>.npy``.
|
|
272
|
+
|
|
273
|
+
save_to : Pathish
|
|
274
|
+
Path to an existing directory where the file is saved.
|
|
275
|
+
|
|
276
|
+
Notes
|
|
277
|
+
-----
|
|
278
|
+
:func:`numpy.save` and :meth:`pandas.DataFrame.to_csv` overwrite an
|
|
279
|
+
existing file of the same name.
|
|
280
|
+
|
|
281
|
+
Raises
|
|
282
|
+
------
|
|
283
|
+
TypeError
|
|
284
|
+
If `data` type does not match any of the supported formats.
|
|
285
|
+
ValueError
|
|
286
|
+
If `data` is a dict and one of its keys does not contain exactly one
|
|
287
|
+
hyphen.
|
|
288
|
+
|
|
289
|
+
Examples
|
|
290
|
+
--------
|
|
291
|
+
>>> import tempfile
|
|
292
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
293
|
+
|
|
294
|
+
Save a NumPy array to a ``.npy`` file:
|
|
295
|
+
|
|
296
|
+
>>> save_artifact('w-bug-rhoMon', np.array([1.0, 2.0]), tmp.name)
|
|
297
|
+
>>> np.load(Path(tmp.name) / 'w-bug-rhoMon.npy')
|
|
298
|
+
array([1., 2.])
|
|
299
|
+
|
|
300
|
+
Save a DataFrame to a ``.csv`` file:
|
|
301
|
+
|
|
302
|
+
>>> save_artifact('w-bug-rhoMon', pd.DataFrame({'A': [1, 2]}), tmp.name)
|
|
303
|
+
>>> pd.read_csv(Path(tmp.name) / 'w-bug-rhoMon.csv')
|
|
304
|
+
A
|
|
305
|
+
0 1
|
|
306
|
+
1 2
|
|
307
|
+
|
|
308
|
+
Save each entry of a dict to its own file, named by its measure:
|
|
309
|
+
|
|
310
|
+
>>> save_artifact('w-bug-rhoMon', {'rhoMon-mean': np.array([1.5])},
|
|
311
|
+
... tmp.name)
|
|
312
|
+
>>> sorted(path.name for path in Path(tmp.name).iterdir())
|
|
313
|
+
['w-bug-rhoMon-mean.npy', 'w-bug-rhoMon.csv', 'w-bug-rhoMon.npy']
|
|
314
|
+
>>> tmp.cleanup()
|
|
315
|
+
"""
|
|
316
|
+
file_path = os.path.join(save_to, filename)
|
|
317
|
+
|
|
318
|
+
# Save data based on type
|
|
319
|
+
if isinstance(data, pd.DataFrame):
|
|
320
|
+
data.to_csv(f"{file_path}.csv", index=False)
|
|
321
|
+
elif isinstance(data, np.ndarray):
|
|
322
|
+
np.save(f"{file_path}.npy", data)
|
|
323
|
+
elif isinstance(data, dict):
|
|
324
|
+
for prop_key, prop_data in data.items():
|
|
325
|
+
_, prop_measure = prop_key.split("-")
|
|
326
|
+
np.save(f"{file_path}-{prop_measure}.npy", prop_data)
|
|
327
|
+
else:
|
|
328
|
+
raise TypeError(
|
|
329
|
+
f"Unsupported data type {type(data).__name__}."
|
|
330
|
+
+ "Expected pd.DataFrame, np.ndarray, or dict of np.ndarray."
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def make_database(
|
|
335
|
+
old_database: Pathish,
|
|
336
|
+
phase: PhaseT,
|
|
337
|
+
stage: StageT,
|
|
338
|
+
group: GroupT | None,
|
|
339
|
+
) -> str:
|
|
340
|
+
"""
|
|
341
|
+
Create, if needed, the stage directory
|
|
342
|
+
``<root>/<phase>/<prefix>-<group>-<stage>/`` of the sweep that
|
|
343
|
+
`old_database` belongs to.
|
|
344
|
+
|
|
345
|
+
`old_database` is a directory of the same sweep that sits directly in a
|
|
346
|
+
phase directory, ``<root>/<old_phase>/<old_dir>``. The new directory
|
|
347
|
+
keeps ``<root>``, goes into `phase`, and is named
|
|
348
|
+
``<prefix>-<group>-<stage>``, with ``<prefix>`` the part of
|
|
349
|
+
``<old_dir>`` before its first hyphen (or ``*``) and ``<group>`` left
|
|
350
|
+
out when `group` is None. The :doc:`/lineage` page defines phases and
|
|
351
|
+
stages.
|
|
352
|
+
|
|
353
|
+
Parameters
|
|
354
|
+
----------
|
|
355
|
+
old_database : Pathish
|
|
356
|
+
Path to a directory of the sweep, directly inside a phase directory;
|
|
357
|
+
it need not exist.
|
|
358
|
+
phase : :py:obj:`~.PhaseT`
|
|
359
|
+
The new phase name for the directory, specifying its role in the
|
|
360
|
+
workflow: one of ``'simsAll'``, ``'simsCont'``, ``'logs'``,
|
|
361
|
+
``'trjs'``, ``'probes'``, ``'analysis'``, ``'viz'``, ``'galaxy'``,
|
|
362
|
+
or ``'allInOne'``.
|
|
363
|
+
stage : :py:obj:`~.StageT`
|
|
364
|
+
The stage of processing or data type represented in the new
|
|
365
|
+
directory: one of ``'segment'``, ``'wholeSim'``, ``'ens'``,
|
|
366
|
+
``'ensAvg'``, ``'space'``, or ``'galaxy'``.
|
|
367
|
+
group : GroupT or None
|
|
368
|
+
Particle group related to the data in the directory; left out of the
|
|
369
|
+
name when None. It cannot contain ``-``, the field separator of the
|
|
370
|
+
name.
|
|
371
|
+
|
|
372
|
+
Returns
|
|
373
|
+
-------
|
|
374
|
+
str
|
|
375
|
+
The path to the new directory, ending with ``/``. If the directory
|
|
376
|
+
already exists, the same path is returned and an INFO record naming
|
|
377
|
+
it is logged to the module's logger.
|
|
378
|
+
|
|
379
|
+
Raises
|
|
380
|
+
------
|
|
381
|
+
ValueError
|
|
382
|
+
If `phase` is not a value of `PhaseT`, `stage` not a value of
|
|
383
|
+
`StageT`, `group` contains ``-``, or the parent of `old_database` is
|
|
384
|
+
not named for a value of `PhaseT`; no directory is made.
|
|
385
|
+
|
|
386
|
+
Examples
|
|
387
|
+
--------
|
|
388
|
+
The new directory sits next to the old phase directory:
|
|
389
|
+
|
|
390
|
+
- Input: ``root/parent1/.../parentN/old_phase/old_dir``
|
|
391
|
+
- Result: ``root/parent1/.../parentN/phase/new_dir``
|
|
392
|
+
|
|
393
|
+
>>> import tempfile
|
|
394
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
395
|
+
>>> old = Path(tmp.name) / 'probes' / 'N20-bug-segment'
|
|
396
|
+
>>> new = make_database(old, 'analysis', 'ens', 'bug')
|
|
397
|
+
>>> Path(new).relative_to(tmp.name).as_posix()
|
|
398
|
+
'analysis/N20-bug-ens'
|
|
399
|
+
>>> Path(new).is_dir()
|
|
400
|
+
True
|
|
401
|
+
>>> tmp.cleanup()
|
|
402
|
+
|
|
403
|
+
A name outside the phase vocabulary is refused:
|
|
404
|
+
|
|
405
|
+
>>> make_database(old, 'simAll', 'ens', 'bug')
|
|
406
|
+
Traceback (most recent call last):
|
|
407
|
+
...
|
|
408
|
+
ValueError: 'simAll' is an invalid option. Please select one of: \
|
|
409
|
+
simsAll, simsCont, logs, trjs, probes, analysis, viz, galaxy, allInOne.
|
|
410
|
+
"""
|
|
411
|
+
# The type aliases are the one declaration of both vocabularies.
|
|
412
|
+
invalid_keyword(phase, get_args(PhaseT))
|
|
413
|
+
invalid_keyword(stage, get_args(StageT))
|
|
414
|
+
if group is not None and "-" in group:
|
|
415
|
+
raise ValueError(
|
|
416
|
+
f"group {group!r} contains '-', the field separator of a "
|
|
417
|
+
"directory name.")
|
|
418
|
+
|
|
419
|
+
old_path = pathlib.Path(old_database)
|
|
420
|
+
old_phase = old_path.parent.name
|
|
421
|
+
if old_phase not in get_args(PhaseT):
|
|
422
|
+
raise ValueError(
|
|
423
|
+
f"{old_phase!r} is not a phase: the parent of {str(old_path)!r} "
|
|
424
|
+
"must be a phase directory, one of "
|
|
425
|
+
f"{', '.join(get_args(PhaseT))}.")
|
|
426
|
+
|
|
427
|
+
# Common prefix derived from old_database to create new directory name
|
|
428
|
+
prefix = old_path.parts[-1].split("*")[0].split("-")[0]
|
|
429
|
+
new_directory = "-".join([part for part in [prefix, group, stage] if part])
|
|
430
|
+
|
|
431
|
+
# Construct the new database path in the same parent directory level as
|
|
432
|
+
# the old one
|
|
433
|
+
new_database_parts = list(old_path.parts[:-2]) + [phase, new_directory]
|
|
434
|
+
new_database = pathlib.Path(*new_database_parts)
|
|
435
|
+
|
|
436
|
+
# Avoid double slashes at the start of the path
|
|
437
|
+
if str(new_database).startswith("//"):
|
|
438
|
+
new_database = pathlib.Path(str(new_database)[1:])
|
|
439
|
+
|
|
440
|
+
# Create the new directory if it doesn't already exist
|
|
441
|
+
try:
|
|
442
|
+
new_database.mkdir(parents=True, exist_ok=False)
|
|
443
|
+
except FileExistsError:
|
|
444
|
+
logger.info("Reusing the existing directory %s.", new_database)
|
|
445
|
+
|
|
446
|
+
return str(new_database) + "/"
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _as_path(item: Pathish | PathGroup) -> Path:
|
|
450
|
+
"""Normalize a single item into a pathlib.Path.
|
|
451
|
+
|
|
452
|
+
Accepts either:
|
|
453
|
+
- a bare pathish (str, Path, os.PathLike)
|
|
454
|
+
- a sequence of pathish values (we take the first element)
|
|
455
|
+
"""
|
|
456
|
+
# Bare pathish?
|
|
457
|
+
if isinstance(item, (str, os.PathLike)):
|
|
458
|
+
return Path(item)
|
|
459
|
+
|
|
460
|
+
# Sequence of pathish (e.g., tuple[Path, ...] from sort_filenames)
|
|
461
|
+
if isinstance(item, Sequence) and item:
|
|
462
|
+
first = item[0]
|
|
463
|
+
if isinstance(first, (str, os.PathLike)):
|
|
464
|
+
return Path(first)
|
|
465
|
+
|
|
466
|
+
raise TypeError(f"Unsupported path item: {type(item)!r} -> {item!r}")
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _normalize_paths(paths: AnyPathIter) -> Iterator[Path]:
|
|
470
|
+
"""Yield normalized Path objects from either a flat or grouped iterable."""
|
|
471
|
+
for item in paths:
|
|
472
|
+
# item is either Pathish or PathGroup; _as_path handles both
|
|
473
|
+
yield _as_path(item)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _assert_equal_bin_edges(arrays: Sequence[np.ndarray]) -> np.ndarray:
|
|
477
|
+
"""
|
|
478
|
+
Ensure all input arrays of bin edges are identical.
|
|
479
|
+
|
|
480
|
+
Parameters
|
|
481
|
+
----------
|
|
482
|
+
arrays : Sequence[np.ndarray]
|
|
483
|
+
Sequence of 1D arrays representing bin edges from different wholes
|
|
484
|
+
or segments.
|
|
485
|
+
|
|
486
|
+
Returns
|
|
487
|
+
-------
|
|
488
|
+
np.ndarray
|
|
489
|
+
A copy of the first array if all arrays are identical.
|
|
490
|
+
|
|
491
|
+
Raises
|
|
492
|
+
------
|
|
493
|
+
ValueError
|
|
494
|
+
If any array differs from the first one.
|
|
495
|
+
"""
|
|
496
|
+
ref = arrays[0]
|
|
497
|
+
if not all(np.array_equal(ref, a) for a in arrays[1:]):
|
|
498
|
+
raise ValueError("Bin edges are not identical across all arrays.")
|
|
499
|
+
return ref.copy()
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _assert_equal_lengths(ensemble_name: str, lengths: dict[str, int]) -> None:
|
|
503
|
+
"""
|
|
504
|
+
Raise ``ValueError`` unless every whole of `ensemble_name` has the same
|
|
505
|
+
length; `lengths` maps each whole's name to its number of rows or
|
|
506
|
+
values. Replicates of one state point run the same number of steps, so
|
|
507
|
+
a shorter whole is an incomplete run, not one to average over. The cost
|
|
508
|
+
is :math:`O(n)` in the number of wholes.
|
|
509
|
+
"""
|
|
510
|
+
if len(set(lengths.values())) > 1:
|
|
511
|
+
listed = ", ".join(f"{name} {n}" for name, n in lengths.items())
|
|
512
|
+
raise ValueError(
|
|
513
|
+
f"The wholes of ensemble '{ensemble_name}' have unequal "
|
|
514
|
+
f"lengths: {listed}.")
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def whole_from_segments(
|
|
518
|
+
prop: PropertyT,
|
|
519
|
+
segments: AnyPathIter,
|
|
520
|
+
parser: ParserClassT,
|
|
521
|
+
group: GroupT,
|
|
522
|
+
relation: WholeRelationT,
|
|
523
|
+
save_to: Pathish | None = None,
|
|
524
|
+
) -> dict[str, np.ndarray]:
|
|
525
|
+
"""
|
|
526
|
+
Combine the segment files of one property into one array per whole.
|
|
527
|
+
|
|
528
|
+
How the segments of a whole combine is set by `relation`.
|
|
529
|
+
|
|
530
|
+
Parameters
|
|
531
|
+
----------
|
|
532
|
+
prop : :py:obj:`~.PropertyT`
|
|
533
|
+
The property as it appears in file names, such as ``'gyrTMon'``.
|
|
534
|
+
segments : :py:obj:`~.AnyPathIter`
|
|
535
|
+
An iterable of paths to segment ``.npy`` files for `prop`, either
|
|
536
|
+
bare paths or 1-tuples of a path, as
|
|
537
|
+
:func:`~polyphys.manage.utils.sort_filenames` returns them. Each path
|
|
538
|
+
points to a single segment file. Segments of different wholes may be
|
|
539
|
+
interleaved, but the segments of each whole must come in
|
|
540
|
+
consecutive ``segment_id`` order (the ``j<NN>`` keyword of their
|
|
541
|
+
names), with no id skipped.
|
|
542
|
+
parser : :py:obj:`~.ParserClassT`
|
|
543
|
+
A parser class from :mod:`polyphys.manage.parser` that interprets
|
|
544
|
+
file paths and extracts attributes such as the `whole` name and the
|
|
545
|
+
``segment_id``.
|
|
546
|
+
group : :py:obj:`~.GroupT`
|
|
547
|
+
Type of the particle group.
|
|
548
|
+
relation : :py:obj:`~.WholeRelationT`
|
|
549
|
+
Specifies how to combine segments into a whole. Accepted values:
|
|
550
|
+
|
|
551
|
+
- 'histogram'
|
|
552
|
+
'segment' is an N-dimensional histogram-like file, so the segments
|
|
553
|
+
of a 'whole' are summed element-wise. In the two-dimensional
|
|
554
|
+
space, a histogram has a (x_nbins, y_nbins) shape.
|
|
555
|
+
|
|
556
|
+
- 'tseries'
|
|
557
|
+
'segment' is a time-series-like file, so the segments of a 'whole'
|
|
558
|
+
are concatenated along axis 0, in ``segment_id`` order.
|
|
559
|
+
|
|
560
|
+
- 'bin_edge'
|
|
561
|
+
'segment' is a bin-edge file. All the segments of a whole must
|
|
562
|
+
hold identical edges, and one copy of them is the whole's
|
|
563
|
+
bin-edge array.
|
|
564
|
+
|
|
565
|
+
save_to : Pathish, optional
|
|
566
|
+
Directory path where the output files are saved, if specified. Each
|
|
567
|
+
whole is saved as ``<whole>-<group>-<prop>.npy``, overwriting a file
|
|
568
|
+
of the same name.
|
|
569
|
+
|
|
570
|
+
Returns
|
|
571
|
+
-------
|
|
572
|
+
dict[str, np.ndarray]
|
|
573
|
+
Dictionary with whole names as keys and the combined whole data as
|
|
574
|
+
NumPy arrays, in the order in which the wholes first appear in
|
|
575
|
+
`segments`.
|
|
576
|
+
|
|
577
|
+
Raises
|
|
578
|
+
------
|
|
579
|
+
ValueError
|
|
580
|
+
If `relation` is not one of the accepted values; if the segments of
|
|
581
|
+
a whole do not arrive in strictly increasing ``segment_id`` order,
|
|
582
|
+
which includes a segment passed twice; if they skip a
|
|
583
|
+
``segment_id``, such as ``j01`` then ``j03``, which would shift the
|
|
584
|
+
time of every later frame or leave a part out of a sum; or, for
|
|
585
|
+
'bin_edge', if the segments of a whole hold different edges.
|
|
586
|
+
TypeError
|
|
587
|
+
If an item of `segments` is neither a path nor a sequence whose first
|
|
588
|
+
element is a path.
|
|
589
|
+
|
|
590
|
+
Notes
|
|
591
|
+
-----
|
|
592
|
+
The order of a whole's segments is part of the data's meaning for
|
|
593
|
+
'tseries', and a duplicated segment would be counted twice by
|
|
594
|
+
'histogram'. This function checks the order and that no id is skipped;
|
|
595
|
+
it does not require the first id to be 1. Sort the paths with
|
|
596
|
+
:func:`~polyphys.manage.utils.sort_filenames` first, which orders the
|
|
597
|
+
``j<NN>`` keyword naturally.
|
|
598
|
+
|
|
599
|
+
Let n_segments be the number of segment files and m the number of
|
|
600
|
+
values in one segment. Time is O(n_segments * m). Every segment of
|
|
601
|
+
every whole is loaded and held in memory before any whole is reduced,
|
|
602
|
+
so peak memory is O(n_segments * m), not the size of one whole; the
|
|
603
|
+
returned wholes add O(n_wholes * m) for 'histogram' and 'bin_edge' and
|
|
604
|
+
O(n_segments * m) for 'tseries', where n_wholes is the number of
|
|
605
|
+
wholes.
|
|
606
|
+
|
|
607
|
+
Examples
|
|
608
|
+
--------
|
|
609
|
+
Two segments of one whole, written as the probe phase names them, join
|
|
610
|
+
into one time series in ``segment_id`` order:
|
|
611
|
+
|
|
612
|
+
>>> import tempfile
|
|
613
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
614
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
615
|
+
>>> whole = ('N20r5.5lz25.0ac1.0nc100dt0.005bdump1000'
|
|
616
|
+
... 'adump5000ens1')
|
|
617
|
+
>>> paths = []
|
|
618
|
+
>>> for seg_id in (1, 2):
|
|
619
|
+
... path = Path(tmp.name) / f'{whole}.j0{seg_id}.bug.gyrTMon.npy'
|
|
620
|
+
... np.save(path, np.array([10.0, 11.0]) * seg_id)
|
|
621
|
+
... paths.append(path)
|
|
622
|
+
>>> wholes = whole_from_segments('gyrTMon', paths, ChainCyl, 'bug',
|
|
623
|
+
... 'tseries')
|
|
624
|
+
>>> wholes[whole]
|
|
625
|
+
array([10., 11., 20., 22.])
|
|
626
|
+
|
|
627
|
+
'histogram' sums the same segments element-wise:
|
|
628
|
+
|
|
629
|
+
>>> whole_from_segments('gyrTMon', paths, ChainCyl, 'bug',
|
|
630
|
+
... 'histogram')[whole]
|
|
631
|
+
array([30., 33.])
|
|
632
|
+
|
|
633
|
+
Segments out of order raise rather than giving a reversed series:
|
|
634
|
+
|
|
635
|
+
>>> whole_from_segments('gyrTMon', paths[::-1], ChainCyl, 'bug',
|
|
636
|
+
... 'tseries') # doctest: +ELLIPSIS
|
|
637
|
+
Traceback (most recent call last):
|
|
638
|
+
...
|
|
639
|
+
ValueError: The segments of whole '...ens1' are not in ... [2, 1]...
|
|
640
|
+
>>> tmp.cleanup()
|
|
641
|
+
"""
|
|
642
|
+
invalid_keyword(relation, ["histogram", "tseries", "bin_edge"])
|
|
643
|
+
# Combine segments into wholes based on segment names and relation type,
|
|
644
|
+
# checking that each whole's segment_id values strictly increase
|
|
645
|
+
grouped_segments: dict[str, list[np.ndarray]] = {}
|
|
646
|
+
segment_ids: dict[str, list[int]] = {}
|
|
647
|
+
for seg_path in _normalize_paths(segments):
|
|
648
|
+
seg_info = parser(seg_path, "segment", group, ispath=True)
|
|
649
|
+
whole_name = getattr(seg_info, "whole")
|
|
650
|
+
seg_id = getattr(seg_info, "segment_id")
|
|
651
|
+
whole_ids = segment_ids.setdefault(whole_name, [])
|
|
652
|
+
if whole_ids and seg_id <= whole_ids[-1]:
|
|
653
|
+
raise ValueError(
|
|
654
|
+
f"The segments of whole '{whole_name}' are not in strictly "
|
|
655
|
+
f"increasing segment_id order: {whole_ids + [seg_id]}. Sort "
|
|
656
|
+
"them with sort_filenames first."
|
|
657
|
+
)
|
|
658
|
+
if whole_ids and seg_id != whole_ids[-1] + 1:
|
|
659
|
+
raise ValueError(
|
|
660
|
+
f"The segments of whole '{whole_name}' skip a segment_id: "
|
|
661
|
+
f"{whole_ids + [seg_id]}."
|
|
662
|
+
)
|
|
663
|
+
whole_ids.append(seg_id)
|
|
664
|
+
grouped_segments.setdefault(whole_name, []).append(np.load(seg_path))
|
|
665
|
+
|
|
666
|
+
# Define reduction strategies for each relation type
|
|
667
|
+
reducers: dict[str, Callable[[Sequence[np.ndarray]], np.ndarray]] = {
|
|
668
|
+
"histogram": lambda arrs: np.sum(arrs, axis=0),
|
|
669
|
+
"tseries": lambda arrs: np.concatenate(arrs, axis=0),
|
|
670
|
+
"bin_edge": _assert_equal_bin_edges,
|
|
671
|
+
}
|
|
672
|
+
reducer = reducers[relation]
|
|
673
|
+
|
|
674
|
+
# Apply the appropriate mapping function to merge each group of segments
|
|
675
|
+
# into a whole.
|
|
676
|
+
wholes: dict[str, np.ndarray] = \
|
|
677
|
+
{name: reducer(vals) for name, vals in grouped_segments.items()}
|
|
678
|
+
|
|
679
|
+
if save_to is not None:
|
|
680
|
+
for name, data in wholes.items():
|
|
681
|
+
save_artifact(create_fullname(name, group, prop), data, save_to)
|
|
682
|
+
|
|
683
|
+
return wholes
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def whole_from_file(
|
|
687
|
+
whole_paths: AnyPathIter,
|
|
688
|
+
parser: ParserClassT,
|
|
689
|
+
group: GroupT,
|
|
690
|
+
) -> dict[str, np.ndarray]:
|
|
691
|
+
"""
|
|
692
|
+
Load the whole ``.npy`` files of one property of a particle `group`.
|
|
693
|
+
|
|
694
|
+
Each path in `whole_paths` points to one whole file.
|
|
695
|
+
|
|
696
|
+
Parameters
|
|
697
|
+
----------
|
|
698
|
+
whole_paths : AnyPathIter
|
|
699
|
+
Paths to whole files, either bare paths or 1-tuples of a path, as
|
|
700
|
+
:func:`~polyphys.manage.utils.sort_filenames` returns them. Each file
|
|
701
|
+
is loaded independently.
|
|
702
|
+
parser : ParserClassT
|
|
703
|
+
A parser class from :mod:`polyphys.manage.parser` that interprets
|
|
704
|
+
file paths and extracts attributes such as the `whole` name.
|
|
705
|
+
group : GroupT
|
|
706
|
+
The particle group to which the data pertains.
|
|
707
|
+
|
|
708
|
+
Returns
|
|
709
|
+
-------
|
|
710
|
+
dict[str, np.ndarray]
|
|
711
|
+
Dictionary where keys are the `whole` names (derived from each file
|
|
712
|
+
path) and values are NumPy arrays containing the loaded data.
|
|
713
|
+
|
|
714
|
+
Raises
|
|
715
|
+
------
|
|
716
|
+
TypeError
|
|
717
|
+
If an item of `whole_paths` is neither a path nor a sequence whose
|
|
718
|
+
first element is a path.
|
|
719
|
+
ValueError
|
|
720
|
+
If two files parse to the same whole name; the message names both.
|
|
721
|
+
|
|
722
|
+
Notes
|
|
723
|
+
-----
|
|
724
|
+
This function assumes that each file in `whole_paths` contains data
|
|
725
|
+
related to a complete simulation, referred to as a "whole." For further
|
|
726
|
+
information on terminology, see the `organizer` module documentation.
|
|
727
|
+
|
|
728
|
+
Let n_wholes be the number of files and m the number of values in one
|
|
729
|
+
file. Time and memory are O(n_wholes * m): every whole is loaded and
|
|
730
|
+
held in memory at once.
|
|
731
|
+
|
|
732
|
+
Examples
|
|
733
|
+
--------
|
|
734
|
+
>>> import tempfile
|
|
735
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
736
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
737
|
+
>>> whole = ('N20r5.5lz25.0ac1.0nc100dt0.005bdump1000'
|
|
738
|
+
... 'adump5000ens1')
|
|
739
|
+
>>> path = Path(tmp.name) / f'{whole}-bug-gyrTMon.npy'
|
|
740
|
+
>>> np.save(path, np.array([1.0, 2.0, 3.0]))
|
|
741
|
+
>>> whole_from_file([path], ChainCyl, 'bug')[whole]
|
|
742
|
+
array([1., 2., 3.])
|
|
743
|
+
>>> tmp.cleanup()
|
|
744
|
+
"""
|
|
745
|
+
wholes: dict[str, np.ndarray] = {}
|
|
746
|
+
sources: dict[str, Path] = {}
|
|
747
|
+
for whole_path in _normalize_paths(whole_paths):
|
|
748
|
+
whole_info = parser(whole_path, "whole", group, ispath=True)
|
|
749
|
+
name = getattr(whole_info, "whole")
|
|
750
|
+
if name in sources:
|
|
751
|
+
raise ValueError(
|
|
752
|
+
f"'{sources[name]}' and '{whole_path}' are both whole "
|
|
753
|
+
f"'{name}'.")
|
|
754
|
+
sources[name] = whole_path
|
|
755
|
+
wholes[name] = np.load(whole_path)
|
|
756
|
+
|
|
757
|
+
return wholes
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def ens_from_bin_edge(
|
|
761
|
+
ens_data: tuple[str, dict[str, np.ndarray]],
|
|
762
|
+
) -> tuple[str, np.ndarray]:
|
|
763
|
+
"""
|
|
764
|
+
Reduce the bin edges of the 'whole' datasets in an 'ensemble' to the
|
|
765
|
+
single array they share.
|
|
766
|
+
|
|
767
|
+
Every 'whole' in an ensemble must be binned identically, so that the
|
|
768
|
+
histograms paired with these edges stay aligned bin by bin. The edges are
|
|
769
|
+
checked for equality with ``_assert_equal_bin_edges``, the check
|
|
770
|
+
:func:`whole_from_segments` applies to segments, rather than merged.
|
|
771
|
+
|
|
772
|
+
Parameters
|
|
773
|
+
----------
|
|
774
|
+
ens_data : tuple[str, dict[str, np.ndarray]]
|
|
775
|
+
A tuple where the first element is the ensemble name (str) and the
|
|
776
|
+
second element is a non-empty dictionary of bin edge arrays from
|
|
777
|
+
different 'whole' datasets within that ensemble. Within the
|
|
778
|
+
dictionary, keys are whole names while values are 1D arrays of whole
|
|
779
|
+
bin edges, each of shape ``(n_bins + 1,)``.
|
|
780
|
+
|
|
781
|
+
Returns
|
|
782
|
+
-------
|
|
783
|
+
tuple[str, np.ndarray]
|
|
784
|
+
A tuple where the first element is the ensemble name, and the second
|
|
785
|
+
is a copy of the shared 1D array of bin edges, of shape
|
|
786
|
+
``(n_bins + 1,)``.
|
|
787
|
+
|
|
788
|
+
Raises
|
|
789
|
+
------
|
|
790
|
+
ValueError
|
|
791
|
+
If the bin edges of any 'whole' differ from those of the first one,
|
|
792
|
+
in length or in value.
|
|
793
|
+
|
|
794
|
+
Notes
|
|
795
|
+
-----
|
|
796
|
+
Time and space complexity are O(n_wholes * n_edges) and O(n_edges).
|
|
797
|
+
|
|
798
|
+
Examples
|
|
799
|
+
--------
|
|
800
|
+
>>> ens_from_bin_edge(
|
|
801
|
+
... ('ens1', {'w1': np.array([0., 1., 2.]),
|
|
802
|
+
... 'w2': np.array([0., 1., 2.])})
|
|
803
|
+
... )
|
|
804
|
+
('ens1', array([0., 1., 2.]))
|
|
805
|
+
|
|
806
|
+
Differing edges raise instead of being merged:
|
|
807
|
+
|
|
808
|
+
>>> ens_from_bin_edge(
|
|
809
|
+
... ('ens1', {'w1': np.array([0., 1., 2.]),
|
|
810
|
+
... 'w2': np.array([0., 1., 3.])})
|
|
811
|
+
... )
|
|
812
|
+
Traceback (most recent call last):
|
|
813
|
+
...
|
|
814
|
+
ValueError: Bin edges are not identical across all arrays.
|
|
815
|
+
"""
|
|
816
|
+
ensemble_name, whole_edges = ens_data
|
|
817
|
+
shared_bin_edges = _assert_equal_bin_edges(list(whole_edges.values()))
|
|
818
|
+
return ensemble_name, shared_bin_edges
|
|
819
|
+
|
|
820
|
+
|
|
821
|
+
def ens_from_vec(
|
|
822
|
+
ens_data: tuple[str, dict[str, np.ndarray]],
|
|
823
|
+
) -> tuple[str, pd.DataFrame]:
|
|
824
|
+
"""
|
|
825
|
+
Merge the 1D arrays of the wholes of one ensemble into a DataFrame.
|
|
826
|
+
|
|
827
|
+
Each whole becomes one column, such as a time series or a histogram of
|
|
828
|
+
the whole.
|
|
829
|
+
|
|
830
|
+
Parameters
|
|
831
|
+
----------
|
|
832
|
+
ens_data : tuple[str, dict[str, np.ndarray]]
|
|
833
|
+
A tuple where the first element is the ensemble name, and the second
|
|
834
|
+
is a dictionary mapping each 'whole' dataset name to a 1D NumPy array
|
|
835
|
+
of measurements.
|
|
836
|
+
|
|
837
|
+
Returns
|
|
838
|
+
-------
|
|
839
|
+
tuple[str, pd.DataFrame]
|
|
840
|
+
A tuple where the first element is the ensemble name, and the second
|
|
841
|
+
is a pandas DataFrame. Each column represents a 'whole' dataset, in
|
|
842
|
+
the order of the dictionary, and rows contain corresponding vector
|
|
843
|
+
values from each dataset.
|
|
844
|
+
|
|
845
|
+
Raises
|
|
846
|
+
------
|
|
847
|
+
ValueError
|
|
848
|
+
If the vectors have unequal lengths; the message names the ensemble
|
|
849
|
+
and each whole's length.
|
|
850
|
+
|
|
851
|
+
Notes
|
|
852
|
+
-----
|
|
853
|
+
Let n_wholes be the number of wholes and m the length of one vector.
|
|
854
|
+
Time and extra memory are O(n_wholes * m): the vectors are copied into
|
|
855
|
+
the DataFrame.
|
|
856
|
+
|
|
857
|
+
Examples
|
|
858
|
+
--------
|
|
859
|
+
>>> ens_name, ensemble_df = ens_from_vec(
|
|
860
|
+
... ('ensemble1', {'whole1': np.array([0.1, 0.2, 0.3]),
|
|
861
|
+
... 'whole2': np.array([0.4, 0.5, 0.6])})
|
|
862
|
+
... )
|
|
863
|
+
>>> ens_name
|
|
864
|
+
'ensemble1'
|
|
865
|
+
>>> ensemble_df
|
|
866
|
+
whole1 whole2
|
|
867
|
+
0 0.1 0.4
|
|
868
|
+
1 0.2 0.5
|
|
869
|
+
2 0.3 0.6
|
|
870
|
+
"""
|
|
871
|
+
ensemble_name, whole_vectors = ens_data
|
|
872
|
+
_assert_equal_lengths(
|
|
873
|
+
ensemble_name, {name: len(v) for name, v in whole_vectors.items()})
|
|
874
|
+
ensemble_df = pd.DataFrame.from_dict(whole_vectors, orient="columns")
|
|
875
|
+
return ensemble_name, ensemble_df
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def ens_from_mat_t(
|
|
879
|
+
ens_data: tuple[str, dict[str, np.ndarray]],
|
|
880
|
+
) -> tuple[str, np.ndarray]:
|
|
881
|
+
"""
|
|
882
|
+
Stack the 2D arrays of the wholes of one ensemble into a 3D array.
|
|
883
|
+
|
|
884
|
+
Each whole becomes one slice along the first axis.
|
|
885
|
+
|
|
886
|
+
Parameters
|
|
887
|
+
----------
|
|
888
|
+
ens_data : tuple[str, dict[str, np.ndarray]]
|
|
889
|
+
A tuple where the first element is the ensemble name (str) and the
|
|
890
|
+
second element is a dictionary with 'whole' names as keys and 2D NumPy
|
|
891
|
+
arrays as values.
|
|
892
|
+
|
|
893
|
+
Returns
|
|
894
|
+
-------
|
|
895
|
+
tuple[str, np.ndarray]
|
|
896
|
+
A tuple where the first element is the ensemble name and the second
|
|
897
|
+
is a 3D NumPy array of shape ``(n_wholes, n_rows, n_cols)``, formed
|
|
898
|
+
by stacking the 2D arrays along the first axis in the order of the
|
|
899
|
+
dictionary.
|
|
900
|
+
|
|
901
|
+
Notes
|
|
902
|
+
-----
|
|
903
|
+
Let n_wholes be the number of wholes and m the number of elements of one
|
|
904
|
+
matrix. Time and extra memory are O(n_wholes * m): the matrices are
|
|
905
|
+
copied into the stacked array.
|
|
906
|
+
|
|
907
|
+
Examples
|
|
908
|
+
--------
|
|
909
|
+
>>> ens_name, combined = ens_from_mat_t(
|
|
910
|
+
... ('ensemble1', {'whole1': np.array([[1, 2], [3, 4]]),
|
|
911
|
+
... 'whole2': np.array([[5, 6], [7, 8]])})
|
|
912
|
+
... )
|
|
913
|
+
>>> combined.shape
|
|
914
|
+
(2, 2, 2)
|
|
915
|
+
>>> combined[1]
|
|
916
|
+
array([[5, 6],
|
|
917
|
+
[7, 8]])
|
|
918
|
+
"""
|
|
919
|
+
ensemble_name, whole_matrices = ens_data
|
|
920
|
+
combined_matrix = np.stack(list(whole_matrices.values()), axis=0)
|
|
921
|
+
return ensemble_name, combined_matrix
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def ens_from_df(
|
|
925
|
+
ens_data: tuple[str, dict[str, pd.DataFrame]],
|
|
926
|
+
) -> tuple[str, pd.DataFrame]:
|
|
927
|
+
"""
|
|
928
|
+
Average the DataFrames of the wholes of one ensemble into one
|
|
929
|
+
DataFrame.
|
|
930
|
+
|
|
931
|
+
Each DataFrame is one whole of the ensemble.
|
|
932
|
+
|
|
933
|
+
In a 'whole' DataFrame, headers are 'elements' of a 'matrix' or 2D
|
|
934
|
+
quantity and columns are the values of a given property.
|
|
935
|
+
|
|
936
|
+
In each 'whole' DataFrame, headers represent 'elements' of a 2D quantity
|
|
937
|
+
(matrix-like structure), and columns contain the values for a specified
|
|
938
|
+
property. Each 'whole' DataFrame may be either a 'histogram' or a
|
|
939
|
+
'time-series':
|
|
940
|
+
|
|
941
|
+
- In a 'histogram' DataFrame, an additional column, `bin_center`, holds
|
|
942
|
+
bin center values that are invariant under averaging and are retained
|
|
943
|
+
in the output DataFrame. The length of histogram DataFrame is equal to
|
|
944
|
+
the number of bins.
|
|
945
|
+
|
|
946
|
+
- In a 'time-series' DataFrame, headers (columns) are elements of a matrix,
|
|
947
|
+
and rows represent the values of each element over time. The length of
|
|
948
|
+
timeseries DataFrame is equal to the number of time frames.
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
Parameters
|
|
952
|
+
----------
|
|
953
|
+
ens_data : tuple[str, dict[str, pd.DataFrame]]
|
|
954
|
+
A tuple where the first element is the ensemble name and the second
|
|
955
|
+
is a dictionary with 'whole' names as keys and DataFrames, sharing
|
|
956
|
+
their columns and index labels, as values.
|
|
957
|
+
|
|
958
|
+
Returns
|
|
959
|
+
-------
|
|
960
|
+
tuple[str, pd.DataFrame]
|
|
961
|
+
A tuple with:
|
|
962
|
+
|
|
963
|
+
- The ensemble name (str).
|
|
964
|
+
- A DataFrame with the averaged values across all 'whole'
|
|
965
|
+
DataFrames.
|
|
966
|
+
|
|
967
|
+
Raises
|
|
968
|
+
------
|
|
969
|
+
ValueError
|
|
970
|
+
If the DataFrames have unequal numbers of rows; the message names
|
|
971
|
+
the ensemble and each whole's length. A shorter whole is not
|
|
972
|
+
averaged over the rows it has.
|
|
973
|
+
|
|
974
|
+
Notes
|
|
975
|
+
-----
|
|
976
|
+
A clear distinction exists between ensembles of 'vector' or 'matrix' types
|
|
977
|
+
and ensembles of 'dataframe' type. In vector or matrix types, each column
|
|
978
|
+
in the resulting ensemble DataFrame is named after a 'whole' and represents
|
|
979
|
+
that whole's values for a specific property. In contrast, for DataFrame
|
|
980
|
+
type ensembles, the columns are elements of a 2D quantity (e.g., matrix or
|
|
981
|
+
time-series values) and hold averaged values across all whole DataFrames.
|
|
982
|
+
|
|
983
|
+
This function assumes that averaging non-element headers, such as
|
|
984
|
+
'bin_center,' does not change their values, and these are carried forward
|
|
985
|
+
without modification.
|
|
986
|
+
|
|
987
|
+
Let n_wholes be the number of wholes, n_rows the rows and n_cols the
|
|
988
|
+
columns of one DataFrame. Time and extra memory are
|
|
989
|
+
O(n_wholes * n_rows * n_cols): the DataFrames are concatenated into one
|
|
990
|
+
before they are grouped by index label and averaged.
|
|
991
|
+
|
|
992
|
+
Examples
|
|
993
|
+
--------
|
|
994
|
+
>>> df1 = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
|
|
995
|
+
>>> df2 = pd.DataFrame({'A': [2, 3, 4], 'B': [5, 6, 7]})
|
|
996
|
+
>>> ens_name, averaged_df = ens_from_df(
|
|
997
|
+
... ('ensemble1', {'whole1': df1, 'whole2': df2})
|
|
998
|
+
... )
|
|
999
|
+
>>> averaged_df
|
|
1000
|
+
A B
|
|
1001
|
+
0 1.5 4.5
|
|
1002
|
+
1 2.5 5.5
|
|
1003
|
+
2 3.5 6.5
|
|
1004
|
+
"""
|
|
1005
|
+
ensemble_name, whole_data = ens_data
|
|
1006
|
+
_assert_equal_lengths(
|
|
1007
|
+
ensemble_name, {name: len(df) for name, df in whole_data.items()})
|
|
1008
|
+
ensemble_df = pd.concat(list(whole_data.values())).groupby(level=0).mean()
|
|
1009
|
+
return ensemble_name, ensemble_df
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def ensemble(
|
|
1013
|
+
prop: PropertyT,
|
|
1014
|
+
wholes: dict[str, np.ndarray] | dict[str, pd.DataFrame],
|
|
1015
|
+
parser: ParserClassT,
|
|
1016
|
+
group: GroupT,
|
|
1017
|
+
whole_type: Literal["vector", "matrix", "dataframe", "bin_edge"],
|
|
1018
|
+
whole_edges: dict[str, np.ndarray] | None = None,
|
|
1019
|
+
save_to: Pathish | None = None,
|
|
1020
|
+
) -> dict[str, np.ndarray | pd.DataFrame]:
|
|
1021
|
+
"""
|
|
1022
|
+
Merge the wholes of one property into one ensemble per state point.
|
|
1023
|
+
|
|
1024
|
+
`whole_type` sets the structure of each whole and how the wholes of one
|
|
1025
|
+
ensemble merge.
|
|
1026
|
+
|
|
1027
|
+
Parameters
|
|
1028
|
+
----------
|
|
1029
|
+
prop : :py:obj:`~.PropertyT`
|
|
1030
|
+
The property as it appears in file names, such as ``'rhoMon'``.
|
|
1031
|
+
wholes : dict[str, np.ndarray] | dict[str, pd.DataFrame]
|
|
1032
|
+
Dictionary containing 'whole' names as keys and their corresponding
|
|
1033
|
+
data as values, represented as NumPy arrays or DataFrames.
|
|
1034
|
+
parser : ParserClassT
|
|
1035
|
+
Parser class used to interpret whole filenames, such as the ensemble
|
|
1036
|
+
name of each 'whole'.
|
|
1037
|
+
group : GroupT
|
|
1038
|
+
Type of the particle group being processed.
|
|
1039
|
+
whole_type : {'vector', 'matrix', 'dataframe', 'bin_edge'}
|
|
1040
|
+
Specifies the type of each 'whole' and defines the merging method:
|
|
1041
|
+
|
|
1042
|
+
- 'vector': A 1D NumPy array, such as a time series or histogram.
|
|
1043
|
+
- 'matrix': A 2D NumPy array, such as a gyration tensor.
|
|
1044
|
+
- 'dataframe': A DataFrame representing either a time series (with
|
|
1045
|
+
headers as matrix elements and rows as values over time) or a
|
|
1046
|
+
histogram (headers as elements, rows as counts or frequencies,
|
|
1047
|
+
and an additional 'bin_center' column for bin centers).
|
|
1048
|
+
- 'bin_edge': A 1D array representing bin edges. Assumes all bin edges
|
|
1049
|
+
are identical and selects one to represent the ensemble.
|
|
1050
|
+
|
|
1051
|
+
whole_edges : dict[str, np.ndarray], optional
|
|
1052
|
+
Dictionary with 'whole' names as keys and 1D bin edge arrays, each of
|
|
1053
|
+
shape ``(n_bins + 1,)``, as values. Only valid when `whole_type` is
|
|
1054
|
+
'vector'. The wholes of an ensemble share their bin edges by design,
|
|
1055
|
+
so the edges of the first whole of each ensemble are used and the
|
|
1056
|
+
others are not compared. Each merged ensemble gets a 'bin_center'
|
|
1057
|
+
column of length ``n_bins`` holding the midpoints of those edges.
|
|
1058
|
+
Must contain every key of `wholes`.
|
|
1059
|
+
save_to : Pathish, optional
|
|
1060
|
+
Directory path where output files are saved, if specified; see
|
|
1061
|
+
Notes.
|
|
1062
|
+
|
|
1063
|
+
Returns
|
|
1064
|
+
-------
|
|
1065
|
+
dict[str, np.ndarray | pd.DataFrame]
|
|
1066
|
+
The ensembles, keyed by their ``ensemble_long`` names, as NumPy
|
|
1067
|
+
arrays or DataFrames.
|
|
1068
|
+
|
|
1069
|
+
Raises
|
|
1070
|
+
------
|
|
1071
|
+
ValueError
|
|
1072
|
+
If any parameter has an invalid value; if `whole_edges` is given and
|
|
1073
|
+
`whole_type` is not 'vector'; or if the number of bins differs from
|
|
1074
|
+
the number of rows of the merged ensemble.
|
|
1075
|
+
|
|
1076
|
+
Notes
|
|
1077
|
+
-----
|
|
1078
|
+
Different averaging methods apply based on `whole_type`. When `whole_type`
|
|
1079
|
+
is "DataFrame," the resulting DataFrame averages the values of each matrix
|
|
1080
|
+
element across wholes, retaining headers that are mean-invariant (e.g.,
|
|
1081
|
+
'bin_center'). For 'vector' and 'matrix' types, columns represent
|
|
1082
|
+
individual wholes in the ensemble DataFrame.
|
|
1083
|
+
|
|
1084
|
+
The bin center of bin ``i`` is the midpoint of its two edges,
|
|
1085
|
+
``0.5 * (edges[i] + edges[i + 1])``. A center is invariant under
|
|
1086
|
+
averaging over wholes, which is why :func:`ensemble_avg` excludes
|
|
1087
|
+
'bin_center' from its statistics by default.
|
|
1088
|
+
|
|
1089
|
+
Let n_wholes be the number of wholes, m the number of values in one
|
|
1090
|
+
whole, n_ensembles the number of ensembles, and n_edges the length of
|
|
1091
|
+
one bin-edge array. Grouping parses each whole name once and merging
|
|
1092
|
+
copies every whole once, so time and extra memory are
|
|
1093
|
+
O(n_wholes * m): the caller's `wholes` and the merged ensembles are both
|
|
1094
|
+
held in memory. With `whole_edges`, computing the centers adds
|
|
1095
|
+
O(n_ensembles * n_edges) time and memory.
|
|
1096
|
+
|
|
1097
|
+
With `save_to`, each ensemble is saved by :func:`save_artifact` under
|
|
1098
|
+
``<ensemble_long>-<group>-<prop>``, as ``.csv`` for a DataFrame and
|
|
1099
|
+
``.npy`` for an array, overwriting a file of the same name.
|
|
1100
|
+
|
|
1101
|
+
Examples
|
|
1102
|
+
--------
|
|
1103
|
+
Two replicates of one state point, with shared edges, merge into one
|
|
1104
|
+
ensemble that carries the edge midpoints as 'bin_center':
|
|
1105
|
+
|
|
1106
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
1107
|
+
>>> base = ('N2000r10.5lz300.0ac1.0nc117000dt0.005bdump5000'
|
|
1108
|
+
... 'adump10000ens')
|
|
1109
|
+
>>> wholes = {base + '1': np.array([1., 2., 3.]),
|
|
1110
|
+
... base + '2': np.array([3., 4., 5.])}
|
|
1111
|
+
>>> edges = {name: np.array([0., 1., 2., 4.]) for name in wholes}
|
|
1112
|
+
>>> ens = ensemble('rhoMon', wholes, ChainCyl, 'bug', 'vector',
|
|
1113
|
+
... whole_edges=edges)
|
|
1114
|
+
>>> ens_df = next(iter(ens.values()))
|
|
1115
|
+
>>> ens_df['bin_center'].tolist()
|
|
1116
|
+
[0.5, 1.5, 3.0]
|
|
1117
|
+
|
|
1118
|
+
Without `whole_edges`, the ensemble has one column per whole only:
|
|
1119
|
+
|
|
1120
|
+
>>> ens = ensemble('rhoMon', wholes, ChainCyl, 'bug', 'vector')
|
|
1121
|
+
>>> 'bin_center' in next(iter(ens.values())).columns
|
|
1122
|
+
False
|
|
1123
|
+
"""
|
|
1124
|
+
invalid_keyword(whole_type, ["vector", "matrix", "dataframe", "bin_edge"])
|
|
1125
|
+
if whole_edges is not None and whole_type != "vector":
|
|
1126
|
+
raise ValueError(
|
|
1127
|
+
"'whole_edges' is only valid with whole_type 'vector', not "
|
|
1128
|
+
f"'{whole_type}'."
|
|
1129
|
+
)
|
|
1130
|
+
|
|
1131
|
+
merging_func: dict[
|
|
1132
|
+
str, Callable[..., tuple[str, np.ndarray | pd.DataFrame]]
|
|
1133
|
+
] = {
|
|
1134
|
+
"vector": ens_from_vec,
|
|
1135
|
+
"matrix": ens_from_mat_t,
|
|
1136
|
+
"dataframe": ens_from_df,
|
|
1137
|
+
"bin_edge": ens_from_bin_edge,
|
|
1138
|
+
}
|
|
1139
|
+
# Collecting wholes for each ensemble; the wholes of an ensemble share
|
|
1140
|
+
# their bin edges by design, so the centers come from its first whole
|
|
1141
|
+
ens_wholes: dict[str, dict[str, np.ndarray | pd.DataFrame]] = {}
|
|
1142
|
+
bin_centers: dict[str, np.ndarray] = {}
|
|
1143
|
+
for whole_name, whole_data in wholes.items():
|
|
1144
|
+
whole_info = parser(whole_name, "whole", group)
|
|
1145
|
+
ens_name = getattr(whole_info, "ensemble_long")
|
|
1146
|
+
ens_wholes.setdefault(ens_name, {})[whole_name] = whole_data
|
|
1147
|
+
if whole_edges is not None and ens_name not in bin_centers:
|
|
1148
|
+
edges = np.asarray(whole_edges[whole_name])
|
|
1149
|
+
bin_centers[ens_name] = 0.5 * (edges[:-1] + edges[1:])
|
|
1150
|
+
|
|
1151
|
+
# Merging wholes into a single ensemble data object
|
|
1152
|
+
ensembles: dict[str, np.ndarray | pd.DataFrame] = dict(
|
|
1153
|
+
map(merging_func[whole_type], ens_wholes.items())
|
|
1154
|
+
)
|
|
1155
|
+
for ens_name, centers in bin_centers.items():
|
|
1156
|
+
ensembles[ens_name]["bin_center"] = centers
|
|
1157
|
+
|
|
1158
|
+
# Save ensemble data object if save_to is specified
|
|
1159
|
+
if save_to:
|
|
1160
|
+
for ens_name, data in ensembles.items():
|
|
1161
|
+
ens_fullname = create_fullname(ens_name, group, prop)
|
|
1162
|
+
save_artifact(ens_fullname, data, save_to)
|
|
1163
|
+
|
|
1164
|
+
return ensembles
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def ens_avg_from_bin_edge(
|
|
1168
|
+
ens_data: np.ndarray | Sequence[np.ndarray],
|
|
1169
|
+
) -> np.ndarray:
|
|
1170
|
+
"""
|
|
1171
|
+
Return the bin edges of an 'ensemble' for its ensemble-averaged
|
|
1172
|
+
artifact.
|
|
1173
|
+
|
|
1174
|
+
Bin edges are not averaged: every 'whole' in the ensemble must share
|
|
1175
|
+
them. A single 1D array, the form :func:`ens_from_bin_edge` produces and
|
|
1176
|
+
:func:`ensemble_avg` passes here, is returned as a copy. A sequence of
|
|
1177
|
+
arrays, one per 'whole', is checked for equality with
|
|
1178
|
+
``_assert_equal_bin_edges``, and a copy of the shared array is
|
|
1179
|
+
returned.
|
|
1180
|
+
|
|
1181
|
+
Parameters
|
|
1182
|
+
----------
|
|
1183
|
+
ens_data : np.ndarray or Sequence[np.ndarray]
|
|
1184
|
+
Either the ensemble's 1D array of bin edges, of shape
|
|
1185
|
+
``(n_bins + 1,)``, or a non-empty sequence of such arrays, one per
|
|
1186
|
+
'whole'. A 2D array of shape ``(n_wholes, n_bins + 1)`` counts as a
|
|
1187
|
+
sequence of its rows.
|
|
1188
|
+
|
|
1189
|
+
Returns
|
|
1190
|
+
-------
|
|
1191
|
+
np.ndarray
|
|
1192
|
+
A copy of the shared 1D array of bin edges, of shape
|
|
1193
|
+
``(n_bins + 1,)``.
|
|
1194
|
+
|
|
1195
|
+
Raises
|
|
1196
|
+
------
|
|
1197
|
+
ValueError
|
|
1198
|
+
If `ens_data` is a sequence and any array in it differs from the
|
|
1199
|
+
first one, in length or in value.
|
|
1200
|
+
|
|
1201
|
+
Notes
|
|
1202
|
+
-----
|
|
1203
|
+
This function performs no averaging or statistical calculation. Time and
|
|
1204
|
+
space complexity are O(n_wholes * n_edges) and O(n_edges).
|
|
1205
|
+
|
|
1206
|
+
Examples
|
|
1207
|
+
--------
|
|
1208
|
+
>>> ens_avg_from_bin_edge(np.array([0.0, 0.1, 0.2]))
|
|
1209
|
+
array([0. , 0.1, 0.2])
|
|
1210
|
+
>>> ens_avg_from_bin_edge(
|
|
1211
|
+
... [np.array([0.0, 0.1, 0.2]), np.array([0.0, 0.1, 0.2])]
|
|
1212
|
+
... )
|
|
1213
|
+
array([0. , 0.1, 0.2])
|
|
1214
|
+
>>> ens_avg_from_bin_edge(
|
|
1215
|
+
... [np.array([0.0, 0.1, 0.2]), np.array([0.0, 0.1, 0.3])]
|
|
1216
|
+
... )
|
|
1217
|
+
Traceback (most recent call last):
|
|
1218
|
+
...
|
|
1219
|
+
ValueError: Bin edges are not identical across all arrays.
|
|
1220
|
+
"""
|
|
1221
|
+
if isinstance(ens_data, np.ndarray) and ens_data.ndim == 1:
|
|
1222
|
+
return ens_data.copy()
|
|
1223
|
+
return _assert_equal_bin_edges([np.asarray(edges) for edges in ens_data])
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
def ens_avg_from_ndarray(
|
|
1227
|
+
ens_prop: PropertyT, ens_data: np.ndarray
|
|
1228
|
+
) -> dict[str, np.ndarray]:
|
|
1229
|
+
"""
|
|
1230
|
+
Compute the mean, variance, and standard error of the mean (SEM)
|
|
1231
|
+
across the wholes of an ensemble 3D array.
|
|
1232
|
+
|
|
1233
|
+
Each slice along the first axis (axis=0) is one whole's matrix, and the
|
|
1234
|
+
statistics are taken along that axis.
|
|
1235
|
+
|
|
1236
|
+
Parameters
|
|
1237
|
+
----------
|
|
1238
|
+
ens_prop : PropertyT
|
|
1239
|
+
The name of the property to which this 'ensemble' data belongs. Used
|
|
1240
|
+
as a prefix for the computed statistics.
|
|
1241
|
+
ens_data : np.ndarray
|
|
1242
|
+
A 3D NumPy array where each matrix slice along axis=0 represents a
|
|
1243
|
+
'whole' dataset.
|
|
1244
|
+
|
|
1245
|
+
Returns
|
|
1246
|
+
-------
|
|
1247
|
+
dict[str, np.ndarray]
|
|
1248
|
+
A dictionary where keys are statistical measures (mean, variance, SEM)
|
|
1249
|
+
with `ens_prop` as a prefix while values are NumPy arrays of the
|
|
1250
|
+
computed statistics across 'whole' matrices.
|
|
1251
|
+
|
|
1252
|
+
Notes
|
|
1253
|
+
-----
|
|
1254
|
+
- The SEM is the sample standard deviation with ``ddof=1`` divided by
|
|
1255
|
+
the square root of the number of wholes, which are assumed
|
|
1256
|
+
independent. Bessel's correction makes the variance unbiased; the
|
|
1257
|
+
standard deviation is still biased, though less (the
|
|
1258
|
+
:func:`numpy.std` documentation, Notes), and so is the SEM.
|
|
1259
|
+
- This function performs averaging along axis=0 for each element across
|
|
1260
|
+
the 'whole' data matrices.
|
|
1261
|
+
- Let n_wholes be the length of axis 0 and m the number of elements of
|
|
1262
|
+
one whole. Time is O(n_wholes * m). NumPy's ``var`` and ``std`` build
|
|
1263
|
+
temporaries of the size of `ens_data`, so peak extra memory is
|
|
1264
|
+
O(n_wholes * m); the three results take O(m).
|
|
1265
|
+
|
|
1266
|
+
Examples
|
|
1267
|
+
--------
|
|
1268
|
+
>>> ens_data = np.array([
|
|
1269
|
+
... [[1.0, 2.0], [3.0, 4.0]],
|
|
1270
|
+
... [[2.0, 3.0], [4.0, 5.0]],
|
|
1271
|
+
... ])
|
|
1272
|
+
>>> ens_avg = ens_avg_from_ndarray('gyrTMon', ens_data)
|
|
1273
|
+
>>> list(ens_avg)
|
|
1274
|
+
['gyrTMon-mean', 'gyrTMon-var', 'gyrTMon-sem']
|
|
1275
|
+
>>> ens_avg['gyrTMon-mean']
|
|
1276
|
+
array([[1.5, 2.5],
|
|
1277
|
+
[3.5, 4.5]])
|
|
1278
|
+
>>> ens_avg['gyrTMon-sem']
|
|
1279
|
+
array([[0.5, 0.5],
|
|
1280
|
+
[0.5, 0.5]])
|
|
1281
|
+
"""
|
|
1282
|
+
n_wholes_sqrt = ens_data.shape[0] ** 0.5
|
|
1283
|
+
ens_avg = {
|
|
1284
|
+
f"{ens_prop}-mean": np.mean(ens_data, axis=0),
|
|
1285
|
+
f"{ens_prop}-var": np.var(ens_data, axis=0, ddof=1),
|
|
1286
|
+
f"{ens_prop}-sem": np.std(ens_data, axis=0, ddof=1) / n_wholes_sqrt,
|
|
1287
|
+
}
|
|
1288
|
+
return ens_avg
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
def ens_avg_from_df(
|
|
1292
|
+
ens_prop: PropertyT, ens_data: pd.DataFrame, exclude: list[str]
|
|
1293
|
+
) -> pd.DataFrame:
|
|
1294
|
+
"""
|
|
1295
|
+
Compute the mean, variance, and standard error of the mean (SEM) across
|
|
1296
|
+
the whole columns of an ensemble DataFrame.
|
|
1297
|
+
|
|
1298
|
+
The result keeps the excluded columns, such as ``bin_center``, and
|
|
1299
|
+
replaces the whole columns with the statistics.
|
|
1300
|
+
|
|
1301
|
+
Parameters
|
|
1302
|
+
----------
|
|
1303
|
+
ens_prop : PropertyT
|
|
1304
|
+
The name of the property to which this 'ensemble' data belongs, used
|
|
1305
|
+
as a prefix for the generated statistics columns.
|
|
1306
|
+
ens_data : pd.DataFrame
|
|
1307
|
+
A DataFrame containing ensemble data. Each 'whole' dataset is in its
|
|
1308
|
+
own column, and any columns specified in `exclude` remain unaffected.
|
|
1309
|
+
exclude : list of str
|
|
1310
|
+
A list of column names to exclude from averaging. For example,
|
|
1311
|
+
`exclude` might contain 'bin_center' for histogram-type ensembles.
|
|
1312
|
+
|
|
1313
|
+
Returns
|
|
1314
|
+
-------
|
|
1315
|
+
pd.DataFrame
|
|
1316
|
+
A new DataFrame with the excluded columns of `ens_data`, followed by
|
|
1317
|
+
the columns ``<ens_prop>-mean``, ``<ens_prop>-var``, and
|
|
1318
|
+
``<ens_prop>-sem``: the mean, variance, and SEM of each row across
|
|
1319
|
+
the 'whole' data columns. It has the same index as `ens_data`.
|
|
1320
|
+
|
|
1321
|
+
Notes
|
|
1322
|
+
-----
|
|
1323
|
+
- `ens_data` is not modified; the statistics are written to a new
|
|
1324
|
+
DataFrame.
|
|
1325
|
+
- The function averages only the 'whole' columns, excluding any specified
|
|
1326
|
+
in `exclude`.
|
|
1327
|
+
- The standard error of the mean (SEM) is computed from the sample
|
|
1328
|
+
standard deviation with ``ddof=1``, the wholes assumed independent.
|
|
1329
|
+
Bessel's correction makes the variance unbiased; the standard
|
|
1330
|
+
deviation is still biased, though less (the :func:`numpy.std`
|
|
1331
|
+
documentation, Notes), and so is the SEM.
|
|
1332
|
+
- Let n_rows be the number of rows of `ens_data` and n_wholes the number
|
|
1333
|
+
of 'whole' columns. Time is O(n_rows * n_wholes). Each statistic
|
|
1334
|
+
selects the 'whole' columns as a new DataFrame, so peak extra memory
|
|
1335
|
+
is O(n_rows * n_wholes), not O(n_rows); the result takes O(n_rows).
|
|
1336
|
+
|
|
1337
|
+
Examples
|
|
1338
|
+
--------
|
|
1339
|
+
>>> ens_data = pd.DataFrame({
|
|
1340
|
+
... 'bin_center': [1, 2, 3],
|
|
1341
|
+
... 'whole1': [4.0, 5.0, 6.0],
|
|
1342
|
+
... 'whole2': [5.0, 6.0, 7.0],
|
|
1343
|
+
... })
|
|
1344
|
+
>>> ens_avg_from_df('density', ens_data, ['bin_center'])
|
|
1345
|
+
bin_center density-mean density-var density-sem
|
|
1346
|
+
0 1 4.5 0.5 0.5
|
|
1347
|
+
1 2 5.5 0.5 0.5
|
|
1348
|
+
2 3 6.5 0.5 0.5
|
|
1349
|
+
>>> list(ens_data.columns)
|
|
1350
|
+
['bin_center', 'whole1', 'whole2']
|
|
1351
|
+
"""
|
|
1352
|
+
wholes = [col for col in ens_data.columns if col not in exclude]
|
|
1353
|
+
|
|
1354
|
+
ens_avg = ens_data.drop(columns=wholes)
|
|
1355
|
+
ens_avg[ens_prop + "-mean"] = ens_data[wholes].mean(axis=1)
|
|
1356
|
+
ens_avg[ens_prop + "-var"] = ens_data[wholes].var(axis=1, ddof=1)
|
|
1357
|
+
ens_avg[ens_prop + "-sem"] = ens_data[wholes].sem(axis=1, ddof=1)
|
|
1358
|
+
return ens_avg
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def ensemble_avg(
|
|
1362
|
+
prop: PropertyT,
|
|
1363
|
+
ensembles: dict[str, np.ndarray | pd.DataFrame],
|
|
1364
|
+
group: GroupT,
|
|
1365
|
+
ens_type: Literal["dataframe", "ndarray", "bin_edge"],
|
|
1366
|
+
exclude: list[str] | None = None,
|
|
1367
|
+
save_to: Pathish | None = None,
|
|
1368
|
+
) -> dict[str, np.ndarray] | dict[str, pd.DataFrame]:
|
|
1369
|
+
"""
|
|
1370
|
+
Average the wholes of each ensemble of one property.
|
|
1371
|
+
|
|
1372
|
+
Columns listed in `exclude` are left out of the averages.
|
|
1373
|
+
|
|
1374
|
+
Parameters
|
|
1375
|
+
----------
|
|
1376
|
+
prop : :py:obj:`~.PropertyT`
|
|
1377
|
+
The property as it appears in file names, such as ``'rhoMon'``.
|
|
1378
|
+
ensembles : dict[str, np.ndarray | pd.DataFrame]
|
|
1379
|
+
A dictionary where each key is an ensemble name and each value is
|
|
1380
|
+
either a DataFrame or ndarray representing 'whole' data.
|
|
1381
|
+
group : GroupT
|
|
1382
|
+
Type of the particle group.
|
|
1383
|
+
ens_type : {'dataframe', 'ndarray', 'bin_edge'}
|
|
1384
|
+
The data format for each ensemble:
|
|
1385
|
+
|
|
1386
|
+
- 'dataframe': DataFrame format, with columns as 'whole' simulations
|
|
1387
|
+
or items in `exclude`.
|
|
1388
|
+
- 'ndarray': 3D ndarray format, with each slice along axis=0
|
|
1389
|
+
representing a 'whole' matrix.
|
|
1390
|
+
- 'bin_edge': 1D ndarray format for bin edges, with bin edges assumed
|
|
1391
|
+
to be identical across wholes.
|
|
1392
|
+
|
|
1393
|
+
exclude : list[str], default None
|
|
1394
|
+
List of column names or items not included in ensemble averaging,
|
|
1395
|
+
such as 'bin_center' for histograms.
|
|
1396
|
+
save_to : Pathish, optional
|
|
1397
|
+
Path to the directory where the averaged ensemble data is saved, if
|
|
1398
|
+
specified; see Notes.
|
|
1399
|
+
|
|
1400
|
+
Returns
|
|
1401
|
+
-------
|
|
1402
|
+
dict[str, np.ndarray] | dict[str, pd.DataFrame]
|
|
1403
|
+
A dictionary where each key is an ensemble name, and each value is a
|
|
1404
|
+
DataFrame or ndarray of ensemble-averaged data.
|
|
1405
|
+
|
|
1406
|
+
Raises
|
|
1407
|
+
------
|
|
1408
|
+
ValueError
|
|
1409
|
+
If `ens_type` is not one of the accepted values.
|
|
1410
|
+
|
|
1411
|
+
Notes
|
|
1412
|
+
-----
|
|
1413
|
+
With `save_to`, each result is saved by :func:`save_artifact` under
|
|
1414
|
+
``<ensemble>-<group>-<prop>-ensAvg``: a DataFrame as ``.csv``, bin edges
|
|
1415
|
+
as ``.npy``, and the dictionary of an 'ndarray' ensemble as one
|
|
1416
|
+
``...-ensAvg-<measure>.npy`` file per statistic, each overwriting a file
|
|
1417
|
+
of the same name.
|
|
1418
|
+
|
|
1419
|
+
Let n_ensembles be the number of ensembles. Ensembles are averaged one
|
|
1420
|
+
after another, with the time and peak extra memory of
|
|
1421
|
+
:func:`ens_avg_from_df`, :func:`ens_avg_from_ndarray`, or
|
|
1422
|
+
:func:`ens_avg_from_bin_edge` for the largest one; the results for all
|
|
1423
|
+
n_ensembles ensembles are held in memory together.
|
|
1424
|
+
|
|
1425
|
+
Examples
|
|
1426
|
+
--------
|
|
1427
|
+
>>> ensembles = {
|
|
1428
|
+
... 'ensemble1': pd.DataFrame({'whole1': [1.0, 2.0],
|
|
1429
|
+
... 'whole2': [3.0, 4.0]}),
|
|
1430
|
+
... 'ensemble2': pd.DataFrame({'whole1': [5.0, 6.0],
|
|
1431
|
+
... 'whole2': [5.0, 8.0]}),
|
|
1432
|
+
... }
|
|
1433
|
+
>>> avg = ensemble_avg('rhoMon', ensembles, 'bug', 'dataframe')
|
|
1434
|
+
>>> avg['ensemble2']
|
|
1435
|
+
rhoMon-mean rhoMon-var rhoMon-sem
|
|
1436
|
+
0 5.0 0.0 0.0
|
|
1437
|
+
1 7.0 2.0 1.0
|
|
1438
|
+
"""
|
|
1439
|
+
if exclude is None:
|
|
1440
|
+
exclude = ["bin_center"]
|
|
1441
|
+
|
|
1442
|
+
# Compute ensemble averages
|
|
1443
|
+
ens_avgs: dict[str, np.ndarray] | dict[str, pd.DataFrame] = {}
|
|
1444
|
+
if ens_type == "bin_edge":
|
|
1445
|
+
for ens_name, ens_data in ensembles.items():
|
|
1446
|
+
ens_avg = ens_avg_from_bin_edge(ens_data) # type: ignore
|
|
1447
|
+
ens_avgs[ens_name] = ens_avg # type: ignore
|
|
1448
|
+
elif ens_type == "ndarray":
|
|
1449
|
+
for ens_name, ens_data in ensembles.items():
|
|
1450
|
+
ens_avg = ens_avg_from_ndarray(prop, ens_data) # type: ignore
|
|
1451
|
+
ens_avgs[ens_name] = ens_avg # type: ignore
|
|
1452
|
+
elif ens_type == "dataframe":
|
|
1453
|
+
for ens_name, ens_data in ensembles.items():
|
|
1454
|
+
ens_avg = ens_avg_from_df(prop, ens_data, exclude) # type: ignore
|
|
1455
|
+
ens_avgs[ens_name] = ens_avg # type: ignore
|
|
1456
|
+
else:
|
|
1457
|
+
invalid_keyword(ens_type, ["dataframe", "ndarray", "bin_edge"])
|
|
1458
|
+
|
|
1459
|
+
# Save averaged ensemble data if save_to path is provided
|
|
1460
|
+
if save_to is not None:
|
|
1461
|
+
save_property = prop + "-ensAvg"
|
|
1462
|
+
for ens_name, data in ens_avgs.items():
|
|
1463
|
+
ens_fullname = create_fullname(ens_name, group, save_property)
|
|
1464
|
+
save_artifact(ens_fullname, data, save_to)
|
|
1465
|
+
|
|
1466
|
+
return ens_avgs
|
|
1467
|
+
|
|
1468
|
+
|
|
1469
|
+
def _group_from_stamp_names(stamps: list[tuple[str]]) -> str:
|
|
1470
|
+
"""
|
|
1471
|
+
The particle group in the names of stamp files,
|
|
1472
|
+
``<lineage name>-<group>-stamps.csv``, which must agree.
|
|
1473
|
+
|
|
1474
|
+
Parameters
|
|
1475
|
+
----------
|
|
1476
|
+
stamps : list[tuple[str]]
|
|
1477
|
+
List of 1-tuples, each holding the path to a 'stamp' CSV file.
|
|
1478
|
+
|
|
1479
|
+
Returns
|
|
1480
|
+
-------
|
|
1481
|
+
str
|
|
1482
|
+
The group the file names share.
|
|
1483
|
+
|
|
1484
|
+
Raises
|
|
1485
|
+
------
|
|
1486
|
+
ValueError
|
|
1487
|
+
If a file name is not ``<name>-<group>-stamps.csv``, or the names
|
|
1488
|
+
give more than one group.
|
|
1489
|
+
"""
|
|
1490
|
+
groups: set[str] = set()
|
|
1491
|
+
for (path,) in stamps:
|
|
1492
|
+
name = Path(path).name
|
|
1493
|
+
parts = name.removesuffix(".csv").rsplit("-", 2)
|
|
1494
|
+
if len(parts) != 3 or parts[2] != "stamps":
|
|
1495
|
+
raise ValueError(
|
|
1496
|
+
f"'{name}' is not a '<name>-<group>-stamps.csv' stamp file, "
|
|
1497
|
+
"and the stamps have no 'group' column.")
|
|
1498
|
+
groups.add(parts[1])
|
|
1499
|
+
if len(groups) != 1:
|
|
1500
|
+
raise ValueError(
|
|
1501
|
+
f"The stamp file names give {sorted(groups)}, not one group.")
|
|
1502
|
+
return groups.pop()
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
def children_stamps(
|
|
1506
|
+
stamps: list[tuple[str]],
|
|
1507
|
+
child_lineage: PrimitiveLineageT,
|
|
1508
|
+
save_to: Pathish | None = None,
|
|
1509
|
+
) -> pd.DataFrame:
|
|
1510
|
+
"""
|
|
1511
|
+
Combine the stamp files of the artifacts of one lineage, `child_lineage`,
|
|
1512
|
+
into one DataFrame for a *space*.
|
|
1513
|
+
|
|
1514
|
+
With `save_to`, the DataFrame is also saved as a CSV file.
|
|
1515
|
+
|
|
1516
|
+
Parameters
|
|
1517
|
+
----------
|
|
1518
|
+
stamps : list[tuple[str]]
|
|
1519
|
+
List of 1-tuples, as :func:`~polyphys.manage.utils.sort_filenames`
|
|
1520
|
+
returns them, each holding the path to a 'stamp' CSV file. Rows are
|
|
1521
|
+
concatenated in this order.
|
|
1522
|
+
child_lineage : {'segment', 'whole'}
|
|
1523
|
+
Lineage of the stamp files, and so of the rows of the result.
|
|
1524
|
+
save_to : Pathish, optional
|
|
1525
|
+
Directory path where the combined DataFrame is saved, if specified,
|
|
1526
|
+
as ``<space>-<group>-stamps-<child_lineage>.csv``, with ``space`` and
|
|
1527
|
+
``group`` read from the first row; a file of the same name is
|
|
1528
|
+
overwritten.
|
|
1529
|
+
|
|
1530
|
+
Returns
|
|
1531
|
+
-------
|
|
1532
|
+
pd.DataFrame
|
|
1533
|
+
Combined DataFrame containing data from all the 'stamp' files, with
|
|
1534
|
+
a fresh ``0..n-1`` index, and a ``group`` column (see Notes).
|
|
1535
|
+
|
|
1536
|
+
Raises
|
|
1537
|
+
------
|
|
1538
|
+
ValueError
|
|
1539
|
+
If `child_lineage` is not one of the accepted values; or if the
|
|
1540
|
+
stamps have no ``group`` column and a file name is not
|
|
1541
|
+
``<name>-<group>-stamps.csv``, or the names give more than one
|
|
1542
|
+
group.
|
|
1543
|
+
|
|
1544
|
+
Warns
|
|
1545
|
+
-----
|
|
1546
|
+
UserWarning
|
|
1547
|
+
If `child_lineage` is 'whole' and the 'segment' and 'segment_id'
|
|
1548
|
+
columns are dropped.
|
|
1549
|
+
|
|
1550
|
+
Notes
|
|
1551
|
+
-----
|
|
1552
|
+
- It is assumed that all the stamp files have belong to the same *group* in
|
|
1553
|
+
a given *space*.
|
|
1554
|
+
- If `child_lineage` is 'whole', columns 'segment' and 'segment_id' are
|
|
1555
|
+
removed, as they are redundant for whole-lineage data, with a
|
|
1556
|
+
`UserWarning`.
|
|
1557
|
+
Either may be absent, as in stamps the probe phase writes for a whole;
|
|
1558
|
+
if neither is present, nothing is dropped and a DEBUG record is
|
|
1559
|
+
logged.
|
|
1560
|
+
- The stamps :func:`~polyphys.probe.core.write_stamps` writes carry a
|
|
1561
|
+
``group`` column; the research code's stamps did not, and held the
|
|
1562
|
+
group in the file name, ``<lineage name>-<group>-stamps.csv``. When
|
|
1563
|
+
the column is missing it is added from the file names, after
|
|
1564
|
+
``space`` when that column is present and last otherwise, so the
|
|
1565
|
+
result and the parent stamps built from it carry the group either
|
|
1566
|
+
way.
|
|
1567
|
+
- Let n_files be the number of stamp files, n_rows the total number of
|
|
1568
|
+
rows, and n_cols the number of columns. Time is O(n_rows * n_cols).
|
|
1569
|
+
Every file is read before the frames are concatenated, so peak memory
|
|
1570
|
+
is about twice the combined data, O(n_rows * n_cols).
|
|
1571
|
+
|
|
1572
|
+
Examples
|
|
1573
|
+
--------
|
|
1574
|
+
>>> import tempfile
|
|
1575
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
1576
|
+
>>> paths = []
|
|
1577
|
+
>>> for seg_id in (1, 2):
|
|
1578
|
+
... path = Path(tmp.name) / f'w1.j0{seg_id}-bug-stamps.csv'
|
|
1579
|
+
... pd.DataFrame({'segment': [f'w1.j0{seg_id}'], 'whole': ['w1'],
|
|
1580
|
+
... 'segment_id': [seg_id], 'n_frames': [10]}
|
|
1581
|
+
... ).to_csv(path, index=False)
|
|
1582
|
+
... paths.append((str(path),))
|
|
1583
|
+
>>> children_stamps(paths, 'segment')
|
|
1584
|
+
segment whole segment_id n_frames group
|
|
1585
|
+
0 w1.j01 w1 1 10 bug
|
|
1586
|
+
1 w1.j02 w1 2 10 bug
|
|
1587
|
+
>>> tmp.cleanup()
|
|
1588
|
+
|
|
1589
|
+
The stamps have no ``group`` column, so it comes from the file names
|
|
1590
|
+
and, with no ``space`` column either, goes last.
|
|
1591
|
+
"""
|
|
1592
|
+
invalid_keyword(child_lineage, ["segment", "whole"])
|
|
1593
|
+
|
|
1594
|
+
# Load and concatenate all stamps
|
|
1595
|
+
space_stamps = pd.concat(
|
|
1596
|
+
[pd.read_csv(stamp[0]) for stamp in stamps], ignore_index=True
|
|
1597
|
+
)
|
|
1598
|
+
|
|
1599
|
+
# The research code's stamps hold the group in the file name only.
|
|
1600
|
+
if "group" not in space_stamps.columns:
|
|
1601
|
+
group = _group_from_stamp_names(stamps)
|
|
1602
|
+
columns = list(space_stamps.columns)
|
|
1603
|
+
position = (columns.index("space") + 1 if "space" in columns
|
|
1604
|
+
else len(columns))
|
|
1605
|
+
space_stamps.insert(position, "group", group)
|
|
1606
|
+
|
|
1607
|
+
# Drop specific columns for 'whole' lineage
|
|
1608
|
+
cols_to_drop = ["segment", "segment_id"]
|
|
1609
|
+
if child_lineage == "whole":
|
|
1610
|
+
dropped = [col for col in cols_to_drop if col in space_stamps.columns]
|
|
1611
|
+
if dropped:
|
|
1612
|
+
space_stamps = space_stamps.drop(columns=dropped)
|
|
1613
|
+
warnings.warn(
|
|
1614
|
+
"'segment' and 'segment_id' columns are dropped when"
|
|
1615
|
+
" individual 'whole' stamps combined to create a single"
|
|
1616
|
+
" dataframe of 'whole' stamps by 'children_stamps'.",
|
|
1617
|
+
UserWarning,
|
|
1618
|
+
)
|
|
1619
|
+
else:
|
|
1620
|
+
logger.debug("No %s columns to drop from the whole stamps.",
|
|
1621
|
+
cols_to_drop)
|
|
1622
|
+
|
|
1623
|
+
# Save the resulting DataFrame to a CSV if specified
|
|
1624
|
+
if save_to is not None:
|
|
1625
|
+
space_name = str(space_stamps.loc[0, "space"])
|
|
1626
|
+
group = str(space_stamps.loc[0, "group"])
|
|
1627
|
+
filename = f"{space_name}-{group}-stamps-{child_lineage}.csv"
|
|
1628
|
+
space_stamps.to_csv(os.path.join(save_to, filename), index=False)
|
|
1629
|
+
|
|
1630
|
+
return space_stamps
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
def parents_stamps(
|
|
1634
|
+
stamps: pd.DataFrame,
|
|
1635
|
+
child_lineage: PrimitiveLineageT,
|
|
1636
|
+
properties: dict[str, Callable[..., Any]] | None = None,
|
|
1637
|
+
save_to: Pathish | None = None,
|
|
1638
|
+
) -> pd.DataFrame:
|
|
1639
|
+
"""
|
|
1640
|
+
Aggregate the child stamps of a *space* into parent stamps.
|
|
1641
|
+
|
|
1642
|
+
The aggregation functions are given by `properties`; with `save_to`, the
|
|
1643
|
+
result is also saved as a CSV file.
|
|
1644
|
+
|
|
1645
|
+
Parameters
|
|
1646
|
+
----------
|
|
1647
|
+
stamps : pd.DataFrame
|
|
1648
|
+
DataFrame containing child stamp data, the lineage of whose rows is
|
|
1649
|
+
`child_lineage`. It must have a 'whole' and a 'segment_id' column for
|
|
1650
|
+
'segment' stamps, and an 'ensemble_long' and an 'ensemble_id' column
|
|
1651
|
+
for 'whole' stamps, and an 'n_frames' column for both.
|
|
1652
|
+
child_lineage : {'segment', 'whole'}
|
|
1653
|
+
Lineage of the child stamps, the rows of `stamps`; the parents are
|
|
1654
|
+
one level up, 'whole' or 'ensemble_long'.
|
|
1655
|
+
properties : dict of str -> Callable, optional
|
|
1656
|
+
Dictionary specifying aggregation functions for specific columns,
|
|
1657
|
+
replacing the default 'last'. It cannot name the parent column
|
|
1658
|
+
('whole' or 'ensemble_long'), the counted column ('segment_id' or
|
|
1659
|
+
'ensemble_id'), or 'n_frames', which the function aggregates itself.
|
|
1660
|
+
save_to : Pathish, optional
|
|
1661
|
+
Directory path where the parent DataFrame is saved, if specified,
|
|
1662
|
+
as ``<space>-<group>-stamps-<level>.csv``, with ``<level>`` the
|
|
1663
|
+
lineage of the parent rows, ``whole`` for 'segment' stamps and
|
|
1664
|
+
``ensemble_long`` for 'whole' stamps, and with ``space`` and
|
|
1665
|
+
``group`` read from the first row; a file of the same name is
|
|
1666
|
+
overwritten.
|
|
1667
|
+
|
|
1668
|
+
Returns
|
|
1669
|
+
-------
|
|
1670
|
+
pd.DataFrame
|
|
1671
|
+
Aggregated parent DataFrame, one row per parent:
|
|
1672
|
+
|
|
1673
|
+
- 'segment': one row per 'whole', with 'n_segments' (the number of
|
|
1674
|
+
segments) and 'n_frames' (their sum).
|
|
1675
|
+
- 'whole': one row per 'ensemble_long', with 'n_ensembles' (the
|
|
1676
|
+
number of wholes) and 'n_frames' (the last whole's); 'segment',
|
|
1677
|
+
'segment_id', and 'n_segments' are dropped if present.
|
|
1678
|
+
|
|
1679
|
+
Every other column takes its last value in each group unless
|
|
1680
|
+
`properties` says otherwise; 'lineage_name' and the `child_lineage`
|
|
1681
|
+
column are dropped.
|
|
1682
|
+
|
|
1683
|
+
Raises
|
|
1684
|
+
------
|
|
1685
|
+
ValueError
|
|
1686
|
+
If `child_lineage` is not one of the accepted values; if a row has no
|
|
1687
|
+
parent name (NaN in 'whole' for 'segment' stamps, in
|
|
1688
|
+
'ensemble_long' for 'whole' stamps), which the group-by would drop;
|
|
1689
|
+
if `properties` names a column the function aggregates itself; or
|
|
1690
|
+
if `save_to` is given and the stamps lack a 'space' or a 'group'
|
|
1691
|
+
column, which the file name needs.
|
|
1692
|
+
|
|
1693
|
+
Notes
|
|
1694
|
+
-----
|
|
1695
|
+
- It is assumed that all the stamp files have belong to the same *group* in
|
|
1696
|
+
a given *space*.
|
|
1697
|
+
- If `child_lineage` is 'segment', stamps correspond to individual
|
|
1698
|
+
segments, each with a unique `segment_id`.
|
|
1699
|
+
- If `child_lineage` is 'whole', stamps correspond to whole simulations,
|
|
1700
|
+
each identified by an `ensemble_id`.
|
|
1701
|
+
- The 'lineage_name' column and the `child_lineage` column are not
|
|
1702
|
+
carried into the parent stamps. Either may be absent, as
|
|
1703
|
+
'lineage_name' is from the stamps the probe phase writes; an absent
|
|
1704
|
+
one is skipped with a DEBUG record.
|
|
1705
|
+
- Let n_rows be the number of child stamps and n_cols the number of
|
|
1706
|
+
columns. The group-by is hash-based, so expected time is
|
|
1707
|
+
O(n_rows * n_cols); memory is one copy of the parent stamps.
|
|
1708
|
+
|
|
1709
|
+
Examples
|
|
1710
|
+
--------
|
|
1711
|
+
>>> stamps = pd.DataFrame({
|
|
1712
|
+
... 'lineage_name': ['w1.j01', 'w1.j02', 'w2.j01'],
|
|
1713
|
+
... 'segment': ['w1.j01', 'w1.j02', 'w2.j01'],
|
|
1714
|
+
... 'whole': ['w1', 'w1', 'w2'],
|
|
1715
|
+
... 'segment_id': [1, 2, 1],
|
|
1716
|
+
... 'n_frames': [10, 20, 30],
|
|
1717
|
+
... })
|
|
1718
|
+
>>> parents_stamps(stamps, 'segment')
|
|
1719
|
+
whole n_segments n_frames
|
|
1720
|
+
0 w1 2 30
|
|
1721
|
+
1 w2 1 30
|
|
1722
|
+
"""
|
|
1723
|
+
invalid_keyword(child_lineage, ["segment", "whole"])
|
|
1724
|
+
|
|
1725
|
+
# Base aggregation functions for columns
|
|
1726
|
+
base_columns = list(stamps.columns)
|
|
1727
|
+
for column in ("lineage_name", child_lineage):
|
|
1728
|
+
if column in base_columns:
|
|
1729
|
+
base_columns.remove(column)
|
|
1730
|
+
else:
|
|
1731
|
+
logger.debug("Column %r is not among the stamps columns.", column)
|
|
1732
|
+
agg_funcs: dict[str, str | Callable[..., Any]] = {
|
|
1733
|
+
col: "last" for col in base_columns
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
# Define grouping column and lineage-specific aggregations
|
|
1737
|
+
if child_lineage == "whole":
|
|
1738
|
+
parent_groupby = "ensemble_long"
|
|
1739
|
+
own_aggs: dict[str, str] = {"ensemble_id": "count",
|
|
1740
|
+
"n_frames": "last"}
|
|
1741
|
+
else:
|
|
1742
|
+
parent_groupby = "whole"
|
|
1743
|
+
own_aggs = {"segment_id": "count", "n_frames": "sum"}
|
|
1744
|
+
for column in (properties or {}):
|
|
1745
|
+
if column == parent_groupby or column in own_aggs:
|
|
1746
|
+
raise ValueError(
|
|
1747
|
+
f"properties cannot set {column!r}, which parents_stamps "
|
|
1748
|
+
"aggregates itself.")
|
|
1749
|
+
missing = int(stamps[parent_groupby].isna().sum())
|
|
1750
|
+
if missing:
|
|
1751
|
+
rows = "row has" if missing == 1 else "rows have"
|
|
1752
|
+
raise ValueError(
|
|
1753
|
+
f"{missing} stamps {rows} no {parent_groupby!r}; the group-by "
|
|
1754
|
+
"would drop them.")
|
|
1755
|
+
if properties is not None:
|
|
1756
|
+
agg_funcs.update(properties)
|
|
1757
|
+
agg_funcs.update(own_aggs)
|
|
1758
|
+
|
|
1759
|
+
# Perform the groupby and aggregate
|
|
1760
|
+
agg_funcs.pop(parent_groupby)
|
|
1761
|
+
parent_stamps = stamps.groupby(parent_groupby).agg(agg_funcs).reset_index()
|
|
1762
|
+
|
|
1763
|
+
# Rename and drop columns based on lineage
|
|
1764
|
+
if child_lineage == "whole":
|
|
1765
|
+
parent_stamps = parent_stamps.rename(
|
|
1766
|
+
columns={"ensemble_id": "n_ensembles"}
|
|
1767
|
+
)
|
|
1768
|
+
# If the 'whole' stamps are generated directly in the 'probe' phase,
|
|
1769
|
+
# then 'segment' and 'segment_id' columns are "N/A" and are removed
|
|
1770
|
+
# from the list of stamps columns that are added to the parents
|
|
1771
|
+
# stamps.
|
|
1772
|
+
# There is no need to have the 'n_segment' column in the parent
|
|
1773
|
+
# stamps, so it is removed. The 'whole' stamps directly generated
|
|
1774
|
+
# in the 'probe' phase do not have such a column, but those generated
|
|
1775
|
+
# from 'segment' stamps have.
|
|
1776
|
+
# Dropping redundant columns silently:
|
|
1777
|
+
parent_stamps = parent_stamps.drop(
|
|
1778
|
+
columns=["n_segments", "segment_id", "segment"],
|
|
1779
|
+
errors="ignore",
|
|
1780
|
+
)
|
|
1781
|
+
else:
|
|
1782
|
+
parent_stamps = parent_stamps.rename(
|
|
1783
|
+
columns={"segment_id": "n_segments"}
|
|
1784
|
+
)
|
|
1785
|
+
|
|
1786
|
+
if save_to is not None:
|
|
1787
|
+
for column in ("space", "group"):
|
|
1788
|
+
if column not in parent_stamps.columns:
|
|
1789
|
+
raise ValueError(
|
|
1790
|
+
f"Saving the parent stamps needs a {column!r} column "
|
|
1791
|
+
"for the file name.")
|
|
1792
|
+
space_name = str(parent_stamps.loc[0, "space"])
|
|
1793
|
+
group = str(parent_stamps.loc[0, "group"])
|
|
1794
|
+
level = parent_groupby
|
|
1795
|
+
filename = f"{space_name}-{group}-stamps-{level}.csv"
|
|
1796
|
+
parent_stamps.to_csv(os.path.join(save_to, filename), index=False)
|
|
1797
|
+
|
|
1798
|
+
return parent_stamps
|
|
1799
|
+
|
|
1800
|
+
|
|
1801
|
+
def space_tseries(
|
|
1802
|
+
input_database: str,
|
|
1803
|
+
prop: PropertyT,
|
|
1804
|
+
parser: ParserClassT,
|
|
1805
|
+
hierarchy: str,
|
|
1806
|
+
physical_attrs: list[str],
|
|
1807
|
+
group: GroupT,
|
|
1808
|
+
*,
|
|
1809
|
+
divisor: float = 0.025,
|
|
1810
|
+
round_to: int = 3,
|
|
1811
|
+
is_save: bool | None = False,
|
|
1812
|
+
crowder_attr: str = "phi_bulk_c",
|
|
1813
|
+
) -> pd.DataFrame:
|
|
1814
|
+
"""
|
|
1815
|
+
Stack the ensemble-averaged time series of one property across the
|
|
1816
|
+
state points of a space into one DataFrame.
|
|
1817
|
+
|
|
1818
|
+
The physical attributes in `physical_attrs` are added as columns.
|
|
1819
|
+
|
|
1820
|
+
Parameters
|
|
1821
|
+
----------
|
|
1822
|
+
input_database : str
|
|
1823
|
+
Path to the directory containing the ensemble-averaged time-series
|
|
1824
|
+
files, named ``<ensemble_long>-<group>-<prop>-ensAvg.csv``, with or
|
|
1825
|
+
without a trailing path separator.
|
|
1826
|
+
prop : str
|
|
1827
|
+
Name of the physical property of interest.
|
|
1828
|
+
parser : ParserClassT
|
|
1829
|
+
Parser class to infer file-specific information from filenames or
|
|
1830
|
+
paths.
|
|
1831
|
+
hierarchy : str
|
|
1832
|
+
Pattern prefix for the time-series filenames (e.g., `"N*"`).
|
|
1833
|
+
physical_attrs : list[str]
|
|
1834
|
+
List of physical attributes to add as new columns in the output
|
|
1835
|
+
DataFrame.
|
|
1836
|
+
group : GroupT
|
|
1837
|
+
Particle group type. Its trajectory's dump period, in time steps,
|
|
1838
|
+
is the `parser` attribute named by ``dump_keyword``.
|
|
1839
|
+
divisor : float, default 0.025
|
|
1840
|
+
Rounding step for the crowder volume fraction named by
|
|
1841
|
+
`crowder_attr`.
|
|
1842
|
+
round_to : int, default 3
|
|
1843
|
+
Number of decimal places for rounding the crowder volume fraction.
|
|
1844
|
+
is_save : bool, default False
|
|
1845
|
+
If True, saves the concatenated DataFrame to a CSV file named
|
|
1846
|
+
``<space>-<group>-<prop>-space.csv`` in the ``analysis`` phase's
|
|
1847
|
+
``space`` directory, which :func:`make_database` creates next to
|
|
1848
|
+
`input_database`; a file of the same name is overwritten.
|
|
1849
|
+
crowder_attr : str, default 'phi_bulk_c'
|
|
1850
|
+
Name of the `parser` attribute that holds the bulk volume fraction
|
|
1851
|
+
of crowders. Its value is added as a column of that name, whether or
|
|
1852
|
+
not it is listed in `physical_attrs`, and rounded into the
|
|
1853
|
+
``phi_bulk_c_round`` column.
|
|
1854
|
+
|
|
1855
|
+
Returns
|
|
1856
|
+
-------
|
|
1857
|
+
pd.DataFrame
|
|
1858
|
+
Concatenated DataFrame of all time-series with added physical
|
|
1859
|
+
attributes. Besides the columns of each file, it has ``t_index``
|
|
1860
|
+
(the frame index times the dump period of the `group` trajectory),
|
|
1861
|
+
``time`` (``t_index`` times the parser's ``dt``), one column per
|
|
1862
|
+
entry in `physical_attrs`, a `crowder_attr` column, and
|
|
1863
|
+
``phi_bulk_c_round``.
|
|
1864
|
+
|
|
1865
|
+
Examples
|
|
1866
|
+
--------
|
|
1867
|
+
One ensemble-averaged time series of two frames of the ``bug``
|
|
1868
|
+
trajectory, dumped every 1000 steps (``bdump``) of 0.005 time units:
|
|
1869
|
+
|
|
1870
|
+
>>> import tempfile
|
|
1871
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
1872
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
1873
|
+
>>> database = Path(tmp.name) / 'analysis' / 'N20-bug-ensAvg'
|
|
1874
|
+
>>> database.mkdir(parents=True)
|
|
1875
|
+
>>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
|
|
1876
|
+
>>> pd.DataFrame({'gyrTMon-mean': [1.0, 2.0]}).to_csv(
|
|
1877
|
+
... database / f'{state}-bug-gyrTMon-ensAvg.csv', index=False)
|
|
1878
|
+
>>> space = space_tseries(str(database) + '/', 'gyrTMon', ChainCyl,
|
|
1879
|
+
... 'N*', ['nmon'], 'bug')
|
|
1880
|
+
>>> space[['t_index', 'time', 'gyrTMon-mean', 'nmon']]
|
|
1881
|
+
t_index time gyrTMon-mean nmon
|
|
1882
|
+
0 0 0.0 1.0 20
|
|
1883
|
+
1 1000 5.0 2.0 20
|
|
1884
|
+
>>> tmp.cleanup()
|
|
1885
|
+
|
|
1886
|
+
Notes
|
|
1887
|
+
-----
|
|
1888
|
+
- Files are selected by globbing
|
|
1889
|
+
``os.path.join(input_database, hierarchy + '-<prop>-ensAvg.csv')``
|
|
1890
|
+
and sorted alphanumerically by their full names.
|
|
1891
|
+
- Requires a parser class with methods to retrieve attribute information
|
|
1892
|
+
for each file.
|
|
1893
|
+
- Let n_files be the number of matched files, n_rows the total number
|
|
1894
|
+
of rows across them, and n_cols the number of columns of the result.
|
|
1895
|
+
Time is O(n_rows * n_cols + n_files log n_files). Every file is read
|
|
1896
|
+
and kept before the frames are concatenated, so peak memory is about
|
|
1897
|
+
twice the result, O(n_rows * n_cols).
|
|
1898
|
+
"""
|
|
1899
|
+
prop_ext = f"-{prop}-ensAvg.csv"
|
|
1900
|
+
ens_avg_csvs = sort_filenames(
|
|
1901
|
+
glob(os.path.join(input_database, hierarchy + prop_ext)), ["csv"]
|
|
1902
|
+
)
|
|
1903
|
+
prop_csvs = []
|
|
1904
|
+
|
|
1905
|
+
for ens_avg_csv in ens_avg_csvs:
|
|
1906
|
+
ens_avg = pd.read_csv(ens_avg_csv[0], header=0)
|
|
1907
|
+
prop_info = parser(
|
|
1908
|
+
ens_avg_csv[0], "ensemble_long", group, ispath=True
|
|
1909
|
+
)
|
|
1910
|
+
|
|
1911
|
+
ens_avg = ens_avg.reset_index()
|
|
1912
|
+
ens_avg = ens_avg.rename(columns={"index": "t_index"})
|
|
1913
|
+
|
|
1914
|
+
# Scale frame indices by the dump period of the group's trajectory
|
|
1915
|
+
ens_avg["t_index"] *= getattr(prop_info, prop_info.dump_keyword)
|
|
1916
|
+
ens_avg["time"] = ens_avg["t_index"] * getattr(prop_info, "dt")
|
|
1917
|
+
|
|
1918
|
+
# Add physical attributes
|
|
1919
|
+
for attr_name in physical_attrs:
|
|
1920
|
+
ens_avg[attr_name] = getattr(prop_info, attr_name)
|
|
1921
|
+
|
|
1922
|
+
# Add and round the bulk volume fraction of crowders
|
|
1923
|
+
ens_avg[crowder_attr] = getattr(prop_info, crowder_attr)
|
|
1924
|
+
ens_avg["phi_bulk_c_round"] = ens_avg[crowder_attr].apply(
|
|
1925
|
+
lambda x: round_to_multiple(x, divisor, round_to)
|
|
1926
|
+
)
|
|
1927
|
+
|
|
1928
|
+
prop_csvs.append(ens_avg)
|
|
1929
|
+
|
|
1930
|
+
# Concatenate all time-series DataFrames
|
|
1931
|
+
prop_db = pd.concat(prop_csvs, axis=0)
|
|
1932
|
+
prop_db = prop_db.reset_index(drop=True)
|
|
1933
|
+
|
|
1934
|
+
# Optionally save to file
|
|
1935
|
+
if is_save:
|
|
1936
|
+
save_to_space = make_database(
|
|
1937
|
+
input_database, "analysis", stage="space", group=group
|
|
1938
|
+
)
|
|
1939
|
+
space = save_to_space.split("/")[-2].split("-")[0]
|
|
1940
|
+
filepath = (
|
|
1941
|
+
save_to_space + f"{create_fullname(space, group, prop)}-space.csv"
|
|
1942
|
+
)
|
|
1943
|
+
prop_db.to_csv(filepath, index=False)
|
|
1944
|
+
|
|
1945
|
+
return prop_db
|
|
1946
|
+
|
|
1947
|
+
|
|
1948
|
+
def space_hists(
|
|
1949
|
+
input_database: str,
|
|
1950
|
+
prop: PropertyT,
|
|
1951
|
+
parser: ParserClassT,
|
|
1952
|
+
hierarchy: str,
|
|
1953
|
+
physical_attrs: list[str],
|
|
1954
|
+
group: GroupT,
|
|
1955
|
+
bin_center: np.ndarray | None = None,
|
|
1956
|
+
normalize: bool | None = False,
|
|
1957
|
+
divisor: float = 0.025,
|
|
1958
|
+
round_to: int = 3,
|
|
1959
|
+
is_save: bool | None = False,
|
|
1960
|
+
crowder_attr: str = "phi_bulk_c",
|
|
1961
|
+
) -> pd.DataFrame:
|
|
1962
|
+
"""
|
|
1963
|
+
Stack the ensemble-averaged histograms of one property across the state
|
|
1964
|
+
points of a space into one DataFrame.
|
|
1965
|
+
|
|
1966
|
+
The histograms are normalized if `normalize` asks for it.
|
|
1967
|
+
|
|
1968
|
+
Parameters
|
|
1969
|
+
----------
|
|
1970
|
+
input_database : str
|
|
1971
|
+
Path to the directory containing the ensemble-averaged histogram
|
|
1972
|
+
files, named ``<ensemble_long>-<group>-<prop>-ensAvg.csv``, with or
|
|
1973
|
+
without a trailing path separator.
|
|
1974
|
+
prop : :py:obj:`~.PropertyT`
|
|
1975
|
+
Name of the physical property of interest.
|
|
1976
|
+
parser : ParserClassT
|
|
1977
|
+
Parser class to infer file-specific information from filenames or
|
|
1978
|
+
paths.
|
|
1979
|
+
hierarchy : str
|
|
1980
|
+
Pattern prefix for the histogram filenames (e.g., `"N*"`).
|
|
1981
|
+
physical_attrs : list[str]
|
|
1982
|
+
List of physical attributes to add as new columns in the output
|
|
1983
|
+
DataFrame.
|
|
1984
|
+
group : GroupT
|
|
1985
|
+
Particle group type.
|
|
1986
|
+
bin_center : np.ndarray, optional
|
|
1987
|
+
Array of bin centers. If not provided, must be present in the
|
|
1988
|
+
DataFrames.
|
|
1989
|
+
normalize : bool, default False
|
|
1990
|
+
If True, normalizes the histogram data.
|
|
1991
|
+
divisor : float, default 0.025
|
|
1992
|
+
Rounding step for the crowder volume fraction named by
|
|
1993
|
+
`crowder_attr`.
|
|
1994
|
+
round_to : int, default 3
|
|
1995
|
+
Number of decimal places for rounding the crowder volume fraction.
|
|
1996
|
+
is_save : bool, default False
|
|
1997
|
+
If True, saves the concatenated DataFrame to a CSV file named
|
|
1998
|
+
``<space>-<group>-<prop>-space.csv`` in the ``analysis`` phase's
|
|
1999
|
+
``space`` directory, which :func:`make_database` creates next to
|
|
2000
|
+
`input_database`; a file of the same name is overwritten.
|
|
2001
|
+
crowder_attr : str, default 'phi_bulk_c'
|
|
2002
|
+
Name of the `parser` attribute that holds the bulk volume fraction
|
|
2003
|
+
of crowders. Its value is added as a column of that name, whether or
|
|
2004
|
+
not it is listed in `physical_attrs`, and rounded into the
|
|
2005
|
+
``phi_bulk_c_round`` column.
|
|
2006
|
+
|
|
2007
|
+
Returns
|
|
2008
|
+
-------
|
|
2009
|
+
pd.DataFrame
|
|
2010
|
+
Concatenated DataFrame of all histograms with added physical
|
|
2011
|
+
attributes: one column per entry in `physical_attrs`, a
|
|
2012
|
+
`crowder_attr` column, and ``phi_bulk_c_round``.
|
|
2013
|
+
|
|
2014
|
+
Warns
|
|
2015
|
+
-----
|
|
2016
|
+
UserWarning
|
|
2017
|
+
If `normalize` is True and the ``<prop>-mean`` column of a file sums
|
|
2018
|
+
to zero; its ``<prop>-norm`` column is then set to zero.
|
|
2019
|
+
|
|
2020
|
+
Examples
|
|
2021
|
+
--------
|
|
2022
|
+
One ensemble-averaged histogram of three bins, normalized to sum to one:
|
|
2023
|
+
|
|
2024
|
+
>>> import tempfile
|
|
2025
|
+
>>> from polyphys.tests.synthetic import ChainCyl
|
|
2026
|
+
>>> tmp = tempfile.TemporaryDirectory()
|
|
2027
|
+
>>> database = Path(tmp.name) / 'analysis' / 'N20-bug-ensAvg'
|
|
2028
|
+
>>> database.mkdir(parents=True)
|
|
2029
|
+
>>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
|
|
2030
|
+
>>> pd.DataFrame({'rhoMon-mean': [1.0, 2.0, 1.0]}).to_csv(
|
|
2031
|
+
... database / f'{state}-bug-rhoMon-ensAvg.csv', index=False)
|
|
2032
|
+
>>> space = space_hists(str(database) + '/', 'rhoMon', ChainCyl, 'N*',
|
|
2033
|
+
... ['nmon'], 'bug',
|
|
2034
|
+
... bin_center=np.array([0.5, 1.5, 2.5]),
|
|
2035
|
+
... normalize=True)
|
|
2036
|
+
>>> space[['bin_center', 'rhoMon-mean', 'rhoMon-norm', 'nmon']]
|
|
2037
|
+
bin_center rhoMon-mean rhoMon-norm nmon
|
|
2038
|
+
0 0.5 1.0 0.25 20
|
|
2039
|
+
1 1.5 2.0 0.50 20
|
|
2040
|
+
2 2.5 1.0 0.25 20
|
|
2041
|
+
>>> tmp.cleanup()
|
|
2042
|
+
|
|
2043
|
+
Notes
|
|
2044
|
+
-----
|
|
2045
|
+
- Files are selected by globbing
|
|
2046
|
+
``os.path.join(input_database, hierarchy + '-<prop>-ensAvg.csv')``
|
|
2047
|
+
and sorted alphanumerically by their full names.
|
|
2048
|
+
- If `normalize` is True, histogram values will be scaled to sum to 1.
|
|
2049
|
+
- Let n_files be the number of matched files, n_rows the total number
|
|
2050
|
+
of rows across them, and n_cols the number of columns of the result.
|
|
2051
|
+
Time is O(n_rows * n_cols + n_files log n_files). Every file is read
|
|
2052
|
+
and kept before the frames are concatenated, so peak memory is about
|
|
2053
|
+
twice the result, O(n_rows * n_cols).
|
|
2054
|
+
- The `bin_center` should be provided if it is not available in the input
|
|
2055
|
+
data. When all the ensemble-averaged DataFrames have the same bin
|
|
2056
|
+
center, like 'clustersHistTFoci' or 'bondsHistTFoci' properties, the
|
|
2057
|
+
`bin_center` may not be provided.
|
|
2058
|
+
"""
|
|
2059
|
+
prop_ext = f"-{prop}-ensAvg.csv"
|
|
2060
|
+
ens_avg_csvs_ungrouped = glob(
|
|
2061
|
+
os.path.join(input_database, hierarchy + prop_ext))
|
|
2062
|
+
ens_avg_csvs = sort_filenames(ens_avg_csvs_ungrouped, formats=["csv"])
|
|
2063
|
+
prop_csvs = []
|
|
2064
|
+
|
|
2065
|
+
for ens_avg_csv in ens_avg_csvs:
|
|
2066
|
+
ens_avg = pd.read_csv(ens_avg_csv[0], header=0)
|
|
2067
|
+
prop_info = parser(
|
|
2068
|
+
ens_avg_csv[0], "ensemble_long", group, ispath=True
|
|
2069
|
+
)
|
|
2070
|
+
|
|
2071
|
+
# Handle bin_center if provided
|
|
2072
|
+
if bin_center is not None:
|
|
2073
|
+
ens_avg["bin_center"] = bin_center.tolist()
|
|
2074
|
+
ens_avg["bin_center-norm"] = (
|
|
2075
|
+
ens_avg["bin_center"] / ens_avg["bin_center"].max()
|
|
2076
|
+
)
|
|
2077
|
+
|
|
2078
|
+
# Normalize data if specified
|
|
2079
|
+
if normalize:
|
|
2080
|
+
normalizer = ens_avg[prop + "-mean"].sum()
|
|
2081
|
+
if normalizer != 0:
|
|
2082
|
+
ens_avg[prop + "-norm"] = ens_avg[prop + "-mean"] / normalizer
|
|
2083
|
+
else:
|
|
2084
|
+
warnings.warn(
|
|
2085
|
+
"All values are zero; normalized values set to zero.",
|
|
2086
|
+
UserWarning,
|
|
2087
|
+
)
|
|
2088
|
+
ens_avg[prop + "-norm"] = 0
|
|
2089
|
+
|
|
2090
|
+
# Add physical attributes
|
|
2091
|
+
for attr_name in physical_attrs:
|
|
2092
|
+
ens_avg[attr_name] = getattr(prop_info, attr_name)
|
|
2093
|
+
|
|
2094
|
+
# Add and round the bulk volume fraction of crowders
|
|
2095
|
+
ens_avg[crowder_attr] = getattr(prop_info, crowder_attr)
|
|
2096
|
+
ens_avg["phi_bulk_c_round"] = ens_avg[crowder_attr].apply(
|
|
2097
|
+
lambda x: round_to_multiple(x, divisor, round_to)
|
|
2098
|
+
)
|
|
2099
|
+
|
|
2100
|
+
prop_csvs.append(ens_avg)
|
|
2101
|
+
|
|
2102
|
+
# Concatenate all histograms
|
|
2103
|
+
prop_db = pd.concat(prop_csvs, axis=0)
|
|
2104
|
+
prop_db = prop_db.reset_index(drop=True)
|
|
2105
|
+
|
|
2106
|
+
# Optionally save to file
|
|
2107
|
+
if is_save:
|
|
2108
|
+
save_to_space = make_database(
|
|
2109
|
+
input_database, "analysis", stage="space", group=group
|
|
2110
|
+
)
|
|
2111
|
+
space = save_to_space.split("/")[-2].split("-")[0]
|
|
2112
|
+
filepath = (
|
|
2113
|
+
save_to_space + f"{create_fullname(space, group, prop)}-space.csv"
|
|
2114
|
+
)
|
|
2115
|
+
prop_db.to_csv(filepath, index=False)
|
|
2116
|
+
|
|
2117
|
+
return prop_db
|
|
2118
|
+
|
|
2119
|
+
|
|
2120
|
+
BULK_NAME_MAP: Mapping[str, str] = MappingProxyType({
|
|
2121
|
+
"phi_m_bulk": "phi_bulk_m",
|
|
2122
|
+
"rho_m_bulk": "rho_bulk_m",
|
|
2123
|
+
"phi_c_bulk": "phi_bulk_c",
|
|
2124
|
+
"rho_c_bulk": "rho_bulk_c",
|
|
2125
|
+
"phi_c_bulk_one": "phi_bulk_c_one",
|
|
2126
|
+
"phi_c_bulk_two": "phi_bulk_c_two",
|
|
2127
|
+
"rho_c_bulk_one": "rho_bulk_c_one",
|
|
2128
|
+
"rho_c_bulk_two": "rho_bulk_c_two",
|
|
2129
|
+
"phi_hns_bulk": "phi_bulk_hns",
|
|
2130
|
+
"rho_hns_bulk": "rho_bulk_hns",
|
|
2131
|
+
"phi_c_bulk_round": "phi_bulk_c_round",
|
|
2132
|
+
})
|
|
2133
|
+
"""
|
|
2134
|
+
Each bulk-quantity name of the research code mapped to its PolyPhys name.
|
|
2135
|
+
|
|
2136
|
+
The research code named a bulk quantity ``<quantity>_<species>_bulk``, such
|
|
2137
|
+
as ``phi_c_bulk``; PolyPhys names it ``<quantity>_bulk_<species>``, such as
|
|
2138
|
+
``phi_bulk_c``. The research code wrote every parser attribute as a column
|
|
2139
|
+
of its ``*-stamps.csv`` files, and those columns carry into its
|
|
2140
|
+
ensemble-average and ``space`` tables, so a table it wrote carries the old
|
|
2141
|
+
names. The map is read-only; :func:`rename_bulk_columns` applies it.
|
|
2142
|
+
|
|
2143
|
+
.. versionadded:: 0.7.0
|
|
2144
|
+
"""
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
def rename_bulk_columns(table: pd.DataFrame) -> pd.DataFrame:
|
|
2148
|
+
"""
|
|
2149
|
+
Return a copy of `table` whose bulk-quantity columns carry the PolyPhys
|
|
2150
|
+
names.
|
|
2151
|
+
|
|
2152
|
+
Each column named by a key of :data:`BULK_NAME_MAP` is renamed to that
|
|
2153
|
+
key's value; every other column, every value, and the column order are
|
|
2154
|
+
unchanged. A table that already carries the PolyPhys names comes back
|
|
2155
|
+
equal to `table`.
|
|
2156
|
+
|
|
2157
|
+
Parameters
|
|
2158
|
+
----------
|
|
2159
|
+
table : pd.DataFrame
|
|
2160
|
+
A table read from a stamp, ensemble-average, or ``space`` file that
|
|
2161
|
+
the research code wrote.
|
|
2162
|
+
|
|
2163
|
+
Returns
|
|
2164
|
+
-------
|
|
2165
|
+
pd.DataFrame
|
|
2166
|
+
A new table; `table` is not modified.
|
|
2167
|
+
|
|
2168
|
+
Notes
|
|
2169
|
+
-----
|
|
2170
|
+
Runs in :math:`O(n_\\text{col})` time for the renaming and
|
|
2171
|
+
:math:`O(n_\\text{row} n_\\text{col})` space for the copy.
|
|
2172
|
+
|
|
2173
|
+
Examples
|
|
2174
|
+
--------
|
|
2175
|
+
A row of a research-code stamp file:
|
|
2176
|
+
|
|
2177
|
+
>>> import pandas as pd
|
|
2178
|
+
>>> old = pd.DataFrame({'nmon': [20], 'phi_c_bulk': [0.1],
|
|
2179
|
+
... 'rho_c_bulk': [0.191], 'n_frames': [100]})
|
|
2180
|
+
>>> new = rename_bulk_columns(old)
|
|
2181
|
+
>>> list(new.columns)
|
|
2182
|
+
['nmon', 'phi_bulk_c', 'rho_bulk_c', 'n_frames']
|
|
2183
|
+
>>> list(old.columns)
|
|
2184
|
+
['nmon', 'phi_c_bulk', 'rho_c_bulk', 'n_frames']
|
|
2185
|
+
|
|
2186
|
+
.. versionadded:: 0.7.0
|
|
2187
|
+
"""
|
|
2188
|
+
return table.rename(columns=dict(BULK_NAME_MAP))
|