polyphys 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. polyphys/__init__.py +7 -0
  2. polyphys/__version__.py +22 -0
  3. polyphys/analyze/__init__.py +51 -0
  4. polyphys/analyze/clusters.py +1738 -0
  5. polyphys/analyze/correlations.py +995 -0
  6. polyphys/analyze/density.py +1138 -0
  7. polyphys/analyze/measurer.py +1338 -0
  8. polyphys/analyze/pipeline.py +1868 -0
  9. polyphys/analyze/stats.py +810 -0
  10. polyphys/analyze/sumrule.py +1159 -0
  11. polyphys/manage/__init__.py +11 -0
  12. polyphys/manage/organizer.py +2188 -0
  13. polyphys/manage/parser.py +1264 -0
  14. polyphys/manage/types.py +171 -0
  15. polyphys/manage/utils.py +777 -0
  16. polyphys/probe/__init__.py +25 -0
  17. polyphys/probe/binders.py +711 -0
  18. polyphys/probe/chain_size.py +583 -0
  19. polyphys/probe/core.py +437 -0
  20. polyphys/probe/histograms.py +863 -0
  21. polyphys/probe/pairs.py +555 -0
  22. polyphys/simulate/__init__.py +48 -0
  23. polyphys/simulate/build.py +1231 -0
  24. polyphys/simulate/core.py +237 -0
  25. polyphys/simulate/data.py +977 -0
  26. polyphys/simulate/dump.py +752 -0
  27. polyphys/simulate/log.py +788 -0
  28. polyphys/spec.py +1359 -0
  29. polyphys/tests/analyze/test_clusters.py +1058 -0
  30. polyphys/tests/analyze/test_correlations.py +708 -0
  31. polyphys/tests/analyze/test_density.py +813 -0
  32. polyphys/tests/analyze/test_measurer.py +785 -0
  33. polyphys/tests/analyze/test_pipeline.py +1881 -0
  34. polyphys/tests/analyze/test_stats.py +732 -0
  35. polyphys/tests/analyze/test_sumrule.py +917 -0
  36. polyphys/tests/conftest.py +753 -0
  37. polyphys/tests/data/lammps_build/make_starts.py +70 -0
  38. polyphys/tests/manage/test_organizer.py +1495 -0
  39. polyphys/tests/manage/test_parser.py +1222 -0
  40. polyphys/tests/manage/test_utils.py +498 -0
  41. polyphys/tests/probe/conftest.py +407 -0
  42. polyphys/tests/probe/test_binders.py +686 -0
  43. polyphys/tests/probe/test_chain_size.py +557 -0
  44. polyphys/tests/probe/test_core.py +391 -0
  45. polyphys/tests/probe/test_histograms.py +779 -0
  46. polyphys/tests/probe/test_pairs.py +475 -0
  47. polyphys/tests/simulate/test_build.py +1042 -0
  48. polyphys/tests/simulate/test_core.py +106 -0
  49. polyphys/tests/simulate/test_data.py +858 -0
  50. polyphys/tests/simulate/test_dump.py +487 -0
  51. polyphys/tests/simulate/test_log.py +667 -0
  52. polyphys/tests/strategies.py +120 -0
  53. polyphys/tests/synthetic.py +410 -0
  54. polyphys/tests/test_conftest.py +414 -0
  55. polyphys/tests/test_docs.py +82 -0
  56. polyphys/tests/test_packaging.py +97 -0
  57. polyphys/tests/test_spec.py +772 -0
  58. polyphys/tests/viz/conftest.py +32 -0
  59. polyphys/tests/viz/test_colors.py +113 -0
  60. polyphys/tests/viz/test_helpers.py +292 -0
  61. polyphys/viz/__init__.py +15 -0
  62. polyphys/viz/colors.py +204 -0
  63. polyphys/viz/helpers.py +549 -0
  64. polyphys-1.0.0.dist-info/METADATA +339 -0
  65. polyphys-1.0.0.dist-info/RECORD +69 -0
  66. polyphys-1.0.0.dist-info/WHEEL +5 -0
  67. polyphys-1.0.0.dist-info/licenses/AUTHORS.rst +13 -0
  68. polyphys-1.0.0.dist-info/licenses/LICENSE +22 -0
  69. polyphys-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2188 @@
1
+ """\
2
+ ==========================================================
3
+ :mod:`polyphys.manage.organizer`
4
+ ==========================================================
5
+ The :mod:`polyphys.manage.organizer` module provides tools for organizing and
6
+ aggregating simulation data based on filename patterns, file structures, and
7
+ processing stages. It facilitates handling segmented data, grouping experiments
8
+ into ensembles, and consolidating ensembles into spaces.
9
+
10
+ The module is tailored for use with LAMMPS and similar simulation tools, where
11
+ it automates data consolidation across hierarchical levels. It is particularly
12
+ useful for organizing data generated in high-throughput simulations, enabling
13
+ efficient post-processing, ensemble averaging, and preparation of data for
14
+ statistical and comparative analysis.
15
+
16
+ Terminology
17
+ ===========
18
+ The following terms are used throughout the module to structure and organize
19
+ simulation output data:
20
+
21
+ - **Experiment**: A complete simulation run, typically using tools like LAMMPS,
22
+ representing a specific set of conditions.
23
+
24
+ - **Lineage**: The level of an artifact in the hierarchy
25
+ ``segment → whole → ensemble_long → ensemble → space``, the values of
26
+ :py:obj:`~.LineageT`; each level aggregates the one
27
+ before it:
28
+
29
+ - **segment**: a chunk of one simulation run.
30
+ - **whole**: one run, its segments joined.
31
+ - **ensemble_long**: one state point, every input parameter in the name.
32
+ - **ensemble**: one state point, its replicates (wholes with different
33
+ random seeds) collapsed into a short name.
34
+ - **space**: the parameter sweep, the state points of one study.
35
+
36
+ - **Attributes**: Macroscopic characteristics such as box size or polymer
37
+ topology that remain constant within a space.
38
+
39
+ - **Properties**: Physical features measured during the simulation, like
40
+ density or radius of gyration, with optional associated measures (e.g.,
41
+ autocorrelation).
42
+
43
+ - **Species**: The particle or molecular entity type, such as 'Mon' for
44
+ monomers, each with specific behaviors or attributes as needed.
45
+
46
+ - **Phases**: Directories organized by processing phase: `simsAll` and
47
+ `simsCont` (simulation inputs), `logs`, `trjs` (trajectories), `probes`
48
+ (probing data), `analysis` (post-processing), `viz` (visualization-ready
49
+ data), `galaxy`, and `allInOne`; the values of
50
+ :py:obj:`~.PhaseT`.
51
+
52
+ - **Stages**: Directories inside a phase, each holding the artifacts of one
53
+ lineage level or one step of the reduction: `segment`, `wholeSim`, `ens`,
54
+ `ensAvg`, `space`, and `galaxy`, the values of
55
+ :py:obj:`~.StageT`. The organizer writes them in the
56
+ `analysis` phase.
57
+
58
+ The words `segment` and `space` are both lineage levels and stages, and
59
+ `galaxy` is both a phase and a stage; which one a word is follows from where
60
+ it appears: a lineage argument, a phase directory, or the last field of a
61
+ stage directory's name. `galaxy` is not a lineage level.
62
+
63
+ Naming Conventions
64
+ ==================
65
+ The :doc:`/lineage` page is the reference; in short:
66
+
67
+ - A stage directory is ``<root>/<phase>/<prefix>-<group>-<stage>``, as
68
+ :func:`make_database` builds it.
69
+ - A file is ``<lineage name>-<group>-<property>``, as
70
+ :func:`create_fullname` builds it, with ``-ensAvg`` or ``-space`` added
71
+ where two stages hold files of one level (``ens`` and ``ensAvg``) or the
72
+ level is ``space``, and then the extension. A stamps table is
73
+ ``<space>-<group>-stamps-<level>.csv``, with ``<level>`` the lineage of
74
+ its rows.
75
+ - Fields are joined by ``-``. A lineage name never contains ``-``, and a
76
+ group cannot; a property may, to join a measure (``gyrTMon-acf``), so it
77
+ is everything between the group and the stage suffix or the extension.
78
+
79
+ Dependencies
80
+ ============
81
+ - `numpy`: For numerical operations on arrays, such as summing or concatenating
82
+ segments.
83
+ - `pandas`: For handling tabular data and DataFrames used in data aggregation.
84
+
85
+ Contents
86
+ ========
87
+
88
+ Naming and persistence:
89
+ :func:`create_fullname`, :func:`save_artifact`, and :func:`make_database`.
90
+
91
+ Building wholes from their parts:
92
+ :func:`whole_from_segments` and :func:`whole_from_file`.
93
+
94
+ Grouping wholes into ensembles:
95
+ :func:`ens_from_bin_edge`, :func:`ens_from_vec`, :func:`ens_from_mat_t`,
96
+ :func:`ens_from_df`, and :func:`ensemble`.
97
+
98
+ Averaging over an ensemble:
99
+ :func:`ens_avg_from_bin_edge`, :func:`ens_avg_from_ndarray`,
100
+ :func:`ens_avg_from_df`, and :func:`ensemble_avg`.
101
+
102
+ Space-level collection and bookkeeping:
103
+ :func:`children_stamps`, :func:`parents_stamps`, :func:`space_tseries`, and
104
+ :func:`space_hists`.
105
+
106
+ Reading tables the research code wrote:
107
+ :data:`BULK_NAME_MAP` and :func:`rename_bulk_columns`.
108
+
109
+ The density sum rule on the ``ensAvg`` profiles these reducers build is in
110
+ :mod:`polyphys.analyze.sumrule`.
111
+
112
+ Usage ===== The :mod:`~polyphys.manage.organizer` module streamlines
113
+ organization and aggregation of simulation data. Users can assemble segmented
114
+ data, organize data into ensembles, and generate ensemble-averaged data at
115
+ various aggregation stages. Its multi-level organization supports comparative
116
+ analysis across varied conditions within spaces and galaxies.
117
+
118
+ Examples
119
+ ========
120
+ Two replicates (``ens1`` and ``ens2``) of one state point, each probed in
121
+ two segments (``j01`` and ``j02``), reduce to one ensemble-averaged time
122
+ series. Segment ``s`` of replicate ``e`` holds two frames of the value
123
+ ``10 * e + s``:
124
+
125
+ >>> import tempfile
126
+ >>> from polyphys.tests.synthetic import ChainCyl
127
+ >>> tmp = tempfile.TemporaryDirectory()
128
+ >>> root = Path(tmp.name)
129
+ >>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
130
+ >>> paths = []
131
+ >>> for e in (1, 2):
132
+ ... for s in (1, 2):
133
+ ... path = root / f'{state}ens{e}.j0{s}.bug.gyrTMon.npy'
134
+ ... np.save(path, np.full(2, 10.0 * e + s))
135
+ ... paths.append(path)
136
+
137
+ Join each replicate's segments into a whole, collect the wholes of the
138
+ state point into an ensemble, and average over the replicates:
139
+
140
+ >>> wholes = whole_from_segments('gyrTMon', paths, ChainCyl, 'bug',
141
+ ... 'tseries')
142
+ >>> wholes[state + 'ens1']
143
+ array([11., 11., 12., 12.])
144
+ >>> ensembles = ensemble('gyrTMon', wholes, ChainCyl, 'bug', 'vector')
145
+ >>> averages = ensemble_avg('gyrTMon', ensembles, 'bug', 'dataframe')
146
+ >>> averages[state]
147
+ gyrTMon-mean gyrTMon-var gyrTMon-sem
148
+ 0 16.0 50.0 5.0
149
+ 1 16.0 50.0 5.0
150
+ 2 17.0 50.0 5.0
151
+ 3 17.0 50.0 5.0
152
+ >>> tmp.cleanup()
153
+
154
+ Notes
155
+ =====
156
+ - Filenames must follow consistent patterns for the `organizer` module to
157
+ accurately parse and organize data.
158
+ - This module supports only cylindrical, slit, or cubic geometries.
159
+ - Functions in this module rely on consistent naming conventions for groups,
160
+ properties, species, and stages.
161
+ """
162
+
163
+ import warnings
164
+ import logging
165
+ from typing import (
166
+ Any, Callable, Literal, Mapping, Sequence, Iterator, TYPE_CHECKING,
167
+ get_args,
168
+ )
169
+ import os
170
+ import pathlib
171
+ from pathlib import Path
172
+ from glob import glob
173
+ from types import MappingProxyType
174
+ import numpy as np
175
+ import pandas as pd
176
+ from .types import (
177
+ Pathish,
178
+ PathGroup,
179
+ AnyPathIter,
180
+ GroupT,
181
+ PropertyT,
182
+ ParserClassT,
183
+ StageT,
184
+ PhaseT,
185
+ WholeRelationT,
186
+ PrimitiveLineageT,
187
+ )
188
+ from .utils import invalid_keyword, sort_filenames, round_to_multiple
189
+
190
+ if TYPE_CHECKING:
191
+ # Unused here, but sphinx_autodoc_typehints runs this block to resolve
192
+ # the "ParserBase" forward reference inside ParserClassT.
193
+ from .parser import ParserBase # noqa: F401
194
+
195
+ __all__ = [
196
+ 'BULK_NAME_MAP',
197
+ 'children_stamps',
198
+ 'create_fullname',
199
+ 'ens_avg_from_bin_edge',
200
+ 'ens_avg_from_df',
201
+ 'ens_avg_from_ndarray',
202
+ 'ens_from_bin_edge',
203
+ 'ens_from_df',
204
+ 'ens_from_mat_t',
205
+ 'ens_from_vec',
206
+ 'ensemble',
207
+ 'ensemble_avg',
208
+ 'make_database',
209
+ 'parents_stamps',
210
+ 'rename_bulk_columns',
211
+ 'save_artifact',
212
+ 'space_hists',
213
+ 'space_tseries',
214
+ 'whole_from_file',
215
+ 'whole_from_segments',
216
+ ]
217
+
218
+ logger = logging.getLogger(__name__)
219
+
220
+
221
+ def create_fullname(name: str, group: GroupT, prop: PropertyT) -> str:
222
+ """
223
+ Join a base name, a particle group, and a property into an artifact's
224
+ full name.
225
+
226
+ Parameters
227
+ ----------
228
+ name : str
229
+ The base name.
230
+ group : :py:obj:`~.GroupT`
231
+ Particle group (e.g., 'bug' or 'nucleoid').
232
+ prop : :py:obj:`~.PropertyT`
233
+ Physical property name (e.g., 'density').
234
+
235
+ Returns
236
+ -------
237
+ str
238
+ A string that combines `name`, `group`, and `prop` in a
239
+ hyphen-separated format.
240
+
241
+ Examples
242
+ --------
243
+ >>> create_fullname('sample', 'bug', 'density')
244
+ 'sample-bug-density'
245
+ """
246
+ return "-".join([name, group, prop])
247
+
248
+
249
+ def save_artifact(
250
+ filename: str,
251
+ data: np.ndarray | pd.DataFrame | dict[str, np.ndarray],
252
+ save_to: Pathish,
253
+ ) -> None:
254
+ """
255
+ Save `data` as ``<save_to>/<filename>``, a ``.npy`` or ``.csv`` file by
256
+ its type.
257
+
258
+ An existing file of the same name is overwritten.
259
+
260
+ Parameters
261
+ ----------
262
+ filename : str
263
+ The output file name.
264
+ data : np.ndarray | pd.DataFrame | dict[str, np.ndarray]
265
+ Data to be saved. Accepted types:
266
+
267
+ - `np.ndarray`: Saved as ``<filename>.npy``.
268
+ - `pd.DataFrame`: Saved as ``<filename>.csv``, without the index.
269
+ - `dict` of `np.ndarray`: Each key must have the form
270
+ ``<prop>-<measure>``, with exactly one hyphen, and each entry is
271
+ saved as ``<filename>-<measure>.npy``.
272
+
273
+ save_to : Pathish
274
+ Path to an existing directory where the file is saved.
275
+
276
+ Notes
277
+ -----
278
+ :func:`numpy.save` and :meth:`pandas.DataFrame.to_csv` overwrite an
279
+ existing file of the same name.
280
+
281
+ Raises
282
+ ------
283
+ TypeError
284
+ If `data` type does not match any of the supported formats.
285
+ ValueError
286
+ If `data` is a dict and one of its keys does not contain exactly one
287
+ hyphen.
288
+
289
+ Examples
290
+ --------
291
+ >>> import tempfile
292
+ >>> tmp = tempfile.TemporaryDirectory()
293
+
294
+ Save a NumPy array to a ``.npy`` file:
295
+
296
+ >>> save_artifact('w-bug-rhoMon', np.array([1.0, 2.0]), tmp.name)
297
+ >>> np.load(Path(tmp.name) / 'w-bug-rhoMon.npy')
298
+ array([1., 2.])
299
+
300
+ Save a DataFrame to a ``.csv`` file:
301
+
302
+ >>> save_artifact('w-bug-rhoMon', pd.DataFrame({'A': [1, 2]}), tmp.name)
303
+ >>> pd.read_csv(Path(tmp.name) / 'w-bug-rhoMon.csv')
304
+ A
305
+ 0 1
306
+ 1 2
307
+
308
+ Save each entry of a dict to its own file, named by its measure:
309
+
310
+ >>> save_artifact('w-bug-rhoMon', {'rhoMon-mean': np.array([1.5])},
311
+ ... tmp.name)
312
+ >>> sorted(path.name for path in Path(tmp.name).iterdir())
313
+ ['w-bug-rhoMon-mean.npy', 'w-bug-rhoMon.csv', 'w-bug-rhoMon.npy']
314
+ >>> tmp.cleanup()
315
+ """
316
+ file_path = os.path.join(save_to, filename)
317
+
318
+ # Save data based on type
319
+ if isinstance(data, pd.DataFrame):
320
+ data.to_csv(f"{file_path}.csv", index=False)
321
+ elif isinstance(data, np.ndarray):
322
+ np.save(f"{file_path}.npy", data)
323
+ elif isinstance(data, dict):
324
+ for prop_key, prop_data in data.items():
325
+ _, prop_measure = prop_key.split("-")
326
+ np.save(f"{file_path}-{prop_measure}.npy", prop_data)
327
+ else:
328
+ raise TypeError(
329
+ f"Unsupported data type {type(data).__name__}."
330
+ + "Expected pd.DataFrame, np.ndarray, or dict of np.ndarray."
331
+ )
332
+
333
+
334
+ def make_database(
335
+ old_database: Pathish,
336
+ phase: PhaseT,
337
+ stage: StageT,
338
+ group: GroupT | None,
339
+ ) -> str:
340
+ """
341
+ Create, if needed, the stage directory
342
+ ``<root>/<phase>/<prefix>-<group>-<stage>/`` of the sweep that
343
+ `old_database` belongs to.
344
+
345
+ `old_database` is a directory of the same sweep that sits directly in a
346
+ phase directory, ``<root>/<old_phase>/<old_dir>``. The new directory
347
+ keeps ``<root>``, goes into `phase`, and is named
348
+ ``<prefix>-<group>-<stage>``, with ``<prefix>`` the part of
349
+ ``<old_dir>`` before its first hyphen (or ``*``) and ``<group>`` left
350
+ out when `group` is None. The :doc:`/lineage` page defines phases and
351
+ stages.
352
+
353
+ Parameters
354
+ ----------
355
+ old_database : Pathish
356
+ Path to a directory of the sweep, directly inside a phase directory;
357
+ it need not exist.
358
+ phase : :py:obj:`~.PhaseT`
359
+ The new phase name for the directory, specifying its role in the
360
+ workflow: one of ``'simsAll'``, ``'simsCont'``, ``'logs'``,
361
+ ``'trjs'``, ``'probes'``, ``'analysis'``, ``'viz'``, ``'galaxy'``,
362
+ or ``'allInOne'``.
363
+ stage : :py:obj:`~.StageT`
364
+ The stage of processing or data type represented in the new
365
+ directory: one of ``'segment'``, ``'wholeSim'``, ``'ens'``,
366
+ ``'ensAvg'``, ``'space'``, or ``'galaxy'``.
367
+ group : GroupT or None
368
+ Particle group related to the data in the directory; left out of the
369
+ name when None. It cannot contain ``-``, the field separator of the
370
+ name.
371
+
372
+ Returns
373
+ -------
374
+ str
375
+ The path to the new directory, ending with ``/``. If the directory
376
+ already exists, the same path is returned and an INFO record naming
377
+ it is logged to the module's logger.
378
+
379
+ Raises
380
+ ------
381
+ ValueError
382
+ If `phase` is not a value of `PhaseT`, `stage` not a value of
383
+ `StageT`, `group` contains ``-``, or the parent of `old_database` is
384
+ not named for a value of `PhaseT`; no directory is made.
385
+
386
+ Examples
387
+ --------
388
+ The new directory sits next to the old phase directory:
389
+
390
+ - Input: ``root/parent1/.../parentN/old_phase/old_dir``
391
+ - Result: ``root/parent1/.../parentN/phase/new_dir``
392
+
393
+ >>> import tempfile
394
+ >>> tmp = tempfile.TemporaryDirectory()
395
+ >>> old = Path(tmp.name) / 'probes' / 'N20-bug-segment'
396
+ >>> new = make_database(old, 'analysis', 'ens', 'bug')
397
+ >>> Path(new).relative_to(tmp.name).as_posix()
398
+ 'analysis/N20-bug-ens'
399
+ >>> Path(new).is_dir()
400
+ True
401
+ >>> tmp.cleanup()
402
+
403
+ A name outside the phase vocabulary is refused:
404
+
405
+ >>> make_database(old, 'simAll', 'ens', 'bug')
406
+ Traceback (most recent call last):
407
+ ...
408
+ ValueError: 'simAll' is an invalid option. Please select one of: \
409
+ simsAll, simsCont, logs, trjs, probes, analysis, viz, galaxy, allInOne.
410
+ """
411
+ # The type aliases are the one declaration of both vocabularies.
412
+ invalid_keyword(phase, get_args(PhaseT))
413
+ invalid_keyword(stage, get_args(StageT))
414
+ if group is not None and "-" in group:
415
+ raise ValueError(
416
+ f"group {group!r} contains '-', the field separator of a "
417
+ "directory name.")
418
+
419
+ old_path = pathlib.Path(old_database)
420
+ old_phase = old_path.parent.name
421
+ if old_phase not in get_args(PhaseT):
422
+ raise ValueError(
423
+ f"{old_phase!r} is not a phase: the parent of {str(old_path)!r} "
424
+ "must be a phase directory, one of "
425
+ f"{', '.join(get_args(PhaseT))}.")
426
+
427
+ # Common prefix derived from old_database to create new directory name
428
+ prefix = old_path.parts[-1].split("*")[0].split("-")[0]
429
+ new_directory = "-".join([part for part in [prefix, group, stage] if part])
430
+
431
+ # Construct the new database path in the same parent directory level as
432
+ # the old one
433
+ new_database_parts = list(old_path.parts[:-2]) + [phase, new_directory]
434
+ new_database = pathlib.Path(*new_database_parts)
435
+
436
+ # Avoid double slashes at the start of the path
437
+ if str(new_database).startswith("//"):
438
+ new_database = pathlib.Path(str(new_database)[1:])
439
+
440
+ # Create the new directory if it doesn't already exist
441
+ try:
442
+ new_database.mkdir(parents=True, exist_ok=False)
443
+ except FileExistsError:
444
+ logger.info("Reusing the existing directory %s.", new_database)
445
+
446
+ return str(new_database) + "/"
447
+
448
+
449
+ def _as_path(item: Pathish | PathGroup) -> Path:
450
+ """Normalize a single item into a pathlib.Path.
451
+
452
+ Accepts either:
453
+ - a bare pathish (str, Path, os.PathLike)
454
+ - a sequence of pathish values (we take the first element)
455
+ """
456
+ # Bare pathish?
457
+ if isinstance(item, (str, os.PathLike)):
458
+ return Path(item)
459
+
460
+ # Sequence of pathish (e.g., tuple[Path, ...] from sort_filenames)
461
+ if isinstance(item, Sequence) and item:
462
+ first = item[0]
463
+ if isinstance(first, (str, os.PathLike)):
464
+ return Path(first)
465
+
466
+ raise TypeError(f"Unsupported path item: {type(item)!r} -> {item!r}")
467
+
468
+
469
+ def _normalize_paths(paths: AnyPathIter) -> Iterator[Path]:
470
+ """Yield normalized Path objects from either a flat or grouped iterable."""
471
+ for item in paths:
472
+ # item is either Pathish or PathGroup; _as_path handles both
473
+ yield _as_path(item)
474
+
475
+
476
+ def _assert_equal_bin_edges(arrays: Sequence[np.ndarray]) -> np.ndarray:
477
+ """
478
+ Ensure all input arrays of bin edges are identical.
479
+
480
+ Parameters
481
+ ----------
482
+ arrays : Sequence[np.ndarray]
483
+ Sequence of 1D arrays representing bin edges from different wholes
484
+ or segments.
485
+
486
+ Returns
487
+ -------
488
+ np.ndarray
489
+ A copy of the first array if all arrays are identical.
490
+
491
+ Raises
492
+ ------
493
+ ValueError
494
+ If any array differs from the first one.
495
+ """
496
+ ref = arrays[0]
497
+ if not all(np.array_equal(ref, a) for a in arrays[1:]):
498
+ raise ValueError("Bin edges are not identical across all arrays.")
499
+ return ref.copy()
500
+
501
+
502
+ def _assert_equal_lengths(ensemble_name: str, lengths: dict[str, int]) -> None:
503
+ """
504
+ Raise ``ValueError`` unless every whole of `ensemble_name` has the same
505
+ length; `lengths` maps each whole's name to its number of rows or
506
+ values. Replicates of one state point run the same number of steps, so
507
+ a shorter whole is an incomplete run, not one to average over. The cost
508
+ is :math:`O(n)` in the number of wholes.
509
+ """
510
+ if len(set(lengths.values())) > 1:
511
+ listed = ", ".join(f"{name} {n}" for name, n in lengths.items())
512
+ raise ValueError(
513
+ f"The wholes of ensemble '{ensemble_name}' have unequal "
514
+ f"lengths: {listed}.")
515
+
516
+
517
+ def whole_from_segments(
518
+ prop: PropertyT,
519
+ segments: AnyPathIter,
520
+ parser: ParserClassT,
521
+ group: GroupT,
522
+ relation: WholeRelationT,
523
+ save_to: Pathish | None = None,
524
+ ) -> dict[str, np.ndarray]:
525
+ """
526
+ Combine the segment files of one property into one array per whole.
527
+
528
+ How the segments of a whole combine is set by `relation`.
529
+
530
+ Parameters
531
+ ----------
532
+ prop : :py:obj:`~.PropertyT`
533
+ The property as it appears in file names, such as ``'gyrTMon'``.
534
+ segments : :py:obj:`~.AnyPathIter`
535
+ An iterable of paths to segment ``.npy`` files for `prop`, either
536
+ bare paths or 1-tuples of a path, as
537
+ :func:`~polyphys.manage.utils.sort_filenames` returns them. Each path
538
+ points to a single segment file. Segments of different wholes may be
539
+ interleaved, but the segments of each whole must come in
540
+ consecutive ``segment_id`` order (the ``j<NN>`` keyword of their
541
+ names), with no id skipped.
542
+ parser : :py:obj:`~.ParserClassT`
543
+ A parser class from :mod:`polyphys.manage.parser` that interprets
544
+ file paths and extracts attributes such as the `whole` name and the
545
+ ``segment_id``.
546
+ group : :py:obj:`~.GroupT`
547
+ Type of the particle group.
548
+ relation : :py:obj:`~.WholeRelationT`
549
+ Specifies how to combine segments into a whole. Accepted values:
550
+
551
+ - 'histogram'
552
+ 'segment' is an N-dimensional histogram-like file, so the segments
553
+ of a 'whole' are summed element-wise. In the two-dimensional
554
+ space, a histogram has a (x_nbins, y_nbins) shape.
555
+
556
+ - 'tseries'
557
+ 'segment' is a time-series-like file, so the segments of a 'whole'
558
+ are concatenated along axis 0, in ``segment_id`` order.
559
+
560
+ - 'bin_edge'
561
+ 'segment' is a bin-edge file. All the segments of a whole must
562
+ hold identical edges, and one copy of them is the whole's
563
+ bin-edge array.
564
+
565
+ save_to : Pathish, optional
566
+ Directory path where the output files are saved, if specified. Each
567
+ whole is saved as ``<whole>-<group>-<prop>.npy``, overwriting a file
568
+ of the same name.
569
+
570
+ Returns
571
+ -------
572
+ dict[str, np.ndarray]
573
+ Dictionary with whole names as keys and the combined whole data as
574
+ NumPy arrays, in the order in which the wholes first appear in
575
+ `segments`.
576
+
577
+ Raises
578
+ ------
579
+ ValueError
580
+ If `relation` is not one of the accepted values; if the segments of
581
+ a whole do not arrive in strictly increasing ``segment_id`` order,
582
+ which includes a segment passed twice; if they skip a
583
+ ``segment_id``, such as ``j01`` then ``j03``, which would shift the
584
+ time of every later frame or leave a part out of a sum; or, for
585
+ 'bin_edge', if the segments of a whole hold different edges.
586
+ TypeError
587
+ If an item of `segments` is neither a path nor a sequence whose first
588
+ element is a path.
589
+
590
+ Notes
591
+ -----
592
+ The order of a whole's segments is part of the data's meaning for
593
+ 'tseries', and a duplicated segment would be counted twice by
594
+ 'histogram'. This function checks the order and that no id is skipped;
595
+ it does not require the first id to be 1. Sort the paths with
596
+ :func:`~polyphys.manage.utils.sort_filenames` first, which orders the
597
+ ``j<NN>`` keyword naturally.
598
+
599
+ Let n_segments be the number of segment files and m the number of
600
+ values in one segment. Time is O(n_segments * m). Every segment of
601
+ every whole is loaded and held in memory before any whole is reduced,
602
+ so peak memory is O(n_segments * m), not the size of one whole; the
603
+ returned wholes add O(n_wholes * m) for 'histogram' and 'bin_edge' and
604
+ O(n_segments * m) for 'tseries', where n_wholes is the number of
605
+ wholes.
606
+
607
+ Examples
608
+ --------
609
+ Two segments of one whole, written as the probe phase names them, join
610
+ into one time series in ``segment_id`` order:
611
+
612
+ >>> import tempfile
613
+ >>> from polyphys.tests.synthetic import ChainCyl
614
+ >>> tmp = tempfile.TemporaryDirectory()
615
+ >>> whole = ('N20r5.5lz25.0ac1.0nc100dt0.005bdump1000'
616
+ ... 'adump5000ens1')
617
+ >>> paths = []
618
+ >>> for seg_id in (1, 2):
619
+ ... path = Path(tmp.name) / f'{whole}.j0{seg_id}.bug.gyrTMon.npy'
620
+ ... np.save(path, np.array([10.0, 11.0]) * seg_id)
621
+ ... paths.append(path)
622
+ >>> wholes = whole_from_segments('gyrTMon', paths, ChainCyl, 'bug',
623
+ ... 'tseries')
624
+ >>> wholes[whole]
625
+ array([10., 11., 20., 22.])
626
+
627
+ 'histogram' sums the same segments element-wise:
628
+
629
+ >>> whole_from_segments('gyrTMon', paths, ChainCyl, 'bug',
630
+ ... 'histogram')[whole]
631
+ array([30., 33.])
632
+
633
+ Segments out of order raise rather than giving a reversed series:
634
+
635
+ >>> whole_from_segments('gyrTMon', paths[::-1], ChainCyl, 'bug',
636
+ ... 'tseries') # doctest: +ELLIPSIS
637
+ Traceback (most recent call last):
638
+ ...
639
+ ValueError: The segments of whole '...ens1' are not in ... [2, 1]...
640
+ >>> tmp.cleanup()
641
+ """
642
+ invalid_keyword(relation, ["histogram", "tseries", "bin_edge"])
643
+ # Combine segments into wholes based on segment names and relation type,
644
+ # checking that each whole's segment_id values strictly increase
645
+ grouped_segments: dict[str, list[np.ndarray]] = {}
646
+ segment_ids: dict[str, list[int]] = {}
647
+ for seg_path in _normalize_paths(segments):
648
+ seg_info = parser(seg_path, "segment", group, ispath=True)
649
+ whole_name = getattr(seg_info, "whole")
650
+ seg_id = getattr(seg_info, "segment_id")
651
+ whole_ids = segment_ids.setdefault(whole_name, [])
652
+ if whole_ids and seg_id <= whole_ids[-1]:
653
+ raise ValueError(
654
+ f"The segments of whole '{whole_name}' are not in strictly "
655
+ f"increasing segment_id order: {whole_ids + [seg_id]}. Sort "
656
+ "them with sort_filenames first."
657
+ )
658
+ if whole_ids and seg_id != whole_ids[-1] + 1:
659
+ raise ValueError(
660
+ f"The segments of whole '{whole_name}' skip a segment_id: "
661
+ f"{whole_ids + [seg_id]}."
662
+ )
663
+ whole_ids.append(seg_id)
664
+ grouped_segments.setdefault(whole_name, []).append(np.load(seg_path))
665
+
666
+ # Define reduction strategies for each relation type
667
+ reducers: dict[str, Callable[[Sequence[np.ndarray]], np.ndarray]] = {
668
+ "histogram": lambda arrs: np.sum(arrs, axis=0),
669
+ "tseries": lambda arrs: np.concatenate(arrs, axis=0),
670
+ "bin_edge": _assert_equal_bin_edges,
671
+ }
672
+ reducer = reducers[relation]
673
+
674
+ # Apply the appropriate mapping function to merge each group of segments
675
+ # into a whole.
676
+ wholes: dict[str, np.ndarray] = \
677
+ {name: reducer(vals) for name, vals in grouped_segments.items()}
678
+
679
+ if save_to is not None:
680
+ for name, data in wholes.items():
681
+ save_artifact(create_fullname(name, group, prop), data, save_to)
682
+
683
+ return wholes
684
+
685
+
686
+ def whole_from_file(
687
+ whole_paths: AnyPathIter,
688
+ parser: ParserClassT,
689
+ group: GroupT,
690
+ ) -> dict[str, np.ndarray]:
691
+ """
692
+ Load the whole ``.npy`` files of one property of a particle `group`.
693
+
694
+ Each path in `whole_paths` points to one whole file.
695
+
696
+ Parameters
697
+ ----------
698
+ whole_paths : AnyPathIter
699
+ Paths to whole files, either bare paths or 1-tuples of a path, as
700
+ :func:`~polyphys.manage.utils.sort_filenames` returns them. Each file
701
+ is loaded independently.
702
+ parser : ParserClassT
703
+ A parser class from :mod:`polyphys.manage.parser` that interprets
704
+ file paths and extracts attributes such as the `whole` name.
705
+ group : GroupT
706
+ The particle group to which the data pertains.
707
+
708
+ Returns
709
+ -------
710
+ dict[str, np.ndarray]
711
+ Dictionary where keys are the `whole` names (derived from each file
712
+ path) and values are NumPy arrays containing the loaded data.
713
+
714
+ Raises
715
+ ------
716
+ TypeError
717
+ If an item of `whole_paths` is neither a path nor a sequence whose
718
+ first element is a path.
719
+ ValueError
720
+ If two files parse to the same whole name; the message names both.
721
+
722
+ Notes
723
+ -----
724
+ This function assumes that each file in `whole_paths` contains data
725
+ related to a complete simulation, referred to as a "whole." For further
726
+ information on terminology, see the `organizer` module documentation.
727
+
728
+ Let n_wholes be the number of files and m the number of values in one
729
+ file. Time and memory are O(n_wholes * m): every whole is loaded and
730
+ held in memory at once.
731
+
732
+ Examples
733
+ --------
734
+ >>> import tempfile
735
+ >>> from polyphys.tests.synthetic import ChainCyl
736
+ >>> tmp = tempfile.TemporaryDirectory()
737
+ >>> whole = ('N20r5.5lz25.0ac1.0nc100dt0.005bdump1000'
738
+ ... 'adump5000ens1')
739
+ >>> path = Path(tmp.name) / f'{whole}-bug-gyrTMon.npy'
740
+ >>> np.save(path, np.array([1.0, 2.0, 3.0]))
741
+ >>> whole_from_file([path], ChainCyl, 'bug')[whole]
742
+ array([1., 2., 3.])
743
+ >>> tmp.cleanup()
744
+ """
745
+ wholes: dict[str, np.ndarray] = {}
746
+ sources: dict[str, Path] = {}
747
+ for whole_path in _normalize_paths(whole_paths):
748
+ whole_info = parser(whole_path, "whole", group, ispath=True)
749
+ name = getattr(whole_info, "whole")
750
+ if name in sources:
751
+ raise ValueError(
752
+ f"'{sources[name]}' and '{whole_path}' are both whole "
753
+ f"'{name}'.")
754
+ sources[name] = whole_path
755
+ wholes[name] = np.load(whole_path)
756
+
757
+ return wholes
758
+
759
+
760
+ def ens_from_bin_edge(
761
+ ens_data: tuple[str, dict[str, np.ndarray]],
762
+ ) -> tuple[str, np.ndarray]:
763
+ """
764
+ Reduce the bin edges of the 'whole' datasets in an 'ensemble' to the
765
+ single array they share.
766
+
767
+ Every 'whole' in an ensemble must be binned identically, so that the
768
+ histograms paired with these edges stay aligned bin by bin. The edges are
769
+ checked for equality with ``_assert_equal_bin_edges``, the check
770
+ :func:`whole_from_segments` applies to segments, rather than merged.
771
+
772
+ Parameters
773
+ ----------
774
+ ens_data : tuple[str, dict[str, np.ndarray]]
775
+ A tuple where the first element is the ensemble name (str) and the
776
+ second element is a non-empty dictionary of bin edge arrays from
777
+ different 'whole' datasets within that ensemble. Within the
778
+ dictionary, keys are whole names while values are 1D arrays of whole
779
+ bin edges, each of shape ``(n_bins + 1,)``.
780
+
781
+ Returns
782
+ -------
783
+ tuple[str, np.ndarray]
784
+ A tuple where the first element is the ensemble name, and the second
785
+ is a copy of the shared 1D array of bin edges, of shape
786
+ ``(n_bins + 1,)``.
787
+
788
+ Raises
789
+ ------
790
+ ValueError
791
+ If the bin edges of any 'whole' differ from those of the first one,
792
+ in length or in value.
793
+
794
+ Notes
795
+ -----
796
+ Time and space complexity are O(n_wholes * n_edges) and O(n_edges).
797
+
798
+ Examples
799
+ --------
800
+ >>> ens_from_bin_edge(
801
+ ... ('ens1', {'w1': np.array([0., 1., 2.]),
802
+ ... 'w2': np.array([0., 1., 2.])})
803
+ ... )
804
+ ('ens1', array([0., 1., 2.]))
805
+
806
+ Differing edges raise instead of being merged:
807
+
808
+ >>> ens_from_bin_edge(
809
+ ... ('ens1', {'w1': np.array([0., 1., 2.]),
810
+ ... 'w2': np.array([0., 1., 3.])})
811
+ ... )
812
+ Traceback (most recent call last):
813
+ ...
814
+ ValueError: Bin edges are not identical across all arrays.
815
+ """
816
+ ensemble_name, whole_edges = ens_data
817
+ shared_bin_edges = _assert_equal_bin_edges(list(whole_edges.values()))
818
+ return ensemble_name, shared_bin_edges
819
+
820
+
821
+ def ens_from_vec(
822
+ ens_data: tuple[str, dict[str, np.ndarray]],
823
+ ) -> tuple[str, pd.DataFrame]:
824
+ """
825
+ Merge the 1D arrays of the wholes of one ensemble into a DataFrame.
826
+
827
+ Each whole becomes one column, such as a time series or a histogram of
828
+ the whole.
829
+
830
+ Parameters
831
+ ----------
832
+ ens_data : tuple[str, dict[str, np.ndarray]]
833
+ A tuple where the first element is the ensemble name, and the second
834
+ is a dictionary mapping each 'whole' dataset name to a 1D NumPy array
835
+ of measurements.
836
+
837
+ Returns
838
+ -------
839
+ tuple[str, pd.DataFrame]
840
+ A tuple where the first element is the ensemble name, and the second
841
+ is a pandas DataFrame. Each column represents a 'whole' dataset, in
842
+ the order of the dictionary, and rows contain corresponding vector
843
+ values from each dataset.
844
+
845
+ Raises
846
+ ------
847
+ ValueError
848
+ If the vectors have unequal lengths; the message names the ensemble
849
+ and each whole's length.
850
+
851
+ Notes
852
+ -----
853
+ Let n_wholes be the number of wholes and m the length of one vector.
854
+ Time and extra memory are O(n_wholes * m): the vectors are copied into
855
+ the DataFrame.
856
+
857
+ Examples
858
+ --------
859
+ >>> ens_name, ensemble_df = ens_from_vec(
860
+ ... ('ensemble1', {'whole1': np.array([0.1, 0.2, 0.3]),
861
+ ... 'whole2': np.array([0.4, 0.5, 0.6])})
862
+ ... )
863
+ >>> ens_name
864
+ 'ensemble1'
865
+ >>> ensemble_df
866
+ whole1 whole2
867
+ 0 0.1 0.4
868
+ 1 0.2 0.5
869
+ 2 0.3 0.6
870
+ """
871
+ ensemble_name, whole_vectors = ens_data
872
+ _assert_equal_lengths(
873
+ ensemble_name, {name: len(v) for name, v in whole_vectors.items()})
874
+ ensemble_df = pd.DataFrame.from_dict(whole_vectors, orient="columns")
875
+ return ensemble_name, ensemble_df
876
+
877
+
878
+ def ens_from_mat_t(
879
+ ens_data: tuple[str, dict[str, np.ndarray]],
880
+ ) -> tuple[str, np.ndarray]:
881
+ """
882
+ Stack the 2D arrays of the wholes of one ensemble into a 3D array.
883
+
884
+ Each whole becomes one slice along the first axis.
885
+
886
+ Parameters
887
+ ----------
888
+ ens_data : tuple[str, dict[str, np.ndarray]]
889
+ A tuple where the first element is the ensemble name (str) and the
890
+ second element is a dictionary with 'whole' names as keys and 2D NumPy
891
+ arrays as values.
892
+
893
+ Returns
894
+ -------
895
+ tuple[str, np.ndarray]
896
+ A tuple where the first element is the ensemble name and the second
897
+ is a 3D NumPy array of shape ``(n_wholes, n_rows, n_cols)``, formed
898
+ by stacking the 2D arrays along the first axis in the order of the
899
+ dictionary.
900
+
901
+ Notes
902
+ -----
903
+ Let n_wholes be the number of wholes and m the number of elements of one
904
+ matrix. Time and extra memory are O(n_wholes * m): the matrices are
905
+ copied into the stacked array.
906
+
907
+ Examples
908
+ --------
909
+ >>> ens_name, combined = ens_from_mat_t(
910
+ ... ('ensemble1', {'whole1': np.array([[1, 2], [3, 4]]),
911
+ ... 'whole2': np.array([[5, 6], [7, 8]])})
912
+ ... )
913
+ >>> combined.shape
914
+ (2, 2, 2)
915
+ >>> combined[1]
916
+ array([[5, 6],
917
+ [7, 8]])
918
+ """
919
+ ensemble_name, whole_matrices = ens_data
920
+ combined_matrix = np.stack(list(whole_matrices.values()), axis=0)
921
+ return ensemble_name, combined_matrix
922
+
923
+
924
+ def ens_from_df(
925
+ ens_data: tuple[str, dict[str, pd.DataFrame]],
926
+ ) -> tuple[str, pd.DataFrame]:
927
+ """
928
+ Average the DataFrames of the wholes of one ensemble into one
929
+ DataFrame.
930
+
931
+ Each DataFrame is one whole of the ensemble.
932
+
933
+ In a 'whole' DataFrame, headers are 'elements' of a 'matrix' or 2D
934
+ quantity and columns are the values of a given property.
935
+
936
+ In each 'whole' DataFrame, headers represent 'elements' of a 2D quantity
937
+ (matrix-like structure), and columns contain the values for a specified
938
+ property. Each 'whole' DataFrame may be either a 'histogram' or a
939
+ 'time-series':
940
+
941
+ - In a 'histogram' DataFrame, an additional column, `bin_center`, holds
942
+ bin center values that are invariant under averaging and are retained
943
+ in the output DataFrame. The length of histogram DataFrame is equal to
944
+ the number of bins.
945
+
946
+ - In a 'time-series' DataFrame, headers (columns) are elements of a matrix,
947
+ and rows represent the values of each element over time. The length of
948
+ timeseries DataFrame is equal to the number of time frames.
949
+
950
+
951
+ Parameters
952
+ ----------
953
+ ens_data : tuple[str, dict[str, pd.DataFrame]]
954
+ A tuple where the first element is the ensemble name and the second
955
+ is a dictionary with 'whole' names as keys and DataFrames, sharing
956
+ their columns and index labels, as values.
957
+
958
+ Returns
959
+ -------
960
+ tuple[str, pd.DataFrame]
961
+ A tuple with:
962
+
963
+ - The ensemble name (str).
964
+ - A DataFrame with the averaged values across all 'whole'
965
+ DataFrames.
966
+
967
+ Raises
968
+ ------
969
+ ValueError
970
+ If the DataFrames have unequal numbers of rows; the message names
971
+ the ensemble and each whole's length. A shorter whole is not
972
+ averaged over the rows it has.
973
+
974
+ Notes
975
+ -----
976
+ A clear distinction exists between ensembles of 'vector' or 'matrix' types
977
+ and ensembles of 'dataframe' type. In vector or matrix types, each column
978
+ in the resulting ensemble DataFrame is named after a 'whole' and represents
979
+ that whole's values for a specific property. In contrast, for DataFrame
980
+ type ensembles, the columns are elements of a 2D quantity (e.g., matrix or
981
+ time-series values) and hold averaged values across all whole DataFrames.
982
+
983
+ This function assumes that averaging non-element headers, such as
984
+ 'bin_center,' does not change their values, and these are carried forward
985
+ without modification.
986
+
987
+ Let n_wholes be the number of wholes, n_rows the rows and n_cols the
988
+ columns of one DataFrame. Time and extra memory are
989
+ O(n_wholes * n_rows * n_cols): the DataFrames are concatenated into one
990
+ before they are grouped by index label and averaged.
991
+
992
+ Examples
993
+ --------
994
+ >>> df1 = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
995
+ >>> df2 = pd.DataFrame({'A': [2, 3, 4], 'B': [5, 6, 7]})
996
+ >>> ens_name, averaged_df = ens_from_df(
997
+ ... ('ensemble1', {'whole1': df1, 'whole2': df2})
998
+ ... )
999
+ >>> averaged_df
1000
+ A B
1001
+ 0 1.5 4.5
1002
+ 1 2.5 5.5
1003
+ 2 3.5 6.5
1004
+ """
1005
+ ensemble_name, whole_data = ens_data
1006
+ _assert_equal_lengths(
1007
+ ensemble_name, {name: len(df) for name, df in whole_data.items()})
1008
+ ensemble_df = pd.concat(list(whole_data.values())).groupby(level=0).mean()
1009
+ return ensemble_name, ensemble_df
1010
+
1011
+
1012
+ def ensemble(
1013
+ prop: PropertyT,
1014
+ wholes: dict[str, np.ndarray] | dict[str, pd.DataFrame],
1015
+ parser: ParserClassT,
1016
+ group: GroupT,
1017
+ whole_type: Literal["vector", "matrix", "dataframe", "bin_edge"],
1018
+ whole_edges: dict[str, np.ndarray] | None = None,
1019
+ save_to: Pathish | None = None,
1020
+ ) -> dict[str, np.ndarray | pd.DataFrame]:
1021
+ """
1022
+ Merge the wholes of one property into one ensemble per state point.
1023
+
1024
+ `whole_type` sets the structure of each whole and how the wholes of one
1025
+ ensemble merge.
1026
+
1027
+ Parameters
1028
+ ----------
1029
+ prop : :py:obj:`~.PropertyT`
1030
+ The property as it appears in file names, such as ``'rhoMon'``.
1031
+ wholes : dict[str, np.ndarray] | dict[str, pd.DataFrame]
1032
+ Dictionary containing 'whole' names as keys and their corresponding
1033
+ data as values, represented as NumPy arrays or DataFrames.
1034
+ parser : ParserClassT
1035
+ Parser class used to interpret whole filenames, such as the ensemble
1036
+ name of each 'whole'.
1037
+ group : GroupT
1038
+ Type of the particle group being processed.
1039
+ whole_type : {'vector', 'matrix', 'dataframe', 'bin_edge'}
1040
+ Specifies the type of each 'whole' and defines the merging method:
1041
+
1042
+ - 'vector': A 1D NumPy array, such as a time series or histogram.
1043
+ - 'matrix': A 2D NumPy array, such as a gyration tensor.
1044
+ - 'dataframe': A DataFrame representing either a time series (with
1045
+ headers as matrix elements and rows as values over time) or a
1046
+ histogram (headers as elements, rows as counts or frequencies,
1047
+ and an additional 'bin_center' column for bin centers).
1048
+ - 'bin_edge': A 1D array representing bin edges. Assumes all bin edges
1049
+ are identical and selects one to represent the ensemble.
1050
+
1051
+ whole_edges : dict[str, np.ndarray], optional
1052
+ Dictionary with 'whole' names as keys and 1D bin edge arrays, each of
1053
+ shape ``(n_bins + 1,)``, as values. Only valid when `whole_type` is
1054
+ 'vector'. The wholes of an ensemble share their bin edges by design,
1055
+ so the edges of the first whole of each ensemble are used and the
1056
+ others are not compared. Each merged ensemble gets a 'bin_center'
1057
+ column of length ``n_bins`` holding the midpoints of those edges.
1058
+ Must contain every key of `wholes`.
1059
+ save_to : Pathish, optional
1060
+ Directory path where output files are saved, if specified; see
1061
+ Notes.
1062
+
1063
+ Returns
1064
+ -------
1065
+ dict[str, np.ndarray | pd.DataFrame]
1066
+ The ensembles, keyed by their ``ensemble_long`` names, as NumPy
1067
+ arrays or DataFrames.
1068
+
1069
+ Raises
1070
+ ------
1071
+ ValueError
1072
+ If any parameter has an invalid value; if `whole_edges` is given and
1073
+ `whole_type` is not 'vector'; or if the number of bins differs from
1074
+ the number of rows of the merged ensemble.
1075
+
1076
+ Notes
1077
+ -----
1078
+ Different averaging methods apply based on `whole_type`. When `whole_type`
1079
+ is "DataFrame," the resulting DataFrame averages the values of each matrix
1080
+ element across wholes, retaining headers that are mean-invariant (e.g.,
1081
+ 'bin_center'). For 'vector' and 'matrix' types, columns represent
1082
+ individual wholes in the ensemble DataFrame.
1083
+
1084
+ The bin center of bin ``i`` is the midpoint of its two edges,
1085
+ ``0.5 * (edges[i] + edges[i + 1])``. A center is invariant under
1086
+ averaging over wholes, which is why :func:`ensemble_avg` excludes
1087
+ 'bin_center' from its statistics by default.
1088
+
1089
+ Let n_wholes be the number of wholes, m the number of values in one
1090
+ whole, n_ensembles the number of ensembles, and n_edges the length of
1091
+ one bin-edge array. Grouping parses each whole name once and merging
1092
+ copies every whole once, so time and extra memory are
1093
+ O(n_wholes * m): the caller's `wholes` and the merged ensembles are both
1094
+ held in memory. With `whole_edges`, computing the centers adds
1095
+ O(n_ensembles * n_edges) time and memory.
1096
+
1097
+ With `save_to`, each ensemble is saved by :func:`save_artifact` under
1098
+ ``<ensemble_long>-<group>-<prop>``, as ``.csv`` for a DataFrame and
1099
+ ``.npy`` for an array, overwriting a file of the same name.
1100
+
1101
+ Examples
1102
+ --------
1103
+ Two replicates of one state point, with shared edges, merge into one
1104
+ ensemble that carries the edge midpoints as 'bin_center':
1105
+
1106
+ >>> from polyphys.tests.synthetic import ChainCyl
1107
+ >>> base = ('N2000r10.5lz300.0ac1.0nc117000dt0.005bdump5000'
1108
+ ... 'adump10000ens')
1109
+ >>> wholes = {base + '1': np.array([1., 2., 3.]),
1110
+ ... base + '2': np.array([3., 4., 5.])}
1111
+ >>> edges = {name: np.array([0., 1., 2., 4.]) for name in wholes}
1112
+ >>> ens = ensemble('rhoMon', wholes, ChainCyl, 'bug', 'vector',
1113
+ ... whole_edges=edges)
1114
+ >>> ens_df = next(iter(ens.values()))
1115
+ >>> ens_df['bin_center'].tolist()
1116
+ [0.5, 1.5, 3.0]
1117
+
1118
+ Without `whole_edges`, the ensemble has one column per whole only:
1119
+
1120
+ >>> ens = ensemble('rhoMon', wholes, ChainCyl, 'bug', 'vector')
1121
+ >>> 'bin_center' in next(iter(ens.values())).columns
1122
+ False
1123
+ """
1124
+ invalid_keyword(whole_type, ["vector", "matrix", "dataframe", "bin_edge"])
1125
+ if whole_edges is not None and whole_type != "vector":
1126
+ raise ValueError(
1127
+ "'whole_edges' is only valid with whole_type 'vector', not "
1128
+ f"'{whole_type}'."
1129
+ )
1130
+
1131
+ merging_func: dict[
1132
+ str, Callable[..., tuple[str, np.ndarray | pd.DataFrame]]
1133
+ ] = {
1134
+ "vector": ens_from_vec,
1135
+ "matrix": ens_from_mat_t,
1136
+ "dataframe": ens_from_df,
1137
+ "bin_edge": ens_from_bin_edge,
1138
+ }
1139
+ # Collecting wholes for each ensemble; the wholes of an ensemble share
1140
+ # their bin edges by design, so the centers come from its first whole
1141
+ ens_wholes: dict[str, dict[str, np.ndarray | pd.DataFrame]] = {}
1142
+ bin_centers: dict[str, np.ndarray] = {}
1143
+ for whole_name, whole_data in wholes.items():
1144
+ whole_info = parser(whole_name, "whole", group)
1145
+ ens_name = getattr(whole_info, "ensemble_long")
1146
+ ens_wholes.setdefault(ens_name, {})[whole_name] = whole_data
1147
+ if whole_edges is not None and ens_name not in bin_centers:
1148
+ edges = np.asarray(whole_edges[whole_name])
1149
+ bin_centers[ens_name] = 0.5 * (edges[:-1] + edges[1:])
1150
+
1151
+ # Merging wholes into a single ensemble data object
1152
+ ensembles: dict[str, np.ndarray | pd.DataFrame] = dict(
1153
+ map(merging_func[whole_type], ens_wholes.items())
1154
+ )
1155
+ for ens_name, centers in bin_centers.items():
1156
+ ensembles[ens_name]["bin_center"] = centers
1157
+
1158
+ # Save ensemble data object if save_to is specified
1159
+ if save_to:
1160
+ for ens_name, data in ensembles.items():
1161
+ ens_fullname = create_fullname(ens_name, group, prop)
1162
+ save_artifact(ens_fullname, data, save_to)
1163
+
1164
+ return ensembles
1165
+
1166
+
1167
+ def ens_avg_from_bin_edge(
1168
+ ens_data: np.ndarray | Sequence[np.ndarray],
1169
+ ) -> np.ndarray:
1170
+ """
1171
+ Return the bin edges of an 'ensemble' for its ensemble-averaged
1172
+ artifact.
1173
+
1174
+ Bin edges are not averaged: every 'whole' in the ensemble must share
1175
+ them. A single 1D array, the form :func:`ens_from_bin_edge` produces and
1176
+ :func:`ensemble_avg` passes here, is returned as a copy. A sequence of
1177
+ arrays, one per 'whole', is checked for equality with
1178
+ ``_assert_equal_bin_edges``, and a copy of the shared array is
1179
+ returned.
1180
+
1181
+ Parameters
1182
+ ----------
1183
+ ens_data : np.ndarray or Sequence[np.ndarray]
1184
+ Either the ensemble's 1D array of bin edges, of shape
1185
+ ``(n_bins + 1,)``, or a non-empty sequence of such arrays, one per
1186
+ 'whole'. A 2D array of shape ``(n_wholes, n_bins + 1)`` counts as a
1187
+ sequence of its rows.
1188
+
1189
+ Returns
1190
+ -------
1191
+ np.ndarray
1192
+ A copy of the shared 1D array of bin edges, of shape
1193
+ ``(n_bins + 1,)``.
1194
+
1195
+ Raises
1196
+ ------
1197
+ ValueError
1198
+ If `ens_data` is a sequence and any array in it differs from the
1199
+ first one, in length or in value.
1200
+
1201
+ Notes
1202
+ -----
1203
+ This function performs no averaging or statistical calculation. Time and
1204
+ space complexity are O(n_wholes * n_edges) and O(n_edges).
1205
+
1206
+ Examples
1207
+ --------
1208
+ >>> ens_avg_from_bin_edge(np.array([0.0, 0.1, 0.2]))
1209
+ array([0. , 0.1, 0.2])
1210
+ >>> ens_avg_from_bin_edge(
1211
+ ... [np.array([0.0, 0.1, 0.2]), np.array([0.0, 0.1, 0.2])]
1212
+ ... )
1213
+ array([0. , 0.1, 0.2])
1214
+ >>> ens_avg_from_bin_edge(
1215
+ ... [np.array([0.0, 0.1, 0.2]), np.array([0.0, 0.1, 0.3])]
1216
+ ... )
1217
+ Traceback (most recent call last):
1218
+ ...
1219
+ ValueError: Bin edges are not identical across all arrays.
1220
+ """
1221
+ if isinstance(ens_data, np.ndarray) and ens_data.ndim == 1:
1222
+ return ens_data.copy()
1223
+ return _assert_equal_bin_edges([np.asarray(edges) for edges in ens_data])
1224
+
1225
+
1226
+ def ens_avg_from_ndarray(
1227
+ ens_prop: PropertyT, ens_data: np.ndarray
1228
+ ) -> dict[str, np.ndarray]:
1229
+ """
1230
+ Compute the mean, variance, and standard error of the mean (SEM)
1231
+ across the wholes of an ensemble 3D array.
1232
+
1233
+ Each slice along the first axis (axis=0) is one whole's matrix, and the
1234
+ statistics are taken along that axis.
1235
+
1236
+ Parameters
1237
+ ----------
1238
+ ens_prop : PropertyT
1239
+ The name of the property to which this 'ensemble' data belongs. Used
1240
+ as a prefix for the computed statistics.
1241
+ ens_data : np.ndarray
1242
+ A 3D NumPy array where each matrix slice along axis=0 represents a
1243
+ 'whole' dataset.
1244
+
1245
+ Returns
1246
+ -------
1247
+ dict[str, np.ndarray]
1248
+ A dictionary where keys are statistical measures (mean, variance, SEM)
1249
+ with `ens_prop` as a prefix while values are NumPy arrays of the
1250
+ computed statistics across 'whole' matrices.
1251
+
1252
+ Notes
1253
+ -----
1254
+ - The SEM is the sample standard deviation with ``ddof=1`` divided by
1255
+ the square root of the number of wholes, which are assumed
1256
+ independent. Bessel's correction makes the variance unbiased; the
1257
+ standard deviation is still biased, though less (the
1258
+ :func:`numpy.std` documentation, Notes), and so is the SEM.
1259
+ - This function performs averaging along axis=0 for each element across
1260
+ the 'whole' data matrices.
1261
+ - Let n_wholes be the length of axis 0 and m the number of elements of
1262
+ one whole. Time is O(n_wholes * m). NumPy's ``var`` and ``std`` build
1263
+ temporaries of the size of `ens_data`, so peak extra memory is
1264
+ O(n_wholes * m); the three results take O(m).
1265
+
1266
+ Examples
1267
+ --------
1268
+ >>> ens_data = np.array([
1269
+ ... [[1.0, 2.0], [3.0, 4.0]],
1270
+ ... [[2.0, 3.0], [4.0, 5.0]],
1271
+ ... ])
1272
+ >>> ens_avg = ens_avg_from_ndarray('gyrTMon', ens_data)
1273
+ >>> list(ens_avg)
1274
+ ['gyrTMon-mean', 'gyrTMon-var', 'gyrTMon-sem']
1275
+ >>> ens_avg['gyrTMon-mean']
1276
+ array([[1.5, 2.5],
1277
+ [3.5, 4.5]])
1278
+ >>> ens_avg['gyrTMon-sem']
1279
+ array([[0.5, 0.5],
1280
+ [0.5, 0.5]])
1281
+ """
1282
+ n_wholes_sqrt = ens_data.shape[0] ** 0.5
1283
+ ens_avg = {
1284
+ f"{ens_prop}-mean": np.mean(ens_data, axis=0),
1285
+ f"{ens_prop}-var": np.var(ens_data, axis=0, ddof=1),
1286
+ f"{ens_prop}-sem": np.std(ens_data, axis=0, ddof=1) / n_wholes_sqrt,
1287
+ }
1288
+ return ens_avg
1289
+
1290
+
1291
+ def ens_avg_from_df(
1292
+ ens_prop: PropertyT, ens_data: pd.DataFrame, exclude: list[str]
1293
+ ) -> pd.DataFrame:
1294
+ """
1295
+ Compute the mean, variance, and standard error of the mean (SEM) across
1296
+ the whole columns of an ensemble DataFrame.
1297
+
1298
+ The result keeps the excluded columns, such as ``bin_center``, and
1299
+ replaces the whole columns with the statistics.
1300
+
1301
+ Parameters
1302
+ ----------
1303
+ ens_prop : PropertyT
1304
+ The name of the property to which this 'ensemble' data belongs, used
1305
+ as a prefix for the generated statistics columns.
1306
+ ens_data : pd.DataFrame
1307
+ A DataFrame containing ensemble data. Each 'whole' dataset is in its
1308
+ own column, and any columns specified in `exclude` remain unaffected.
1309
+ exclude : list of str
1310
+ A list of column names to exclude from averaging. For example,
1311
+ `exclude` might contain 'bin_center' for histogram-type ensembles.
1312
+
1313
+ Returns
1314
+ -------
1315
+ pd.DataFrame
1316
+ A new DataFrame with the excluded columns of `ens_data`, followed by
1317
+ the columns ``<ens_prop>-mean``, ``<ens_prop>-var``, and
1318
+ ``<ens_prop>-sem``: the mean, variance, and SEM of each row across
1319
+ the 'whole' data columns. It has the same index as `ens_data`.
1320
+
1321
+ Notes
1322
+ -----
1323
+ - `ens_data` is not modified; the statistics are written to a new
1324
+ DataFrame.
1325
+ - The function averages only the 'whole' columns, excluding any specified
1326
+ in `exclude`.
1327
+ - The standard error of the mean (SEM) is computed from the sample
1328
+ standard deviation with ``ddof=1``, the wholes assumed independent.
1329
+ Bessel's correction makes the variance unbiased; the standard
1330
+ deviation is still biased, though less (the :func:`numpy.std`
1331
+ documentation, Notes), and so is the SEM.
1332
+ - Let n_rows be the number of rows of `ens_data` and n_wholes the number
1333
+ of 'whole' columns. Time is O(n_rows * n_wholes). Each statistic
1334
+ selects the 'whole' columns as a new DataFrame, so peak extra memory
1335
+ is O(n_rows * n_wholes), not O(n_rows); the result takes O(n_rows).
1336
+
1337
+ Examples
1338
+ --------
1339
+ >>> ens_data = pd.DataFrame({
1340
+ ... 'bin_center': [1, 2, 3],
1341
+ ... 'whole1': [4.0, 5.0, 6.0],
1342
+ ... 'whole2': [5.0, 6.0, 7.0],
1343
+ ... })
1344
+ >>> ens_avg_from_df('density', ens_data, ['bin_center'])
1345
+ bin_center density-mean density-var density-sem
1346
+ 0 1 4.5 0.5 0.5
1347
+ 1 2 5.5 0.5 0.5
1348
+ 2 3 6.5 0.5 0.5
1349
+ >>> list(ens_data.columns)
1350
+ ['bin_center', 'whole1', 'whole2']
1351
+ """
1352
+ wholes = [col for col in ens_data.columns if col not in exclude]
1353
+
1354
+ ens_avg = ens_data.drop(columns=wholes)
1355
+ ens_avg[ens_prop + "-mean"] = ens_data[wholes].mean(axis=1)
1356
+ ens_avg[ens_prop + "-var"] = ens_data[wholes].var(axis=1, ddof=1)
1357
+ ens_avg[ens_prop + "-sem"] = ens_data[wholes].sem(axis=1, ddof=1)
1358
+ return ens_avg
1359
+
1360
+
1361
+ def ensemble_avg(
1362
+ prop: PropertyT,
1363
+ ensembles: dict[str, np.ndarray | pd.DataFrame],
1364
+ group: GroupT,
1365
+ ens_type: Literal["dataframe", "ndarray", "bin_edge"],
1366
+ exclude: list[str] | None = None,
1367
+ save_to: Pathish | None = None,
1368
+ ) -> dict[str, np.ndarray] | dict[str, pd.DataFrame]:
1369
+ """
1370
+ Average the wholes of each ensemble of one property.
1371
+
1372
+ Columns listed in `exclude` are left out of the averages.
1373
+
1374
+ Parameters
1375
+ ----------
1376
+ prop : :py:obj:`~.PropertyT`
1377
+ The property as it appears in file names, such as ``'rhoMon'``.
1378
+ ensembles : dict[str, np.ndarray | pd.DataFrame]
1379
+ A dictionary where each key is an ensemble name and each value is
1380
+ either a DataFrame or ndarray representing 'whole' data.
1381
+ group : GroupT
1382
+ Type of the particle group.
1383
+ ens_type : {'dataframe', 'ndarray', 'bin_edge'}
1384
+ The data format for each ensemble:
1385
+
1386
+ - 'dataframe': DataFrame format, with columns as 'whole' simulations
1387
+ or items in `exclude`.
1388
+ - 'ndarray': 3D ndarray format, with each slice along axis=0
1389
+ representing a 'whole' matrix.
1390
+ - 'bin_edge': 1D ndarray format for bin edges, with bin edges assumed
1391
+ to be identical across wholes.
1392
+
1393
+ exclude : list[str], default None
1394
+ List of column names or items not included in ensemble averaging,
1395
+ such as 'bin_center' for histograms.
1396
+ save_to : Pathish, optional
1397
+ Path to the directory where the averaged ensemble data is saved, if
1398
+ specified; see Notes.
1399
+
1400
+ Returns
1401
+ -------
1402
+ dict[str, np.ndarray] | dict[str, pd.DataFrame]
1403
+ A dictionary where each key is an ensemble name, and each value is a
1404
+ DataFrame or ndarray of ensemble-averaged data.
1405
+
1406
+ Raises
1407
+ ------
1408
+ ValueError
1409
+ If `ens_type` is not one of the accepted values.
1410
+
1411
+ Notes
1412
+ -----
1413
+ With `save_to`, each result is saved by :func:`save_artifact` under
1414
+ ``<ensemble>-<group>-<prop>-ensAvg``: a DataFrame as ``.csv``, bin edges
1415
+ as ``.npy``, and the dictionary of an 'ndarray' ensemble as one
1416
+ ``...-ensAvg-<measure>.npy`` file per statistic, each overwriting a file
1417
+ of the same name.
1418
+
1419
+ Let n_ensembles be the number of ensembles. Ensembles are averaged one
1420
+ after another, with the time and peak extra memory of
1421
+ :func:`ens_avg_from_df`, :func:`ens_avg_from_ndarray`, or
1422
+ :func:`ens_avg_from_bin_edge` for the largest one; the results for all
1423
+ n_ensembles ensembles are held in memory together.
1424
+
1425
+ Examples
1426
+ --------
1427
+ >>> ensembles = {
1428
+ ... 'ensemble1': pd.DataFrame({'whole1': [1.0, 2.0],
1429
+ ... 'whole2': [3.0, 4.0]}),
1430
+ ... 'ensemble2': pd.DataFrame({'whole1': [5.0, 6.0],
1431
+ ... 'whole2': [5.0, 8.0]}),
1432
+ ... }
1433
+ >>> avg = ensemble_avg('rhoMon', ensembles, 'bug', 'dataframe')
1434
+ >>> avg['ensemble2']
1435
+ rhoMon-mean rhoMon-var rhoMon-sem
1436
+ 0 5.0 0.0 0.0
1437
+ 1 7.0 2.0 1.0
1438
+ """
1439
+ if exclude is None:
1440
+ exclude = ["bin_center"]
1441
+
1442
+ # Compute ensemble averages
1443
+ ens_avgs: dict[str, np.ndarray] | dict[str, pd.DataFrame] = {}
1444
+ if ens_type == "bin_edge":
1445
+ for ens_name, ens_data in ensembles.items():
1446
+ ens_avg = ens_avg_from_bin_edge(ens_data) # type: ignore
1447
+ ens_avgs[ens_name] = ens_avg # type: ignore
1448
+ elif ens_type == "ndarray":
1449
+ for ens_name, ens_data in ensembles.items():
1450
+ ens_avg = ens_avg_from_ndarray(prop, ens_data) # type: ignore
1451
+ ens_avgs[ens_name] = ens_avg # type: ignore
1452
+ elif ens_type == "dataframe":
1453
+ for ens_name, ens_data in ensembles.items():
1454
+ ens_avg = ens_avg_from_df(prop, ens_data, exclude) # type: ignore
1455
+ ens_avgs[ens_name] = ens_avg # type: ignore
1456
+ else:
1457
+ invalid_keyword(ens_type, ["dataframe", "ndarray", "bin_edge"])
1458
+
1459
+ # Save averaged ensemble data if save_to path is provided
1460
+ if save_to is not None:
1461
+ save_property = prop + "-ensAvg"
1462
+ for ens_name, data in ens_avgs.items():
1463
+ ens_fullname = create_fullname(ens_name, group, save_property)
1464
+ save_artifact(ens_fullname, data, save_to)
1465
+
1466
+ return ens_avgs
1467
+
1468
+
1469
+ def _group_from_stamp_names(stamps: list[tuple[str]]) -> str:
1470
+ """
1471
+ The particle group in the names of stamp files,
1472
+ ``<lineage name>-<group>-stamps.csv``, which must agree.
1473
+
1474
+ Parameters
1475
+ ----------
1476
+ stamps : list[tuple[str]]
1477
+ List of 1-tuples, each holding the path to a 'stamp' CSV file.
1478
+
1479
+ Returns
1480
+ -------
1481
+ str
1482
+ The group the file names share.
1483
+
1484
+ Raises
1485
+ ------
1486
+ ValueError
1487
+ If a file name is not ``<name>-<group>-stamps.csv``, or the names
1488
+ give more than one group.
1489
+ """
1490
+ groups: set[str] = set()
1491
+ for (path,) in stamps:
1492
+ name = Path(path).name
1493
+ parts = name.removesuffix(".csv").rsplit("-", 2)
1494
+ if len(parts) != 3 or parts[2] != "stamps":
1495
+ raise ValueError(
1496
+ f"'{name}' is not a '<name>-<group>-stamps.csv' stamp file, "
1497
+ "and the stamps have no 'group' column.")
1498
+ groups.add(parts[1])
1499
+ if len(groups) != 1:
1500
+ raise ValueError(
1501
+ f"The stamp file names give {sorted(groups)}, not one group.")
1502
+ return groups.pop()
1503
+
1504
+
1505
+ def children_stamps(
1506
+ stamps: list[tuple[str]],
1507
+ child_lineage: PrimitiveLineageT,
1508
+ save_to: Pathish | None = None,
1509
+ ) -> pd.DataFrame:
1510
+ """
1511
+ Combine the stamp files of the artifacts of one lineage, `child_lineage`,
1512
+ into one DataFrame for a *space*.
1513
+
1514
+ With `save_to`, the DataFrame is also saved as a CSV file.
1515
+
1516
+ Parameters
1517
+ ----------
1518
+ stamps : list[tuple[str]]
1519
+ List of 1-tuples, as :func:`~polyphys.manage.utils.sort_filenames`
1520
+ returns them, each holding the path to a 'stamp' CSV file. Rows are
1521
+ concatenated in this order.
1522
+ child_lineage : {'segment', 'whole'}
1523
+ Lineage of the stamp files, and so of the rows of the result.
1524
+ save_to : Pathish, optional
1525
+ Directory path where the combined DataFrame is saved, if specified,
1526
+ as ``<space>-<group>-stamps-<child_lineage>.csv``, with ``space`` and
1527
+ ``group`` read from the first row; a file of the same name is
1528
+ overwritten.
1529
+
1530
+ Returns
1531
+ -------
1532
+ pd.DataFrame
1533
+ Combined DataFrame containing data from all the 'stamp' files, with
1534
+ a fresh ``0..n-1`` index, and a ``group`` column (see Notes).
1535
+
1536
+ Raises
1537
+ ------
1538
+ ValueError
1539
+ If `child_lineage` is not one of the accepted values; or if the
1540
+ stamps have no ``group`` column and a file name is not
1541
+ ``<name>-<group>-stamps.csv``, or the names give more than one
1542
+ group.
1543
+
1544
+ Warns
1545
+ -----
1546
+ UserWarning
1547
+ If `child_lineage` is 'whole' and the 'segment' and 'segment_id'
1548
+ columns are dropped.
1549
+
1550
+ Notes
1551
+ -----
1552
+ - It is assumed that all the stamp files have belong to the same *group* in
1553
+ a given *space*.
1554
+ - If `child_lineage` is 'whole', columns 'segment' and 'segment_id' are
1555
+ removed, as they are redundant for whole-lineage data, with a
1556
+ `UserWarning`.
1557
+ Either may be absent, as in stamps the probe phase writes for a whole;
1558
+ if neither is present, nothing is dropped and a DEBUG record is
1559
+ logged.
1560
+ - The stamps :func:`~polyphys.probe.core.write_stamps` writes carry a
1561
+ ``group`` column; the research code's stamps did not, and held the
1562
+ group in the file name, ``<lineage name>-<group>-stamps.csv``. When
1563
+ the column is missing it is added from the file names, after
1564
+ ``space`` when that column is present and last otherwise, so the
1565
+ result and the parent stamps built from it carry the group either
1566
+ way.
1567
+ - Let n_files be the number of stamp files, n_rows the total number of
1568
+ rows, and n_cols the number of columns. Time is O(n_rows * n_cols).
1569
+ Every file is read before the frames are concatenated, so peak memory
1570
+ is about twice the combined data, O(n_rows * n_cols).
1571
+
1572
+ Examples
1573
+ --------
1574
+ >>> import tempfile
1575
+ >>> tmp = tempfile.TemporaryDirectory()
1576
+ >>> paths = []
1577
+ >>> for seg_id in (1, 2):
1578
+ ... path = Path(tmp.name) / f'w1.j0{seg_id}-bug-stamps.csv'
1579
+ ... pd.DataFrame({'segment': [f'w1.j0{seg_id}'], 'whole': ['w1'],
1580
+ ... 'segment_id': [seg_id], 'n_frames': [10]}
1581
+ ... ).to_csv(path, index=False)
1582
+ ... paths.append((str(path),))
1583
+ >>> children_stamps(paths, 'segment')
1584
+ segment whole segment_id n_frames group
1585
+ 0 w1.j01 w1 1 10 bug
1586
+ 1 w1.j02 w1 2 10 bug
1587
+ >>> tmp.cleanup()
1588
+
1589
+ The stamps have no ``group`` column, so it comes from the file names
1590
+ and, with no ``space`` column either, goes last.
1591
+ """
1592
+ invalid_keyword(child_lineage, ["segment", "whole"])
1593
+
1594
+ # Load and concatenate all stamps
1595
+ space_stamps = pd.concat(
1596
+ [pd.read_csv(stamp[0]) for stamp in stamps], ignore_index=True
1597
+ )
1598
+
1599
+ # The research code's stamps hold the group in the file name only.
1600
+ if "group" not in space_stamps.columns:
1601
+ group = _group_from_stamp_names(stamps)
1602
+ columns = list(space_stamps.columns)
1603
+ position = (columns.index("space") + 1 if "space" in columns
1604
+ else len(columns))
1605
+ space_stamps.insert(position, "group", group)
1606
+
1607
+ # Drop specific columns for 'whole' lineage
1608
+ cols_to_drop = ["segment", "segment_id"]
1609
+ if child_lineage == "whole":
1610
+ dropped = [col for col in cols_to_drop if col in space_stamps.columns]
1611
+ if dropped:
1612
+ space_stamps = space_stamps.drop(columns=dropped)
1613
+ warnings.warn(
1614
+ "'segment' and 'segment_id' columns are dropped when"
1615
+ " individual 'whole' stamps combined to create a single"
1616
+ " dataframe of 'whole' stamps by 'children_stamps'.",
1617
+ UserWarning,
1618
+ )
1619
+ else:
1620
+ logger.debug("No %s columns to drop from the whole stamps.",
1621
+ cols_to_drop)
1622
+
1623
+ # Save the resulting DataFrame to a CSV if specified
1624
+ if save_to is not None:
1625
+ space_name = str(space_stamps.loc[0, "space"])
1626
+ group = str(space_stamps.loc[0, "group"])
1627
+ filename = f"{space_name}-{group}-stamps-{child_lineage}.csv"
1628
+ space_stamps.to_csv(os.path.join(save_to, filename), index=False)
1629
+
1630
+ return space_stamps
1631
+
1632
+
1633
+ def parents_stamps(
1634
+ stamps: pd.DataFrame,
1635
+ child_lineage: PrimitiveLineageT,
1636
+ properties: dict[str, Callable[..., Any]] | None = None,
1637
+ save_to: Pathish | None = None,
1638
+ ) -> pd.DataFrame:
1639
+ """
1640
+ Aggregate the child stamps of a *space* into parent stamps.
1641
+
1642
+ The aggregation functions are given by `properties`; with `save_to`, the
1643
+ result is also saved as a CSV file.
1644
+
1645
+ Parameters
1646
+ ----------
1647
+ stamps : pd.DataFrame
1648
+ DataFrame containing child stamp data, the lineage of whose rows is
1649
+ `child_lineage`. It must have a 'whole' and a 'segment_id' column for
1650
+ 'segment' stamps, and an 'ensemble_long' and an 'ensemble_id' column
1651
+ for 'whole' stamps, and an 'n_frames' column for both.
1652
+ child_lineage : {'segment', 'whole'}
1653
+ Lineage of the child stamps, the rows of `stamps`; the parents are
1654
+ one level up, 'whole' or 'ensemble_long'.
1655
+ properties : dict of str -> Callable, optional
1656
+ Dictionary specifying aggregation functions for specific columns,
1657
+ replacing the default 'last'. It cannot name the parent column
1658
+ ('whole' or 'ensemble_long'), the counted column ('segment_id' or
1659
+ 'ensemble_id'), or 'n_frames', which the function aggregates itself.
1660
+ save_to : Pathish, optional
1661
+ Directory path where the parent DataFrame is saved, if specified,
1662
+ as ``<space>-<group>-stamps-<level>.csv``, with ``<level>`` the
1663
+ lineage of the parent rows, ``whole`` for 'segment' stamps and
1664
+ ``ensemble_long`` for 'whole' stamps, and with ``space`` and
1665
+ ``group`` read from the first row; a file of the same name is
1666
+ overwritten.
1667
+
1668
+ Returns
1669
+ -------
1670
+ pd.DataFrame
1671
+ Aggregated parent DataFrame, one row per parent:
1672
+
1673
+ - 'segment': one row per 'whole', with 'n_segments' (the number of
1674
+ segments) and 'n_frames' (their sum).
1675
+ - 'whole': one row per 'ensemble_long', with 'n_ensembles' (the
1676
+ number of wholes) and 'n_frames' (the last whole's); 'segment',
1677
+ 'segment_id', and 'n_segments' are dropped if present.
1678
+
1679
+ Every other column takes its last value in each group unless
1680
+ `properties` says otherwise; 'lineage_name' and the `child_lineage`
1681
+ column are dropped.
1682
+
1683
+ Raises
1684
+ ------
1685
+ ValueError
1686
+ If `child_lineage` is not one of the accepted values; if a row has no
1687
+ parent name (NaN in 'whole' for 'segment' stamps, in
1688
+ 'ensemble_long' for 'whole' stamps), which the group-by would drop;
1689
+ if `properties` names a column the function aggregates itself; or
1690
+ if `save_to` is given and the stamps lack a 'space' or a 'group'
1691
+ column, which the file name needs.
1692
+
1693
+ Notes
1694
+ -----
1695
+ - It is assumed that all the stamp files have belong to the same *group* in
1696
+ a given *space*.
1697
+ - If `child_lineage` is 'segment', stamps correspond to individual
1698
+ segments, each with a unique `segment_id`.
1699
+ - If `child_lineage` is 'whole', stamps correspond to whole simulations,
1700
+ each identified by an `ensemble_id`.
1701
+ - The 'lineage_name' column and the `child_lineage` column are not
1702
+ carried into the parent stamps. Either may be absent, as
1703
+ 'lineage_name' is from the stamps the probe phase writes; an absent
1704
+ one is skipped with a DEBUG record.
1705
+ - Let n_rows be the number of child stamps and n_cols the number of
1706
+ columns. The group-by is hash-based, so expected time is
1707
+ O(n_rows * n_cols); memory is one copy of the parent stamps.
1708
+
1709
+ Examples
1710
+ --------
1711
+ >>> stamps = pd.DataFrame({
1712
+ ... 'lineage_name': ['w1.j01', 'w1.j02', 'w2.j01'],
1713
+ ... 'segment': ['w1.j01', 'w1.j02', 'w2.j01'],
1714
+ ... 'whole': ['w1', 'w1', 'w2'],
1715
+ ... 'segment_id': [1, 2, 1],
1716
+ ... 'n_frames': [10, 20, 30],
1717
+ ... })
1718
+ >>> parents_stamps(stamps, 'segment')
1719
+ whole n_segments n_frames
1720
+ 0 w1 2 30
1721
+ 1 w2 1 30
1722
+ """
1723
+ invalid_keyword(child_lineage, ["segment", "whole"])
1724
+
1725
+ # Base aggregation functions for columns
1726
+ base_columns = list(stamps.columns)
1727
+ for column in ("lineage_name", child_lineage):
1728
+ if column in base_columns:
1729
+ base_columns.remove(column)
1730
+ else:
1731
+ logger.debug("Column %r is not among the stamps columns.", column)
1732
+ agg_funcs: dict[str, str | Callable[..., Any]] = {
1733
+ col: "last" for col in base_columns
1734
+ }
1735
+
1736
+ # Define grouping column and lineage-specific aggregations
1737
+ if child_lineage == "whole":
1738
+ parent_groupby = "ensemble_long"
1739
+ own_aggs: dict[str, str] = {"ensemble_id": "count",
1740
+ "n_frames": "last"}
1741
+ else:
1742
+ parent_groupby = "whole"
1743
+ own_aggs = {"segment_id": "count", "n_frames": "sum"}
1744
+ for column in (properties or {}):
1745
+ if column == parent_groupby or column in own_aggs:
1746
+ raise ValueError(
1747
+ f"properties cannot set {column!r}, which parents_stamps "
1748
+ "aggregates itself.")
1749
+ missing = int(stamps[parent_groupby].isna().sum())
1750
+ if missing:
1751
+ rows = "row has" if missing == 1 else "rows have"
1752
+ raise ValueError(
1753
+ f"{missing} stamps {rows} no {parent_groupby!r}; the group-by "
1754
+ "would drop them.")
1755
+ if properties is not None:
1756
+ agg_funcs.update(properties)
1757
+ agg_funcs.update(own_aggs)
1758
+
1759
+ # Perform the groupby and aggregate
1760
+ agg_funcs.pop(parent_groupby)
1761
+ parent_stamps = stamps.groupby(parent_groupby).agg(agg_funcs).reset_index()
1762
+
1763
+ # Rename and drop columns based on lineage
1764
+ if child_lineage == "whole":
1765
+ parent_stamps = parent_stamps.rename(
1766
+ columns={"ensemble_id": "n_ensembles"}
1767
+ )
1768
+ # If the 'whole' stamps are generated directly in the 'probe' phase,
1769
+ # then 'segment' and 'segment_id' columns are "N/A" and are removed
1770
+ # from the list of stamps columns that are added to the parents
1771
+ # stamps.
1772
+ # There is no need to have the 'n_segment' column in the parent
1773
+ # stamps, so it is removed. The 'whole' stamps directly generated
1774
+ # in the 'probe' phase do not have such a column, but those generated
1775
+ # from 'segment' stamps have.
1776
+ # Dropping redundant columns silently:
1777
+ parent_stamps = parent_stamps.drop(
1778
+ columns=["n_segments", "segment_id", "segment"],
1779
+ errors="ignore",
1780
+ )
1781
+ else:
1782
+ parent_stamps = parent_stamps.rename(
1783
+ columns={"segment_id": "n_segments"}
1784
+ )
1785
+
1786
+ if save_to is not None:
1787
+ for column in ("space", "group"):
1788
+ if column not in parent_stamps.columns:
1789
+ raise ValueError(
1790
+ f"Saving the parent stamps needs a {column!r} column "
1791
+ "for the file name.")
1792
+ space_name = str(parent_stamps.loc[0, "space"])
1793
+ group = str(parent_stamps.loc[0, "group"])
1794
+ level = parent_groupby
1795
+ filename = f"{space_name}-{group}-stamps-{level}.csv"
1796
+ parent_stamps.to_csv(os.path.join(save_to, filename), index=False)
1797
+
1798
+ return parent_stamps
1799
+
1800
+
1801
+ def space_tseries(
1802
+ input_database: str,
1803
+ prop: PropertyT,
1804
+ parser: ParserClassT,
1805
+ hierarchy: str,
1806
+ physical_attrs: list[str],
1807
+ group: GroupT,
1808
+ *,
1809
+ divisor: float = 0.025,
1810
+ round_to: int = 3,
1811
+ is_save: bool | None = False,
1812
+ crowder_attr: str = "phi_bulk_c",
1813
+ ) -> pd.DataFrame:
1814
+ """
1815
+ Stack the ensemble-averaged time series of one property across the
1816
+ state points of a space into one DataFrame.
1817
+
1818
+ The physical attributes in `physical_attrs` are added as columns.
1819
+
1820
+ Parameters
1821
+ ----------
1822
+ input_database : str
1823
+ Path to the directory containing the ensemble-averaged time-series
1824
+ files, named ``<ensemble_long>-<group>-<prop>-ensAvg.csv``, with or
1825
+ without a trailing path separator.
1826
+ prop : str
1827
+ Name of the physical property of interest.
1828
+ parser : ParserClassT
1829
+ Parser class to infer file-specific information from filenames or
1830
+ paths.
1831
+ hierarchy : str
1832
+ Pattern prefix for the time-series filenames (e.g., `"N*"`).
1833
+ physical_attrs : list[str]
1834
+ List of physical attributes to add as new columns in the output
1835
+ DataFrame.
1836
+ group : GroupT
1837
+ Particle group type. Its trajectory's dump period, in time steps,
1838
+ is the `parser` attribute named by ``dump_keyword``.
1839
+ divisor : float, default 0.025
1840
+ Rounding step for the crowder volume fraction named by
1841
+ `crowder_attr`.
1842
+ round_to : int, default 3
1843
+ Number of decimal places for rounding the crowder volume fraction.
1844
+ is_save : bool, default False
1845
+ If True, saves the concatenated DataFrame to a CSV file named
1846
+ ``<space>-<group>-<prop>-space.csv`` in the ``analysis`` phase's
1847
+ ``space`` directory, which :func:`make_database` creates next to
1848
+ `input_database`; a file of the same name is overwritten.
1849
+ crowder_attr : str, default 'phi_bulk_c'
1850
+ Name of the `parser` attribute that holds the bulk volume fraction
1851
+ of crowders. Its value is added as a column of that name, whether or
1852
+ not it is listed in `physical_attrs`, and rounded into the
1853
+ ``phi_bulk_c_round`` column.
1854
+
1855
+ Returns
1856
+ -------
1857
+ pd.DataFrame
1858
+ Concatenated DataFrame of all time-series with added physical
1859
+ attributes. Besides the columns of each file, it has ``t_index``
1860
+ (the frame index times the dump period of the `group` trajectory),
1861
+ ``time`` (``t_index`` times the parser's ``dt``), one column per
1862
+ entry in `physical_attrs`, a `crowder_attr` column, and
1863
+ ``phi_bulk_c_round``.
1864
+
1865
+ Examples
1866
+ --------
1867
+ One ensemble-averaged time series of two frames of the ``bug``
1868
+ trajectory, dumped every 1000 steps (``bdump``) of 0.005 time units:
1869
+
1870
+ >>> import tempfile
1871
+ >>> from polyphys.tests.synthetic import ChainCyl
1872
+ >>> tmp = tempfile.TemporaryDirectory()
1873
+ >>> database = Path(tmp.name) / 'analysis' / 'N20-bug-ensAvg'
1874
+ >>> database.mkdir(parents=True)
1875
+ >>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
1876
+ >>> pd.DataFrame({'gyrTMon-mean': [1.0, 2.0]}).to_csv(
1877
+ ... database / f'{state}-bug-gyrTMon-ensAvg.csv', index=False)
1878
+ >>> space = space_tseries(str(database) + '/', 'gyrTMon', ChainCyl,
1879
+ ... 'N*', ['nmon'], 'bug')
1880
+ >>> space[['t_index', 'time', 'gyrTMon-mean', 'nmon']]
1881
+ t_index time gyrTMon-mean nmon
1882
+ 0 0 0.0 1.0 20
1883
+ 1 1000 5.0 2.0 20
1884
+ >>> tmp.cleanup()
1885
+
1886
+ Notes
1887
+ -----
1888
+ - Files are selected by globbing
1889
+ ``os.path.join(input_database, hierarchy + '-<prop>-ensAvg.csv')``
1890
+ and sorted alphanumerically by their full names.
1891
+ - Requires a parser class with methods to retrieve attribute information
1892
+ for each file.
1893
+ - Let n_files be the number of matched files, n_rows the total number
1894
+ of rows across them, and n_cols the number of columns of the result.
1895
+ Time is O(n_rows * n_cols + n_files log n_files). Every file is read
1896
+ and kept before the frames are concatenated, so peak memory is about
1897
+ twice the result, O(n_rows * n_cols).
1898
+ """
1899
+ prop_ext = f"-{prop}-ensAvg.csv"
1900
+ ens_avg_csvs = sort_filenames(
1901
+ glob(os.path.join(input_database, hierarchy + prop_ext)), ["csv"]
1902
+ )
1903
+ prop_csvs = []
1904
+
1905
+ for ens_avg_csv in ens_avg_csvs:
1906
+ ens_avg = pd.read_csv(ens_avg_csv[0], header=0)
1907
+ prop_info = parser(
1908
+ ens_avg_csv[0], "ensemble_long", group, ispath=True
1909
+ )
1910
+
1911
+ ens_avg = ens_avg.reset_index()
1912
+ ens_avg = ens_avg.rename(columns={"index": "t_index"})
1913
+
1914
+ # Scale frame indices by the dump period of the group's trajectory
1915
+ ens_avg["t_index"] *= getattr(prop_info, prop_info.dump_keyword)
1916
+ ens_avg["time"] = ens_avg["t_index"] * getattr(prop_info, "dt")
1917
+
1918
+ # Add physical attributes
1919
+ for attr_name in physical_attrs:
1920
+ ens_avg[attr_name] = getattr(prop_info, attr_name)
1921
+
1922
+ # Add and round the bulk volume fraction of crowders
1923
+ ens_avg[crowder_attr] = getattr(prop_info, crowder_attr)
1924
+ ens_avg["phi_bulk_c_round"] = ens_avg[crowder_attr].apply(
1925
+ lambda x: round_to_multiple(x, divisor, round_to)
1926
+ )
1927
+
1928
+ prop_csvs.append(ens_avg)
1929
+
1930
+ # Concatenate all time-series DataFrames
1931
+ prop_db = pd.concat(prop_csvs, axis=0)
1932
+ prop_db = prop_db.reset_index(drop=True)
1933
+
1934
+ # Optionally save to file
1935
+ if is_save:
1936
+ save_to_space = make_database(
1937
+ input_database, "analysis", stage="space", group=group
1938
+ )
1939
+ space = save_to_space.split("/")[-2].split("-")[0]
1940
+ filepath = (
1941
+ save_to_space + f"{create_fullname(space, group, prop)}-space.csv"
1942
+ )
1943
+ prop_db.to_csv(filepath, index=False)
1944
+
1945
+ return prop_db
1946
+
1947
+
1948
+ def space_hists(
1949
+ input_database: str,
1950
+ prop: PropertyT,
1951
+ parser: ParserClassT,
1952
+ hierarchy: str,
1953
+ physical_attrs: list[str],
1954
+ group: GroupT,
1955
+ bin_center: np.ndarray | None = None,
1956
+ normalize: bool | None = False,
1957
+ divisor: float = 0.025,
1958
+ round_to: int = 3,
1959
+ is_save: bool | None = False,
1960
+ crowder_attr: str = "phi_bulk_c",
1961
+ ) -> pd.DataFrame:
1962
+ """
1963
+ Stack the ensemble-averaged histograms of one property across the state
1964
+ points of a space into one DataFrame.
1965
+
1966
+ The histograms are normalized if `normalize` asks for it.
1967
+
1968
+ Parameters
1969
+ ----------
1970
+ input_database : str
1971
+ Path to the directory containing the ensemble-averaged histogram
1972
+ files, named ``<ensemble_long>-<group>-<prop>-ensAvg.csv``, with or
1973
+ without a trailing path separator.
1974
+ prop : :py:obj:`~.PropertyT`
1975
+ Name of the physical property of interest.
1976
+ parser : ParserClassT
1977
+ Parser class to infer file-specific information from filenames or
1978
+ paths.
1979
+ hierarchy : str
1980
+ Pattern prefix for the histogram filenames (e.g., `"N*"`).
1981
+ physical_attrs : list[str]
1982
+ List of physical attributes to add as new columns in the output
1983
+ DataFrame.
1984
+ group : GroupT
1985
+ Particle group type.
1986
+ bin_center : np.ndarray, optional
1987
+ Array of bin centers. If not provided, must be present in the
1988
+ DataFrames.
1989
+ normalize : bool, default False
1990
+ If True, normalizes the histogram data.
1991
+ divisor : float, default 0.025
1992
+ Rounding step for the crowder volume fraction named by
1993
+ `crowder_attr`.
1994
+ round_to : int, default 3
1995
+ Number of decimal places for rounding the crowder volume fraction.
1996
+ is_save : bool, default False
1997
+ If True, saves the concatenated DataFrame to a CSV file named
1998
+ ``<space>-<group>-<prop>-space.csv`` in the ``analysis`` phase's
1999
+ ``space`` directory, which :func:`make_database` creates next to
2000
+ `input_database`; a file of the same name is overwritten.
2001
+ crowder_attr : str, default 'phi_bulk_c'
2002
+ Name of the `parser` attribute that holds the bulk volume fraction
2003
+ of crowders. Its value is added as a column of that name, whether or
2004
+ not it is listed in `physical_attrs`, and rounded into the
2005
+ ``phi_bulk_c_round`` column.
2006
+
2007
+ Returns
2008
+ -------
2009
+ pd.DataFrame
2010
+ Concatenated DataFrame of all histograms with added physical
2011
+ attributes: one column per entry in `physical_attrs`, a
2012
+ `crowder_attr` column, and ``phi_bulk_c_round``.
2013
+
2014
+ Warns
2015
+ -----
2016
+ UserWarning
2017
+ If `normalize` is True and the ``<prop>-mean`` column of a file sums
2018
+ to zero; its ``<prop>-norm`` column is then set to zero.
2019
+
2020
+ Examples
2021
+ --------
2022
+ One ensemble-averaged histogram of three bins, normalized to sum to one:
2023
+
2024
+ >>> import tempfile
2025
+ >>> from polyphys.tests.synthetic import ChainCyl
2026
+ >>> tmp = tempfile.TemporaryDirectory()
2027
+ >>> database = Path(tmp.name) / 'analysis' / 'N20-bug-ensAvg'
2028
+ >>> database.mkdir(parents=True)
2029
+ >>> state = 'N20r5.5lz25.0ac1.0nc100dt0.005bdump1000adump5000'
2030
+ >>> pd.DataFrame({'rhoMon-mean': [1.0, 2.0, 1.0]}).to_csv(
2031
+ ... database / f'{state}-bug-rhoMon-ensAvg.csv', index=False)
2032
+ >>> space = space_hists(str(database) + '/', 'rhoMon', ChainCyl, 'N*',
2033
+ ... ['nmon'], 'bug',
2034
+ ... bin_center=np.array([0.5, 1.5, 2.5]),
2035
+ ... normalize=True)
2036
+ >>> space[['bin_center', 'rhoMon-mean', 'rhoMon-norm', 'nmon']]
2037
+ bin_center rhoMon-mean rhoMon-norm nmon
2038
+ 0 0.5 1.0 0.25 20
2039
+ 1 1.5 2.0 0.50 20
2040
+ 2 2.5 1.0 0.25 20
2041
+ >>> tmp.cleanup()
2042
+
2043
+ Notes
2044
+ -----
2045
+ - Files are selected by globbing
2046
+ ``os.path.join(input_database, hierarchy + '-<prop>-ensAvg.csv')``
2047
+ and sorted alphanumerically by their full names.
2048
+ - If `normalize` is True, histogram values will be scaled to sum to 1.
2049
+ - Let n_files be the number of matched files, n_rows the total number
2050
+ of rows across them, and n_cols the number of columns of the result.
2051
+ Time is O(n_rows * n_cols + n_files log n_files). Every file is read
2052
+ and kept before the frames are concatenated, so peak memory is about
2053
+ twice the result, O(n_rows * n_cols).
2054
+ - The `bin_center` should be provided if it is not available in the input
2055
+ data. When all the ensemble-averaged DataFrames have the same bin
2056
+ center, like 'clustersHistTFoci' or 'bondsHistTFoci' properties, the
2057
+ `bin_center` may not be provided.
2058
+ """
2059
+ prop_ext = f"-{prop}-ensAvg.csv"
2060
+ ens_avg_csvs_ungrouped = glob(
2061
+ os.path.join(input_database, hierarchy + prop_ext))
2062
+ ens_avg_csvs = sort_filenames(ens_avg_csvs_ungrouped, formats=["csv"])
2063
+ prop_csvs = []
2064
+
2065
+ for ens_avg_csv in ens_avg_csvs:
2066
+ ens_avg = pd.read_csv(ens_avg_csv[0], header=0)
2067
+ prop_info = parser(
2068
+ ens_avg_csv[0], "ensemble_long", group, ispath=True
2069
+ )
2070
+
2071
+ # Handle bin_center if provided
2072
+ if bin_center is not None:
2073
+ ens_avg["bin_center"] = bin_center.tolist()
2074
+ ens_avg["bin_center-norm"] = (
2075
+ ens_avg["bin_center"] / ens_avg["bin_center"].max()
2076
+ )
2077
+
2078
+ # Normalize data if specified
2079
+ if normalize:
2080
+ normalizer = ens_avg[prop + "-mean"].sum()
2081
+ if normalizer != 0:
2082
+ ens_avg[prop + "-norm"] = ens_avg[prop + "-mean"] / normalizer
2083
+ else:
2084
+ warnings.warn(
2085
+ "All values are zero; normalized values set to zero.",
2086
+ UserWarning,
2087
+ )
2088
+ ens_avg[prop + "-norm"] = 0
2089
+
2090
+ # Add physical attributes
2091
+ for attr_name in physical_attrs:
2092
+ ens_avg[attr_name] = getattr(prop_info, attr_name)
2093
+
2094
+ # Add and round the bulk volume fraction of crowders
2095
+ ens_avg[crowder_attr] = getattr(prop_info, crowder_attr)
2096
+ ens_avg["phi_bulk_c_round"] = ens_avg[crowder_attr].apply(
2097
+ lambda x: round_to_multiple(x, divisor, round_to)
2098
+ )
2099
+
2100
+ prop_csvs.append(ens_avg)
2101
+
2102
+ # Concatenate all histograms
2103
+ prop_db = pd.concat(prop_csvs, axis=0)
2104
+ prop_db = prop_db.reset_index(drop=True)
2105
+
2106
+ # Optionally save to file
2107
+ if is_save:
2108
+ save_to_space = make_database(
2109
+ input_database, "analysis", stage="space", group=group
2110
+ )
2111
+ space = save_to_space.split("/")[-2].split("-")[0]
2112
+ filepath = (
2113
+ save_to_space + f"{create_fullname(space, group, prop)}-space.csv"
2114
+ )
2115
+ prop_db.to_csv(filepath, index=False)
2116
+
2117
+ return prop_db
2118
+
2119
+
2120
+ BULK_NAME_MAP: Mapping[str, str] = MappingProxyType({
2121
+ "phi_m_bulk": "phi_bulk_m",
2122
+ "rho_m_bulk": "rho_bulk_m",
2123
+ "phi_c_bulk": "phi_bulk_c",
2124
+ "rho_c_bulk": "rho_bulk_c",
2125
+ "phi_c_bulk_one": "phi_bulk_c_one",
2126
+ "phi_c_bulk_two": "phi_bulk_c_two",
2127
+ "rho_c_bulk_one": "rho_bulk_c_one",
2128
+ "rho_c_bulk_two": "rho_bulk_c_two",
2129
+ "phi_hns_bulk": "phi_bulk_hns",
2130
+ "rho_hns_bulk": "rho_bulk_hns",
2131
+ "phi_c_bulk_round": "phi_bulk_c_round",
2132
+ })
2133
+ """
2134
+ Each bulk-quantity name of the research code mapped to its PolyPhys name.
2135
+
2136
+ The research code named a bulk quantity ``<quantity>_<species>_bulk``, such
2137
+ as ``phi_c_bulk``; PolyPhys names it ``<quantity>_bulk_<species>``, such as
2138
+ ``phi_bulk_c``. The research code wrote every parser attribute as a column
2139
+ of its ``*-stamps.csv`` files, and those columns carry into its
2140
+ ensemble-average and ``space`` tables, so a table it wrote carries the old
2141
+ names. The map is read-only; :func:`rename_bulk_columns` applies it.
2142
+
2143
+ .. versionadded:: 0.7.0
2144
+ """
2145
+
2146
+
2147
+ def rename_bulk_columns(table: pd.DataFrame) -> pd.DataFrame:
2148
+ """
2149
+ Return a copy of `table` whose bulk-quantity columns carry the PolyPhys
2150
+ names.
2151
+
2152
+ Each column named by a key of :data:`BULK_NAME_MAP` is renamed to that
2153
+ key's value; every other column, every value, and the column order are
2154
+ unchanged. A table that already carries the PolyPhys names comes back
2155
+ equal to `table`.
2156
+
2157
+ Parameters
2158
+ ----------
2159
+ table : pd.DataFrame
2160
+ A table read from a stamp, ensemble-average, or ``space`` file that
2161
+ the research code wrote.
2162
+
2163
+ Returns
2164
+ -------
2165
+ pd.DataFrame
2166
+ A new table; `table` is not modified.
2167
+
2168
+ Notes
2169
+ -----
2170
+ Runs in :math:`O(n_\\text{col})` time for the renaming and
2171
+ :math:`O(n_\\text{row} n_\\text{col})` space for the copy.
2172
+
2173
+ Examples
2174
+ --------
2175
+ A row of a research-code stamp file:
2176
+
2177
+ >>> import pandas as pd
2178
+ >>> old = pd.DataFrame({'nmon': [20], 'phi_c_bulk': [0.1],
2179
+ ... 'rho_c_bulk': [0.191], 'n_frames': [100]})
2180
+ >>> new = rename_bulk_columns(old)
2181
+ >>> list(new.columns)
2182
+ ['nmon', 'phi_bulk_c', 'rho_bulk_c', 'n_frames']
2183
+ >>> list(old.columns)
2184
+ ['nmon', 'phi_c_bulk', 'rho_c_bulk', 'n_frames']
2185
+
2186
+ .. versionadded:: 0.7.0
2187
+ """
2188
+ return table.rename(columns=dict(BULK_NAME_MAP))