jcmsuite-xarray 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jcmsuite_xarray/__init__.py +4 -0
- jcmsuite_xarray/export.py +809 -0
- jcmsuite_xarray/fields.py +245 -0
- jcmsuite_xarray/grouped.py +609 -0
- jcmsuite_xarray/headers.py +291 -0
- jcmsuite_xarray/runtime.py +99 -0
- jcmsuite_xarray/shared.py +246 -0
- jcmsuite_xarray/single.py +71 -0
- jcmsuite_xarray/tables.py +62 -0
- jcmsuite_xarray/tabular.py +305 -0
- jcmsuite_xarray-1.0.0.dist-info/METADATA +18 -0
- jcmsuite_xarray-1.0.0.dist-info/RECORD +13 -0
- jcmsuite_xarray-1.0.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,809 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# Copyright (c) 2001-2026 JCMwave GmbH
|
|
3
|
+
|
|
4
|
+
"""Helpers for `xarray` / `DataTree` exports of structured result payloads."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Mapping, Sequence
|
|
9
|
+
from typing import TYPE_CHECKING, cast
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
from jcmsuite.exceptions import InvalidConfigurationError, UnsupportedExportError
|
|
14
|
+
from jcmsuite.results.artifacts import (
|
|
15
|
+
ComputationalCosts,
|
|
16
|
+
Eigenvalues,
|
|
17
|
+
PostprocessCollection,
|
|
18
|
+
)
|
|
19
|
+
from jcmsuite.results.internal.export.xarray.types import XarrayDataset, XarrayDataTree
|
|
20
|
+
|
|
21
|
+
from .grouped import (
|
|
22
|
+
_common_ordered_child_names,
|
|
23
|
+
_GroupedPostprocessTensor,
|
|
24
|
+
_split_structured_payload,
|
|
25
|
+
_stack_batch_grouped_tensors,
|
|
26
|
+
_StructuredPayloadParts,
|
|
27
|
+
grouped_tensor_to_xarray_dataset,
|
|
28
|
+
postprocess_payload_to_grouped_tensor,
|
|
29
|
+
)
|
|
30
|
+
from .headers import (
|
|
31
|
+
_append_stacked_header_data_variable,
|
|
32
|
+
_flatten_header_fields,
|
|
33
|
+
_header_payload_to_dataset,
|
|
34
|
+
)
|
|
35
|
+
from .runtime import (
|
|
36
|
+
_create_data_tree,
|
|
37
|
+
_DataTreeModule,
|
|
38
|
+
_load_datatree_module,
|
|
39
|
+
_load_xarray_module,
|
|
40
|
+
_XarrayModule,
|
|
41
|
+
)
|
|
42
|
+
from .shared import _build_batch_coord_variable, _ensure_generated_companion_dim_coords
|
|
43
|
+
from .tabular import (
|
|
44
|
+
_BatchIndexedKeys,
|
|
45
|
+
_stack_batch_tabular_tables,
|
|
46
|
+
_TabularXarrayTable,
|
|
47
|
+
tabular_payload_to_xarray_table,
|
|
48
|
+
tabular_table_to_xarray_dataset,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
if TYPE_CHECKING:
|
|
52
|
+
from jcmsuite.results.artifacts import PostprocessRecord
|
|
53
|
+
from jcmsuite.results.internal.models.batch import BatchExportRecord
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def batch_keys_to_xarray_dataset(
|
|
57
|
+
records: Sequence["BatchExportRecord"],
|
|
58
|
+
) -> XarrayDataTree:
|
|
59
|
+
"""Convert batch key metadata to a strict root-only `DataTree`."""
|
|
60
|
+
|
|
61
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
62
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
63
|
+
key_names: tuple[str, ...] = _strict_batch_key_names(records)
|
|
64
|
+
data_vars: dict[str, object] = {}
|
|
65
|
+
key_name: str
|
|
66
|
+
for key_name in key_names:
|
|
67
|
+
values: list[object] = [record.keys[key_name] for record in records]
|
|
68
|
+
data_vars[key_name] = (
|
|
69
|
+
("batch_index",),
|
|
70
|
+
_strict_batch_key_values(key_name, values),
|
|
71
|
+
)
|
|
72
|
+
dataset: XarrayDataset = xarray_module.Dataset(
|
|
73
|
+
data_vars=data_vars,
|
|
74
|
+
coords={"batch_index": [record.index for record in records]},
|
|
75
|
+
)
|
|
76
|
+
return _create_data_tree(datatree_module=datatree_module, dataset=dataset)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def batch_costs_to_xarray_dataset(
|
|
80
|
+
records: Sequence["BatchExportRecord"],
|
|
81
|
+
) -> XarrayDataTree:
|
|
82
|
+
"""Convert batch computational-cost tables into one `DataTree`."""
|
|
83
|
+
|
|
84
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
85
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
86
|
+
tables: list[_TabularXarrayTable | None] = [
|
|
87
|
+
(
|
|
88
|
+
tabular_payload_to_xarray_table(
|
|
89
|
+
cast(ComputationalCosts, record.payload).table,
|
|
90
|
+
payload_name="Computational-cost payload",
|
|
91
|
+
)
|
|
92
|
+
if record.payload is not None
|
|
93
|
+
else None
|
|
94
|
+
)
|
|
95
|
+
for record in records
|
|
96
|
+
]
|
|
97
|
+
dataset: XarrayDataset = _stack_batch_tabular_tables(
|
|
98
|
+
records,
|
|
99
|
+
tables,
|
|
100
|
+
xarray_module=xarray_module,
|
|
101
|
+
batch_axis_name="batch_index",
|
|
102
|
+
)
|
|
103
|
+
dataset.attrs.pop("header", None)
|
|
104
|
+
return _create_data_tree(datatree_module=datatree_module, dataset=dataset)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _strict_batch_key_names(
|
|
108
|
+
records: Sequence["BatchExportRecord"],
|
|
109
|
+
) -> tuple[str, ...]:
|
|
110
|
+
"""Validate and return the common key-name set for strict export."""
|
|
111
|
+
|
|
112
|
+
if not records:
|
|
113
|
+
return ()
|
|
114
|
+
first_names: set[str] = set(records[0].keys)
|
|
115
|
+
record: BatchExportRecord
|
|
116
|
+
for record in records[1:]:
|
|
117
|
+
if set(record.keys) != first_names:
|
|
118
|
+
raise InvalidConfigurationError(
|
|
119
|
+
"Batch key payloads must have the same key names for direct "
|
|
120
|
+
"xarray export."
|
|
121
|
+
)
|
|
122
|
+
if "index" in first_names:
|
|
123
|
+
raise InvalidConfigurationError(
|
|
124
|
+
"Batch key name 'index' conflicts with the xarray batch index "
|
|
125
|
+
"coordinate."
|
|
126
|
+
)
|
|
127
|
+
return tuple(sorted(first_names))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _strict_batch_key_values(key_name: str, values: Sequence[object]) -> np.ndarray:
|
|
131
|
+
"""Validate scalar key values without applying shape coercion."""
|
|
132
|
+
|
|
133
|
+
categories: set[str] = set()
|
|
134
|
+
value: object
|
|
135
|
+
for value in values:
|
|
136
|
+
if value is None or not np.isscalar(value):
|
|
137
|
+
raise InvalidConfigurationError(
|
|
138
|
+
f"Batch key {key_name!r} is not directly representable in "
|
|
139
|
+
"xarray without coercion."
|
|
140
|
+
)
|
|
141
|
+
categories.add(_batch_key_scalar_category(value))
|
|
142
|
+
if len(categories) > 1:
|
|
143
|
+
raise InvalidConfigurationError(
|
|
144
|
+
f"Batch key {key_name!r} has incompatible scalar types for direct "
|
|
145
|
+
"xarray export."
|
|
146
|
+
)
|
|
147
|
+
return np.asarray(values)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _batch_key_scalar_category(value: object) -> str:
|
|
151
|
+
"""Return the strict xarray scalar category for one key value."""
|
|
152
|
+
|
|
153
|
+
if isinstance(value, (bool, np.bool_)):
|
|
154
|
+
return "bool"
|
|
155
|
+
if isinstance(value, (int, np.integer)):
|
|
156
|
+
return "integer"
|
|
157
|
+
if isinstance(value, (float, np.floating)):
|
|
158
|
+
return "floating"
|
|
159
|
+
if isinstance(value, (complex, np.complexfloating)):
|
|
160
|
+
return "complex"
|
|
161
|
+
if isinstance(value, str):
|
|
162
|
+
return "string"
|
|
163
|
+
if isinstance(value, (bytes, np.bytes_)):
|
|
164
|
+
return "bytes"
|
|
165
|
+
raise InvalidConfigurationError(
|
|
166
|
+
"Batch key value is not directly representable in xarray without " "coercion."
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def postprocess_payload_to_xarray_dataset(
|
|
171
|
+
payload: object,
|
|
172
|
+
*,
|
|
173
|
+
postprocess_title: str | None,
|
|
174
|
+
source_file: object,
|
|
175
|
+
) -> XarrayDataTree:
|
|
176
|
+
"""Convert one supported postprocess payload into a `DataTree`."""
|
|
177
|
+
|
|
178
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
179
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
180
|
+
if isinstance(payload, Mapping):
|
|
181
|
+
return _postprocess_mapping_to_data_tree(
|
|
182
|
+
payload,
|
|
183
|
+
postprocess_title=postprocess_title,
|
|
184
|
+
source_file=source_file,
|
|
185
|
+
xarray_module=xarray_module,
|
|
186
|
+
datatree_module=datatree_module,
|
|
187
|
+
)
|
|
188
|
+
raise InvalidConfigurationError(
|
|
189
|
+
"Postprocess payload is not mapping-like and cannot be exported via to_xarray()."
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def eigenvalues_payload_to_xarray_dataset(payload: object) -> XarrayDataTree:
|
|
194
|
+
"""Convert one tabular eigenvalue payload into a root-only `DataTree`."""
|
|
195
|
+
|
|
196
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
197
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
198
|
+
normalized_payload: object = payload
|
|
199
|
+
title_value: object | None = None
|
|
200
|
+
if isinstance(payload, Mapping):
|
|
201
|
+
normalized_payload = dict(payload)
|
|
202
|
+
title_value = normalized_payload.pop("title", None)
|
|
203
|
+
table: _TabularXarrayTable = tabular_payload_to_xarray_table(
|
|
204
|
+
normalized_payload,
|
|
205
|
+
payload_name="Eigenvalue payload",
|
|
206
|
+
)
|
|
207
|
+
if title_value is not None:
|
|
208
|
+
table.attrs["title"] = title_value
|
|
209
|
+
dataset: XarrayDataset = tabular_table_to_xarray_dataset(
|
|
210
|
+
table, xarray_module=xarray_module
|
|
211
|
+
)
|
|
212
|
+
return _create_data_tree(datatree_module=datatree_module, dataset=dataset)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def header_payload_to_xarray_dataset(payload: object) -> XarrayDataTree:
|
|
216
|
+
"""Convert one postprocess header payload into a root-only `DataTree`."""
|
|
217
|
+
|
|
218
|
+
if not isinstance(payload, Mapping):
|
|
219
|
+
raise InvalidConfigurationError(
|
|
220
|
+
"Postprocess header payload is not mapping-like and cannot be exported via to_xarray()."
|
|
221
|
+
)
|
|
222
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
223
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
224
|
+
dataset: XarrayDataset = _header_payload_to_dataset(
|
|
225
|
+
payload,
|
|
226
|
+
xarray_module=xarray_module,
|
|
227
|
+
)
|
|
228
|
+
return _create_data_tree(datatree_module=datatree_module, dataset=dataset)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def batch_selected_postprocess_headers_to_xarray_dataset(
|
|
232
|
+
entries: Sequence["BatchExportRecord"],
|
|
233
|
+
*,
|
|
234
|
+
index: int,
|
|
235
|
+
) -> XarrayDataTree:
|
|
236
|
+
"""Convert one selected batch postprocess header stream into a `DataTree`."""
|
|
237
|
+
|
|
238
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
239
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
240
|
+
selected_entries: list[BatchExportRecord] = []
|
|
241
|
+
header_payloads: list[Mapping[object, object] | None] = []
|
|
242
|
+
entry: BatchExportRecord
|
|
243
|
+
for entry in entries:
|
|
244
|
+
if not isinstance(entry.payload, PostprocessCollection):
|
|
245
|
+
continue
|
|
246
|
+
record = _select_batch_postprocess_record(entry, index=index)
|
|
247
|
+
selected_entries.append(entry)
|
|
248
|
+
header_payloads.append(None if record.header is None else record.header.table)
|
|
249
|
+
|
|
250
|
+
dataset: XarrayDataset = _stack_batch_header_payloads(
|
|
251
|
+
selected_entries,
|
|
252
|
+
header_payloads,
|
|
253
|
+
xarray_module=xarray_module,
|
|
254
|
+
)
|
|
255
|
+
return _create_data_tree(datatree_module=datatree_module, dataset=dataset)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def batch_selected_postprocesses_to_xarray_dataset(
|
|
259
|
+
entries: Sequence["BatchExportRecord"],
|
|
260
|
+
*,
|
|
261
|
+
index: int,
|
|
262
|
+
) -> XarrayDataTree:
|
|
263
|
+
"""Convert one selected batch postprocess stream into a stacked `DataTree`."""
|
|
264
|
+
|
|
265
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
266
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
267
|
+
selected_entries: list[BatchExportRecord] = []
|
|
268
|
+
payloads: list[Mapping[object, object]] = []
|
|
269
|
+
selected_title: str | None = None
|
|
270
|
+
entry: BatchExportRecord
|
|
271
|
+
for entry in entries:
|
|
272
|
+
if not isinstance(entry.payload, PostprocessCollection):
|
|
273
|
+
continue
|
|
274
|
+
record = _select_batch_postprocess_record(entry, index=index)
|
|
275
|
+
if selected_title is None:
|
|
276
|
+
selected_title = record.title
|
|
277
|
+
payload_object: object = _record_xarray_payload(record)
|
|
278
|
+
if not isinstance(payload_object, Mapping):
|
|
279
|
+
raise InvalidConfigurationError(
|
|
280
|
+
"Postprocess payload is not mapping-like and cannot be exported via to_xarray()."
|
|
281
|
+
)
|
|
282
|
+
payloads.append(payload_object)
|
|
283
|
+
selected_entries.append(entry)
|
|
284
|
+
|
|
285
|
+
if not payloads:
|
|
286
|
+
raise InvalidConfigurationError(
|
|
287
|
+
"No successful batch entries with the selected postprocess payload are available for to_xarray()."
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
return _stack_batch_postprocess_payloads_to_data_tree(
|
|
291
|
+
selected_entries,
|
|
292
|
+
payloads,
|
|
293
|
+
index=index,
|
|
294
|
+
title=selected_title,
|
|
295
|
+
xarray_module=xarray_module,
|
|
296
|
+
datatree_module=datatree_module,
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def batch_eigenvalues_to_xarray_dataset(
|
|
301
|
+
records: Sequence["BatchExportRecord"],
|
|
302
|
+
) -> XarrayDataTree:
|
|
303
|
+
"""Convert batch eigenvalue tables into one stacked root-only `DataTree`."""
|
|
304
|
+
|
|
305
|
+
xarray_module: _XarrayModule = _load_xarray_module()
|
|
306
|
+
datatree_module: _DataTreeModule = _load_datatree_module()
|
|
307
|
+
tables: list[_TabularXarrayTable | None] = []
|
|
308
|
+
record: BatchExportRecord
|
|
309
|
+
for record in records:
|
|
310
|
+
if not isinstance(record.payload, Eigenvalues):
|
|
311
|
+
tables.append(None)
|
|
312
|
+
continue
|
|
313
|
+
table: _TabularXarrayTable = tabular_payload_to_xarray_table(
|
|
314
|
+
record.payload.table,
|
|
315
|
+
payload_name="Eigenvalue payload",
|
|
316
|
+
)
|
|
317
|
+
tables.append(table)
|
|
318
|
+
|
|
319
|
+
if not any(table is not None for table in tables):
|
|
320
|
+
raise InvalidConfigurationError(
|
|
321
|
+
"No successful batch entries with eigenvalue payloads are available for to_xarray()."
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
dataset: XarrayDataset = _stack_batch_tabular_tables(
|
|
325
|
+
records,
|
|
326
|
+
tables,
|
|
327
|
+
xarray_module=xarray_module,
|
|
328
|
+
batch_axis_name="batch_index",
|
|
329
|
+
)
|
|
330
|
+
return _create_data_tree(datatree_module=datatree_module, dataset=dataset)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _postprocess_mapping_to_data_tree(
|
|
334
|
+
payload: Mapping[object, object],
|
|
335
|
+
*,
|
|
336
|
+
postprocess_title: str | None,
|
|
337
|
+
source_file: object,
|
|
338
|
+
xarray_module: _XarrayModule,
|
|
339
|
+
datatree_module: _DataTreeModule,
|
|
340
|
+
node_name: str | None = None,
|
|
341
|
+
) -> XarrayDataTree:
|
|
342
|
+
"""Convert one structured mapping payload into a `DataTree`."""
|
|
343
|
+
|
|
344
|
+
payload_parts: _StructuredPayloadParts = _split_structured_payload(payload)
|
|
345
|
+
root_dataset: XarrayDataset = _current_postprocess_level_to_dataset(
|
|
346
|
+
payload_parts,
|
|
347
|
+
postprocess_title=postprocess_title,
|
|
348
|
+
source_file=source_file,
|
|
349
|
+
xarray_module=xarray_module,
|
|
350
|
+
)
|
|
351
|
+
children: dict[str, XarrayDataTree] = {}
|
|
352
|
+
child_name: str
|
|
353
|
+
child_payload: Mapping[object, object]
|
|
354
|
+
for child_name, child_payload in payload_parts.nested_fields.items():
|
|
355
|
+
children[child_name] = _postprocess_mapping_to_data_tree(
|
|
356
|
+
child_payload,
|
|
357
|
+
postprocess_title=postprocess_title,
|
|
358
|
+
source_file=source_file,
|
|
359
|
+
xarray_module=xarray_module,
|
|
360
|
+
datatree_module=datatree_module,
|
|
361
|
+
node_name=child_name,
|
|
362
|
+
)
|
|
363
|
+
return _create_data_tree(
|
|
364
|
+
datatree_module=datatree_module,
|
|
365
|
+
dataset=root_dataset,
|
|
366
|
+
children=children,
|
|
367
|
+
name=node_name,
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _stack_batch_header_payloads(
|
|
372
|
+
entries: Sequence[_BatchIndexedKeys],
|
|
373
|
+
payloads: Sequence[Mapping[object, object] | None],
|
|
374
|
+
*,
|
|
375
|
+
xarray_module: _XarrayModule,
|
|
376
|
+
) -> XarrayDataset:
|
|
377
|
+
"""Stack selected header payloads across a batch `index` axis."""
|
|
378
|
+
|
|
379
|
+
flattened_payloads: list[dict[str, object]] = []
|
|
380
|
+
payload: Mapping[object, object] | None
|
|
381
|
+
for payload in payloads:
|
|
382
|
+
flattened_fields: dict[str, object] = {}
|
|
383
|
+
if payload is not None:
|
|
384
|
+
_flatten_header_fields(
|
|
385
|
+
payload,
|
|
386
|
+
flattened_fields=flattened_fields,
|
|
387
|
+
field_prefix="",
|
|
388
|
+
)
|
|
389
|
+
flattened_payloads.append(flattened_fields)
|
|
390
|
+
|
|
391
|
+
data_vars: dict[str, object] = {}
|
|
392
|
+
coords: dict[str, object] = {"index": [entry.index for entry in entries]}
|
|
393
|
+
attrs: dict[str, object] = {}
|
|
394
|
+
if entries:
|
|
395
|
+
_append_batch_key_coords(
|
|
396
|
+
coords=coords,
|
|
397
|
+
attrs=attrs,
|
|
398
|
+
keys_by_entry=[entry.keys for entry in entries],
|
|
399
|
+
key_names=sorted(entries[0].keys),
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
all_field_names: set[str] = set()
|
|
403
|
+
flattened_payload: dict[str, object]
|
|
404
|
+
for flattened_payload in flattened_payloads:
|
|
405
|
+
all_field_names.update(flattened_payload)
|
|
406
|
+
|
|
407
|
+
field_name: str
|
|
408
|
+
for field_name in sorted(all_field_names):
|
|
409
|
+
values: list[object | None] = [
|
|
410
|
+
flattened_fields.get(field_name) for flattened_fields in flattened_payloads
|
|
411
|
+
]
|
|
412
|
+
present_values: list[object] = [value for value in values if value is not None]
|
|
413
|
+
if not present_values:
|
|
414
|
+
continue
|
|
415
|
+
if all(isinstance(value, str) for value in present_values):
|
|
416
|
+
first_value: str = cast(str, present_values[0])
|
|
417
|
+
if all(cast(str, value) == first_value for value in present_values[1:]):
|
|
418
|
+
attrs[field_name] = first_value
|
|
419
|
+
continue
|
|
420
|
+
_append_stacked_header_data_variable(
|
|
421
|
+
data_vars=data_vars,
|
|
422
|
+
coords=coords,
|
|
423
|
+
field_name=field_name,
|
|
424
|
+
values=values,
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
return xarray_module.Dataset(
|
|
428
|
+
data_vars=data_vars,
|
|
429
|
+
coords=coords or None,
|
|
430
|
+
attrs=attrs or None,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _stack_batch_postprocess_payloads_to_data_tree(
|
|
435
|
+
entries: Sequence[_BatchIndexedKeys],
|
|
436
|
+
payloads: Sequence[Mapping[object, object]],
|
|
437
|
+
*,
|
|
438
|
+
index: int | None,
|
|
439
|
+
title: str | None,
|
|
440
|
+
xarray_module: _XarrayModule,
|
|
441
|
+
datatree_module: _DataTreeModule,
|
|
442
|
+
node_name: str | None = None,
|
|
443
|
+
batch_axis_name: str = "index",
|
|
444
|
+
available: Sequence[bool] | None = None,
|
|
445
|
+
) -> XarrayDataTree:
|
|
446
|
+
"""Convert structured batch-selected payloads into a stacked `DataTree`."""
|
|
447
|
+
|
|
448
|
+
payload_parts_list: list[_StructuredPayloadParts] = [
|
|
449
|
+
_split_structured_payload(payload) for payload in payloads
|
|
450
|
+
]
|
|
451
|
+
root_dataset: XarrayDataset = _stack_batch_postprocess_level_to_dataset(
|
|
452
|
+
entries,
|
|
453
|
+
payload_parts_list,
|
|
454
|
+
index=index,
|
|
455
|
+
title=title,
|
|
456
|
+
xarray_module=xarray_module,
|
|
457
|
+
batch_axis_name=batch_axis_name,
|
|
458
|
+
available=available,
|
|
459
|
+
)
|
|
460
|
+
common_child_names: tuple[str, ...] = _common_ordered_child_names(
|
|
461
|
+
payload_parts_list
|
|
462
|
+
)
|
|
463
|
+
children: dict[str, XarrayDataTree] = {}
|
|
464
|
+
child_name: str
|
|
465
|
+
for child_name in common_child_names:
|
|
466
|
+
child_payloads: list[Mapping[object, object]] = [
|
|
467
|
+
payload_parts.nested_fields[child_name]
|
|
468
|
+
for payload_parts in payload_parts_list
|
|
469
|
+
]
|
|
470
|
+
children[child_name] = _stack_batch_postprocess_payloads_to_data_tree(
|
|
471
|
+
entries,
|
|
472
|
+
child_payloads,
|
|
473
|
+
index=index,
|
|
474
|
+
title=title,
|
|
475
|
+
xarray_module=xarray_module,
|
|
476
|
+
datatree_module=datatree_module,
|
|
477
|
+
node_name=child_name,
|
|
478
|
+
batch_axis_name=batch_axis_name,
|
|
479
|
+
available=available,
|
|
480
|
+
)
|
|
481
|
+
return _create_data_tree(
|
|
482
|
+
datatree_module=datatree_module,
|
|
483
|
+
dataset=root_dataset,
|
|
484
|
+
children=children,
|
|
485
|
+
name=node_name,
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _current_postprocess_level_to_dataset(
|
|
490
|
+
payload_parts: _StructuredPayloadParts,
|
|
491
|
+
*,
|
|
492
|
+
postprocess_title: str | None,
|
|
493
|
+
source_file: object,
|
|
494
|
+
xarray_module: _XarrayModule,
|
|
495
|
+
) -> XarrayDataset:
|
|
496
|
+
"""Build one dataset for the current postprocess tree node."""
|
|
497
|
+
|
|
498
|
+
if payload_parts.grouped_fields:
|
|
499
|
+
tensor: _GroupedPostprocessTensor = postprocess_payload_to_grouped_tensor(
|
|
500
|
+
payload_parts,
|
|
501
|
+
postprocess_title=postprocess_title,
|
|
502
|
+
source_file=source_file,
|
|
503
|
+
)
|
|
504
|
+
return grouped_tensor_to_xarray_dataset(tensor, xarray_module=xarray_module)
|
|
505
|
+
|
|
506
|
+
if payload_parts.companion_fields:
|
|
507
|
+
table: _TabularXarrayTable = tabular_payload_to_xarray_table(
|
|
508
|
+
payload_parts.companion_fields,
|
|
509
|
+
payload_name="Postprocess payload",
|
|
510
|
+
)
|
|
511
|
+
table_attrs: dict[str, object] = dict(table.attrs)
|
|
512
|
+
table_attrs["postprocess_title"] = postprocess_title
|
|
513
|
+
table_attrs["source_file"] = None if source_file is None else str(source_file)
|
|
514
|
+
return tabular_table_to_xarray_dataset(
|
|
515
|
+
_TabularXarrayTable(
|
|
516
|
+
row_count=table.row_count,
|
|
517
|
+
column_names=table.column_names,
|
|
518
|
+
column_arrays=table.column_arrays,
|
|
519
|
+
column_dims=table.column_dims,
|
|
520
|
+
attrs=table_attrs,
|
|
521
|
+
),
|
|
522
|
+
xarray_module=xarray_module,
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
return xarray_module.Dataset(
|
|
526
|
+
data_vars={},
|
|
527
|
+
coords={},
|
|
528
|
+
attrs={
|
|
529
|
+
"postprocess_title": postprocess_title,
|
|
530
|
+
"source_file": None if source_file is None else str(source_file),
|
|
531
|
+
},
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _stack_batch_postprocess_level_to_dataset(
|
|
536
|
+
entries: Sequence[_BatchIndexedKeys],
|
|
537
|
+
payload_parts_list: Sequence[_StructuredPayloadParts],
|
|
538
|
+
*,
|
|
539
|
+
index: int | None,
|
|
540
|
+
title: str | None,
|
|
541
|
+
xarray_module: _XarrayModule,
|
|
542
|
+
batch_axis_name: str = "index",
|
|
543
|
+
available: Sequence[bool] | None = None,
|
|
544
|
+
) -> XarrayDataset:
|
|
545
|
+
"""Build one stacked dataset for the current batch postprocess tree node."""
|
|
546
|
+
|
|
547
|
+
first_parts: _StructuredPayloadParts = payload_parts_list[0]
|
|
548
|
+
inferred_postprocess_title: str | None = title
|
|
549
|
+
if inferred_postprocess_title is None:
|
|
550
|
+
raw_title: object | None = first_parts.companion_fields.get("title")
|
|
551
|
+
if isinstance(raw_title, str):
|
|
552
|
+
inferred_postprocess_title = raw_title
|
|
553
|
+
if any(
|
|
554
|
+
tuple(parts.grouped_fields.keys()) != tuple(first_parts.grouped_fields.keys())
|
|
555
|
+
or tuple(parts.companion_fields.keys())
|
|
556
|
+
!= tuple(first_parts.companion_fields.keys())
|
|
557
|
+
for parts in payload_parts_list[1:]
|
|
558
|
+
):
|
|
559
|
+
raise InvalidConfigurationError(
|
|
560
|
+
"Selected batch postprocess stream mixes incompatible structured payload node shapes and cannot be stacked via to_xarray()."
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
if first_parts.grouped_fields:
|
|
564
|
+
tensors: list[_GroupedPostprocessTensor] = [
|
|
565
|
+
postprocess_payload_to_grouped_tensor(
|
|
566
|
+
parts,
|
|
567
|
+
postprocess_title=inferred_postprocess_title,
|
|
568
|
+
source_file=None,
|
|
569
|
+
)
|
|
570
|
+
for parts in payload_parts_list
|
|
571
|
+
]
|
|
572
|
+
return _stack_batch_grouped_tensors(
|
|
573
|
+
entries,
|
|
574
|
+
tensors,
|
|
575
|
+
index=index,
|
|
576
|
+
title=title,
|
|
577
|
+
xarray_module=xarray_module,
|
|
578
|
+
batch_axis_name=batch_axis_name,
|
|
579
|
+
available=available,
|
|
580
|
+
)
|
|
581
|
+
|
|
582
|
+
if first_parts.companion_fields:
|
|
583
|
+
tables: list[_TabularXarrayTable] = [
|
|
584
|
+
tabular_payload_to_xarray_table(
|
|
585
|
+
parts.companion_fields,
|
|
586
|
+
payload_name="Postprocess payload",
|
|
587
|
+
)
|
|
588
|
+
for parts in payload_parts_list
|
|
589
|
+
]
|
|
590
|
+
return _stack_batch_tabular_tables(
|
|
591
|
+
entries,
|
|
592
|
+
tables,
|
|
593
|
+
xarray_module=xarray_module,
|
|
594
|
+
batch_axis_name=batch_axis_name,
|
|
595
|
+
valid_entries=available,
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
coords: dict[str, object] = {batch_axis_name: [entry.index for entry in entries]}
|
|
599
|
+
attrs: dict[str, object] = {}
|
|
600
|
+
common_key_names: set[str] = set(entries[0].keys)
|
|
601
|
+
entry: _BatchIndexedKeys
|
|
602
|
+
for entry in entries[1:]:
|
|
603
|
+
common_key_names &= set(entry.keys)
|
|
604
|
+
_append_batch_key_coords(
|
|
605
|
+
coords=coords,
|
|
606
|
+
attrs=attrs,
|
|
607
|
+
keys_by_entry=[entry.keys for entry in entries],
|
|
608
|
+
key_names=sorted(common_key_names),
|
|
609
|
+
)
|
|
610
|
+
data_vars: dict[str, object] = {}
|
|
611
|
+
if available is not None:
|
|
612
|
+
data_vars["is_valid"] = (
|
|
613
|
+
(batch_axis_name,),
|
|
614
|
+
np.asarray(available, dtype=bool),
|
|
615
|
+
)
|
|
616
|
+
return xarray_module.Dataset(
|
|
617
|
+
data_vars=data_vars, coords=coords, attrs=attrs or None
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _append_batch_key_coords(
|
|
622
|
+
*,
|
|
623
|
+
coords: dict[str, object],
|
|
624
|
+
attrs: dict[str, object],
|
|
625
|
+
keys_by_entry: Sequence[Mapping[str, object]],
|
|
626
|
+
key_names: Sequence[str],
|
|
627
|
+
) -> None:
|
|
628
|
+
"""Append supported batch key coords and note keys that were skipped."""
|
|
629
|
+
|
|
630
|
+
dropped_key_names: list[str] = []
|
|
631
|
+
key_name: str
|
|
632
|
+
for key_name in key_names:
|
|
633
|
+
coord_variable: tuple[tuple[str, ...], np.ndarray] | None = (
|
|
634
|
+
_build_batch_coord_variable(
|
|
635
|
+
key_name,
|
|
636
|
+
[entry_keys.get(key_name) for entry_keys in keys_by_entry],
|
|
637
|
+
)
|
|
638
|
+
)
|
|
639
|
+
if coord_variable is None:
|
|
640
|
+
dropped_key_names.append(key_name)
|
|
641
|
+
continue
|
|
642
|
+
coord_dims: tuple[str, ...]
|
|
643
|
+
coord_array: np.ndarray
|
|
644
|
+
coord_dims, coord_array = coord_variable
|
|
645
|
+
coords[key_name] = (coord_dims, coord_array)
|
|
646
|
+
if len(coord_dims) > 1:
|
|
647
|
+
_ensure_generated_companion_dim_coords(
|
|
648
|
+
coords,
|
|
649
|
+
coord_dims=coord_dims,
|
|
650
|
+
coord_array=coord_array,
|
|
651
|
+
)
|
|
652
|
+
if dropped_key_names:
|
|
653
|
+
attrs["dropped_batch_key_coords"] = tuple(dropped_key_names)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _select_batch_postprocess_record(
|
|
657
|
+
entry: "BatchExportRecord",
|
|
658
|
+
*,
|
|
659
|
+
index: int,
|
|
660
|
+
) -> "PostprocessRecord":
|
|
661
|
+
"""Select one postprocess record from one successful batch entry."""
|
|
662
|
+
|
|
663
|
+
if not isinstance(entry.payload, PostprocessCollection):
|
|
664
|
+
raise InvalidConfigurationError(
|
|
665
|
+
f"Batch entry {entry.index} has no solve result for xarray export."
|
|
666
|
+
)
|
|
667
|
+
if index < 0 or index >= len(entry.payload):
|
|
668
|
+
raise InvalidConfigurationError(
|
|
669
|
+
f"Selected postprocess index {index!r} is missing for batch entry {entry.index}."
|
|
670
|
+
)
|
|
671
|
+
record: PostprocessRecord | None = entry.payload[index]
|
|
672
|
+
if record is None:
|
|
673
|
+
raise InvalidConfigurationError(
|
|
674
|
+
f"Selected postprocess index {index!r} is unavailable for batch entry {entry.index}."
|
|
675
|
+
)
|
|
676
|
+
return record
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def _record_xarray_payload(record: "PostprocessRecord") -> object:
|
|
680
|
+
"""Return the exact payload object used for record-level xarray export."""
|
|
681
|
+
|
|
682
|
+
try:
|
|
683
|
+
payload_object: object = record.to_dict()
|
|
684
|
+
except UnsupportedExportError:
|
|
685
|
+
payload_object = record._raw
|
|
686
|
+
return payload_object
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def stack_batch_field_trees(
|
|
690
|
+
trees: list[XarrayDataTree],
|
|
691
|
+
*,
|
|
692
|
+
available: list[bool],
|
|
693
|
+
batch_indices: list[int],
|
|
694
|
+
xarray_module: _XarrayModule,
|
|
695
|
+
datatree_module: _DataTreeModule,
|
|
696
|
+
) -> XarrayDataTree:
|
|
697
|
+
"""Stack compatible field tree datasets with a leading batch dimension."""
|
|
698
|
+
|
|
699
|
+
def dimensions_and_values(value: object) -> tuple[tuple[str, ...], np.ndarray]:
|
|
700
|
+
"""Normalize real xarray DataArrays and the lightweight test adapter."""
|
|
701
|
+
|
|
702
|
+
if isinstance(value, tuple) and len(value) == 2:
|
|
703
|
+
dimensions, values = value
|
|
704
|
+
if isinstance(dimensions, tuple):
|
|
705
|
+
return tuple(str(dimension) for dimension in dimensions), np.asarray(
|
|
706
|
+
values
|
|
707
|
+
)
|
|
708
|
+
if isinstance(dimensions, str):
|
|
709
|
+
return (dimensions,), np.asarray(values)
|
|
710
|
+
dimensions = getattr(value, "dims", None)
|
|
711
|
+
values = getattr(value, "values", None)
|
|
712
|
+
if dimensions is None or values is None:
|
|
713
|
+
raise InvalidConfigurationError(
|
|
714
|
+
"Field tree contains an invalid array node."
|
|
715
|
+
)
|
|
716
|
+
return tuple(str(dimension) for dimension in dimensions), np.asarray(values)
|
|
717
|
+
|
|
718
|
+
first = trees[0]
|
|
719
|
+
first_dataset = getattr(first, "ds", None)
|
|
720
|
+
if first_dataset is None:
|
|
721
|
+
raise InvalidConfigurationError("Field tree has no root dataset.")
|
|
722
|
+
data_vars: dict[str, object] = {}
|
|
723
|
+
coords: dict[str, object] = {
|
|
724
|
+
"batch_index": batch_indices,
|
|
725
|
+
}
|
|
726
|
+
for name, value in first_dataset.data_vars.items():
|
|
727
|
+
dims, _first_array = dimensions_and_values(value)
|
|
728
|
+
arrays = [
|
|
729
|
+
dimensions_and_values(getattr(tree, "ds").data_vars[name])[1] # noqa: B009
|
|
730
|
+
for tree in trees
|
|
731
|
+
]
|
|
732
|
+
shape = tuple(
|
|
733
|
+
max(array.shape[axis] for array in arrays) for axis in range(arrays[0].ndim)
|
|
734
|
+
)
|
|
735
|
+
dtype = np.result_type(*[array.dtype for array in arrays], np.float64)
|
|
736
|
+
stacked = np.full((len(arrays), *shape), np.nan, dtype=dtype)
|
|
737
|
+
padded = np.zeros((len(arrays), *shape), dtype=bool)
|
|
738
|
+
for position, array in enumerate(arrays):
|
|
739
|
+
if not available[position]:
|
|
740
|
+
continue
|
|
741
|
+
target = stacked[position]
|
|
742
|
+
target[tuple(slice(0, size) for size in array.shape)] = array
|
|
743
|
+
padding = padded[position]
|
|
744
|
+
padding[...] = True
|
|
745
|
+
padding[tuple(slice(0, size) for size in array.shape)] = False
|
|
746
|
+
data_vars[name] = (("batch_index", *dims), stacked)
|
|
747
|
+
if name == "field":
|
|
748
|
+
data_vars["is_padded"] = (("batch_index", *dims), padded)
|
|
749
|
+
for name, value in first_dataset.coords.items():
|
|
750
|
+
if isinstance(value, (list, np.ndarray)):
|
|
751
|
+
dims, array = (name,), np.asarray(value)
|
|
752
|
+
else:
|
|
753
|
+
dims, array = dimensions_and_values(value)
|
|
754
|
+
coordinate_arrays = [
|
|
755
|
+
(
|
|
756
|
+
np.asarray(getattr(tree, "ds").coords[name]) # noqa: B009
|
|
757
|
+
if isinstance(
|
|
758
|
+
getattr(tree, "ds").coords[name],
|
|
759
|
+
(list, np.ndarray), # noqa: B009
|
|
760
|
+
)
|
|
761
|
+
else dimensions_and_values(
|
|
762
|
+
getattr(tree, "ds").coords[name] # noqa: B009
|
|
763
|
+
)[1]
|
|
764
|
+
)
|
|
765
|
+
for tree in trees
|
|
766
|
+
]
|
|
767
|
+
coordinate_dtype = np.result_type(
|
|
768
|
+
*[coordinate.dtype for coordinate in coordinate_arrays]
|
|
769
|
+
)
|
|
770
|
+
if not np.issubdtype(coordinate_dtype, np.number):
|
|
771
|
+
coords[name] = (dims, array)
|
|
772
|
+
continue
|
|
773
|
+
shape = tuple(
|
|
774
|
+
max(coordinate.shape[axis] for coordinate in coordinate_arrays)
|
|
775
|
+
for axis in range(coordinate_arrays[0].ndim)
|
|
776
|
+
)
|
|
777
|
+
stacked_coordinates = np.full(
|
|
778
|
+
(len(coordinate_arrays), *shape),
|
|
779
|
+
np.nan,
|
|
780
|
+
dtype=np.result_type(coordinate_dtype, np.float64),
|
|
781
|
+
)
|
|
782
|
+
for position, coordinate in enumerate(coordinate_arrays):
|
|
783
|
+
if not available[position]:
|
|
784
|
+
continue
|
|
785
|
+
stacked_coordinates[position][
|
|
786
|
+
tuple(slice(0, size) for size in coordinate.shape)
|
|
787
|
+
] = coordinate
|
|
788
|
+
coords[name] = (("batch_index", *dims), stacked_coordinates)
|
|
789
|
+
valid = np.asarray(available, dtype=bool)
|
|
790
|
+
data_vars["is_valid"] = (("batch_index",), valid)
|
|
791
|
+
root = xarray_module.Dataset(
|
|
792
|
+
data_vars=data_vars, coords=coords, attrs=dict(first_dataset.attrs)
|
|
793
|
+
)
|
|
794
|
+
child_names = tuple(getattr(first, "children", {}))
|
|
795
|
+
children = {
|
|
796
|
+
name: stack_batch_field_trees(
|
|
797
|
+
[getattr(tree, "children")[name] for tree in trees], # noqa: B009
|
|
798
|
+
available=available,
|
|
799
|
+
batch_indices=batch_indices,
|
|
800
|
+
xarray_module=xarray_module,
|
|
801
|
+
datatree_module=datatree_module,
|
|
802
|
+
)
|
|
803
|
+
for name in child_names
|
|
804
|
+
}
|
|
805
|
+
return _create_data_tree(
|
|
806
|
+
datatree_module=datatree_module,
|
|
807
|
+
dataset=root,
|
|
808
|
+
children=children,
|
|
809
|
+
)
|