jcmsuite 7.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of jcmsuite might be problematic. Click here for more details.
- jcmsuite/__init__.py +22 -0
- jcmsuite/_build_info.py +4 -0
- jcmsuite/context/__init__.py +44 -0
- jcmsuite/context/interface_metadata.py +158 -0
- jcmsuite/context/models.py +213 -0
- jcmsuite/context/scope.py +210 -0
- jcmsuite/context/solver_context.py +569 -0
- jcmsuite/context/subprocess_tools.py +172 -0
- jcmsuite/contract.py +47 -0
- jcmsuite/daemon/__init__.py +19 -0
- jcmsuite/daemon/handler.py +925 -0
- jcmsuite/daemon/internal/__init__.py +4 -0
- jcmsuite/daemon/internal/bundle_protocol.py +64 -0
- jcmsuite/daemon/internal/handler_helpers.py +368 -0
- jcmsuite/daemon/internal/handler_resource_jobs.py +227 -0
- jcmsuite/daemon/internal/handler_wait.py +564 -0
- jcmsuite/daemon/models.py +349 -0
- jcmsuite/daemon/namespaces.py +250 -0
- jcmsuite/daemon/protocol.py +801 -0
- jcmsuite/daemon/resources.py +407 -0
- jcmsuite/daemon/scope.py +53 -0
- jcmsuite/daemon/transport.py +223 -0
- jcmsuite/editing/__init__.py +10 -0
- jcmsuite/editing/editor.py +274 -0
- jcmsuite/exceptions/__init__.py +32 -0
- jcmsuite/exceptions/errors.py +173 -0
- jcmsuite/geo/__init__.py +10 -0
- jcmsuite/geo/mesh.py +302 -0
- jcmsuite/info/__init__.py +10 -0
- jcmsuite/info/runtime_info.py +45 -0
- jcmsuite/internal/__init__.py +2 -0
- jcmsuite/internal/diagnostics.py +74 -0
- jcmsuite/internal/progress.py +80 -0
- jcmsuite/loaders/__init__.py +18 -0
- jcmsuite/loaders/blob_header.py +315 -0
- jcmsuite/loaders/cartesian_field_reader.py +249 -0
- jcmsuite/loaders/cartesian_fields.py +324 -0
- jcmsuite/loaders/nested_mapping.py +69 -0
- jcmsuite/loaders/powerflux.py +136 -0
- jcmsuite/loaders/table.py +637 -0
- jcmsuite/logging/__init__.py +22 -0
- jcmsuite/logging/runtime.py +210 -0
- jcmsuite/results/__init__.py +126 -0
- jcmsuite/results/artifacts.py +806 -0
- jcmsuite/results/discovery.py +82 -0
- jcmsuite/results/field_exports.py +1062 -0
- jcmsuite/results/field_payloads.py +453 -0
- jcmsuite/results/internal/__init__.py +4 -0
- jcmsuite/results/internal/export/__init__.py +4 -0
- jcmsuite/results/internal/export/common.py +56 -0
- jcmsuite/results/internal/export/pandas/__init__.py +4 -0
- jcmsuite/results/internal/export/pandas/common.py +165 -0
- jcmsuite/results/internal/export/pandas/payload.py +480 -0
- jcmsuite/results/internal/export/pandas/rows.py +484 -0
- jcmsuite/results/internal/export/preview.py +178 -0
- jcmsuite/results/internal/export/xarray/__init__.py +4 -0
- jcmsuite/results/internal/export/xarray/export.py +688 -0
- jcmsuite/results/internal/export/xarray/grouped.py +608 -0
- jcmsuite/results/internal/export/xarray/headers.py +290 -0
- jcmsuite/results/internal/export/xarray/runtime.py +99 -0
- jcmsuite/results/internal/export/xarray/shared.py +246 -0
- jcmsuite/results/internal/export/xarray/tabular.py +304 -0
- jcmsuite/results/internal/export/xarray/types.py +28 -0
- jcmsuite/results/internal/models/__init__.py +4 -0
- jcmsuite/results/internal/models/batch.py +1081 -0
- jcmsuite/results/internal/models/batch_postprocess.py +1728 -0
- jcmsuite/results/internal/models/bundle.py +343 -0
- jcmsuite/results/internal/models/core.py +556 -0
- jcmsuite/results/internal/models/submission.py +1157 -0
- jcmsuite/results/internal/resultbag/__init__.py +4 -0
- jcmsuite/results/internal/resultbag/access.py +489 -0
- jcmsuite/results/internal/resultbag/iteration.py +199 -0
- jcmsuite/results/internal/resultbag/query.py +226 -0
- jcmsuite/results/internal/resultbag/raw.py +407 -0
- jcmsuite/results/internal/resultbag/source_files.py +142 -0
- jcmsuite/results/internal/resultbag/store.py +881 -0
- jcmsuite/results/internal/resultbag/utils.py +258 -0
- jcmsuite/results/models.py +76 -0
- jcmsuite/results/opening.py +55 -0
- jcmsuite/results/pandas_export.py +325 -0
- jcmsuite/results/payloads.py +478 -0
- jcmsuite/results/postprocess.py +97 -0
- jcmsuite/results/postprocess_payloads.py +757 -0
- jcmsuite/results/postprocess_record.py +418 -0
- jcmsuite/results/resultbag.py +279 -0
- jcmsuite/results/resultbag_management.py +433 -0
- jcmsuite/results/structured_derivatives.py +387 -0
- jcmsuite/solve/__init__.py +10 -0
- jcmsuite/solve/daemon_bundle_staging.py +100 -0
- jcmsuite/solve/daemon_bundle_wait.py +89 -0
- jcmsuite/solve/orchestration.py +1576 -0
- jcmsuite/solve/orchestration_execution.py +180 -0
- jcmsuite/solve/orchestration_routing.py +128 -0
- jcmsuite/solve/orchestration_support.py +323 -0
- jcmsuite/solve/path_resolution.py +98 -0
- jcmsuite/solve/progress.py +20 -0
- jcmsuite/solve/solver.py +709 -0
- jcmsuite/solve/staging.py +264 -0
- jcmsuite/solve/types.py +25 -0
- jcmsuite/solve/working_dirs.py +227 -0
- jcmsuite/templates/__init__.py +11 -0
- jcmsuite/templates/converter.py +71 -0
- jcmsuite/templates/formatting.py +335 -0
- jcmsuite/templates/formatting_cli.py +47 -0
- jcmsuite/templates/rendering.py +666 -0
- jcmsuite/torch/__init__.py +25 -0
- jcmsuite/torch/autograd.py +488 -0
- jcmsuite/torch/coordinate_policy.py +82 -0
- jcmsuite/torch/core.py +312 -0
- jcmsuite/torch/results/__init__.py +51 -0
- jcmsuite/torch/types.py +84 -0
- jcmsuite/torch/validation.py +208 -0
- jcmsuite/torch/wrappers.py +4332 -0
- jcmsuite/view/__init__.py +10 -0
- jcmsuite/view/viewer.py +122 -0
- jcmsuite-7.4.2.dist-info/METADATA +180 -0
- jcmsuite-7.4.2.dist-info/RECORD +119 -0
- jcmsuite-7.4.2.dist-info/WHEEL +4 -0
- jcmsuite-7.4.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# Copyright (c) 2001-2026 JCMwave GmbH
|
|
3
|
+
|
|
4
|
+
"""Grouped postprocess helpers for internal `xarray` exports."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
from collections.abc import Mapping, Sequence
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import cast
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from .....exceptions import InvalidConfigurationError
|
|
16
|
+
from ..pandas.payload import is_numeric_key_mapping
|
|
17
|
+
from .runtime import _XarrayModule
|
|
18
|
+
from .shared import (
|
|
19
|
+
_array_dtype_for_padding,
|
|
20
|
+
_ensure_generated_companion_dim_coords,
|
|
21
|
+
_ensure_generated_value_dim_coords,
|
|
22
|
+
_field_key_sort_key,
|
|
23
|
+
_normalize_scalar_coord_value,
|
|
24
|
+
_ordered_union,
|
|
25
|
+
_stack_sample_coordinate_arrays,
|
|
26
|
+
)
|
|
27
|
+
from .tabular import _BatchIndexedKeys
|
|
28
|
+
from .types import XarrayDataset
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(f"{__package__}.export")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(slots=True, frozen=True)
|
|
34
|
+
class _GroupedPostprocessTensor:
|
|
35
|
+
"""Normalized grouped postprocess tensor ready for xarray export."""
|
|
36
|
+
|
|
37
|
+
primary_field_name: str
|
|
38
|
+
postprocess_title: str | None
|
|
39
|
+
source_file: str | None
|
|
40
|
+
field_index: tuple[object, ...]
|
|
41
|
+
sample_index: tuple[int, ...]
|
|
42
|
+
grouped_values: dict[str, np.ndarray]
|
|
43
|
+
grouped_dims: dict[str, tuple[str, ...]]
|
|
44
|
+
is_valid: np.ndarray
|
|
45
|
+
is_padded: np.ndarray
|
|
46
|
+
sample_coords: dict[str, tuple[tuple[str, ...], np.ndarray]]
|
|
47
|
+
attrs: dict[str, object]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(slots=True, frozen=True)
|
|
51
|
+
class _StructuredPayloadParts:
|
|
52
|
+
"""Partitioned mapping payload for one tree node."""
|
|
53
|
+
|
|
54
|
+
grouped_fields: dict[str, Mapping[object, object]]
|
|
55
|
+
companion_fields: dict[str, object]
|
|
56
|
+
nested_fields: dict[str, Mapping[object, object]]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def postprocess_payload_to_grouped_tensor(
|
|
60
|
+
payload: object,
|
|
61
|
+
*,
|
|
62
|
+
postprocess_title: str | None,
|
|
63
|
+
source_file: object,
|
|
64
|
+
) -> _GroupedPostprocessTensor:
|
|
65
|
+
"""Normalize one supported grouped postprocess payload into arrays and labels."""
|
|
66
|
+
|
|
67
|
+
payload_parts: _StructuredPayloadParts
|
|
68
|
+
if isinstance(payload, _StructuredPayloadParts):
|
|
69
|
+
payload_parts = payload
|
|
70
|
+
elif isinstance(payload, Mapping):
|
|
71
|
+
payload_parts = _split_structured_payload(payload)
|
|
72
|
+
else:
|
|
73
|
+
raise InvalidConfigurationError(
|
|
74
|
+
"Postprocess payload is not mapping-like and cannot be exported via to_xarray()."
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
grouped_fields: dict[str, Mapping[object, object]] = payload_parts.grouped_fields
|
|
78
|
+
companion_fields: dict[str, object] = payload_parts.companion_fields
|
|
79
|
+
|
|
80
|
+
if not grouped_fields:
|
|
81
|
+
raise InvalidConfigurationError(
|
|
82
|
+
"Postprocess payload does not contain a grouped numeric-key field supported by to_xarray()."
|
|
83
|
+
)
|
|
84
|
+
grouped_field_names: tuple[str, ...] = tuple(grouped_fields.keys())
|
|
85
|
+
first_field_name: str = grouped_field_names[0]
|
|
86
|
+
first_payload: Mapping[object, object] = grouped_fields[first_field_name]
|
|
87
|
+
first_tensor: _GroupedPostprocessTensor = _build_grouped_field_tensor(
|
|
88
|
+
first_payload,
|
|
89
|
+
field_name=first_field_name,
|
|
90
|
+
postprocess_title=postprocess_title,
|
|
91
|
+
source_file=source_file,
|
|
92
|
+
companion_fields=companion_fields,
|
|
93
|
+
multi_field_payload=len(grouped_field_names) > 1,
|
|
94
|
+
)
|
|
95
|
+
grouped_values: dict[str, np.ndarray] = dict(first_tensor.grouped_values)
|
|
96
|
+
grouped_dims: dict[str, tuple[str, ...]] = dict(first_tensor.grouped_dims)
|
|
97
|
+
|
|
98
|
+
grouped_field_name: str
|
|
99
|
+
for grouped_field_name in grouped_field_names[1:]:
|
|
100
|
+
grouped_payload: Mapping[object, object] = grouped_fields[grouped_field_name]
|
|
101
|
+
grouped_tensor: _GroupedPostprocessTensor = _build_grouped_field_tensor(
|
|
102
|
+
grouped_payload,
|
|
103
|
+
field_name=grouped_field_name,
|
|
104
|
+
postprocess_title=postprocess_title,
|
|
105
|
+
source_file=source_file,
|
|
106
|
+
companion_fields=companion_fields,
|
|
107
|
+
multi_field_payload=True,
|
|
108
|
+
)
|
|
109
|
+
if grouped_tensor.field_index != first_tensor.field_index:
|
|
110
|
+
raise InvalidConfigurationError(
|
|
111
|
+
"Postprocess payload contains grouped fields with different field-index keys; to_xarray() requires aligned grouped fields."
|
|
112
|
+
)
|
|
113
|
+
if not np.array_equal(grouped_tensor.is_valid, first_tensor.is_valid):
|
|
114
|
+
raise InvalidConfigurationError(
|
|
115
|
+
"Postprocess payload contains grouped fields with different per-field sample counts; to_xarray() requires aligned grouped fields."
|
|
116
|
+
)
|
|
117
|
+
if not np.array_equal(grouped_tensor.is_padded, first_tensor.is_padded):
|
|
118
|
+
raise InvalidConfigurationError(
|
|
119
|
+
"Postprocess payload contains grouped fields with different padding structure; to_xarray() requires aligned grouped fields."
|
|
120
|
+
)
|
|
121
|
+
grouped_values.update(grouped_tensor.grouped_values)
|
|
122
|
+
grouped_dims.update(grouped_tensor.grouped_dims)
|
|
123
|
+
|
|
124
|
+
attrs: dict[str, object] = dict(first_tensor.attrs)
|
|
125
|
+
if len(grouped_field_names) == 1:
|
|
126
|
+
attrs["field_name"] = first_field_name
|
|
127
|
+
else:
|
|
128
|
+
attrs["field_names"] = grouped_field_names
|
|
129
|
+
|
|
130
|
+
return _GroupedPostprocessTensor(
|
|
131
|
+
primary_field_name=first_field_name,
|
|
132
|
+
postprocess_title=first_tensor.postprocess_title,
|
|
133
|
+
source_file=first_tensor.source_file,
|
|
134
|
+
field_index=first_tensor.field_index,
|
|
135
|
+
sample_index=first_tensor.sample_index,
|
|
136
|
+
grouped_values=grouped_values,
|
|
137
|
+
grouped_dims=grouped_dims,
|
|
138
|
+
is_valid=first_tensor.is_valid,
|
|
139
|
+
is_padded=first_tensor.is_padded,
|
|
140
|
+
sample_coords=first_tensor.sample_coords,
|
|
141
|
+
attrs=attrs,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def grouped_tensor_to_xarray_dataset(
|
|
146
|
+
tensor: _GroupedPostprocessTensor,
|
|
147
|
+
*,
|
|
148
|
+
xarray_module: _XarrayModule,
|
|
149
|
+
) -> XarrayDataset:
|
|
150
|
+
"""Convert one normalized grouped tensor into an `xarray.Dataset`."""
|
|
151
|
+
|
|
152
|
+
coords: dict[str, object] = {
|
|
153
|
+
"field_index": list(tensor.field_index),
|
|
154
|
+
"sample_index": list(tensor.sample_index),
|
|
155
|
+
}
|
|
156
|
+
data_vars: dict[str, object] = {}
|
|
157
|
+
|
|
158
|
+
field_name: str
|
|
159
|
+
for field_name, value_array in tensor.grouped_values.items():
|
|
160
|
+
value_dims: tuple[str, ...] = tensor.grouped_dims[field_name]
|
|
161
|
+
_ensure_generated_value_dim_coords(
|
|
162
|
+
coords,
|
|
163
|
+
value_dims=value_dims,
|
|
164
|
+
value_array=value_array,
|
|
165
|
+
)
|
|
166
|
+
data_vars[field_name] = (value_dims, value_array)
|
|
167
|
+
|
|
168
|
+
coord_name: str
|
|
169
|
+
coord_value: tuple[tuple[str, ...], np.ndarray]
|
|
170
|
+
for coord_name, coord_value in tensor.sample_coords.items():
|
|
171
|
+
coord_dims, coord_array = coord_value
|
|
172
|
+
_ensure_generated_companion_dim_coords(
|
|
173
|
+
coords,
|
|
174
|
+
coord_dims=coord_dims,
|
|
175
|
+
coord_array=coord_array,
|
|
176
|
+
)
|
|
177
|
+
coords[coord_name] = (coord_dims, coord_array)
|
|
178
|
+
|
|
179
|
+
attrs: dict[str, object] = dict(tensor.attrs)
|
|
180
|
+
attrs["postprocess_title"] = tensor.postprocess_title
|
|
181
|
+
attrs["source_file"] = tensor.source_file
|
|
182
|
+
|
|
183
|
+
data_vars["is_valid"] = (("field_index", "sample_index"), tensor.is_valid)
|
|
184
|
+
data_vars["is_padded"] = (("field_index", "sample_index"), tensor.is_padded)
|
|
185
|
+
return xarray_module.Dataset(
|
|
186
|
+
data_vars=data_vars,
|
|
187
|
+
coords=coords,
|
|
188
|
+
attrs=attrs,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _build_grouped_field_tensor(
|
|
193
|
+
grouped_payload: Mapping[object, object],
|
|
194
|
+
*,
|
|
195
|
+
field_name: str,
|
|
196
|
+
postprocess_title: str | None,
|
|
197
|
+
source_file: object,
|
|
198
|
+
companion_fields: Mapping[str, object],
|
|
199
|
+
multi_field_payload: bool,
|
|
200
|
+
) -> _GroupedPostprocessTensor:
|
|
201
|
+
"""Build one normalized tensor for a grouped numeric-key payload."""
|
|
202
|
+
|
|
203
|
+
field_keys: list[object] = sorted(grouped_payload.keys(), key=_field_key_sort_key)
|
|
204
|
+
arrays: list[np.ndarray] = []
|
|
205
|
+
field_key: object
|
|
206
|
+
for field_key in field_keys:
|
|
207
|
+
value_object: object = grouped_payload[field_key]
|
|
208
|
+
if not isinstance(value_object, np.ndarray):
|
|
209
|
+
raise InvalidConfigurationError(
|
|
210
|
+
"Grouped postprocess values must be numpy arrays for to_xarray()."
|
|
211
|
+
)
|
|
212
|
+
if value_object.ndim == 0:
|
|
213
|
+
raise InvalidConfigurationError(
|
|
214
|
+
"Grouped scalar postprocess values are not supported by to_xarray() in this slice."
|
|
215
|
+
)
|
|
216
|
+
arrays.append(value_object)
|
|
217
|
+
|
|
218
|
+
first_array: np.ndarray = arrays[0]
|
|
219
|
+
trailing_shape: tuple[int, ...] = tuple(int(axis) for axis in first_array.shape[1:])
|
|
220
|
+
value_ndim: int = int(first_array.ndim)
|
|
221
|
+
if value_ndim < 1:
|
|
222
|
+
raise InvalidConfigurationError(
|
|
223
|
+
"Grouped postprocess values must have at least one sample dimension for to_xarray()."
|
|
224
|
+
)
|
|
225
|
+
array_value: np.ndarray
|
|
226
|
+
for array_value in arrays[1:]:
|
|
227
|
+
if array_value.ndim != value_ndim:
|
|
228
|
+
raise InvalidConfigurationError(
|
|
229
|
+
"Grouped postprocess arrays must share the same dimensionality for to_xarray()."
|
|
230
|
+
)
|
|
231
|
+
if tuple(int(axis) for axis in array_value.shape[1:]) != trailing_shape:
|
|
232
|
+
raise InvalidConfigurationError(
|
|
233
|
+
"Grouped postprocess arrays must share trailing dimensions for to_xarray()."
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
sample_lengths: list[int] = [int(array.shape[0]) for array in arrays]
|
|
237
|
+
max_samples: int = max(sample_lengths)
|
|
238
|
+
requires_padding: bool = any(length != max_samples for length in sample_lengths)
|
|
239
|
+
if requires_padding:
|
|
240
|
+
logger.debug(
|
|
241
|
+
"Padding grouped postprocess %r to sample length %d from lengths %s.",
|
|
242
|
+
field_name,
|
|
243
|
+
max_samples,
|
|
244
|
+
sample_lengths,
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
value_dtype: np.dtype[np.generic] = _array_dtype_for_padding(
|
|
248
|
+
arrays,
|
|
249
|
+
requires_padding=requires_padding,
|
|
250
|
+
)
|
|
251
|
+
value_shape: tuple[int, ...] = (len(field_keys), max_samples) + trailing_shape
|
|
252
|
+
value_array: np.ndarray = np.full(value_shape, np.nan, dtype=value_dtype)
|
|
253
|
+
is_valid: np.ndarray = np.zeros((len(field_keys), max_samples), dtype=bool)
|
|
254
|
+
is_padded: np.ndarray = np.zeros((len(field_keys), max_samples), dtype=bool)
|
|
255
|
+
|
|
256
|
+
field_position: int
|
|
257
|
+
for field_position, array_value in enumerate(arrays):
|
|
258
|
+
sample_count: int = int(array_value.shape[0])
|
|
259
|
+
value_array[field_position, :sample_count, ...] = array_value.astype(
|
|
260
|
+
value_dtype,
|
|
261
|
+
copy=False,
|
|
262
|
+
)
|
|
263
|
+
is_valid[field_position, :sample_count] = True
|
|
264
|
+
if sample_count < max_samples:
|
|
265
|
+
is_padded[field_position, sample_count:] = True
|
|
266
|
+
|
|
267
|
+
value_dims: tuple[str, ...] = _grouped_value_dims(
|
|
268
|
+
field_name=field_name,
|
|
269
|
+
value_ndim=value_ndim,
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
sample_coords, attrs = _partition_companion_fields(
|
|
273
|
+
companion_fields,
|
|
274
|
+
sample_count=max_samples,
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
return _GroupedPostprocessTensor(
|
|
278
|
+
primary_field_name=field_name,
|
|
279
|
+
postprocess_title=postprocess_title,
|
|
280
|
+
source_file=None if source_file is None else str(source_file),
|
|
281
|
+
field_index=tuple(field_keys),
|
|
282
|
+
sample_index=tuple(range(max_samples)),
|
|
283
|
+
grouped_values={field_name: value_array},
|
|
284
|
+
grouped_dims={field_name: value_dims},
|
|
285
|
+
is_valid=is_valid,
|
|
286
|
+
is_padded=is_padded,
|
|
287
|
+
sample_coords=sample_coords,
|
|
288
|
+
attrs=attrs,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _grouped_value_dims(
|
|
293
|
+
*,
|
|
294
|
+
field_name: str,
|
|
295
|
+
value_ndim: int,
|
|
296
|
+
) -> tuple[str, ...]:
|
|
297
|
+
"""Return xarray dimension names for one grouped postprocess data variable."""
|
|
298
|
+
|
|
299
|
+
if value_ndim == 1:
|
|
300
|
+
return ("field_index", "sample_index")
|
|
301
|
+
if value_ndim == 2:
|
|
302
|
+
return ("field_index", "sample_index", "component")
|
|
303
|
+
trailing_dims: tuple[str, ...] = tuple(
|
|
304
|
+
f"{field_name}_dim_{dimension_index}"
|
|
305
|
+
for dimension_index in range(value_ndim - 1)
|
|
306
|
+
)
|
|
307
|
+
return ("field_index", "sample_index") + trailing_dims
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _stack_batch_grouped_tensors(
|
|
311
|
+
entries: Sequence[_BatchIndexedKeys],
|
|
312
|
+
tensors: Sequence[_GroupedPostprocessTensor],
|
|
313
|
+
*,
|
|
314
|
+
index: int | None,
|
|
315
|
+
title: str | None,
|
|
316
|
+
xarray_module: _XarrayModule,
|
|
317
|
+
batch_axis_name: str = "index",
|
|
318
|
+
available: Sequence[bool] | None = None,
|
|
319
|
+
) -> XarrayDataset:
|
|
320
|
+
"""Stack selected grouped tensors from batch entries onto one batch axis."""
|
|
321
|
+
|
|
322
|
+
first_tensor: _GroupedPostprocessTensor = tensors[0]
|
|
323
|
+
grouped_field_names: tuple[str, ...] = tuple(first_tensor.grouped_values.keys())
|
|
324
|
+
postprocess_title: str | None = first_tensor.postprocess_title
|
|
325
|
+
|
|
326
|
+
tensor: _GroupedPostprocessTensor
|
|
327
|
+
for tensor in tensors[1:]:
|
|
328
|
+
if tuple(tensor.grouped_values.keys()) != grouped_field_names:
|
|
329
|
+
raise InvalidConfigurationError(
|
|
330
|
+
"Selected batch postprocess stream mixes grouped payloads with different grouped field names and cannot be stacked via to_xarray()."
|
|
331
|
+
)
|
|
332
|
+
grouped_name: str
|
|
333
|
+
for grouped_name in grouped_field_names:
|
|
334
|
+
if (
|
|
335
|
+
tensor.grouped_dims[grouped_name]
|
|
336
|
+
!= first_tensor.grouped_dims[grouped_name]
|
|
337
|
+
):
|
|
338
|
+
raise InvalidConfigurationError(
|
|
339
|
+
"Selected batch postprocess stream mixes incompatible grouped field shapes and cannot be stacked via to_xarray()."
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
global_field_index: tuple[object, ...] = _ordered_union(
|
|
343
|
+
[tensor.field_index for tensor in tensors]
|
|
344
|
+
)
|
|
345
|
+
max_samples: int = max(len(tensor.sample_index) for tensor in tensors)
|
|
346
|
+
requires_batch_padding: bool = any(
|
|
347
|
+
len(tensor.field_index) != len(global_field_index)
|
|
348
|
+
or len(tensor.sample_index) != max_samples
|
|
349
|
+
for tensor in tensors
|
|
350
|
+
)
|
|
351
|
+
if requires_batch_padding:
|
|
352
|
+
logger.debug(
|
|
353
|
+
"Padding batch-selected postprocess %r to field/sample shape (%d, %d) from field counts %s and sample counts %s.",
|
|
354
|
+
title if title is not None else first_tensor.primary_field_name,
|
|
355
|
+
len(global_field_index),
|
|
356
|
+
max_samples,
|
|
357
|
+
[len(tensor.field_index) for tensor in tensors],
|
|
358
|
+
[len(tensor.sample_index) for tensor in tensors],
|
|
359
|
+
)
|
|
360
|
+
is_valid: np.ndarray = np.zeros(
|
|
361
|
+
(len(entries), len(global_field_index), max_samples),
|
|
362
|
+
dtype=bool,
|
|
363
|
+
)
|
|
364
|
+
is_padded: np.ndarray = np.ones(
|
|
365
|
+
(len(entries), len(global_field_index), max_samples),
|
|
366
|
+
dtype=bool,
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
field_position_lookup: dict[object, int] = {
|
|
370
|
+
field_value: position for position, field_value in enumerate(global_field_index)
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
stacked_values: dict[str, np.ndarray] = {}
|
|
374
|
+
field_name: str
|
|
375
|
+
for field_name in grouped_field_names:
|
|
376
|
+
value_arrays: list[np.ndarray] = [
|
|
377
|
+
tensor.grouped_values[field_name] for tensor in tensors
|
|
378
|
+
]
|
|
379
|
+
value_dtype: np.dtype[np.generic] = _array_dtype_for_padding(
|
|
380
|
+
value_arrays,
|
|
381
|
+
requires_padding=requires_batch_padding,
|
|
382
|
+
)
|
|
383
|
+
first_value_array: np.ndarray = first_tensor.grouped_values[field_name]
|
|
384
|
+
trailing_shape: tuple[int, ...] = tuple(
|
|
385
|
+
int(axis) for axis in first_value_array.shape[2:]
|
|
386
|
+
)
|
|
387
|
+
value_shape: tuple[int, ...] = (
|
|
388
|
+
len(entries),
|
|
389
|
+
len(global_field_index),
|
|
390
|
+
max_samples,
|
|
391
|
+
) + trailing_shape
|
|
392
|
+
stacked_values[field_name] = np.full(value_shape, np.nan, dtype=value_dtype)
|
|
393
|
+
|
|
394
|
+
entry_position: int
|
|
395
|
+
for entry_position, tensor in enumerate(tensors):
|
|
396
|
+
local_position: int
|
|
397
|
+
field_value: object
|
|
398
|
+
for local_position, field_value in enumerate(tensor.field_index):
|
|
399
|
+
global_position: int = field_position_lookup[field_value]
|
|
400
|
+
sample_count: int = len(tensor.sample_index)
|
|
401
|
+
grouped_data_name: str
|
|
402
|
+
for grouped_data_name in grouped_field_names:
|
|
403
|
+
source_array: np.ndarray = tensor.grouped_values[grouped_data_name]
|
|
404
|
+
target_array: np.ndarray = stacked_values[grouped_data_name]
|
|
405
|
+
target_array[
|
|
406
|
+
(entry_position, global_position, slice(0, sample_count))
|
|
407
|
+
] = source_array[(local_position, slice(0, sample_count))].astype(
|
|
408
|
+
target_array.dtype,
|
|
409
|
+
copy=False,
|
|
410
|
+
)
|
|
411
|
+
is_valid[entry_position, global_position, :sample_count] = tensor.is_valid[
|
|
412
|
+
local_position,
|
|
413
|
+
:sample_count,
|
|
414
|
+
]
|
|
415
|
+
is_padded[entry_position, global_position, :sample_count] = (
|
|
416
|
+
tensor.is_padded[
|
|
417
|
+
local_position,
|
|
418
|
+
:sample_count,
|
|
419
|
+
]
|
|
420
|
+
)
|
|
421
|
+
if available is not None and not available[entry_position]:
|
|
422
|
+
is_valid[entry_position, :, :] = False
|
|
423
|
+
is_padded[entry_position, :, :] = True
|
|
424
|
+
|
|
425
|
+
coords: dict[str, object] = {
|
|
426
|
+
batch_axis_name: [entry.index for entry in entries],
|
|
427
|
+
"field_index": list(global_field_index),
|
|
428
|
+
"sample_index": list(range(max_samples)),
|
|
429
|
+
"source_file": (
|
|
430
|
+
batch_axis_name,
|
|
431
|
+
np.asarray(
|
|
432
|
+
[
|
|
433
|
+
"" if tensor.source_file is None else tensor.source_file
|
|
434
|
+
for tensor in tensors
|
|
435
|
+
],
|
|
436
|
+
dtype=object,
|
|
437
|
+
),
|
|
438
|
+
),
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
for field_name in grouped_field_names:
|
|
442
|
+
_ensure_generated_value_dim_coords(
|
|
443
|
+
coords,
|
|
444
|
+
value_dims=first_tensor.grouped_dims[field_name],
|
|
445
|
+
value_array=first_tensor.grouped_values[field_name],
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
common_key_names: set[str] = set(entries[0].keys)
|
|
449
|
+
entry: _BatchIndexedKeys
|
|
450
|
+
for entry in entries[1:]:
|
|
451
|
+
common_key_names &= set(entry.keys)
|
|
452
|
+
key_name: str
|
|
453
|
+
for key_name in sorted(common_key_names):
|
|
454
|
+
normalized_values: list[object] = []
|
|
455
|
+
supported: bool = True
|
|
456
|
+
entry_value: object
|
|
457
|
+
for entry in entries:
|
|
458
|
+
entry_value = entry.keys[key_name]
|
|
459
|
+
normalized_value: object | None = _normalize_scalar_coord_value(entry_value)
|
|
460
|
+
if normalized_value is None:
|
|
461
|
+
supported = False
|
|
462
|
+
break
|
|
463
|
+
normalized_values.append(normalized_value)
|
|
464
|
+
if supported:
|
|
465
|
+
coords[key_name] = (batch_axis_name, np.asarray(normalized_values))
|
|
466
|
+
|
|
467
|
+
common_sample_coord_names: set[str] = set(tensors[0].sample_coords)
|
|
468
|
+
for tensor in tensors[1:]:
|
|
469
|
+
common_sample_coord_names &= set(tensor.sample_coords)
|
|
470
|
+
coord_name: str
|
|
471
|
+
for coord_name in sorted(common_sample_coord_names):
|
|
472
|
+
first_coord_dims, _first_coord_array = tensors[0].sample_coords[coord_name]
|
|
473
|
+
candidate_tensor: _GroupedPostprocessTensor
|
|
474
|
+
for candidate_tensor in tensors[1:]:
|
|
475
|
+
candidate_dims, _candidate_array = candidate_tensor.sample_coords[
|
|
476
|
+
coord_name
|
|
477
|
+
]
|
|
478
|
+
if candidate_dims != first_coord_dims:
|
|
479
|
+
raise InvalidConfigurationError(
|
|
480
|
+
f"Selected batch postprocess stream mixes incompatible companion coordinate shapes for {coord_name!r} and cannot be stacked via to_xarray()."
|
|
481
|
+
)
|
|
482
|
+
coord_arrays: list[np.ndarray] = [
|
|
483
|
+
tensor.sample_coords[coord_name][1] for tensor in tensors
|
|
484
|
+
]
|
|
485
|
+
_ensure_generated_companion_dim_coords(
|
|
486
|
+
coords,
|
|
487
|
+
coord_dims=first_coord_dims,
|
|
488
|
+
coord_array=coord_arrays[0],
|
|
489
|
+
)
|
|
490
|
+
coords[coord_name] = (
|
|
491
|
+
(batch_axis_name,) + first_coord_dims,
|
|
492
|
+
_stack_sample_coordinate_arrays(coord_arrays, sample_count=max_samples),
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
attrs: dict[str, object] = {
|
|
496
|
+
"postprocess_title": postprocess_title,
|
|
497
|
+
}
|
|
498
|
+
if len(grouped_field_names) == 1:
|
|
499
|
+
attrs["field_name"] = grouped_field_names[0]
|
|
500
|
+
else:
|
|
501
|
+
attrs["field_names"] = grouped_field_names
|
|
502
|
+
if index is not None:
|
|
503
|
+
attrs["postprocess_index"] = index
|
|
504
|
+
if title is not None:
|
|
505
|
+
attrs["postprocess_selector"] = title
|
|
506
|
+
|
|
507
|
+
coord_tensor: _GroupedPostprocessTensor = first_tensor
|
|
508
|
+
for coord_name, coord_value in coord_tensor.attrs.items():
|
|
509
|
+
attrs[coord_name] = coord_value
|
|
510
|
+
|
|
511
|
+
data_vars: dict[str, object] = {}
|
|
512
|
+
for field_name in grouped_field_names:
|
|
513
|
+
data_vars[field_name] = (
|
|
514
|
+
(batch_axis_name,) + first_tensor.grouped_dims[field_name],
|
|
515
|
+
stacked_values[field_name],
|
|
516
|
+
)
|
|
517
|
+
data_vars["is_valid"] = ((batch_axis_name, "field_index", "sample_index"), is_valid)
|
|
518
|
+
data_vars["is_padded"] = (
|
|
519
|
+
(batch_axis_name, "field_index", "sample_index"),
|
|
520
|
+
is_padded,
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
return xarray_module.Dataset(
|
|
524
|
+
data_vars=data_vars,
|
|
525
|
+
coords=coords,
|
|
526
|
+
attrs=attrs,
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _partition_companion_fields(
|
|
531
|
+
companion_fields: Mapping[str, object],
|
|
532
|
+
*,
|
|
533
|
+
sample_count: int,
|
|
534
|
+
) -> tuple[dict[str, tuple[tuple[str, ...], np.ndarray]], dict[str, object]]:
|
|
535
|
+
"""Split companion fields into sample-aligned coords and scalar attrs."""
|
|
536
|
+
|
|
537
|
+
sample_coords: dict[str, tuple[tuple[str, ...], np.ndarray]] = {}
|
|
538
|
+
attrs: dict[str, object] = {}
|
|
539
|
+
field_name: str
|
|
540
|
+
field_value: object
|
|
541
|
+
for field_name, field_value in companion_fields.items():
|
|
542
|
+
if isinstance(field_value, np.ndarray):
|
|
543
|
+
if field_value.ndim == 0:
|
|
544
|
+
attrs[field_name] = cast(object, field_value[()])
|
|
545
|
+
continue
|
|
546
|
+
if int(field_value.shape[0]) == sample_count:
|
|
547
|
+
trailing_dims: tuple[str, ...] = tuple(
|
|
548
|
+
f"{field_name}_dim_{dimension_index}"
|
|
549
|
+
for dimension_index in range(max(field_value.ndim - 1, 0))
|
|
550
|
+
)
|
|
551
|
+
sample_coords[field_name] = (
|
|
552
|
+
("sample_index",) + trailing_dims,
|
|
553
|
+
field_value,
|
|
554
|
+
)
|
|
555
|
+
continue
|
|
556
|
+
raise InvalidConfigurationError(
|
|
557
|
+
f"Companion field {field_name!r} is not supported by to_xarray() in this slice."
|
|
558
|
+
)
|
|
559
|
+
if isinstance(field_value, (str, bytes, bytearray, int, float, complex, bool)):
|
|
560
|
+
attrs[field_name] = field_value
|
|
561
|
+
continue
|
|
562
|
+
raise InvalidConfigurationError(
|
|
563
|
+
f"Companion field {field_name!r} is not supported by to_xarray() in this slice."
|
|
564
|
+
)
|
|
565
|
+
return sample_coords, attrs
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _split_structured_payload(
|
|
569
|
+
payload: Mapping[object, object],
|
|
570
|
+
) -> _StructuredPayloadParts:
|
|
571
|
+
"""Split one mapping payload into node-local and child-node fields."""
|
|
572
|
+
|
|
573
|
+
grouped_fields: dict[str, Mapping[object, object]] = {}
|
|
574
|
+
companion_fields: dict[str, object] = {}
|
|
575
|
+
nested_fields: dict[str, Mapping[object, object]] = {}
|
|
576
|
+
field_key: object
|
|
577
|
+
field_value: object
|
|
578
|
+
for field_key, field_value in payload.items():
|
|
579
|
+
field_name: str = str(field_key)
|
|
580
|
+
if field_name == "header":
|
|
581
|
+
continue
|
|
582
|
+
if isinstance(field_value, Mapping):
|
|
583
|
+
if is_numeric_key_mapping(field_value):
|
|
584
|
+
grouped_fields[field_name] = field_value
|
|
585
|
+
continue
|
|
586
|
+
nested_fields[field_name] = field_value
|
|
587
|
+
continue
|
|
588
|
+
companion_fields[field_name] = field_value
|
|
589
|
+
return _StructuredPayloadParts(
|
|
590
|
+
grouped_fields=grouped_fields,
|
|
591
|
+
companion_fields=companion_fields,
|
|
592
|
+
nested_fields=nested_fields,
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _common_ordered_child_names(
|
|
597
|
+
payload_parts_list: Sequence[_StructuredPayloadParts],
|
|
598
|
+
) -> tuple[str, ...]:
|
|
599
|
+
"""Return shared child-node names, requiring identical nested structure."""
|
|
600
|
+
|
|
601
|
+
first_names: tuple[str, ...] = tuple(payload_parts_list[0].nested_fields.keys())
|
|
602
|
+
payload_parts: _StructuredPayloadParts
|
|
603
|
+
for payload_parts in payload_parts_list[1:]:
|
|
604
|
+
if tuple(payload_parts.nested_fields.keys()) != first_names:
|
|
605
|
+
raise InvalidConfigurationError(
|
|
606
|
+
"Selected batch postprocess stream mixes incompatible nested payload namespaces and cannot be stacked via to_xarray()."
|
|
607
|
+
)
|
|
608
|
+
return first_names
|