jcmsuite 7.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of jcmsuite might be problematic. Click here for more details.
- jcmsuite/__init__.py +22 -0
- jcmsuite/_build_info.py +4 -0
- jcmsuite/context/__init__.py +44 -0
- jcmsuite/context/interface_metadata.py +158 -0
- jcmsuite/context/models.py +213 -0
- jcmsuite/context/scope.py +210 -0
- jcmsuite/context/solver_context.py +569 -0
- jcmsuite/context/subprocess_tools.py +172 -0
- jcmsuite/contract.py +47 -0
- jcmsuite/daemon/__init__.py +19 -0
- jcmsuite/daemon/handler.py +925 -0
- jcmsuite/daemon/internal/__init__.py +4 -0
- jcmsuite/daemon/internal/bundle_protocol.py +64 -0
- jcmsuite/daemon/internal/handler_helpers.py +368 -0
- jcmsuite/daemon/internal/handler_resource_jobs.py +227 -0
- jcmsuite/daemon/internal/handler_wait.py +564 -0
- jcmsuite/daemon/models.py +349 -0
- jcmsuite/daemon/namespaces.py +250 -0
- jcmsuite/daemon/protocol.py +801 -0
- jcmsuite/daemon/resources.py +407 -0
- jcmsuite/daemon/scope.py +53 -0
- jcmsuite/daemon/transport.py +223 -0
- jcmsuite/editing/__init__.py +10 -0
- jcmsuite/editing/editor.py +274 -0
- jcmsuite/exceptions/__init__.py +32 -0
- jcmsuite/exceptions/errors.py +173 -0
- jcmsuite/geo/__init__.py +10 -0
- jcmsuite/geo/mesh.py +302 -0
- jcmsuite/info/__init__.py +10 -0
- jcmsuite/info/runtime_info.py +45 -0
- jcmsuite/internal/__init__.py +2 -0
- jcmsuite/internal/diagnostics.py +74 -0
- jcmsuite/internal/progress.py +80 -0
- jcmsuite/loaders/__init__.py +18 -0
- jcmsuite/loaders/blob_header.py +315 -0
- jcmsuite/loaders/cartesian_field_reader.py +249 -0
- jcmsuite/loaders/cartesian_fields.py +324 -0
- jcmsuite/loaders/nested_mapping.py +69 -0
- jcmsuite/loaders/powerflux.py +136 -0
- jcmsuite/loaders/table.py +637 -0
- jcmsuite/logging/__init__.py +22 -0
- jcmsuite/logging/runtime.py +210 -0
- jcmsuite/results/__init__.py +126 -0
- jcmsuite/results/artifacts.py +806 -0
- jcmsuite/results/discovery.py +82 -0
- jcmsuite/results/field_exports.py +1062 -0
- jcmsuite/results/field_payloads.py +453 -0
- jcmsuite/results/internal/__init__.py +4 -0
- jcmsuite/results/internal/export/__init__.py +4 -0
- jcmsuite/results/internal/export/common.py +56 -0
- jcmsuite/results/internal/export/pandas/__init__.py +4 -0
- jcmsuite/results/internal/export/pandas/common.py +165 -0
- jcmsuite/results/internal/export/pandas/payload.py +480 -0
- jcmsuite/results/internal/export/pandas/rows.py +484 -0
- jcmsuite/results/internal/export/preview.py +178 -0
- jcmsuite/results/internal/export/xarray/__init__.py +4 -0
- jcmsuite/results/internal/export/xarray/export.py +688 -0
- jcmsuite/results/internal/export/xarray/grouped.py +608 -0
- jcmsuite/results/internal/export/xarray/headers.py +290 -0
- jcmsuite/results/internal/export/xarray/runtime.py +99 -0
- jcmsuite/results/internal/export/xarray/shared.py +246 -0
- jcmsuite/results/internal/export/xarray/tabular.py +304 -0
- jcmsuite/results/internal/export/xarray/types.py +28 -0
- jcmsuite/results/internal/models/__init__.py +4 -0
- jcmsuite/results/internal/models/batch.py +1081 -0
- jcmsuite/results/internal/models/batch_postprocess.py +1728 -0
- jcmsuite/results/internal/models/bundle.py +343 -0
- jcmsuite/results/internal/models/core.py +556 -0
- jcmsuite/results/internal/models/submission.py +1157 -0
- jcmsuite/results/internal/resultbag/__init__.py +4 -0
- jcmsuite/results/internal/resultbag/access.py +489 -0
- jcmsuite/results/internal/resultbag/iteration.py +199 -0
- jcmsuite/results/internal/resultbag/query.py +226 -0
- jcmsuite/results/internal/resultbag/raw.py +407 -0
- jcmsuite/results/internal/resultbag/source_files.py +142 -0
- jcmsuite/results/internal/resultbag/store.py +881 -0
- jcmsuite/results/internal/resultbag/utils.py +258 -0
- jcmsuite/results/models.py +76 -0
- jcmsuite/results/opening.py +55 -0
- jcmsuite/results/pandas_export.py +325 -0
- jcmsuite/results/payloads.py +478 -0
- jcmsuite/results/postprocess.py +97 -0
- jcmsuite/results/postprocess_payloads.py +757 -0
- jcmsuite/results/postprocess_record.py +418 -0
- jcmsuite/results/resultbag.py +279 -0
- jcmsuite/results/resultbag_management.py +433 -0
- jcmsuite/results/structured_derivatives.py +387 -0
- jcmsuite/solve/__init__.py +10 -0
- jcmsuite/solve/daemon_bundle_staging.py +100 -0
- jcmsuite/solve/daemon_bundle_wait.py +89 -0
- jcmsuite/solve/orchestration.py +1576 -0
- jcmsuite/solve/orchestration_execution.py +180 -0
- jcmsuite/solve/orchestration_routing.py +128 -0
- jcmsuite/solve/orchestration_support.py +323 -0
- jcmsuite/solve/path_resolution.py +98 -0
- jcmsuite/solve/progress.py +20 -0
- jcmsuite/solve/solver.py +709 -0
- jcmsuite/solve/staging.py +264 -0
- jcmsuite/solve/types.py +25 -0
- jcmsuite/solve/working_dirs.py +227 -0
- jcmsuite/templates/__init__.py +11 -0
- jcmsuite/templates/converter.py +71 -0
- jcmsuite/templates/formatting.py +335 -0
- jcmsuite/templates/formatting_cli.py +47 -0
- jcmsuite/templates/rendering.py +666 -0
- jcmsuite/torch/__init__.py +25 -0
- jcmsuite/torch/autograd.py +488 -0
- jcmsuite/torch/coordinate_policy.py +82 -0
- jcmsuite/torch/core.py +312 -0
- jcmsuite/torch/results/__init__.py +51 -0
- jcmsuite/torch/types.py +84 -0
- jcmsuite/torch/validation.py +208 -0
- jcmsuite/torch/wrappers.py +4332 -0
- jcmsuite/view/__init__.py +10 -0
- jcmsuite/view/viewer.py +122 -0
- jcmsuite-7.4.2.dist-info/METADATA +180 -0
- jcmsuite-7.4.2.dist-info/RECORD +119 -0
- jcmsuite-7.4.2.dist-info/WHEEL +4 -0
- jcmsuite-7.4.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,1728 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# Copyright (c) 2001-2026 JCMwave GmbH
|
|
3
|
+
|
|
4
|
+
"""Postprocess-specific batch result view helpers."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Callable, Mapping
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import TYPE_CHECKING, Protocol, Self, cast
|
|
11
|
+
|
|
12
|
+
from ....exceptions import InvalidConfigurationError, UnsupportedExportError
|
|
13
|
+
from ...artifacts import (
|
|
14
|
+
PostprocessCollection,
|
|
15
|
+
PostprocessHeader,
|
|
16
|
+
PostprocessRecord,
|
|
17
|
+
)
|
|
18
|
+
from ...field_exports import FieldExportPayload, FieldExportView
|
|
19
|
+
from ...field_payloads import resolve_coordinates
|
|
20
|
+
from ...payloads import (
|
|
21
|
+
DataFrameLongExportMixin,
|
|
22
|
+
DictionaryExportMixin,
|
|
23
|
+
Source,
|
|
24
|
+
StructuredPayload,
|
|
25
|
+
TableCollectionPayload,
|
|
26
|
+
TablePayload,
|
|
27
|
+
)
|
|
28
|
+
from ..export.pandas.common import dataframe_rows
|
|
29
|
+
from ..export.pandas.payload import flatten_payload
|
|
30
|
+
from ..export.xarray.types import XarrayDataTree
|
|
31
|
+
from .batch import (
|
|
32
|
+
BatchExportRecord,
|
|
33
|
+
BatchFieldPayloadBase,
|
|
34
|
+
BatchMetadataPayload,
|
|
35
|
+
BatchStructuredPayload,
|
|
36
|
+
BatchTableCollectionPayload,
|
|
37
|
+
BatchTablePayload,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
if TYPE_CHECKING:
|
|
41
|
+
import pandas as pd # pyright: ignore[reportMissingImports]
|
|
42
|
+
|
|
43
|
+
from ..export.xarray.runtime import _DataTreeModule, _XarrayModule
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class _XarrayExportLoaderModule(Protocol):
|
|
47
|
+
"""Typed access to the historical xarray_export loader seam."""
|
|
48
|
+
|
|
49
|
+
def _load_xarray_module(self) -> _XarrayModule: ...
|
|
50
|
+
|
|
51
|
+
def _load_datatree_module(self) -> _DataTreeModule: ...
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(slots=True, frozen=True, repr=False)
|
|
55
|
+
class BatchPostprocessCollection:
|
|
56
|
+
"""Select one postprocess result across a batch.
|
|
57
|
+
|
|
58
|
+
Parameters
|
|
59
|
+
----------
|
|
60
|
+
records : tuple[BatchExportRecord, ...]
|
|
61
|
+
Postprocess results in batched project keys order.
|
|
62
|
+
|
|
63
|
+
Notes
|
|
64
|
+
-----
|
|
65
|
+
Application code receives this collection from
|
|
66
|
+
`BatchSolveResult.postprocesses`; it does not need to construct the
|
|
67
|
+
collection directly.
|
|
68
|
+
|
|
69
|
+
Select a postprocess result by its integer position. Export data from the
|
|
70
|
+
returned `BatchPostprocessRecord`.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
records: tuple[BatchExportRecord, ...]
|
|
74
|
+
|
|
75
|
+
def __repr__(self) -> str:
|
|
76
|
+
"""Return indexed streams with aligned batch availability states."""
|
|
77
|
+
|
|
78
|
+
from ..export.preview import format_table_preview
|
|
79
|
+
|
|
80
|
+
stream_count: int = max(
|
|
81
|
+
(
|
|
82
|
+
len(record.payload)
|
|
83
|
+
for record in self.records
|
|
84
|
+
if isinstance(record.payload, PostprocessCollection)
|
|
85
|
+
),
|
|
86
|
+
default=0,
|
|
87
|
+
)
|
|
88
|
+
titles_preview: list[str] = []
|
|
89
|
+
entries_preview: list[int] = []
|
|
90
|
+
available_preview: list[int] = []
|
|
91
|
+
missing_preview: list[int] = []
|
|
92
|
+
status_preview: list[str] = []
|
|
93
|
+
for index in range(stream_count):
|
|
94
|
+
titles: set[str] = set()
|
|
95
|
+
available_count: int = 0
|
|
96
|
+
for record in self.records:
|
|
97
|
+
if not isinstance(record.payload, PostprocessCollection):
|
|
98
|
+
continue
|
|
99
|
+
if not 0 <= index < len(record.payload):
|
|
100
|
+
continue
|
|
101
|
+
selected: PostprocessRecord = record.payload[index]
|
|
102
|
+
titles.add("<untitled>" if selected.title is None else selected.title)
|
|
103
|
+
if selected.available:
|
|
104
|
+
available_count += 1
|
|
105
|
+
title: str = (
|
|
106
|
+
"<untitled>"
|
|
107
|
+
if not titles
|
|
108
|
+
else next(iter(titles)) if len(titles) == 1 else "<mixed>"
|
|
109
|
+
)
|
|
110
|
+
missing_count: int = len(self.records) - available_count
|
|
111
|
+
status: str = (
|
|
112
|
+
"available"
|
|
113
|
+
if missing_count == 0
|
|
114
|
+
else "unavailable" if available_count == 0 else "partial"
|
|
115
|
+
)
|
|
116
|
+
titles_preview.append(title)
|
|
117
|
+
entries_preview.append(len(self.records))
|
|
118
|
+
available_preview.append(available_count)
|
|
119
|
+
missing_preview.append(missing_count)
|
|
120
|
+
status_preview.append(status)
|
|
121
|
+
preview_values: dict[object, object] = {
|
|
122
|
+
"index": list(range(stream_count)),
|
|
123
|
+
"title": titles_preview,
|
|
124
|
+
"entries": entries_preview,
|
|
125
|
+
"available": available_preview,
|
|
126
|
+
"missing": missing_preview,
|
|
127
|
+
"status": status_preview,
|
|
128
|
+
}
|
|
129
|
+
preview: str = format_table_preview(
|
|
130
|
+
preview_values,
|
|
131
|
+
include_row_index=False,
|
|
132
|
+
quote_strings=False,
|
|
133
|
+
)
|
|
134
|
+
return f"BatchPostprocessCollection(\n{preview}\n)"
|
|
135
|
+
|
|
136
|
+
def titles(self) -> tuple[str, ...]:
|
|
137
|
+
"""Return postprocess titles from the first available batch entry.
|
|
138
|
+
|
|
139
|
+
Returns
|
|
140
|
+
-------
|
|
141
|
+
tuple[str, ...]
|
|
142
|
+
Titles in postprocess order. Duplicate titles remain in the result.
|
|
143
|
+
An empty batch or a batch without a successful entry returns an
|
|
144
|
+
empty tuple.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
record: BatchExportRecord
|
|
148
|
+
for record in self.records:
|
|
149
|
+
if isinstance(record.payload, PostprocessCollection):
|
|
150
|
+
return record.payload.titles()
|
|
151
|
+
return ()
|
|
152
|
+
|
|
153
|
+
def __getitem__(
|
|
154
|
+
self, selector: int
|
|
155
|
+
) -> BatchPostprocessRecord | _BatchPostprocessSelection:
|
|
156
|
+
"""Select one postprocess result across the batch.
|
|
157
|
+
|
|
158
|
+
Parameters
|
|
159
|
+
----------
|
|
160
|
+
selector : int
|
|
161
|
+
Postprocess position shared across all batch entries. Negative
|
|
162
|
+
positions are not supported.
|
|
163
|
+
|
|
164
|
+
Returns
|
|
165
|
+
-------
|
|
166
|
+
BatchPostprocessRecord | _BatchPostprocessSelection
|
|
167
|
+
Batch record when available entries have compatible result data.
|
|
168
|
+
Older result data can return a read-only selection with the same
|
|
169
|
+
export operations.
|
|
170
|
+
|
|
171
|
+
Raises
|
|
172
|
+
------
|
|
173
|
+
TypeError
|
|
174
|
+
If `selector` is not an integer or is a Boolean value.
|
|
175
|
+
InvalidConfigurationError
|
|
176
|
+
If available entries have incompatible result data.
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
if isinstance(selector, bool):
|
|
180
|
+
raise TypeError("Postprocess selector must be int, not bool.")
|
|
181
|
+
if isinstance(selector, int):
|
|
182
|
+
try:
|
|
183
|
+
return _build_batch_postprocess_record(self.records, selector)
|
|
184
|
+
except InvalidConfigurationError as exc:
|
|
185
|
+
if "field payloads are deferred" in str(exc):
|
|
186
|
+
return _BatchPostprocessSelection(
|
|
187
|
+
records=self.records, index=selector
|
|
188
|
+
)
|
|
189
|
+
if "payload families" in str(exc) or "schemas" in str(exc):
|
|
190
|
+
raise
|
|
191
|
+
return _BatchPostprocessSelection(records=self.records, index=selector)
|
|
192
|
+
raise TypeError("Postprocess selector must be int.")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass(slots=True, frozen=True, repr=False)
|
|
196
|
+
class _BatchPostprocessTablePayload(BatchTablePayload):
|
|
197
|
+
"""Aligned table payload for one homogeneous postprocess stream."""
|
|
198
|
+
|
|
199
|
+
records: tuple[BatchExportRecord, ...]
|
|
200
|
+
postprocess_index: int
|
|
201
|
+
|
|
202
|
+
def __post_init__(self) -> None:
|
|
203
|
+
sources: tuple[Source, ...] = tuple(record.source for record in self.records)
|
|
204
|
+
object.__setattr__(self, "_sources", sources)
|
|
205
|
+
|
|
206
|
+
@property
|
|
207
|
+
def sources(self) -> tuple[Source, ...]:
|
|
208
|
+
"""Return source paths aligned with the current batch entries."""
|
|
209
|
+
|
|
210
|
+
return self._sources
|
|
211
|
+
|
|
212
|
+
def to_dict(self) -> Mapping[object, object]:
|
|
213
|
+
"""Return available table mappings keyed by original batch index."""
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
record.index: cast(DictionaryExportMixin, record.payload).to_dict()
|
|
217
|
+
for record in self.records
|
|
218
|
+
if record.payload is not None
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
def _to_df_long(self) -> pd.DataFrame:
|
|
222
|
+
"""Export available table rows with batch and postprocess identity."""
|
|
223
|
+
|
|
224
|
+
from ...pandas_export import _load_pandas_module
|
|
225
|
+
from ..export.pandas.common import normalize_optional_index_columns
|
|
226
|
+
|
|
227
|
+
rows: list[dict[str, object]] = []
|
|
228
|
+
record: BatchExportRecord
|
|
229
|
+
for record in self.records:
|
|
230
|
+
if record.payload is None:
|
|
231
|
+
continue
|
|
232
|
+
payload_mapping: Mapping[object, object] = cast(
|
|
233
|
+
DictionaryExportMixin, record.payload
|
|
234
|
+
).to_dict()
|
|
235
|
+
title: object = payload_mapping.get("title")
|
|
236
|
+
record_rows = dataframe_rows(
|
|
237
|
+
cast(DataFrameLongExportMixin, record.payload)._to_df_long()
|
|
238
|
+
)
|
|
239
|
+
for row in record_rows:
|
|
240
|
+
rows.append(
|
|
241
|
+
{
|
|
242
|
+
"batch_index": record.index,
|
|
243
|
+
"postprocess_index": self.postprocess_index,
|
|
244
|
+
"postprocess_title": title,
|
|
245
|
+
**row,
|
|
246
|
+
}
|
|
247
|
+
)
|
|
248
|
+
dataframe = _load_pandas_module().DataFrame(rows)
|
|
249
|
+
return cast("pd.DataFrame", normalize_optional_index_columns(dataframe))
|
|
250
|
+
|
|
251
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
252
|
+
"""Export aligned tables with validity and padding masks."""
|
|
253
|
+
|
|
254
|
+
from ..export.xarray import export as xarray_export_module
|
|
255
|
+
from ..export.xarray.export import (
|
|
256
|
+
_stack_batch_postprocess_payloads_to_data_tree,
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
loader = cast(_XarrayExportLoaderModule, xarray_export_module)
|
|
260
|
+
|
|
261
|
+
available_mapping: Mapping[object, object] | None = next(
|
|
262
|
+
(
|
|
263
|
+
cast(DictionaryExportMixin, record.payload).to_dict()
|
|
264
|
+
for record in self.records
|
|
265
|
+
if record.payload is not None
|
|
266
|
+
),
|
|
267
|
+
None,
|
|
268
|
+
)
|
|
269
|
+
if available_mapping is None:
|
|
270
|
+
raise InvalidConfigurationError(
|
|
271
|
+
"No batch postprocess payloads are available for to_xarray()."
|
|
272
|
+
)
|
|
273
|
+
payloads: list[Mapping[object, object]] = []
|
|
274
|
+
availability: list[bool] = []
|
|
275
|
+
for record in self.records:
|
|
276
|
+
if record.payload is None:
|
|
277
|
+
payloads.append(_nan_structure(available_mapping))
|
|
278
|
+
availability.append(False)
|
|
279
|
+
else:
|
|
280
|
+
payloads.append(cast(DictionaryExportMixin, record.payload).to_dict())
|
|
281
|
+
availability.append(True)
|
|
282
|
+
|
|
283
|
+
return _stack_batch_postprocess_payloads_to_data_tree(
|
|
284
|
+
self.records,
|
|
285
|
+
payloads,
|
|
286
|
+
index=self.postprocess_index,
|
|
287
|
+
title=None,
|
|
288
|
+
xarray_module=loader._load_xarray_module(),
|
|
289
|
+
datatree_module=loader._load_datatree_module(),
|
|
290
|
+
batch_axis_name="batch_index",
|
|
291
|
+
available=availability,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
@dataclass(slots=True, frozen=True)
|
|
296
|
+
class _BatchStructuredPayload(BatchStructuredPayload):
|
|
297
|
+
"""Aligned structured payload for one homogeneous postprocess stream."""
|
|
298
|
+
|
|
299
|
+
records: tuple[BatchExportRecord, ...]
|
|
300
|
+
postprocess_index: int
|
|
301
|
+
|
|
302
|
+
def __post_init__(self) -> None:
|
|
303
|
+
object.__setattr__(
|
|
304
|
+
self, "_sources", tuple(record.source for record in self.records)
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
@property
|
|
308
|
+
def sources(self) -> tuple[Source, ...]:
|
|
309
|
+
"""Return source paths aligned with current batch entries."""
|
|
310
|
+
|
|
311
|
+
return self._sources
|
|
312
|
+
|
|
313
|
+
def to_dict(self) -> Mapping[object, object]:
|
|
314
|
+
"""Return available structured mappings keyed by batch index."""
|
|
315
|
+
|
|
316
|
+
return {
|
|
317
|
+
record.index: cast(DictionaryExportMixin, record.payload).to_dict()
|
|
318
|
+
for record in self.records
|
|
319
|
+
if record.payload is not None
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
def _to_df_long(self) -> pd.DataFrame:
|
|
323
|
+
"""Export structured paths with batch and postprocess identity."""
|
|
324
|
+
|
|
325
|
+
from ...pandas_export import _load_pandas_module
|
|
326
|
+
from ..export.pandas.common import normalize_optional_index_columns
|
|
327
|
+
|
|
328
|
+
rows: list[dict[str, object]] = []
|
|
329
|
+
record: BatchExportRecord
|
|
330
|
+
for record in self.records:
|
|
331
|
+
if record.payload is None:
|
|
332
|
+
continue
|
|
333
|
+
payload_rows = dataframe_rows(
|
|
334
|
+
cast(DataFrameLongExportMixin, record.payload)._to_df_long()
|
|
335
|
+
)
|
|
336
|
+
rows.extend(
|
|
337
|
+
{
|
|
338
|
+
"batch_index": record.index,
|
|
339
|
+
"postprocess_index": self.postprocess_index,
|
|
340
|
+
**row,
|
|
341
|
+
}
|
|
342
|
+
for row in payload_rows
|
|
343
|
+
)
|
|
344
|
+
dataframe = _load_pandas_module().DataFrame(rows)
|
|
345
|
+
return cast("pd.DataFrame", normalize_optional_index_columns(dataframe))
|
|
346
|
+
|
|
347
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
348
|
+
"""Export the structured tree with batch availability masks."""
|
|
349
|
+
|
|
350
|
+
from ..export.xarray import export as xarray_export_module
|
|
351
|
+
from ..export.xarray.export import (
|
|
352
|
+
_stack_batch_postprocess_payloads_to_data_tree,
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
available_payload: Mapping[object, object] | None = next(
|
|
356
|
+
(
|
|
357
|
+
cast(DictionaryExportMixin, record.payload).to_dict()
|
|
358
|
+
for record in self.records
|
|
359
|
+
if record.payload is not None
|
|
360
|
+
),
|
|
361
|
+
None,
|
|
362
|
+
)
|
|
363
|
+
if available_payload is None:
|
|
364
|
+
raise InvalidConfigurationError(
|
|
365
|
+
"No structured batch postprocess payloads are available for to_xarray()."
|
|
366
|
+
)
|
|
367
|
+
payloads: list[Mapping[object, object]] = []
|
|
368
|
+
availability: list[bool] = []
|
|
369
|
+
for record in self.records:
|
|
370
|
+
if record.payload is None:
|
|
371
|
+
payloads.append(_nan_structure(available_payload))
|
|
372
|
+
availability.append(False)
|
|
373
|
+
else:
|
|
374
|
+
payloads.append(cast(DictionaryExportMixin, record.payload).to_dict())
|
|
375
|
+
availability.append(True)
|
|
376
|
+
loader = cast(_XarrayExportLoaderModule, xarray_export_module)
|
|
377
|
+
return _stack_batch_postprocess_payloads_to_data_tree(
|
|
378
|
+
self.records,
|
|
379
|
+
payloads,
|
|
380
|
+
index=self.postprocess_index,
|
|
381
|
+
title=None,
|
|
382
|
+
xarray_module=loader._load_xarray_module(),
|
|
383
|
+
datatree_module=loader._load_datatree_module(),
|
|
384
|
+
batch_axis_name="batch_index",
|
|
385
|
+
available=availability,
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _nan_structure(payload: Mapping[object, object]) -> Mapping[object, object]:
|
|
390
|
+
"""Create a schema-preserving NaN placeholder for an unavailable entry."""
|
|
391
|
+
|
|
392
|
+
import numpy as np
|
|
393
|
+
|
|
394
|
+
result: dict[object, object] = {}
|
|
395
|
+
for key, value in payload.items():
|
|
396
|
+
if isinstance(value, Mapping):
|
|
397
|
+
result[key] = _nan_structure(value)
|
|
398
|
+
elif isinstance(value, np.ndarray):
|
|
399
|
+
result[key] = np.full(value.shape, np.nan)
|
|
400
|
+
elif isinstance(value, (int, float, complex)):
|
|
401
|
+
result[key] = np.nan
|
|
402
|
+
else:
|
|
403
|
+
result[key] = value
|
|
404
|
+
return result
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
@dataclass(slots=True, frozen=True)
|
|
408
|
+
class _BatchTableCollectionPayload(BatchTableCollectionPayload):
|
|
409
|
+
"""Aligned collection of named batch table payloads."""
|
|
410
|
+
|
|
411
|
+
records: tuple[BatchExportRecord, ...]
|
|
412
|
+
postprocess_index: int
|
|
413
|
+
|
|
414
|
+
def __post_init__(self) -> None:
|
|
415
|
+
object.__setattr__(
|
|
416
|
+
self, "_sources", tuple(record.source for record in self.records)
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
@property
|
|
420
|
+
def sources(self) -> tuple[Source, ...]:
|
|
421
|
+
"""Return source paths aligned with current batch entries."""
|
|
422
|
+
|
|
423
|
+
return self._sources
|
|
424
|
+
|
|
425
|
+
def to_dict(self) -> Mapping[object, object]:
|
|
426
|
+
"""Return available child-table mappings keyed by batch index."""
|
|
427
|
+
|
|
428
|
+
return {
|
|
429
|
+
record.index: cast(DictionaryExportMixin, record.payload).to_dict()
|
|
430
|
+
for record in self.records
|
|
431
|
+
if record.payload is not None
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
def _to_df_long(self) -> pd.DataFrame:
|
|
435
|
+
"""Export child tables with batch, table, and row identity."""
|
|
436
|
+
|
|
437
|
+
from ...pandas_export import _load_pandas_module
|
|
438
|
+
from ..export.pandas.common import normalize_optional_index_columns
|
|
439
|
+
from ..export.pandas.payload import postprocess_payload_to_long_rows
|
|
440
|
+
|
|
441
|
+
names = self._table_names()
|
|
442
|
+
rows: list[dict[str, object]] = []
|
|
443
|
+
for record in self.records:
|
|
444
|
+
if record.payload is None:
|
|
445
|
+
continue
|
|
446
|
+
mapping = cast(DictionaryExportMixin, record.payload).to_dict()
|
|
447
|
+
for table_index, table_name in enumerate(names):
|
|
448
|
+
table = mapping[table_name]
|
|
449
|
+
if not isinstance(table, Mapping):
|
|
450
|
+
raise InvalidConfigurationError(
|
|
451
|
+
"Batch table-collection payload requires named table mappings."
|
|
452
|
+
)
|
|
453
|
+
values = dict(table)
|
|
454
|
+
values.pop("title", None)
|
|
455
|
+
values.pop("header", None)
|
|
456
|
+
table_rows = postprocess_payload_to_long_rows(values)
|
|
457
|
+
if table_rows is None:
|
|
458
|
+
raise InvalidConfigurationError(
|
|
459
|
+
f"Batch table {table_name!r} is not tabular."
|
|
460
|
+
)
|
|
461
|
+
for row in table_rows:
|
|
462
|
+
rows.append(
|
|
463
|
+
{
|
|
464
|
+
"batch_index": record.index,
|
|
465
|
+
"postprocess_index": self.postprocess_index,
|
|
466
|
+
"table": table_name,
|
|
467
|
+
"table_index": table_index,
|
|
468
|
+
**row,
|
|
469
|
+
}
|
|
470
|
+
)
|
|
471
|
+
dataframe = _load_pandas_module().DataFrame(rows)
|
|
472
|
+
return cast("pd.DataFrame", normalize_optional_index_columns(dataframe))
|
|
473
|
+
|
|
474
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
475
|
+
"""Export named child tables as batch-aligned xarray nodes."""
|
|
476
|
+
|
|
477
|
+
from ..export.xarray import export as xarray_export_module
|
|
478
|
+
from ..export.xarray.runtime import _create_data_tree
|
|
479
|
+
from ..export.xarray.tabular import (
|
|
480
|
+
_stack_batch_tabular_tables,
|
|
481
|
+
tabular_payload_to_xarray_table,
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
loader = cast(_XarrayExportLoaderModule, xarray_export_module)
|
|
485
|
+
names = self._table_names()
|
|
486
|
+
children: dict[str, XarrayDataTree] = {}
|
|
487
|
+
for table_name in names:
|
|
488
|
+
first_table = self._first_table(table_name)
|
|
489
|
+
tables = []
|
|
490
|
+
valid: list[bool] = []
|
|
491
|
+
for record in self.records:
|
|
492
|
+
table = (
|
|
493
|
+
None
|
|
494
|
+
if record.payload is None
|
|
495
|
+
else cast(DictionaryExportMixin, record.payload)
|
|
496
|
+
.to_dict()
|
|
497
|
+
.get(table_name)
|
|
498
|
+
)
|
|
499
|
+
if not isinstance(table, Mapping):
|
|
500
|
+
tables.append(
|
|
501
|
+
tabular_payload_to_xarray_table(
|
|
502
|
+
_nan_table(first_table), payload_name="Batch table"
|
|
503
|
+
)
|
|
504
|
+
)
|
|
505
|
+
valid.append(False)
|
|
506
|
+
else:
|
|
507
|
+
tables.append(
|
|
508
|
+
tabular_payload_to_xarray_table(
|
|
509
|
+
table, payload_name="Batch table"
|
|
510
|
+
)
|
|
511
|
+
)
|
|
512
|
+
valid.append(True)
|
|
513
|
+
dataset = _stack_batch_tabular_tables(
|
|
514
|
+
self.records,
|
|
515
|
+
tables,
|
|
516
|
+
xarray_module=loader._load_xarray_module(),
|
|
517
|
+
batch_axis_name="batch_index",
|
|
518
|
+
valid_entries=valid,
|
|
519
|
+
)
|
|
520
|
+
children[table_name] = _create_data_tree(
|
|
521
|
+
datatree_module=loader._load_datatree_module(),
|
|
522
|
+
dataset=dataset,
|
|
523
|
+
name=table_name,
|
|
524
|
+
)
|
|
525
|
+
root = loader._load_xarray_module().Dataset(
|
|
526
|
+
data_vars={}, attrs={"tables": names, "table_count": len(names)}
|
|
527
|
+
)
|
|
528
|
+
return _create_data_tree(
|
|
529
|
+
datatree_module=loader._load_datatree_module(),
|
|
530
|
+
dataset=root,
|
|
531
|
+
children=children,
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
def _table_names(self) -> tuple[str, ...]:
|
|
535
|
+
for record in self.records:
|
|
536
|
+
if record.payload is not None:
|
|
537
|
+
return tuple(
|
|
538
|
+
str(name)
|
|
539
|
+
for name in cast(DictionaryExportMixin, record.payload).to_dict()
|
|
540
|
+
)
|
|
541
|
+
raise InvalidConfigurationError("No table-collection payloads are available.")
|
|
542
|
+
|
|
543
|
+
def _first_table(self, table_name: str) -> Mapping[object, object]:
|
|
544
|
+
for record in self.records:
|
|
545
|
+
if record.payload is not None:
|
|
546
|
+
table = cast(DictionaryExportMixin, record.payload).to_dict()[
|
|
547
|
+
table_name
|
|
548
|
+
]
|
|
549
|
+
if isinstance(table, Mapping):
|
|
550
|
+
return table
|
|
551
|
+
raise InvalidConfigurationError(f"Table {table_name!r} is unavailable.")
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _nan_table(table: Mapping[object, object]) -> Mapping[object, object]:
|
|
555
|
+
"""Return a schema-preserving NaN table for an unavailable entry."""
|
|
556
|
+
|
|
557
|
+
import numpy as np
|
|
558
|
+
|
|
559
|
+
return {
|
|
560
|
+
key: (
|
|
561
|
+
np.full(value.shape, np.nan)
|
|
562
|
+
if isinstance(value, np.ndarray)
|
|
563
|
+
else np.nan if isinstance(value, (int, float, complex)) else value
|
|
564
|
+
)
|
|
565
|
+
for key, value in table.items()
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
@dataclass(slots=True, frozen=True, repr=False)
|
|
570
|
+
class _BatchFieldViewPayload(BatchFieldPayloadBase):
|
|
571
|
+
"""Store and export field data for a batch.
|
|
572
|
+
|
|
573
|
+
Parameters
|
|
574
|
+
----------
|
|
575
|
+
records : tuple[BatchExportRecord, ...]
|
|
576
|
+
Field data in batch order.
|
|
577
|
+
postprocess_index : int
|
|
578
|
+
Position of the postprocess result in each project result.
|
|
579
|
+
|
|
580
|
+
Notes
|
|
581
|
+
-----
|
|
582
|
+
Application code receives this object from `BatchPostprocessRecord.fields`;
|
|
583
|
+
it does not need to construct the object directly.
|
|
584
|
+
"""
|
|
585
|
+
|
|
586
|
+
records: tuple[BatchExportRecord, ...]
|
|
587
|
+
postprocess_index: int
|
|
588
|
+
|
|
589
|
+
def __post_init__(self) -> None:
|
|
590
|
+
object.__setattr__(
|
|
591
|
+
self, "_sources", tuple(record.source for record in self.records)
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
@property
|
|
595
|
+
def sources(self) -> tuple[Source, ...]:
|
|
596
|
+
"""Return one source file for each batch entry.
|
|
597
|
+
|
|
598
|
+
An unavailable source file is `None`. The order matches the current
|
|
599
|
+
batch entries.
|
|
600
|
+
|
|
601
|
+
Returns
|
|
602
|
+
-------
|
|
603
|
+
tuple[Path | None, ...]
|
|
604
|
+
Source files in batched project keys order.
|
|
605
|
+
"""
|
|
606
|
+
|
|
607
|
+
return self._sources
|
|
608
|
+
|
|
609
|
+
def _project_fields(
|
|
610
|
+
self, projection: Callable[[FieldExportView], FieldExportView]
|
|
611
|
+
) -> Self:
|
|
612
|
+
"""Apply one field projection without changing batch entry alignment."""
|
|
613
|
+
|
|
614
|
+
projected_records: tuple[BatchExportRecord, ...] = tuple(
|
|
615
|
+
BatchExportRecord(
|
|
616
|
+
index=record.index,
|
|
617
|
+
keys=dict(record.keys),
|
|
618
|
+
payload=(
|
|
619
|
+
None
|
|
620
|
+
if not isinstance(record.payload, FieldExportView)
|
|
621
|
+
else projection(record.payload)
|
|
622
|
+
),
|
|
623
|
+
source=record.source,
|
|
624
|
+
)
|
|
625
|
+
for record in self.records
|
|
626
|
+
)
|
|
627
|
+
return type(self)(
|
|
628
|
+
records=projected_records,
|
|
629
|
+
postprocess_index=self.postprocess_index,
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
def to_dict(self) -> Mapping[object, object]:
|
|
633
|
+
"""Return one field dictionary for each available batch entry.
|
|
634
|
+
|
|
635
|
+
Returns
|
|
636
|
+
-------
|
|
637
|
+
Mapping[object, object]
|
|
638
|
+
Field dictionaries indexed by their original batch positions.
|
|
639
|
+
"""
|
|
640
|
+
|
|
641
|
+
exported: dict[object, object] = {}
|
|
642
|
+
for record in self.records:
|
|
643
|
+
if record.payload is None:
|
|
644
|
+
continue
|
|
645
|
+
mapping: dict[object, object] = dict(
|
|
646
|
+
cast(DictionaryExportMixin, record.payload).to_dict()
|
|
647
|
+
)
|
|
648
|
+
field_value: object = mapping.get("field")
|
|
649
|
+
if field_value is not None and not isinstance(field_value, (list, tuple)):
|
|
650
|
+
mapping["field"] = [field_value]
|
|
651
|
+
exported[record.index] = mapping
|
|
652
|
+
return exported
|
|
653
|
+
|
|
654
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
655
|
+
"""Return the batch field data as an xarray `DataTree`.
|
|
656
|
+
|
|
657
|
+
The root node contains the selected fields. Derivatives are in child
|
|
658
|
+
nodes. `batch_index` contains the original batch positions.
|
|
659
|
+
|
|
660
|
+
Returns
|
|
661
|
+
-------
|
|
662
|
+
xarray.DataTree
|
|
663
|
+
Batch field data with validity and padding information.
|
|
664
|
+
|
|
665
|
+
Raises
|
|
666
|
+
------
|
|
667
|
+
InvalidConfigurationError
|
|
668
|
+
If no field data is available or the field shapes cannot be
|
|
669
|
+
combined.
|
|
670
|
+
"""
|
|
671
|
+
|
|
672
|
+
from ..export.xarray import export as xarray_export_module
|
|
673
|
+
|
|
674
|
+
loader = cast(_XarrayExportLoaderModule, xarray_export_module)
|
|
675
|
+
available = [record.payload is not None for record in self.records]
|
|
676
|
+
trees: list[XarrayDataTree] = []
|
|
677
|
+
first_payload: FieldExportView | None = None
|
|
678
|
+
for record in self.records:
|
|
679
|
+
if isinstance(record.payload, FieldExportView):
|
|
680
|
+
first_payload = record.payload
|
|
681
|
+
break
|
|
682
|
+
if first_payload is None:
|
|
683
|
+
raise InvalidConfigurationError(
|
|
684
|
+
"No field batch postprocess payloads are available for to_xarray()."
|
|
685
|
+
)
|
|
686
|
+
for record in self.records:
|
|
687
|
+
payload = record.payload
|
|
688
|
+
if not isinstance(payload, FieldExportView):
|
|
689
|
+
payload = first_payload
|
|
690
|
+
trees.append(payload.to_xarray())
|
|
691
|
+
return _stack_field_trees(
|
|
692
|
+
trees,
|
|
693
|
+
available=available,
|
|
694
|
+
batch_indices=[record.index for record in self.records],
|
|
695
|
+
xarray_module=loader._load_xarray_module(),
|
|
696
|
+
datatree_module=loader._load_datatree_module(),
|
|
697
|
+
)
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
@dataclass(slots=True, frozen=True, repr=False)
|
|
701
|
+
class BatchFieldExportPayload(_BatchFieldViewPayload):
|
|
702
|
+
"""Store and export transformable field data for a batch."""
|
|
703
|
+
|
|
704
|
+
@property
|
|
705
|
+
def intensity(self) -> "BatchIntensityFieldExportPayload":
|
|
706
|
+
"""Return the squared-norm intensity view for available fields.
|
|
707
|
+
|
|
708
|
+
Returns
|
|
709
|
+
-------
|
|
710
|
+
BatchIntensityFieldExportPayload
|
|
711
|
+
Non-transformable field-intensity view. The view retains batch
|
|
712
|
+
entry identity, availability information, and field navigation.
|
|
713
|
+
"""
|
|
714
|
+
|
|
715
|
+
projected: _BatchFieldViewPayload = self._project_fields(
|
|
716
|
+
lambda field: cast(FieldExportPayload, field).intensity
|
|
717
|
+
)
|
|
718
|
+
return BatchIntensityFieldExportPayload(
|
|
719
|
+
records=projected.records,
|
|
720
|
+
postprocess_index=self.postprocess_index,
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
def transform(self, name: str) -> "BatchFieldExportPayload":
|
|
724
|
+
"""Apply a mathematical change to all selected field values.
|
|
725
|
+
|
|
726
|
+
Parameters
|
|
727
|
+
----------
|
|
728
|
+
name : {"abs", "real", "imag", "phase"}
|
|
729
|
+
Mathematical change to apply.
|
|
730
|
+
|
|
731
|
+
Returns
|
|
732
|
+
-------
|
|
733
|
+
BatchFieldExportPayload
|
|
734
|
+
New batch field data with the change applied.
|
|
735
|
+
|
|
736
|
+
Raises
|
|
737
|
+
------
|
|
738
|
+
ValueError
|
|
739
|
+
If `name` is not supported.
|
|
740
|
+
"""
|
|
741
|
+
|
|
742
|
+
return self._project_fields(
|
|
743
|
+
lambda field: cast(FieldExportPayload, field).transform(name)
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
@dataclass(slots=True, frozen=True, repr=False)
|
|
748
|
+
class BatchIntensityFieldExportPayload(_BatchFieldViewPayload):
|
|
749
|
+
"""Store and export squared-norm field intensity across a batch.
|
|
750
|
+
|
|
751
|
+
:attr:`BatchFieldExportPayload.intensity` returns this payload. Each
|
|
752
|
+
available entry contains real-valued intensity fields, identified by its original
|
|
753
|
+
batch index. The payload supports base-field and field-position selection,
|
|
754
|
+
dictionary export, and xarray export. It does not support further intensity
|
|
755
|
+
or field transforms. Solver derivative access is not available for intensity
|
|
756
|
+
values.
|
|
757
|
+
|
|
758
|
+
Do not construct this payload directly.
|
|
759
|
+
"""
|
|
760
|
+
|
|
761
|
+
@property
|
|
762
|
+
def derivatives(self) -> Self:
|
|
763
|
+
"""Reject solver-derivative access for intensity fields.
|
|
764
|
+
|
|
765
|
+
Returns
|
|
766
|
+
-------
|
|
767
|
+
Self
|
|
768
|
+
This property does not return.
|
|
769
|
+
|
|
770
|
+
Raises
|
|
771
|
+
------
|
|
772
|
+
UnsupportedExportError
|
|
773
|
+
Always. Intensity fields do not expose solver derivatives.
|
|
774
|
+
"""
|
|
775
|
+
raise UnsupportedExportError(
|
|
776
|
+
"Derivatives are not available for intensity field views."
|
|
777
|
+
)
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def _stack_field_trees(
|
|
781
|
+
trees: list[XarrayDataTree],
|
|
782
|
+
*,
|
|
783
|
+
available: list[bool],
|
|
784
|
+
batch_indices: list[int],
|
|
785
|
+
xarray_module: _XarrayModule,
|
|
786
|
+
datatree_module: _DataTreeModule,
|
|
787
|
+
) -> XarrayDataTree:
|
|
788
|
+
"""Stack compatible field tree datasets with a leading batch dimension."""
|
|
789
|
+
|
|
790
|
+
import numpy as np
|
|
791
|
+
|
|
792
|
+
from ..export.xarray.runtime import _create_data_tree
|
|
793
|
+
|
|
794
|
+
def dimensions_and_values(value: object) -> tuple[tuple[str, ...], np.ndarray]:
|
|
795
|
+
"""Normalize real xarray DataArrays and the lightweight test adapter."""
|
|
796
|
+
|
|
797
|
+
if isinstance(value, tuple) and len(value) == 2:
|
|
798
|
+
dimensions, values = value
|
|
799
|
+
if isinstance(dimensions, tuple):
|
|
800
|
+
return tuple(str(dimension) for dimension in dimensions), np.asarray(
|
|
801
|
+
values
|
|
802
|
+
)
|
|
803
|
+
if isinstance(dimensions, str):
|
|
804
|
+
return (dimensions,), np.asarray(values)
|
|
805
|
+
dimensions = getattr(value, "dims", None)
|
|
806
|
+
values = getattr(value, "values", None)
|
|
807
|
+
if dimensions is None or values is None:
|
|
808
|
+
raise InvalidConfigurationError(
|
|
809
|
+
"Field tree contains an invalid array node."
|
|
810
|
+
)
|
|
811
|
+
return tuple(str(dimension) for dimension in dimensions), np.asarray(values)
|
|
812
|
+
|
|
813
|
+
first = trees[0]
|
|
814
|
+
first_dataset = getattr(first, "ds", None)
|
|
815
|
+
if first_dataset is None:
|
|
816
|
+
raise InvalidConfigurationError("Field tree has no root dataset.")
|
|
817
|
+
data_vars: dict[str, object] = {}
|
|
818
|
+
coords: dict[str, object] = {
|
|
819
|
+
"batch_index": batch_indices,
|
|
820
|
+
}
|
|
821
|
+
for name, value in first_dataset.data_vars.items():
|
|
822
|
+
dims, _first_array = dimensions_and_values(value)
|
|
823
|
+
arrays = [
|
|
824
|
+
dimensions_and_values(getattr(tree, "ds").data_vars[name])[1] # noqa: B009
|
|
825
|
+
for tree in trees
|
|
826
|
+
]
|
|
827
|
+
shape = tuple(
|
|
828
|
+
max(array.shape[axis] for array in arrays) for axis in range(arrays[0].ndim)
|
|
829
|
+
)
|
|
830
|
+
dtype = np.result_type(*[array.dtype for array in arrays], np.float64)
|
|
831
|
+
stacked = np.full((len(arrays), *shape), np.nan, dtype=dtype)
|
|
832
|
+
padded = np.zeros((len(arrays), *shape), dtype=bool)
|
|
833
|
+
for position, array in enumerate(arrays):
|
|
834
|
+
if not available[position]:
|
|
835
|
+
continue
|
|
836
|
+
target = stacked[position]
|
|
837
|
+
target[tuple(slice(0, size) for size in array.shape)] = array
|
|
838
|
+
padding = padded[position]
|
|
839
|
+
padding[...] = True
|
|
840
|
+
padding[tuple(slice(0, size) for size in array.shape)] = False
|
|
841
|
+
data_vars[name] = (("batch_index", *dims), stacked)
|
|
842
|
+
data_vars["is_padded"] = (("batch_index", *dims), padded)
|
|
843
|
+
for name, value in first_dataset.coords.items():
|
|
844
|
+
if isinstance(value, (list, np.ndarray)):
|
|
845
|
+
dims, array = (name,), np.asarray(value)
|
|
846
|
+
else:
|
|
847
|
+
dims, array = dimensions_and_values(value)
|
|
848
|
+
coordinate_arrays = [
|
|
849
|
+
(
|
|
850
|
+
np.asarray(getattr(tree, "ds").coords[name]) # noqa: B009
|
|
851
|
+
if isinstance(
|
|
852
|
+
getattr(tree, "ds").coords[name],
|
|
853
|
+
(list, np.ndarray), # noqa: B009
|
|
854
|
+
)
|
|
855
|
+
else dimensions_and_values(
|
|
856
|
+
getattr(tree, "ds").coords[name] # noqa: B009
|
|
857
|
+
)[1]
|
|
858
|
+
)
|
|
859
|
+
for tree in trees
|
|
860
|
+
]
|
|
861
|
+
coordinate_dtype = np.result_type(
|
|
862
|
+
*[coordinate.dtype for coordinate in coordinate_arrays]
|
|
863
|
+
)
|
|
864
|
+
if not np.issubdtype(coordinate_dtype, np.number):
|
|
865
|
+
coords[name] = (dims, array)
|
|
866
|
+
continue
|
|
867
|
+
shape = tuple(
|
|
868
|
+
max(coordinate.shape[axis] for coordinate in coordinate_arrays)
|
|
869
|
+
for axis in range(coordinate_arrays[0].ndim)
|
|
870
|
+
)
|
|
871
|
+
stacked_coordinates = np.full(
|
|
872
|
+
(len(coordinate_arrays), *shape),
|
|
873
|
+
np.nan,
|
|
874
|
+
dtype=np.result_type(coordinate_dtype, np.float64),
|
|
875
|
+
)
|
|
876
|
+
for position, coordinate in enumerate(coordinate_arrays):
|
|
877
|
+
if not available[position]:
|
|
878
|
+
continue
|
|
879
|
+
stacked_coordinates[position][
|
|
880
|
+
tuple(slice(0, size) for size in coordinate.shape)
|
|
881
|
+
] = coordinate
|
|
882
|
+
coords[name] = (("batch_index", *dims), stacked_coordinates)
|
|
883
|
+
valid = np.asarray(available, dtype=bool)
|
|
884
|
+
data_vars["is_valid"] = (("batch_index",), valid)
|
|
885
|
+
root = xarray_module.Dataset(
|
|
886
|
+
data_vars=data_vars, coords=coords, attrs=dict(first_dataset.attrs)
|
|
887
|
+
)
|
|
888
|
+
child_names = tuple(getattr(first, "children", {}))
|
|
889
|
+
children = {
|
|
890
|
+
name: _stack_field_trees(
|
|
891
|
+
[getattr(tree, "children")[name] for tree in trees], # noqa: B009
|
|
892
|
+
available=available,
|
|
893
|
+
batch_indices=batch_indices,
|
|
894
|
+
xarray_module=xarray_module,
|
|
895
|
+
datatree_module=datatree_module,
|
|
896
|
+
)
|
|
897
|
+
for name in child_names
|
|
898
|
+
}
|
|
899
|
+
return _create_data_tree(
|
|
900
|
+
datatree_module=datatree_module,
|
|
901
|
+
dataset=root,
|
|
902
|
+
children=children,
|
|
903
|
+
)
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
@dataclass(slots=True, frozen=True, repr=False)
|
|
907
|
+
class BatchPostprocessRecord(DictionaryExportMixin):
|
|
908
|
+
"""Provide one type of postprocess result across a batch.
|
|
909
|
+
|
|
910
|
+
Parameters
|
|
911
|
+
----------
|
|
912
|
+
records : tuple[BatchExportRecord, ...]
|
|
913
|
+
Postprocess results in batched project keys order.
|
|
914
|
+
index : int
|
|
915
|
+
Position of the selected postprocess result in each project result.
|
|
916
|
+
payload : BatchTablePayload | BatchStructuredPayload | BatchTableCollectionPayload | BatchFieldExportPayload
|
|
917
|
+
Data for the selected postprocess result.
|
|
918
|
+
|
|
919
|
+
Notes
|
|
920
|
+
-----
|
|
921
|
+
Application code receives this object after selecting a batch postprocess
|
|
922
|
+
result; it does not need to construct the object directly.
|
|
923
|
+
|
|
924
|
+
Available entries must contain the same type and structure of result data.
|
|
925
|
+
"""
|
|
926
|
+
|
|
927
|
+
records: tuple[BatchExportRecord, ...]
|
|
928
|
+
index: int
|
|
929
|
+
payload: (
|
|
930
|
+
BatchTablePayload
|
|
931
|
+
| BatchStructuredPayload
|
|
932
|
+
| BatchTableCollectionPayload
|
|
933
|
+
| BatchFieldExportPayload
|
|
934
|
+
)
|
|
935
|
+
|
|
936
|
+
def __repr__(self) -> str:
|
|
937
|
+
"""Return stream identity, availability, and a payload preview."""
|
|
938
|
+
|
|
939
|
+
titles: set[str] = set()
|
|
940
|
+
available_count: int = 0
|
|
941
|
+
for record in self.records:
|
|
942
|
+
if not isinstance(record.payload, PostprocessCollection):
|
|
943
|
+
continue
|
|
944
|
+
if not 0 <= self.index < len(record.payload):
|
|
945
|
+
continue
|
|
946
|
+
selected: PostprocessRecord = record.payload[self.index]
|
|
947
|
+
titles.add("<untitled>" if selected.title is None else selected.title)
|
|
948
|
+
if selected.available:
|
|
949
|
+
available_count += 1
|
|
950
|
+
title: str = (
|
|
951
|
+
"<untitled>"
|
|
952
|
+
if not titles
|
|
953
|
+
else next(iter(titles)) if len(titles) == 1 else "<mixed>"
|
|
954
|
+
)
|
|
955
|
+
missing_count: int = len(self.records) - available_count
|
|
956
|
+
status: str = (
|
|
957
|
+
"available"
|
|
958
|
+
if missing_count == 0
|
|
959
|
+
else "unavailable" if available_count == 0 else "partial"
|
|
960
|
+
)
|
|
961
|
+
lines: list[str] = [
|
|
962
|
+
"BatchPostprocessRecord(",
|
|
963
|
+
f" index: {self.index}",
|
|
964
|
+
f" title: {title}",
|
|
965
|
+
f" status: {status} ({available_count}/{len(self.records)})",
|
|
966
|
+
]
|
|
967
|
+
if isinstance(self.payload, (BatchTablePayload, BatchFieldExportPayload)):
|
|
968
|
+
lines.append(" preview:")
|
|
969
|
+
lines.extend(f" {line}" for line in self.payload._preview().splitlines())
|
|
970
|
+
return "\n".join((*lines, ")"))
|
|
971
|
+
|
|
972
|
+
@property
|
|
973
|
+
def sources(self) -> tuple[Source, ...]:
|
|
974
|
+
"""Return one source file for each batch entry.
|
|
975
|
+
|
|
976
|
+
An unavailable source file is `None`. The order matches the Batched
|
|
977
|
+
project keys.
|
|
978
|
+
|
|
979
|
+
Returns
|
|
980
|
+
-------
|
|
981
|
+
tuple[Path | None, ...]
|
|
982
|
+
Source files in batched project keys order.
|
|
983
|
+
"""
|
|
984
|
+
|
|
985
|
+
return self.payload.sources
|
|
986
|
+
|
|
987
|
+
def is_field_export(self) -> bool:
|
|
988
|
+
"""Check whether the selected postprocess result contains field data.
|
|
989
|
+
|
|
990
|
+
Returns
|
|
991
|
+
-------
|
|
992
|
+
bool
|
|
993
|
+
`True` if the result contains field data. Otherwise, `False`.
|
|
994
|
+
"""
|
|
995
|
+
|
|
996
|
+
return isinstance(self.payload, BatchFieldExportPayload)
|
|
997
|
+
|
|
998
|
+
@property
|
|
999
|
+
def header(self) -> BatchMetadataPayload:
|
|
1000
|
+
"""Return the postprocess headers in batched project keys order.
|
|
1001
|
+
|
|
1002
|
+
Returns
|
|
1003
|
+
-------
|
|
1004
|
+
BatchMetadataPayload
|
|
1005
|
+
Header information for each batch entry.
|
|
1006
|
+
"""
|
|
1007
|
+
|
|
1008
|
+
header_records: list[BatchExportRecord] = []
|
|
1009
|
+
record: BatchExportRecord
|
|
1010
|
+
for record in self.records:
|
|
1011
|
+
header_mapping: dict[str, object] = {}
|
|
1012
|
+
if isinstance(
|
|
1013
|
+
record.payload, PostprocessCollection
|
|
1014
|
+
) and 0 <= self.index < len(record.payload):
|
|
1015
|
+
selected = record.payload[self.index]
|
|
1016
|
+
header = selected.header
|
|
1017
|
+
if header is not None:
|
|
1018
|
+
header_mapping = {
|
|
1019
|
+
str(key): value for key, value in header.to_dict().items()
|
|
1020
|
+
}
|
|
1021
|
+
header_records.append(
|
|
1022
|
+
BatchExportRecord(
|
|
1023
|
+
index=record.index,
|
|
1024
|
+
keys=header_mapping,
|
|
1025
|
+
source=(
|
|
1026
|
+
None
|
|
1027
|
+
if not isinstance(record.payload, PostprocessCollection)
|
|
1028
|
+
or not 0 <= self.index < len(record.payload)
|
|
1029
|
+
else record.payload[self.index].source_file
|
|
1030
|
+
),
|
|
1031
|
+
)
|
|
1032
|
+
)
|
|
1033
|
+
return BatchPostprocessHeaderPayload(
|
|
1034
|
+
tuple(header_records), source_records=self.records, index=self.index
|
|
1035
|
+
)
|
|
1036
|
+
|
|
1037
|
+
def to_dict(self) -> Mapping[object, object]:
|
|
1038
|
+
"""Return result dictionaries for the available batch entries.
|
|
1039
|
+
|
|
1040
|
+
Returns
|
|
1041
|
+
-------
|
|
1042
|
+
Mapping[object, object]
|
|
1043
|
+
Result dictionaries indexed by their original batch positions.
|
|
1044
|
+
"""
|
|
1045
|
+
|
|
1046
|
+
return cast(DictionaryExportMixin, self.payload).to_dict()
|
|
1047
|
+
|
|
1048
|
+
def to_df(self, orient: str | None = None) -> pd.DataFrame:
|
|
1049
|
+
"""Return the selected batch postprocess stream as a DataFrame.
|
|
1050
|
+
|
|
1051
|
+
Parameters
|
|
1052
|
+
----------
|
|
1053
|
+
orient : {None, "wide", "long"}, optional
|
|
1054
|
+
Select the table layout. Batch postprocess data supports only the
|
|
1055
|
+
long layout. `None` selects the long layout.
|
|
1056
|
+
|
|
1057
|
+
Returns
|
|
1058
|
+
-------
|
|
1059
|
+
pandas.DataFrame
|
|
1060
|
+
Selected postprocess data in long form.
|
|
1061
|
+
|
|
1062
|
+
Raises
|
|
1063
|
+
------
|
|
1064
|
+
UnsupportedExportError
|
|
1065
|
+
If the selected postprocess contains field data or `orient` is
|
|
1066
|
+
`"wide"`.
|
|
1067
|
+
ValueError
|
|
1068
|
+
If `orient` is not `None`, `"wide"`, or `"long"`.
|
|
1069
|
+
"""
|
|
1070
|
+
|
|
1071
|
+
if orient not in (None, "wide", "long"):
|
|
1072
|
+
raise ValueError("orient must be None, 'wide', or 'long'")
|
|
1073
|
+
if isinstance(self.payload, BatchFieldExportPayload):
|
|
1074
|
+
raise UnsupportedExportError(
|
|
1075
|
+
"to_df is not supported for `BatchFieldExportPayload`."
|
|
1076
|
+
)
|
|
1077
|
+
if orient == "wide":
|
|
1078
|
+
raise UnsupportedExportError(
|
|
1079
|
+
"to_df is not supported for wide batch postprocess exports."
|
|
1080
|
+
)
|
|
1081
|
+
return self.payload.to_df(orient="long")
|
|
1082
|
+
|
|
1083
|
+
def _to_df_long(self) -> pd.DataFrame:
|
|
1084
|
+
"""Return available result data as a long-form DataFrame.
|
|
1085
|
+
|
|
1086
|
+
Returns
|
|
1087
|
+
-------
|
|
1088
|
+
pandas.DataFrame
|
|
1089
|
+
Rows that include the original batch position and postprocess
|
|
1090
|
+
position.
|
|
1091
|
+
|
|
1092
|
+
Raises
|
|
1093
|
+
------
|
|
1094
|
+
UnsupportedExportError
|
|
1095
|
+
If the selected postprocess result contains field data.
|
|
1096
|
+
"""
|
|
1097
|
+
|
|
1098
|
+
if isinstance(self.payload, BatchFieldExportPayload):
|
|
1099
|
+
raise UnsupportedExportError(
|
|
1100
|
+
"to_df is not supported for `BatchFieldExportPayload`."
|
|
1101
|
+
)
|
|
1102
|
+
return cast(DataFrameLongExportMixin, self.payload)._to_df_long()
|
|
1103
|
+
|
|
1104
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
1105
|
+
"""Return the selected batch result as an xarray `DataTree`.
|
|
1106
|
+
|
|
1107
|
+
Returns
|
|
1108
|
+
-------
|
|
1109
|
+
xarray.DataTree
|
|
1110
|
+
Result data in batched project keys order.
|
|
1111
|
+
|
|
1112
|
+
Raises
|
|
1113
|
+
------
|
|
1114
|
+
InvalidConfigurationError
|
|
1115
|
+
If available result data cannot be combined in one tree.
|
|
1116
|
+
"""
|
|
1117
|
+
|
|
1118
|
+
payload_mapping: Mapping[object, object] = cast(
|
|
1119
|
+
DictionaryExportMixin, self.payload
|
|
1120
|
+
).to_dict()
|
|
1121
|
+
is_powerflux: bool = any(
|
|
1122
|
+
isinstance(table, Mapping)
|
|
1123
|
+
and any("PowerFluxDensity" in str(key) for key in table)
|
|
1124
|
+
for table in payload_mapping.values()
|
|
1125
|
+
)
|
|
1126
|
+
if is_powerflux:
|
|
1127
|
+
# Preserve the established PowerFlux tree topology while the
|
|
1128
|
+
# record keeps the BatchPostprocessRecord return type.
|
|
1129
|
+
return _BatchPostprocessSelection(
|
|
1130
|
+
records=self.records, index=self.index
|
|
1131
|
+
).to_xarray()
|
|
1132
|
+
return self.payload.to_xarray()
|
|
1133
|
+
|
|
1134
|
+
def to_powerflux(self) -> BatchPostprocessRecord:
|
|
1135
|
+
"""Convert the selected result to PowerFlux data.
|
|
1136
|
+
|
|
1137
|
+
Returns
|
|
1138
|
+
-------
|
|
1139
|
+
BatchPostprocessRecord
|
|
1140
|
+
Converted data in batched project keys order.
|
|
1141
|
+
|
|
1142
|
+
Raises
|
|
1143
|
+
------
|
|
1144
|
+
InvalidConfigurationError
|
|
1145
|
+
If an available entry does not contain the required Fourier table
|
|
1146
|
+
data.
|
|
1147
|
+
"""
|
|
1148
|
+
|
|
1149
|
+
converted_records: list[BatchExportRecord] = []
|
|
1150
|
+
batch_record: BatchExportRecord
|
|
1151
|
+
for batch_record in self.records:
|
|
1152
|
+
payload: PostprocessCollection | None = None
|
|
1153
|
+
if isinstance(batch_record.payload, PostprocessCollection):
|
|
1154
|
+
collection: PostprocessCollection = batch_record.payload
|
|
1155
|
+
if 0 <= self.index < len(collection):
|
|
1156
|
+
converted_record: PostprocessRecord = collection[
|
|
1157
|
+
self.index
|
|
1158
|
+
].to_powerflux()
|
|
1159
|
+
converted_collection_records: tuple[PostprocessRecord, ...] = (
|
|
1160
|
+
*collection.records[: self.index],
|
|
1161
|
+
converted_record,
|
|
1162
|
+
)
|
|
1163
|
+
payload = PostprocessCollection(
|
|
1164
|
+
records=converted_collection_records
|
|
1165
|
+
)
|
|
1166
|
+
else:
|
|
1167
|
+
payload = collection
|
|
1168
|
+
converted_records.append(
|
|
1169
|
+
BatchExportRecord(
|
|
1170
|
+
index=batch_record.index,
|
|
1171
|
+
keys=dict(batch_record.keys),
|
|
1172
|
+
payload=payload,
|
|
1173
|
+
source=batch_record.source,
|
|
1174
|
+
)
|
|
1175
|
+
)
|
|
1176
|
+
return _build_batch_postprocess_record(tuple(converted_records), self.index)
|
|
1177
|
+
|
|
1178
|
+
@property
|
|
1179
|
+
def fields(self) -> BatchFieldExportPayload:
|
|
1180
|
+
"""Return field data for the selected batch result.
|
|
1181
|
+
|
|
1182
|
+
Returns
|
|
1183
|
+
-------
|
|
1184
|
+
BatchFieldExportPayload
|
|
1185
|
+
Field data in batched project keys order.
|
|
1186
|
+
|
|
1187
|
+
Raises
|
|
1188
|
+
------
|
|
1189
|
+
UnsupportedExportError
|
|
1190
|
+
If the selected postprocess result does not contain field data.
|
|
1191
|
+
"""
|
|
1192
|
+
|
|
1193
|
+
if isinstance(self.payload, BatchFieldExportPayload):
|
|
1194
|
+
return self.payload
|
|
1195
|
+
raise UnsupportedExportError(
|
|
1196
|
+
f"fields is not supported for `{type(self.payload).__name__}`."
|
|
1197
|
+
)
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
class BatchPostprocessHeaderPayload(BatchMetadataPayload):
|
|
1201
|
+
"""Batch metadata payload retaining the selected-header export seam."""
|
|
1202
|
+
|
|
1203
|
+
def __init__(
|
|
1204
|
+
self,
|
|
1205
|
+
records: tuple[BatchExportRecord, ...],
|
|
1206
|
+
*,
|
|
1207
|
+
source_records: tuple[BatchExportRecord, ...],
|
|
1208
|
+
index: int,
|
|
1209
|
+
) -> None:
|
|
1210
|
+
super().__init__(records)
|
|
1211
|
+
self._source_records = source_records
|
|
1212
|
+
self._index = index
|
|
1213
|
+
|
|
1214
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
1215
|
+
"""Export selected headers with the established xarray schema."""
|
|
1216
|
+
|
|
1217
|
+
from ..export.xarray.export import (
|
|
1218
|
+
batch_selected_postprocess_headers_to_xarray_dataset,
|
|
1219
|
+
)
|
|
1220
|
+
|
|
1221
|
+
return batch_selected_postprocess_headers_to_xarray_dataset(
|
|
1222
|
+
self._source_records, index=self._index
|
|
1223
|
+
)
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
def _build_batch_postprocess_record(
|
|
1227
|
+
records: tuple[BatchExportRecord, ...], index: int
|
|
1228
|
+
) -> BatchPostprocessRecord:
|
|
1229
|
+
"""Normalize one selected stream and reject family/schema mismatches."""
|
|
1230
|
+
|
|
1231
|
+
selected_records: list[BatchExportRecord] = []
|
|
1232
|
+
first_family: str | None = None
|
|
1233
|
+
first_schema: tuple[object, ...] | None = None
|
|
1234
|
+
for batch_record in records:
|
|
1235
|
+
selected: PostprocessRecord | None = None
|
|
1236
|
+
if isinstance(batch_record.payload, PostprocessCollection) and 0 <= index < len(
|
|
1237
|
+
batch_record.payload
|
|
1238
|
+
):
|
|
1239
|
+
selected = batch_record.payload[index]
|
|
1240
|
+
payload: object | None = None
|
|
1241
|
+
if selected is not None and selected.available:
|
|
1242
|
+
schema: tuple[object, ...]
|
|
1243
|
+
if isinstance(selected.payload, TableCollectionPayload):
|
|
1244
|
+
family = "collection"
|
|
1245
|
+
collection_mapping = cast(
|
|
1246
|
+
DictionaryExportMixin, selected.payload
|
|
1247
|
+
).to_dict()
|
|
1248
|
+
schema = tuple(
|
|
1249
|
+
(str(name), tuple(cast(Mapping[object, object], table)))
|
|
1250
|
+
for name, table in collection_mapping.items()
|
|
1251
|
+
)
|
|
1252
|
+
elif isinstance(selected.payload, TablePayload):
|
|
1253
|
+
family = "table"
|
|
1254
|
+
schema = tuple(
|
|
1255
|
+
str(key)
|
|
1256
|
+
for key in cast(DictionaryExportMixin, selected.payload).to_dict()
|
|
1257
|
+
)
|
|
1258
|
+
elif isinstance(selected.payload, StructuredPayload):
|
|
1259
|
+
family = "structured"
|
|
1260
|
+
schema = tuple(
|
|
1261
|
+
path
|
|
1262
|
+
for path, _ in flatten_payload(
|
|
1263
|
+
cast(DictionaryExportMixin, selected.payload).to_dict(),
|
|
1264
|
+
path_prefix="",
|
|
1265
|
+
)
|
|
1266
|
+
)
|
|
1267
|
+
elif isinstance(selected.payload, FieldExportPayload):
|
|
1268
|
+
family = "field"
|
|
1269
|
+
schema = _field_schema(selected.payload)
|
|
1270
|
+
else:
|
|
1271
|
+
raise InvalidConfigurationError(
|
|
1272
|
+
"Selected batch postprocess stream mixes incompatible payload families."
|
|
1273
|
+
)
|
|
1274
|
+
payload = selected.payload
|
|
1275
|
+
if first_family is None:
|
|
1276
|
+
first_family = family
|
|
1277
|
+
first_schema = schema
|
|
1278
|
+
elif family != first_family or schema != first_schema:
|
|
1279
|
+
raise InvalidConfigurationError(
|
|
1280
|
+
"Selected batch postprocess stream mixes incompatible payload schemas."
|
|
1281
|
+
)
|
|
1282
|
+
selected_records.append(
|
|
1283
|
+
BatchExportRecord(
|
|
1284
|
+
index=batch_record.index,
|
|
1285
|
+
keys=dict(batch_record.keys),
|
|
1286
|
+
payload=payload,
|
|
1287
|
+
source=(None if selected is None else selected.source_file),
|
|
1288
|
+
)
|
|
1289
|
+
)
|
|
1290
|
+
if first_family is None:
|
|
1291
|
+
raise InvalidConfigurationError(
|
|
1292
|
+
f"No supported postprocess records found for index {index!r}."
|
|
1293
|
+
)
|
|
1294
|
+
payload_type: type[
|
|
1295
|
+
BatchTablePayload
|
|
1296
|
+
| BatchStructuredPayload
|
|
1297
|
+
| BatchTableCollectionPayload
|
|
1298
|
+
| BatchFieldExportPayload
|
|
1299
|
+
]
|
|
1300
|
+
if first_family == "table":
|
|
1301
|
+
payload_type = _BatchPostprocessTablePayload
|
|
1302
|
+
elif first_family == "structured":
|
|
1303
|
+
payload_type = _BatchStructuredPayload
|
|
1304
|
+
elif first_family == "collection":
|
|
1305
|
+
payload_type = _BatchTableCollectionPayload
|
|
1306
|
+
else:
|
|
1307
|
+
payload_type = BatchFieldExportPayload
|
|
1308
|
+
payload_object: (
|
|
1309
|
+
BatchTablePayload
|
|
1310
|
+
| BatchStructuredPayload
|
|
1311
|
+
| BatchTableCollectionPayload
|
|
1312
|
+
| BatchFieldExportPayload
|
|
1313
|
+
) = payload_type(tuple(selected_records), postprocess_index=index)
|
|
1314
|
+
return BatchPostprocessRecord(
|
|
1315
|
+
records=records,
|
|
1316
|
+
index=index,
|
|
1317
|
+
payload=payload_object,
|
|
1318
|
+
)
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
def _field_schema(payload: FieldExportPayload) -> tuple[object, ...]:
|
|
1322
|
+
"""Return the compatible field topology without fixing varying axis sizes."""
|
|
1323
|
+
|
|
1324
|
+
import numpy as np
|
|
1325
|
+
|
|
1326
|
+
base_mapping: Mapping[object, object] = payload.base.to_dict()
|
|
1327
|
+
coordinate_topology = resolve_coordinates(base_mapping)
|
|
1328
|
+
coordinate_names: tuple[str, ...] = coordinate_topology.names
|
|
1329
|
+
coordinate_ranks: tuple[int, ...] = tuple(
|
|
1330
|
+
int(np.asarray(base_mapping[name]).ndim) for name in coordinate_names
|
|
1331
|
+
)
|
|
1332
|
+
field_value: object = base_mapping["field"]
|
|
1333
|
+
field_entries: tuple[object, ...] = (
|
|
1334
|
+
tuple(field_value) if isinstance(field_value, (list, tuple)) else (field_value,)
|
|
1335
|
+
)
|
|
1336
|
+
field_ranks: tuple[int, ...] = tuple(
|
|
1337
|
+
int(np.asarray(entry).ndim) for entry in field_entries
|
|
1338
|
+
)
|
|
1339
|
+
base_names: tuple[str, ...] = tuple(
|
|
1340
|
+
str(name)
|
|
1341
|
+
for name in base_mapping
|
|
1342
|
+
if name not in {*coordinate_names, "field", "title"}
|
|
1343
|
+
)
|
|
1344
|
+
# Solver derivative availability is entry-specific. Batch navigation keeps
|
|
1345
|
+
# each payload intact and performs derivative lookup per original index, so
|
|
1346
|
+
# derivative branch differences do not make otherwise compatible field
|
|
1347
|
+
# topologies incompatible.
|
|
1348
|
+
return (
|
|
1349
|
+
coordinate_names,
|
|
1350
|
+
coordinate_ranks,
|
|
1351
|
+
base_names,
|
|
1352
|
+
len(field_entries),
|
|
1353
|
+
field_ranks,
|
|
1354
|
+
)
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
@dataclass(slots=True, frozen=True)
|
|
1358
|
+
class _BatchPostprocessSelection(DictionaryExportMixin):
|
|
1359
|
+
"""Selected batch postprocess dataframe accessor.
|
|
1360
|
+
|
|
1361
|
+
Notes
|
|
1362
|
+
-----
|
|
1363
|
+
Internal API: fallback helper for streams that cannot be normalized into a
|
|
1364
|
+
typed ``BatchPostprocessRecord``.
|
|
1365
|
+
This view is DataFrame-first: use ``to_df(...)`` for aligned batch exports.
|
|
1366
|
+
It intentionally does not expose a dense numeric ``values`` vector because
|
|
1367
|
+
selected payloads can be heterogeneous across entries.
|
|
1368
|
+
"""
|
|
1369
|
+
|
|
1370
|
+
records: tuple[BatchExportRecord, ...]
|
|
1371
|
+
index: int
|
|
1372
|
+
_resolved_records: tuple[_ResolvedBatchPostprocessRecord, ...] | None = field(
|
|
1373
|
+
default=None,
|
|
1374
|
+
repr=False,
|
|
1375
|
+
)
|
|
1376
|
+
|
|
1377
|
+
def to_dict(self) -> Mapping[object, object]:
|
|
1378
|
+
"""Return the selected postprocess payloads keyed by batch entry index."""
|
|
1379
|
+
|
|
1380
|
+
selected_mapping: dict[object, object] = {}
|
|
1381
|
+
resolved_record: _ResolvedBatchPostprocessRecord
|
|
1382
|
+
for resolved_record in self._resolve_selected_records():
|
|
1383
|
+
payload_mapping: Mapping[object, object] | None = (
|
|
1384
|
+
resolved_record.record.to_dict()
|
|
1385
|
+
)
|
|
1386
|
+
selected_mapping[resolved_record.entry.index] = payload_mapping
|
|
1387
|
+
return selected_mapping
|
|
1388
|
+
|
|
1389
|
+
@property
|
|
1390
|
+
def header(self) -> _BatchPostprocessHeaderSelection:
|
|
1391
|
+
"""Return a selected batch-postprocess header export view."""
|
|
1392
|
+
|
|
1393
|
+
return _BatchPostprocessHeaderSelection(
|
|
1394
|
+
records=self.records,
|
|
1395
|
+
index=self.index,
|
|
1396
|
+
_resolved_records=self._resolve_selected_records(),
|
|
1397
|
+
)
|
|
1398
|
+
|
|
1399
|
+
def to_df(
|
|
1400
|
+
self,
|
|
1401
|
+
*,
|
|
1402
|
+
orient: str | None = None,
|
|
1403
|
+
warn_skipped: bool = True,
|
|
1404
|
+
expand_rows: bool = False,
|
|
1405
|
+
) -> pd.DataFrame:
|
|
1406
|
+
"""Export selected batch postprocess values as a DataFrame.
|
|
1407
|
+
|
|
1408
|
+
Parameters
|
|
1409
|
+
----------
|
|
1410
|
+
warn_skipped : bool, optional
|
|
1411
|
+
Emit warnings when non-tabular or field-export payloads are skipped.
|
|
1412
|
+
expand_rows : bool, optional
|
|
1413
|
+
Compatibility flag retained for API stability. Postprocess dataframe
|
|
1414
|
+
exports now use the same long-form row model regardless of this value.
|
|
1415
|
+
|
|
1416
|
+
Returns
|
|
1417
|
+
-------
|
|
1418
|
+
pandas.DataFrame
|
|
1419
|
+
DataFrame export for the selected batch postprocess stream.
|
|
1420
|
+
"""
|
|
1421
|
+
|
|
1422
|
+
if orient not in (None, "wide", "long"):
|
|
1423
|
+
raise ValueError("orient must be None, 'wide', or 'long'")
|
|
1424
|
+
if orient == "wide":
|
|
1425
|
+
raise UnsupportedExportError(
|
|
1426
|
+
"to_df is not supported for wide batch postprocess exports."
|
|
1427
|
+
)
|
|
1428
|
+
if self._resolved_records is not None:
|
|
1429
|
+
from ...pandas_export import _load_pandas_module
|
|
1430
|
+
from ..export.pandas.common import expanded_keys as _expanded_keys
|
|
1431
|
+
from ..export.pandas.common import normalize_optional_index_columns
|
|
1432
|
+
|
|
1433
|
+
pandas_module = _load_pandas_module()
|
|
1434
|
+
rows: list[dict[str, object]] = []
|
|
1435
|
+
resolved_record: _ResolvedBatchPostprocessRecord
|
|
1436
|
+
for resolved_record in self._resolved_records:
|
|
1437
|
+
postprocess_record = resolved_record.record
|
|
1438
|
+
if not postprocess_record.available:
|
|
1439
|
+
raise InvalidConfigurationError(
|
|
1440
|
+
f"Selected postprocess index {resolved_record.postprocess_index!r} is unavailable for batch entry {resolved_record.entry.index}."
|
|
1441
|
+
)
|
|
1442
|
+
tabular_rows = dataframe_rows(postprocess_record.to_df(orient="long"))
|
|
1443
|
+
|
|
1444
|
+
key_columns: dict[str, object] = _expanded_keys(
|
|
1445
|
+
resolved_record.entry.keys
|
|
1446
|
+
)
|
|
1447
|
+
row_data: dict[str, object]
|
|
1448
|
+
for row_data in tabular_rows:
|
|
1449
|
+
merged_row: dict[str, object] = {
|
|
1450
|
+
"index": resolved_record.entry.index,
|
|
1451
|
+
"postprocess_index": resolved_record.postprocess_index,
|
|
1452
|
+
"postprocess_title": postprocess_record.title,
|
|
1453
|
+
"source_file": str(postprocess_record.source_file),
|
|
1454
|
+
}
|
|
1455
|
+
merged_row.update(key_columns)
|
|
1456
|
+
merged_row.update(row_data)
|
|
1457
|
+
rows.append(merged_row)
|
|
1458
|
+
dataframe = pandas_module.DataFrame(rows)
|
|
1459
|
+
return cast("pd.DataFrame", normalize_optional_index_columns(dataframe))
|
|
1460
|
+
|
|
1461
|
+
from ...pandas_export import batch_postprocesses_to_df
|
|
1462
|
+
|
|
1463
|
+
return batch_postprocesses_to_df(
|
|
1464
|
+
self.records,
|
|
1465
|
+
index=self.index,
|
|
1466
|
+
warn_skipped=warn_skipped,
|
|
1467
|
+
expand_rows=expand_rows,
|
|
1468
|
+
)
|
|
1469
|
+
|
|
1470
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
1471
|
+
"""Export the selected batch postprocess stream as one `DataTree`.
|
|
1472
|
+
|
|
1473
|
+
Returns
|
|
1474
|
+
-------
|
|
1475
|
+
xarray.DataTree
|
|
1476
|
+
Uniform stacked tree with leading ``index`` dimension inside each
|
|
1477
|
+
node dataset plus the grouped postprocess dimensions inherited from
|
|
1478
|
+
``PostprocessRecord.to_xarray()``.
|
|
1479
|
+
|
|
1480
|
+
Raises
|
|
1481
|
+
------
|
|
1482
|
+
InvalidConfigurationError
|
|
1483
|
+
If the selected stream is missing on a successful batch entry, or
|
|
1484
|
+
the grouped payloads cannot be stacked under one tensor contract.
|
|
1485
|
+
"""
|
|
1486
|
+
|
|
1487
|
+
if self._resolved_records is not None:
|
|
1488
|
+
from ..export.xarray import export as _xarray_export_module
|
|
1489
|
+
from ..export.xarray.export import (
|
|
1490
|
+
_record_xarray_payload,
|
|
1491
|
+
_stack_batch_postprocess_payloads_to_data_tree,
|
|
1492
|
+
)
|
|
1493
|
+
|
|
1494
|
+
xarray_export_module = cast(
|
|
1495
|
+
_XarrayExportLoaderModule,
|
|
1496
|
+
_xarray_export_module,
|
|
1497
|
+
)
|
|
1498
|
+
xarray_module = xarray_export_module._load_xarray_module()
|
|
1499
|
+
datatree_module = xarray_export_module._load_datatree_module()
|
|
1500
|
+
selected_entries: list[BatchExportRecord] = []
|
|
1501
|
+
payloads: list[Mapping[object, object]] = []
|
|
1502
|
+
selected_title: str | None = None
|
|
1503
|
+
resolved_record: _ResolvedBatchPostprocessRecord
|
|
1504
|
+
for resolved_record in self._resolved_records:
|
|
1505
|
+
postprocess_record = resolved_record.record
|
|
1506
|
+
if not postprocess_record.available:
|
|
1507
|
+
raise InvalidConfigurationError(
|
|
1508
|
+
f"Selected postprocess index {resolved_record.postprocess_index!r} is unavailable for batch entry {resolved_record.entry.index}."
|
|
1509
|
+
)
|
|
1510
|
+
if selected_title is None:
|
|
1511
|
+
selected_title = postprocess_record.title
|
|
1512
|
+
payload_object: object = _record_xarray_payload(postprocess_record)
|
|
1513
|
+
if not isinstance(payload_object, Mapping):
|
|
1514
|
+
raise InvalidConfigurationError(
|
|
1515
|
+
"Postprocess payload is not mapping-like and cannot be exported via to_xarray()."
|
|
1516
|
+
)
|
|
1517
|
+
payloads.append(payload_object)
|
|
1518
|
+
selected_entries.append(resolved_record.entry)
|
|
1519
|
+
|
|
1520
|
+
if not payloads:
|
|
1521
|
+
raise InvalidConfigurationError(
|
|
1522
|
+
"No successful batch entries with the selected postprocess payload are available for to_xarray()."
|
|
1523
|
+
)
|
|
1524
|
+
|
|
1525
|
+
return _stack_batch_postprocess_payloads_to_data_tree(
|
|
1526
|
+
selected_entries,
|
|
1527
|
+
payloads,
|
|
1528
|
+
index=self.index,
|
|
1529
|
+
title=selected_title,
|
|
1530
|
+
xarray_module=xarray_module,
|
|
1531
|
+
datatree_module=datatree_module,
|
|
1532
|
+
)
|
|
1533
|
+
|
|
1534
|
+
from ..export.xarray.export import (
|
|
1535
|
+
batch_selected_postprocesses_to_xarray_dataset,
|
|
1536
|
+
)
|
|
1537
|
+
|
|
1538
|
+
return batch_selected_postprocesses_to_xarray_dataset(
|
|
1539
|
+
self.records,
|
|
1540
|
+
index=self.index,
|
|
1541
|
+
)
|
|
1542
|
+
|
|
1543
|
+
def to_powerflux(self) -> _BatchPostprocessSelection:
|
|
1544
|
+
"""Convert the selected batch postprocess stream to power-flux records."""
|
|
1545
|
+
|
|
1546
|
+
resolved_records: tuple[_ResolvedBatchPostprocessRecord, ...] = (
|
|
1547
|
+
self._resolve_selected_records()
|
|
1548
|
+
)
|
|
1549
|
+
converted_records: list[_ResolvedBatchPostprocessRecord] = []
|
|
1550
|
+
resolved_record: _ResolvedBatchPostprocessRecord
|
|
1551
|
+
for resolved_record in resolved_records:
|
|
1552
|
+
if not resolved_record.record.available:
|
|
1553
|
+
raise InvalidConfigurationError(
|
|
1554
|
+
f"Selected postprocess index {resolved_record.postprocess_index!r} is unavailable for batch entry {resolved_record.entry.index}."
|
|
1555
|
+
)
|
|
1556
|
+
converted_records.append(
|
|
1557
|
+
_ResolvedBatchPostprocessRecord(
|
|
1558
|
+
entry=resolved_record.entry,
|
|
1559
|
+
postprocess_index=resolved_record.postprocess_index,
|
|
1560
|
+
record=resolved_record.record.to_powerflux(),
|
|
1561
|
+
)
|
|
1562
|
+
)
|
|
1563
|
+
return _BatchPostprocessSelection(
|
|
1564
|
+
records=self.records,
|
|
1565
|
+
index=self.index,
|
|
1566
|
+
_resolved_records=tuple(converted_records),
|
|
1567
|
+
)
|
|
1568
|
+
|
|
1569
|
+
def _resolve_selected_records(self) -> tuple[_ResolvedBatchPostprocessRecord, ...]:
|
|
1570
|
+
"""Resolve the selected postprocess record on each successful batch entry."""
|
|
1571
|
+
|
|
1572
|
+
if self._resolved_records is not None:
|
|
1573
|
+
return self._resolved_records
|
|
1574
|
+
|
|
1575
|
+
resolved_records: list[_ResolvedBatchPostprocessRecord] = []
|
|
1576
|
+
batch_record: BatchExportRecord
|
|
1577
|
+
for batch_record in self.records:
|
|
1578
|
+
if not isinstance(batch_record.payload, PostprocessCollection):
|
|
1579
|
+
continue
|
|
1580
|
+
postprocess_index = self.index
|
|
1581
|
+
if postprocess_index < 0 or postprocess_index >= len(batch_record.payload):
|
|
1582
|
+
raise InvalidConfigurationError(
|
|
1583
|
+
f"Selected postprocess index {postprocess_index!r} is missing for batch entry {batch_record.index}."
|
|
1584
|
+
)
|
|
1585
|
+
selected_record = batch_record.payload[postprocess_index]
|
|
1586
|
+
|
|
1587
|
+
resolved_records.append(
|
|
1588
|
+
_ResolvedBatchPostprocessRecord(
|
|
1589
|
+
entry=batch_record,
|
|
1590
|
+
postprocess_index=postprocess_index,
|
|
1591
|
+
record=selected_record,
|
|
1592
|
+
)
|
|
1593
|
+
)
|
|
1594
|
+
return tuple(resolved_records)
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
@dataclass(slots=True, frozen=True)
|
|
1598
|
+
class _ResolvedBatchPostprocessRecord:
|
|
1599
|
+
"""Internal resolved selected postprocess record for one batch entry."""
|
|
1600
|
+
|
|
1601
|
+
entry: BatchExportRecord
|
|
1602
|
+
postprocess_index: int
|
|
1603
|
+
record: PostprocessRecord
|
|
1604
|
+
|
|
1605
|
+
|
|
1606
|
+
@dataclass(slots=True, frozen=True)
|
|
1607
|
+
class _BatchPostprocessHeaderSelection(DictionaryExportMixin):
|
|
1608
|
+
"""Header export view for one selected batch postprocess stream."""
|
|
1609
|
+
|
|
1610
|
+
records: tuple[BatchExportRecord, ...]
|
|
1611
|
+
index: int
|
|
1612
|
+
_resolved_records: tuple[_ResolvedBatchPostprocessRecord, ...] = field(
|
|
1613
|
+
default_factory=tuple,
|
|
1614
|
+
repr=False,
|
|
1615
|
+
)
|
|
1616
|
+
|
|
1617
|
+
def to_dict(self) -> Mapping[object, object]:
|
|
1618
|
+
"""Return the batch-aligned header rows as a mapping by entry index."""
|
|
1619
|
+
|
|
1620
|
+
header_mapping: dict[object, object] = {}
|
|
1621
|
+
resolved_record: _ResolvedBatchPostprocessRecord
|
|
1622
|
+
for resolved_record in self._resolved_records:
|
|
1623
|
+
header_payload: PostprocessHeader | None = resolved_record.record.header
|
|
1624
|
+
header_mapping[resolved_record.entry.index] = {
|
|
1625
|
+
"title": resolved_record.record.title,
|
|
1626
|
+
"header": (
|
|
1627
|
+
None if header_payload is None else header_payload.to_dict()
|
|
1628
|
+
),
|
|
1629
|
+
}
|
|
1630
|
+
return header_mapping
|
|
1631
|
+
|
|
1632
|
+
def items(
|
|
1633
|
+
self,
|
|
1634
|
+
) -> tuple[tuple[int, dict[str, object], PostprocessHeader | None], ...]:
|
|
1635
|
+
"""Return selected headers in batch-entry order."""
|
|
1636
|
+
|
|
1637
|
+
return tuple(
|
|
1638
|
+
(
|
|
1639
|
+
resolved_record.entry.index,
|
|
1640
|
+
resolved_record.entry.keys,
|
|
1641
|
+
resolved_record.record.header,
|
|
1642
|
+
)
|
|
1643
|
+
for resolved_record in self._resolved_records
|
|
1644
|
+
)
|
|
1645
|
+
|
|
1646
|
+
def to_df(self, orient: str | None = None) -> pd.DataFrame:
|
|
1647
|
+
"""Export selected batch postprocess headers as a long-form DataFrame."""
|
|
1648
|
+
|
|
1649
|
+
if orient not in (None, "wide", "long"):
|
|
1650
|
+
raise ValueError("orient must be None, 'wide', or 'long'")
|
|
1651
|
+
if orient == "wide":
|
|
1652
|
+
raise UnsupportedExportError(
|
|
1653
|
+
"to_df is not supported for wide batch postprocess headers."
|
|
1654
|
+
)
|
|
1655
|
+
|
|
1656
|
+
from ...pandas_export import _load_pandas_module
|
|
1657
|
+
from ..export.pandas.common import expanded_keys as _expanded_keys
|
|
1658
|
+
from ..export.pandas.common import normalize_optional_index_columns
|
|
1659
|
+
from ..export.pandas.payload import postprocess_header_to_long_rows
|
|
1660
|
+
|
|
1661
|
+
pandas_module = _load_pandas_module()
|
|
1662
|
+
rows: list[dict[str, object]] = []
|
|
1663
|
+
resolved_record: _ResolvedBatchPostprocessRecord
|
|
1664
|
+
for resolved_record in self._resolved_records:
|
|
1665
|
+
postprocess_record = resolved_record.record
|
|
1666
|
+
if not postprocess_record.available:
|
|
1667
|
+
raise InvalidConfigurationError(
|
|
1668
|
+
f"Selected postprocess index {resolved_record.postprocess_index!r} is unavailable for batch entry {resolved_record.entry.index}."
|
|
1669
|
+
)
|
|
1670
|
+
base_row: dict[str, object] = {
|
|
1671
|
+
"index": resolved_record.entry.index,
|
|
1672
|
+
"postprocess_index": resolved_record.postprocess_index,
|
|
1673
|
+
"postprocess_title": postprocess_record.title,
|
|
1674
|
+
"source_file": str(postprocess_record.source_file),
|
|
1675
|
+
}
|
|
1676
|
+
base_row.update(_expanded_keys(resolved_record.entry.keys))
|
|
1677
|
+
header_payload: PostprocessHeader | None = postprocess_record.header
|
|
1678
|
+
if header_payload is None:
|
|
1679
|
+
merged_row: dict[str, object] = dict(base_row)
|
|
1680
|
+
merged_row["title"] = postprocess_record.title
|
|
1681
|
+
merged_row.update(
|
|
1682
|
+
{
|
|
1683
|
+
"field_name": None,
|
|
1684
|
+
"field_index": None,
|
|
1685
|
+
"value": None,
|
|
1686
|
+
}
|
|
1687
|
+
)
|
|
1688
|
+
rows.append(merged_row)
|
|
1689
|
+
continue
|
|
1690
|
+
|
|
1691
|
+
header_rows: list[dict[str, object]] | None = (
|
|
1692
|
+
postprocess_header_to_long_rows(header_payload.table)
|
|
1693
|
+
)
|
|
1694
|
+
if header_rows is None:
|
|
1695
|
+
raise InvalidConfigurationError(
|
|
1696
|
+
"Selected postprocess header payload is not tabular."
|
|
1697
|
+
)
|
|
1698
|
+
if not header_rows:
|
|
1699
|
+
merged_row = dict(base_row)
|
|
1700
|
+
merged_row.update(
|
|
1701
|
+
{
|
|
1702
|
+
"field_name": None,
|
|
1703
|
+
"field_index": None,
|
|
1704
|
+
"value": None,
|
|
1705
|
+
}
|
|
1706
|
+
)
|
|
1707
|
+
rows.append(merged_row)
|
|
1708
|
+
continue
|
|
1709
|
+
header_row: dict[str, object]
|
|
1710
|
+
for header_row in header_rows:
|
|
1711
|
+
merged_row = dict(base_row)
|
|
1712
|
+
merged_row["title"] = postprocess_record.title
|
|
1713
|
+
merged_row.update(header_row)
|
|
1714
|
+
rows.append(merged_row)
|
|
1715
|
+
dataframe = pandas_module.DataFrame(rows)
|
|
1716
|
+
return cast("pd.DataFrame", normalize_optional_index_columns(dataframe))
|
|
1717
|
+
|
|
1718
|
+
def to_xarray(self) -> XarrayDataTree:
|
|
1719
|
+
"""Export selected batch postprocess headers as a `DataTree`."""
|
|
1720
|
+
|
|
1721
|
+
from ..export.xarray.export import (
|
|
1722
|
+
batch_selected_postprocess_headers_to_xarray_dataset,
|
|
1723
|
+
)
|
|
1724
|
+
|
|
1725
|
+
return batch_selected_postprocess_headers_to_xarray_dataset(
|
|
1726
|
+
self.records,
|
|
1727
|
+
index=self.index,
|
|
1728
|
+
)
|