xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
xarray_sql/df.py
ADDED
|
@@ -0,0 +1,832 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import sys
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
import xarray as xr
|
|
11
|
+
|
|
12
|
+
from . import cftime as cft
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
_CAN_START_THREADS = sys.platform != "emscripten"
|
|
16
|
+
"""Whether this interpreter can start OS threads; Pyodide cannot."""
|
|
17
|
+
|
|
18
|
+
_DEFAULT_BLOCK_ROWS = 2**20
|
|
19
|
+
"""Rows per partition for a Dataset that records no storage chunks."""
|
|
20
|
+
|
|
21
|
+
Block = dict[Hashable, slice]
|
|
22
|
+
Chunks = dict[str, int] | None
|
|
23
|
+
TableNames = Mapping[tuple[str, ...], str] | None
|
|
24
|
+
"""Maps a dimension group's exact dim tuple to the table name it takes."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# Borrowed from Xarray
|
|
28
|
+
def _get_chunk_slicer(
|
|
29
|
+
dim: Hashable, chunk_index: Mapping, chunk_bounds: Mapping
|
|
30
|
+
):
|
|
31
|
+
if dim in chunk_index:
|
|
32
|
+
which_chunk = chunk_index[dim]
|
|
33
|
+
return slice(
|
|
34
|
+
chunk_bounds[dim][which_chunk], chunk_bounds[dim][which_chunk + 1]
|
|
35
|
+
)
|
|
36
|
+
return slice(None)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def compute_chunks(
|
|
40
|
+
ds: xr.Dataset, chunks: dict[str, int]
|
|
41
|
+
) -> dict[Hashable, tuple[int, ...]]:
|
|
42
|
+
"""Per-dim chunk-size tuples matching ``ds.chunk(chunks).chunks``.
|
|
43
|
+
|
|
44
|
+
Pure arithmetic replacement for the dask rechunk round-trip; dask's
|
|
45
|
+
``.chunk()`` eagerly builds a task graph, which dominates
|
|
46
|
+
``block_slices()`` cost on large datasets.
|
|
47
|
+
"""
|
|
48
|
+
existing = dict(ds.chunks) if ds.chunks else {}
|
|
49
|
+
result: dict[Hashable, tuple[int, ...]] = {}
|
|
50
|
+
for dim in ds.dims:
|
|
51
|
+
size = ds.sizes[dim]
|
|
52
|
+
if dim in chunks:
|
|
53
|
+
cs = chunks[dim]
|
|
54
|
+
if cs <= 0 or cs >= size:
|
|
55
|
+
result[dim] = (size,)
|
|
56
|
+
else:
|
|
57
|
+
n_full, rem = divmod(size, cs)
|
|
58
|
+
result[dim] = (cs,) * n_full + ((rem,) if rem else ())
|
|
59
|
+
elif dim in existing:
|
|
60
|
+
result[dim] = tuple(existing[dim])
|
|
61
|
+
else:
|
|
62
|
+
result[dim] = (size,)
|
|
63
|
+
return result
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def resolve_chunks(
|
|
67
|
+
ds: xr.Dataset, chunks: Chunks
|
|
68
|
+
) -> Mapping[Hashable, tuple[int, ...]]:
|
|
69
|
+
"""Normalise the user's ``chunks`` argument to per-dim size tuples.
|
|
70
|
+
|
|
71
|
+
Filters out keys for dims this dataset doesn't have (sub-datasets in a
|
|
72
|
+
heterogeneous group need not contain every dimension named in the
|
|
73
|
+
spec), then rechunks arithmetically via ``compute_chunks``. Without a
|
|
74
|
+
spec, a chunked dataset keeps its chunks; an unchunked one opened from
|
|
75
|
+
storage is split along the storage chunks its backend recorded, and is
|
|
76
|
+
one chunk along other dimensions. A dataset that records none (built in
|
|
77
|
+
memory) is split into blocks of about ``_DEFAULT_BLOCK_ROWS`` rows.
|
|
78
|
+
|
|
79
|
+
Returns an empty mapping for scalar datasets; callers should treat that
|
|
80
|
+
as "one block covering everything".
|
|
81
|
+
"""
|
|
82
|
+
if chunks is not None:
|
|
83
|
+
chunks = {dim: size for dim, size in chunks.items() if dim in ds.sizes}
|
|
84
|
+
if chunks:
|
|
85
|
+
return compute_chunks(ds, chunks)
|
|
86
|
+
if ds.chunks:
|
|
87
|
+
return {d: tuple(c) for d, c in ds.chunks.items()}
|
|
88
|
+
return compute_chunks(ds, _storage_chunks(ds) or _default_chunks(ds))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _storage_chunks(ds: xr.Dataset) -> dict[str, int]:
|
|
92
|
+
"""Chunk sizes from the backend's ``preferred_chunks`` encoding.
|
|
93
|
+
|
|
94
|
+
Zarr and NetCDF backends record each variable's on-disk chunking there.
|
|
95
|
+
Where variables disagree on a dimension, the first one wins.
|
|
96
|
+
"""
|
|
97
|
+
chunks: dict[str, int] = {}
|
|
98
|
+
for var in ds.data_vars.values():
|
|
99
|
+
for dim, size in var.encoding.get("preferred_chunks", {}).items():
|
|
100
|
+
chunks.setdefault(dim, size)
|
|
101
|
+
return chunks
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _default_chunks(ds: xr.Dataset) -> dict[str, int]:
|
|
105
|
+
"""Chunk sizes giving blocks of at most about ``_DEFAULT_BLOCK_ROWS`` rows.
|
|
106
|
+
|
|
107
|
+
Inner dimensions stay whole while they fit, so each block is a
|
|
108
|
+
contiguous run of the row-major grid; the outermost dimension that
|
|
109
|
+
does not fit is split, and the dimensions outside it get size one.
|
|
110
|
+
"""
|
|
111
|
+
chunks: dict[str, int] = {}
|
|
112
|
+
inner = 1
|
|
113
|
+
for dim in reversed(list(ds.sizes)):
|
|
114
|
+
size = ds.sizes[dim]
|
|
115
|
+
step = max(1, min(size, _DEFAULT_BLOCK_ROWS // inner))
|
|
116
|
+
if step < size:
|
|
117
|
+
chunks[str(dim)] = step
|
|
118
|
+
inner *= step
|
|
119
|
+
return chunks
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _ensure_default_indexes(ds: xr.Dataset) -> xr.Dataset:
|
|
123
|
+
"""Attach a default integer index coordinate to every dimension lacking one.
|
|
124
|
+
|
|
125
|
+
xarray allows "dimensions without coordinates"; these are absent from
|
|
126
|
+
``ds.coords``, so they are dropped from the SQL schema and, once a block is
|
|
127
|
+
sliced out with ``isel``, their position is synthesized *relative to the
|
|
128
|
+
block* (restarting at 0 in every partition). Materialising an explicit
|
|
129
|
+
``arange`` index up front turns them into ordinary dimension coordinates, so
|
|
130
|
+
they appear as columns and carry their absolute position through chunked
|
|
131
|
+
reads. Datasets whose dimensions already have coordinates are returned
|
|
132
|
+
unchanged.
|
|
133
|
+
"""
|
|
134
|
+
missing = {
|
|
135
|
+
dim: np.arange(ds.sizes[dim]) for dim in ds.dims if dim not in ds.coords
|
|
136
|
+
}
|
|
137
|
+
return ds.assign_coords(missing) if missing else ds
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _block_slices_from_resolved(
|
|
141
|
+
ds: xr.Dataset, resolved: Mapping[Hashable, tuple[int, ...]]
|
|
142
|
+
) -> Iterator[Block]:
|
|
143
|
+
"""Emit blocks given pre-resolved per-dim chunk tuples."""
|
|
144
|
+
if not resolved:
|
|
145
|
+
# A dimensionless dataset (e.g. scalar metadata variables) is a
|
|
146
|
+
# single block.
|
|
147
|
+
yield {}
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
chunk_bounds = {
|
|
151
|
+
dim: np.cumsum((0,) + tuple(c)) for dim, c in resolved.items()
|
|
152
|
+
}
|
|
153
|
+
ichunk = {dim: range(len(tuple(c))) for dim, c in resolved.items()}
|
|
154
|
+
ick, icv = zip(*ichunk.items()) # Makes same order of keys and val.
|
|
155
|
+
chunk_idxs = (dict(zip(ick, i)) for i in itertools.product(*icv))
|
|
156
|
+
yield from (
|
|
157
|
+
{
|
|
158
|
+
dim: _get_chunk_slicer(dim, chunk_index, chunk_bounds)
|
|
159
|
+
for dim in ds.dims
|
|
160
|
+
}
|
|
161
|
+
for chunk_index in chunk_idxs
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# Adapted from Xarray `map_blocks` implementation.
|
|
166
|
+
def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> Iterator[Block]:
|
|
167
|
+
"""Compute block slices for a chunked Dataset."""
|
|
168
|
+
yield from _block_slices_from_resolved(ds, resolve_chunks(ds, chunks))
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def explode(ds: xr.Dataset, chunks: Chunks = None) -> Iterator[xr.Dataset]:
|
|
172
|
+
"""Explodes a dataset into its chunks."""
|
|
173
|
+
yield from (ds.isel(b) for b in block_slices(ds, chunks=chunks))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def group_vars_by_dims(ds: xr.Dataset) -> dict[tuple[str, ...], list[str]]:
|
|
177
|
+
"""Group a Dataset's data variables by their exact dimension tuple.
|
|
178
|
+
|
|
179
|
+
Variables that share dimensions can share a table; each distinct
|
|
180
|
+
dimension tuple becomes its own table when a mixed-dimension Dataset
|
|
181
|
+
is registered::
|
|
182
|
+
|
|
183
|
+
("time", "lat", "lon"): ["temperature_2m", "wind_speed"],
|
|
184
|
+
("time", "lat", "lon", "level"): ["pressure", "humidity"]
|
|
185
|
+
"""
|
|
186
|
+
groups = defaultdict(list)
|
|
187
|
+
for var_name, var in ds.data_vars.items():
|
|
188
|
+
dims = var.dims
|
|
189
|
+
groups[dims].append(var_name)
|
|
190
|
+
return groups
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def default_table_name(dims: tuple[str, ...]) -> str:
|
|
194
|
+
"""The table name a dimension group gets when the user names none."""
|
|
195
|
+
# Scalar variables group under empty dims, where "_".join(()) is the
|
|
196
|
+
# empty string; fall back to a valid default table name.
|
|
197
|
+
return "_".join(dims) or "scalar"
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def resolve_table_names(
|
|
201
|
+
ds: xr.Dataset,
|
|
202
|
+
table_names: TableNames = None,
|
|
203
|
+
*,
|
|
204
|
+
case_insensitive: bool = False,
|
|
205
|
+
) -> dict[tuple[str, ...], str]:
|
|
206
|
+
"""Name every dimension group of ``ds``, honouring user overrides.
|
|
207
|
+
|
|
208
|
+
``table_names`` maps a group's exact dimension tuple to the name its
|
|
209
|
+
table should carry; groups it does not mention keep
|
|
210
|
+
[default_table_name][xarray_sql.df.default_table_name]. Keys that
|
|
211
|
+
match no group in ``ds`` are ignored, so one naming map can be reused
|
|
212
|
+
across Datasets that hold different subsets of the same variables.
|
|
213
|
+
|
|
214
|
+
Every engine adapter routes through here, which is what makes
|
|
215
|
+
``table_names={('time', 'lat', 'lon'): 'surface'}`` mean the same
|
|
216
|
+
thing on DataFusion, DuckDB, and the pyarrow-dataset engines.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
ds: The Dataset whose dimension groups are being named.
|
|
220
|
+
table_names: User overrides; see above.
|
|
221
|
+
case_insensitive: Fold names before comparing them for
|
|
222
|
+
collisions — DuckDB identifiers are case-insensitive even
|
|
223
|
+
quoted, so ``'surface'`` and ``'SURFACE'`` name the same
|
|
224
|
+
table there even though they are distinct strings. Other
|
|
225
|
+
engines compare case-sensitively.
|
|
226
|
+
|
|
227
|
+
Raises:
|
|
228
|
+
ValueError: if two groups would end up with the same name (or,
|
|
229
|
+
with ``case_insensitive=True``, names differing only in
|
|
230
|
+
case), which would silently register one table over the
|
|
231
|
+
other.
|
|
232
|
+
"""
|
|
233
|
+
overrides = table_names or {}
|
|
234
|
+
names = {
|
|
235
|
+
dims: overrides.get(dims, default_table_name(dims))
|
|
236
|
+
for dims in group_vars_by_dims(ds)
|
|
237
|
+
}
|
|
238
|
+
taken: dict[str, tuple[tuple[str, ...], str]] = {}
|
|
239
|
+
for dims, name in names.items():
|
|
240
|
+
key = name.casefold() if case_insensitive else name
|
|
241
|
+
if key in taken:
|
|
242
|
+
other_dims, other_name = taken[key]
|
|
243
|
+
detail = (
|
|
244
|
+
f"the same table name {name!r}"
|
|
245
|
+
if other_name == name
|
|
246
|
+
else f"names that collide under DuckDB's case-insensitive "
|
|
247
|
+
f"identifiers: {other_name!r} and {name!r}"
|
|
248
|
+
)
|
|
249
|
+
raise ValueError(
|
|
250
|
+
f"table_names maps two dimension groups to {detail}: "
|
|
251
|
+
f"{other_dims} and {dims}. Give each group a distinct name."
|
|
252
|
+
)
|
|
253
|
+
taken[key] = (dims, name)
|
|
254
|
+
return names
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def shared_coord_arrays(ds: xr.Dataset) -> dict[str, np.ndarray]:
|
|
258
|
+
"""Materialise ``ds``'s dimension coordinates once, to share.
|
|
259
|
+
|
|
260
|
+
Splitting a Dataset into per-dimension-group tables otherwise reads
|
|
261
|
+
every shared dim coordinate once per table — a network round-trip
|
|
262
|
+
apiece for Zarr-backed parents like ARCO-ERA5.
|
|
263
|
+
"""
|
|
264
|
+
return {str(dim): ds.coords[dim].values for dim in ds.dims}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _block_len(block: Block) -> int:
|
|
268
|
+
return int(np.prod([v.stop - v.start for v in block.values()]))
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def from_map_batched(
|
|
272
|
+
func: Callable[..., pd.DataFrame],
|
|
273
|
+
*iterables: tuple[Any, ...],
|
|
274
|
+
args: tuple | None = None,
|
|
275
|
+
schema: pa.Schema = None,
|
|
276
|
+
**kwargs: dict[str, Any],
|
|
277
|
+
) -> pa.RecordBatchReader:
|
|
278
|
+
"""Create a PyArrow RecordBatchReader by mapping a function over iterables.
|
|
279
|
+
|
|
280
|
+
This is equivalent to dask's from_map but returns a PyArrow
|
|
281
|
+
RecordBatchReader that can be used with DataFusion. It iterates over
|
|
282
|
+
RecordBatches which are created via the `func` one-at-a-time.
|
|
283
|
+
|
|
284
|
+
Args:
|
|
285
|
+
func: Function to apply to each element of the iterables. Currently, the
|
|
286
|
+
function must return a Pandas DataFrame.
|
|
287
|
+
*iterables: Iterable objects to map the function over.
|
|
288
|
+
schema: Optional schema needed for the RecordBatchReader.
|
|
289
|
+
args: Additional positional arguments to pass to func.
|
|
290
|
+
**kwargs: Additional keyword arguments to pass to func.
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
A PyArrow RecordBatchReader containing the stream of RecordBatches.
|
|
294
|
+
"""
|
|
295
|
+
if args is None:
|
|
296
|
+
args = ()
|
|
297
|
+
|
|
298
|
+
def map_batches():
|
|
299
|
+
for items in zip(*iterables):
|
|
300
|
+
df = func(*items, *args, **kwargs)
|
|
301
|
+
yield pa.RecordBatch.from_pandas(df, schema=schema)
|
|
302
|
+
|
|
303
|
+
return pa.RecordBatchReader.from_batches(schema, map_batches())
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def from_map(
|
|
307
|
+
func: Callable,
|
|
308
|
+
*iterables: tuple[Any, ...],
|
|
309
|
+
args: tuple | None = None,
|
|
310
|
+
**kwargs: dict[str, Any],
|
|
311
|
+
) -> pa.Table:
|
|
312
|
+
"""Create a PyArrow Table by mapping a function over iterables.
|
|
313
|
+
|
|
314
|
+
This is equivalent to dask's from_map but returns a PyArrow Table
|
|
315
|
+
that can be used with DataFusion instead of a Dask DataFrame.
|
|
316
|
+
|
|
317
|
+
Args:
|
|
318
|
+
func: Function to apply to each element of the iterables.
|
|
319
|
+
*iterables: Iterable objects to map the function over.
|
|
320
|
+
args: Additional positional arguments to pass to func.
|
|
321
|
+
**kwargs: Additional keyword arguments to pass to func.
|
|
322
|
+
|
|
323
|
+
Returns:
|
|
324
|
+
A PyArrow Table containing the concatenated results.
|
|
325
|
+
"""
|
|
326
|
+
if args is None:
|
|
327
|
+
args = ()
|
|
328
|
+
|
|
329
|
+
# Apply the function to each combination of iterable elements
|
|
330
|
+
results = []
|
|
331
|
+
for items in zip(*iterables) if len(iterables) > 1 else iterables[0]:
|
|
332
|
+
if isinstance(items, tuple):
|
|
333
|
+
result = func(*items, *args, **kwargs)
|
|
334
|
+
else:
|
|
335
|
+
result = func(items, *args, **kwargs)
|
|
336
|
+
|
|
337
|
+
# Convert result to PyArrow Table
|
|
338
|
+
if isinstance(result, pd.DataFrame):
|
|
339
|
+
pa_table = pa.Table.from_pandas(result)
|
|
340
|
+
elif isinstance(result, pa.Table):
|
|
341
|
+
pa_table = result
|
|
342
|
+
else:
|
|
343
|
+
# Try to convert to pandas first, then to PyArrow
|
|
344
|
+
try:
|
|
345
|
+
df = pd.DataFrame(result)
|
|
346
|
+
pa_table = pa.Table.from_pandas(df)
|
|
347
|
+
except Exception as e:
|
|
348
|
+
raise ValueError(
|
|
349
|
+
f"Cannot convert function result to PyArrow Table: {e}"
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
results.append(pa_table)
|
|
353
|
+
|
|
354
|
+
# Concatenate all results
|
|
355
|
+
if not results:
|
|
356
|
+
raise ValueError("No results to concatenate")
|
|
357
|
+
|
|
358
|
+
return pa.concat_tables(results)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def pivot(ds: xr.Dataset) -> pd.DataFrame:
|
|
362
|
+
"""Converts an xarray Dataset to a pandas DataFrame."""
|
|
363
|
+
return ds.to_dataframe().reset_index() # type: ignore[no-any-return]
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def dataset_to_record_batch(
|
|
367
|
+
ds: xr.Dataset, schema: pa.Schema
|
|
368
|
+
) -> pa.RecordBatch:
|
|
369
|
+
"""Convert an xarray Dataset partition to an Arrow RecordBatch.
|
|
370
|
+
|
|
371
|
+
Builds the RecordBatch directly from numpy arrays, bypassing the pandas
|
|
372
|
+
round-trip (to_dataframe → reset_index → from_pandas) used by pivot().
|
|
373
|
+
For large partitions this reduces peak memory from ~5× to ~2× the
|
|
374
|
+
partition size.
|
|
375
|
+
|
|
376
|
+
Dimension coordinates are broadcast to the full partition shape and
|
|
377
|
+
ravelled. np.broadcast_to() is zero-copy; the ravel() forces one copy
|
|
378
|
+
per coordinate (unavoidable, since broadcast arrays are non-contiguous).
|
|
379
|
+
Data variable arrays are ravelled in-place — a zero-copy view when the
|
|
380
|
+
underlying array is already C-contiguous (the common case for numpy-backed
|
|
381
|
+
xarray datasets).
|
|
382
|
+
|
|
383
|
+
Args:
|
|
384
|
+
ds: A partition-sized xarray Dataset (already sliced via isel).
|
|
385
|
+
schema: The Arrow schema for the output, as produced by _parse_schema.
|
|
386
|
+
Column order in the output matches schema field order.
|
|
387
|
+
|
|
388
|
+
Returns:
|
|
389
|
+
A RecordBatch with one column per dimension coordinate and data
|
|
390
|
+
variable, in schema order.
|
|
391
|
+
"""
|
|
392
|
+
# Use the data variable's dimension order as canonical so coordinate
|
|
393
|
+
# broadcasts and data variable ravels use the same layout. All data
|
|
394
|
+
# variables are validated to share the same dims tuple.
|
|
395
|
+
if ds.data_vars:
|
|
396
|
+
first_var = next(iter(ds.data_vars.values()))
|
|
397
|
+
dim_names = list(first_var.dims)
|
|
398
|
+
shape = first_var.shape
|
|
399
|
+
else:
|
|
400
|
+
dim_names = list(ds.sizes.keys())
|
|
401
|
+
shape = tuple(ds.sizes[d] for d in dim_names)
|
|
402
|
+
|
|
403
|
+
arrays = []
|
|
404
|
+
for field in schema:
|
|
405
|
+
name = field.name
|
|
406
|
+
if name in ds.coords and name in ds.dims:
|
|
407
|
+
# Broadcast 1-D coordinate to the full N-D partition shape, then ravel.
|
|
408
|
+
axis = dim_names.index(name)
|
|
409
|
+
coord = ds.coords[name].values
|
|
410
|
+
if cft.is_cftime(coord):
|
|
411
|
+
coord = cft.convert_for_field(coord, field)
|
|
412
|
+
reshape = [1] * len(shape)
|
|
413
|
+
reshape[axis] = coord.shape[0]
|
|
414
|
+
arr = np.broadcast_to(coord.reshape(reshape), shape).ravel()
|
|
415
|
+
arrays.append(pa.array(arr, type=field.type))
|
|
416
|
+
else:
|
|
417
|
+
# Data variable: ravel to 1-D (zero-copy for C-contiguous arrays).
|
|
418
|
+
raw = ds[name].values.ravel()
|
|
419
|
+
if cft.is_cftime(ds[name].values):
|
|
420
|
+
raw = cft.convert_for_field(ds[name].values, field)
|
|
421
|
+
|
|
422
|
+
# from_pandas=True maps NaN → Arrow null inside the C++ copy kernel,
|
|
423
|
+
# so SQL aggregates (MAX, MIN, AVG) skip missing values correctly.
|
|
424
|
+
arrays.append(pa.array(raw, type=field.type, from_pandas=True))
|
|
425
|
+
|
|
426
|
+
return pa.RecordBatch.from_arrays(arrays, schema=schema)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
DEFAULT_BATCH_SIZE: int = 65_536
|
|
430
|
+
"""Default number of rows per emitted Arrow RecordBatch.
|
|
431
|
+
|
|
432
|
+
64 K rows balances DataFusion pipeline depth against per-batch overhead.
|
|
433
|
+
"""
|
|
434
|
+
|
|
435
|
+
_FULL_PIVOT_MAX_ROWS: int = 8_388_608
|
|
436
|
+
"""Row cap for the whole-partition coordinate fast path in
|
|
437
|
+
iter_record_batches.
|
|
438
|
+
|
|
439
|
+
Below this, coordinate columns are materialised for the full partition
|
|
440
|
+
with repeat/tile (sequential writes, ~3x faster than per-batch index
|
|
441
|
+
arithmetic) and batches are zero-copy slices; the cost is holding every
|
|
442
|
+
coordinate column of the partition in memory at once (rows x 8 bytes x
|
|
443
|
+
n_dims). Above it — e.g. single-time-step reanalysis partitions with
|
|
444
|
+
tens of millions of rows — the per-batch path keeps peak memory at
|
|
445
|
+
O(batch_size) per coordinate instead.
|
|
446
|
+
"""
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _as_single_array(values, type: pa.DataType, *, from_pandas: bool = False):
|
|
450
|
+
"""``pa.array`` that always returns a contiguous ``pa.Array``.
|
|
451
|
+
|
|
452
|
+
``pa.array`` may return a ``ChunkedArray`` instead of an ``Array`` for
|
|
453
|
+
large inputs (observed for numpy fixed-width unicode columns of a few
|
|
454
|
+
million rows — e.g. a string dimension coordinate tiled across a full
|
|
455
|
+
partition). ``RecordBatch.from_arrays`` rejects chunked input, so
|
|
456
|
+
flatten it back to one contiguous array.
|
|
457
|
+
"""
|
|
458
|
+
arr = pa.array(values, type=type, from_pandas=from_pandas)
|
|
459
|
+
if isinstance(arr, pa.ChunkedArray):
|
|
460
|
+
arr = arr.combine_chunks()
|
|
461
|
+
return arr
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def iter_record_batches(
|
|
465
|
+
ds: xr.Dataset,
|
|
466
|
+
schema: pa.Schema,
|
|
467
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
468
|
+
) -> Iterator[pa.RecordBatch]:
|
|
469
|
+
"""Yield RecordBatches of at most *batch_size* rows from a partition Dataset.
|
|
470
|
+
|
|
471
|
+
Unlike `dataset_to_record_batch`, which materialises the entire
|
|
472
|
+
partition as one batch, this generator emits smaller batches so that
|
|
473
|
+
DataFusion can begin filtering and aggregating before the full partition
|
|
474
|
+
is loaded. Peak memory per batch is O(batch_size) for coordinate columns
|
|
475
|
+
and O(partition_size) for data-variable columns (which must be loaded in
|
|
476
|
+
full from storage).
|
|
477
|
+
|
|
478
|
+
Coordinate values are computed per batch via strided index arithmetic —
|
|
479
|
+
no broadcast array spanning the whole partition is ever allocated. Data
|
|
480
|
+
variable flat arrays are loaded once (triggering any remote I/O) and then
|
|
481
|
+
sliced as zero-copy views for each batch.
|
|
482
|
+
|
|
483
|
+
Args:
|
|
484
|
+
ds: A partition-sized xarray Dataset (already sliced via isel).
|
|
485
|
+
schema: The Arrow schema for the output, as produced by _parse_schema.
|
|
486
|
+
batch_size: Maximum number of rows per yielded RecordBatch.
|
|
487
|
+
|
|
488
|
+
Yields:
|
|
489
|
+
RecordBatches in schema column order, covering all rows of the
|
|
490
|
+
partition exactly once.
|
|
491
|
+
"""
|
|
492
|
+
if ds.data_vars:
|
|
493
|
+
first_var = next(iter(ds.data_vars.values()))
|
|
494
|
+
dim_names = list(first_var.dims)
|
|
495
|
+
shape = first_var.shape
|
|
496
|
+
else:
|
|
497
|
+
dim_names = list(ds.sizes.keys())
|
|
498
|
+
shape = tuple(ds.sizes[d] for d in dim_names)
|
|
499
|
+
|
|
500
|
+
total_rows = int(np.prod(shape))
|
|
501
|
+
|
|
502
|
+
# Preload small 1-D coordinate arrays (negligible memory).
|
|
503
|
+
# Convert cftime objects to numeric values matching the schema type.
|
|
504
|
+
# Projected scans may omit dimension columns from the schema; those
|
|
505
|
+
# dims still shape the iteration but never emit a column.
|
|
506
|
+
coord_values = {}
|
|
507
|
+
schema_names = set(schema.names)
|
|
508
|
+
for name in dim_names:
|
|
509
|
+
# A dim the projection dropped (e.g. time under GROUP BY level) is never
|
|
510
|
+
# read in the batch loop below, which only iterates the schema's fields.
|
|
511
|
+
# Skip it so schema.field(name) is not called for a projected-away name
|
|
512
|
+
# (it raises for cftime coords, which take the convert_for_field path).
|
|
513
|
+
if name not in schema_names:
|
|
514
|
+
continue
|
|
515
|
+
vals = ds.coords[name].values
|
|
516
|
+
if cft.is_cftime(vals):
|
|
517
|
+
coord_values[name] = cft.convert_for_field(vals, schema.field(name))
|
|
518
|
+
else:
|
|
519
|
+
coord_values[name] = vals
|
|
520
|
+
|
|
521
|
+
# C-order stride for each dimension: stride[k] = prod(shape[k+1:]).
|
|
522
|
+
# Flat row index i → coordinate index for dim k: (i // stride[k]) % shape[k].
|
|
523
|
+
strides = [int(np.prod(shape[k + 1 :])) for k in range(len(shape))]
|
|
524
|
+
|
|
525
|
+
# Load data-variable arrays fully (triggers Dask/Zarr compute once).
|
|
526
|
+
# ravel() is a zero-copy view for C-contiguous arrays.
|
|
527
|
+
data_arrays = {}
|
|
528
|
+
for field in schema:
|
|
529
|
+
if field.name not in ds.dims:
|
|
530
|
+
raw = ds[field.name].values
|
|
531
|
+
if cft.is_cftime(raw):
|
|
532
|
+
data_arrays[field.name] = cft.convert_for_field(raw, field)
|
|
533
|
+
else:
|
|
534
|
+
data_arrays[field.name] = raw.ravel()
|
|
535
|
+
|
|
536
|
+
if 0 < total_rows <= _FULL_PIVOT_MAX_ROWS:
|
|
537
|
+
# Fast path: build each coordinate column once for the whole
|
|
538
|
+
# partition. In C order, dim k's flat column is its coord values
|
|
539
|
+
# each repeated prod(shape[k+1:]) times, with that pattern tiled
|
|
540
|
+
# prod(shape[:k]) times — two sequential-write kernels, much
|
|
541
|
+
# faster than per-batch division/modulo plus gather. Batches are
|
|
542
|
+
# then zero-copy slices of the full-partition Arrow arrays.
|
|
543
|
+
full_arrays = []
|
|
544
|
+
for field in schema:
|
|
545
|
+
name = field.name
|
|
546
|
+
if name in ds.coords and name in ds.dims:
|
|
547
|
+
k = dim_names.index(name)
|
|
548
|
+
outer = int(np.prod(shape[:k]))
|
|
549
|
+
col: np.ndarray = np.repeat(coord_values[name], strides[k])
|
|
550
|
+
if outer > 1:
|
|
551
|
+
col = np.tile(col, outer)
|
|
552
|
+
full_arrays.append(_as_single_array(col, field.type))
|
|
553
|
+
else:
|
|
554
|
+
full_arrays.append(
|
|
555
|
+
_as_single_array(
|
|
556
|
+
data_arrays[name], field.type, from_pandas=True
|
|
557
|
+
)
|
|
558
|
+
)
|
|
559
|
+
for row_start in range(0, total_rows, batch_size):
|
|
560
|
+
yield pa.RecordBatch.from_arrays(
|
|
561
|
+
[a.slice(row_start, batch_size) for a in full_arrays],
|
|
562
|
+
schema=schema,
|
|
563
|
+
)
|
|
564
|
+
return
|
|
565
|
+
|
|
566
|
+
for row_start in range(0, total_rows, batch_size):
|
|
567
|
+
row_end = min(row_start + batch_size, total_rows)
|
|
568
|
+
row_idx = np.arange(row_start, row_end)
|
|
569
|
+
|
|
570
|
+
arrays = []
|
|
571
|
+
for field in schema:
|
|
572
|
+
name = field.name
|
|
573
|
+
if name in ds.coords and name in ds.dims:
|
|
574
|
+
k = dim_names.index(name)
|
|
575
|
+
coord_idx = (row_idx // strides[k]) % shape[k]
|
|
576
|
+
arrays.append(
|
|
577
|
+
_as_single_array(coord_values[name][coord_idx], field.type)
|
|
578
|
+
)
|
|
579
|
+
else:
|
|
580
|
+
arrays.append(
|
|
581
|
+
_as_single_array(
|
|
582
|
+
data_arrays[name][row_start:row_end],
|
|
583
|
+
field.type,
|
|
584
|
+
from_pandas=True,
|
|
585
|
+
)
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
yield pa.RecordBatch.from_arrays(arrays, schema=schema)
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _arrow_type_for_object(values: np.ndarray) -> pa.DataType:
|
|
592
|
+
"""Infer an Arrow type for a non-cftime object-dtype array.
|
|
593
|
+
|
|
594
|
+
``pa.from_numpy_dtype`` cannot map numpy object dtype, so let pyarrow infer
|
|
595
|
+
the type from the data instead: strings become ``pa.string()``, bytes
|
|
596
|
+
``pa.binary()``, and other representable Python scalars their Arrow
|
|
597
|
+
equivalent. An all-null array stays ``pa.null()``, and a column mixing
|
|
598
|
+
incompatible types (e.g. str and int) raises, surfacing a clear error
|
|
599
|
+
rather than a silent coercion. Object-dtype arrays are never Dask/Zarr
|
|
600
|
+
backed, so this triggers no remote I/O.
|
|
601
|
+
"""
|
|
602
|
+
return pa.array(np.asarray(values).ravel()).type
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _arrow_type_for_categorical(values: np.ndarray) -> pa.DataType:
|
|
606
|
+
"""Return the Arrow value type emitted for a categorical coordinate.
|
|
607
|
+
|
|
608
|
+
xarray preserves a pandas ``CategoricalDtype`` on the coordinate variable,
|
|
609
|
+
but ``.values`` exposes the category labels that are actually written to
|
|
610
|
+
Arrow. Infer from those labels so string categories become UTF-8 and
|
|
611
|
+
numeric categories retain their numeric width.
|
|
612
|
+
"""
|
|
613
|
+
values = np.asarray(values)
|
|
614
|
+
if values.dtype == np.dtype("O"):
|
|
615
|
+
return _arrow_type_for_object(values)
|
|
616
|
+
return pa.from_numpy_dtype(values.dtype)
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _parse_schema(ds: xr.Dataset) -> pa.Schema:
|
|
620
|
+
"""Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns.
|
|
621
|
+
|
|
622
|
+
Only *dimension coordinates* become dimension columns, so a dimension
|
|
623
|
+
without a coordinate would be dropped. Callers must run the Dataset through
|
|
624
|
+
``_ensure_default_indexes`` first (the readers do) so every dimension
|
|
625
|
+
has a coordinate and appears as a column.
|
|
626
|
+
|
|
627
|
+
Uses the xarray index type to detect cftime coordinates without
|
|
628
|
+
materializing their data — important for Dask/Zarr-backed datasets
|
|
629
|
+
where .values would trigger eager computation.
|
|
630
|
+
|
|
631
|
+
cftime coordinates are mapped to one of two Arrow types:
|
|
632
|
+
|
|
633
|
+
* **Gregorian-like calendars** (standard, noleap, all_leap, etc.):
|
|
634
|
+
``pa.timestamp('us')`` so string-based SQL filters work naturally.
|
|
635
|
+
* **Non-Gregorian calendars** (360_day, julian):
|
|
636
|
+
``pa.int64()`` with ``xarray:units`` / ``xarray:calendar`` metadata
|
|
637
|
+
on the field, preserving lossless CF-convention encoding.
|
|
638
|
+
"""
|
|
639
|
+
columns = []
|
|
640
|
+
|
|
641
|
+
for coord_name, coord_var in ds.coords.items():
|
|
642
|
+
# Only include dimension coordinates
|
|
643
|
+
if coord_name in ds.dims:
|
|
644
|
+
if cft.is_cftime_index(ds, coord_name):
|
|
645
|
+
units, calendar = cft.encoding(ds, coord_name)
|
|
646
|
+
columns.append(cft.arrow_field(coord_name, units, calendar))
|
|
647
|
+
elif isinstance(coord_var.dtype, pd.CategoricalDtype):
|
|
648
|
+
columns.append(
|
|
649
|
+
pa.field(
|
|
650
|
+
coord_name,
|
|
651
|
+
_arrow_type_for_categorical(coord_var.values),
|
|
652
|
+
)
|
|
653
|
+
)
|
|
654
|
+
elif coord_var.dtype == np.dtype("O"):
|
|
655
|
+
# Object dtype that isn't cftime (e.g. string station names).
|
|
656
|
+
arrow_type = _arrow_type_for_object(coord_var.values)
|
|
657
|
+
columns.append(pa.field(coord_name, arrow_type))
|
|
658
|
+
else:
|
|
659
|
+
pa_type = pa.from_numpy_dtype(coord_var.dtype)
|
|
660
|
+
columns.append(pa.field(coord_name, pa_type))
|
|
661
|
+
|
|
662
|
+
for var_name, var in ds.data_vars.items():
|
|
663
|
+
# An object-dtype data variable may hold cftime objects (encode it like
|
|
664
|
+
# a cftime coordinate) or strings/other Python scalars (infer the Arrow
|
|
665
|
+
# type from the data). The dtype check keeps the common numeric path off
|
|
666
|
+
# the object branch.
|
|
667
|
+
if var.dtype == np.dtype("O"):
|
|
668
|
+
if cft.is_cftime(var.values):
|
|
669
|
+
# Encode with the same units/calendar as a cftime coordinate.
|
|
670
|
+
cal = var.values.ravel()[0].calendar
|
|
671
|
+
columns.append(
|
|
672
|
+
cft.arrow_field(var_name, cft.DEFAULT_UNITS, cal)
|
|
673
|
+
)
|
|
674
|
+
else:
|
|
675
|
+
arrow_type = _arrow_type_for_object(var.values)
|
|
676
|
+
columns.append(pa.field(var_name, arrow_type))
|
|
677
|
+
else:
|
|
678
|
+
pa_type = pa.from_numpy_dtype(var.dtype)
|
|
679
|
+
columns.append(pa.field(var_name, pa_type))
|
|
680
|
+
|
|
681
|
+
return pa.schema(columns)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
# Type alias for partition metadata: maps dimension name to (min, max, dtype_str) values
|
|
685
|
+
PartitionBounds = dict[str, tuple[Any, Any, str]]
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
_INTEGER_BOUND_TAGS = {
|
|
689
|
+
("i", 1): "int8",
|
|
690
|
+
("i", 2): "int16",
|
|
691
|
+
("i", 4): "int32",
|
|
692
|
+
("i", 8): "int64",
|
|
693
|
+
("u", 1): "uint8",
|
|
694
|
+
("u", 2): "uint16",
|
|
695
|
+
("u", 4): "uint32",
|
|
696
|
+
("u", 8): "uint64",
|
|
697
|
+
}
|
|
698
|
+
_FLOAT_BOUND_TAGS = {2: "float16", 4: "float32", 8: "float64"}
|
|
699
|
+
_TEMPORAL_BOUND_UNITS = {"s", "ms", "us", "ns"}
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def _coordinate_partition_bound(
|
|
703
|
+
coord_values: np.ndarray,
|
|
704
|
+
) -> tuple[Any, Any, str] | None:
|
|
705
|
+
"""Return an exact, Rust-compatible bound for one coordinate slice.
|
|
706
|
+
|
|
707
|
+
Unsupported, null-tainted, or unordered values return ``None``. Partition
|
|
708
|
+
pruning is optional, so omitting unsafe metadata is always preferable to
|
|
709
|
+
coercing a value or failing dataset registration.
|
|
710
|
+
"""
|
|
711
|
+
values = np.asarray(coord_values)
|
|
712
|
+
if values.size == 0:
|
|
713
|
+
return None
|
|
714
|
+
|
|
715
|
+
kind = values.dtype.kind
|
|
716
|
+
|
|
717
|
+
if kind in ("M", "m"):
|
|
718
|
+
if np.isnat(values).any():
|
|
719
|
+
return None
|
|
720
|
+
unit, step = np.datetime_data(values.dtype)
|
|
721
|
+
if unit not in _TEMPORAL_BOUND_UNITS or step != 1:
|
|
722
|
+
return None
|
|
723
|
+
raw = values.astype(np.int64, copy=False)
|
|
724
|
+
prefix = "timestamp" if kind == "M" else "duration"
|
|
725
|
+
return int(raw.min()), int(raw.max()), f"{prefix}_{unit}"
|
|
726
|
+
|
|
727
|
+
if kind == "b":
|
|
728
|
+
return bool(values.min()), bool(values.max()), "bool"
|
|
729
|
+
|
|
730
|
+
if kind in ("i", "u"):
|
|
731
|
+
tag = _INTEGER_BOUND_TAGS.get((kind, values.dtype.itemsize))
|
|
732
|
+
if tag is None:
|
|
733
|
+
return None
|
|
734
|
+
return int(values.min()), int(values.max()), tag
|
|
735
|
+
|
|
736
|
+
if kind == "f":
|
|
737
|
+
tag = _FLOAT_BOUND_TAGS.get(values.dtype.itemsize)
|
|
738
|
+
if tag is None or np.isnan(values).any():
|
|
739
|
+
return None
|
|
740
|
+
return float(values.min()), float(values.max()), tag
|
|
741
|
+
|
|
742
|
+
if kind in ("U", "S", "O"):
|
|
743
|
+
python_values = values.ravel().tolist()
|
|
744
|
+
if all(isinstance(value, str) for value in python_values):
|
|
745
|
+
return min(python_values), max(python_values), "utf8"
|
|
746
|
+
if all(isinstance(value, bytes) for value in python_values):
|
|
747
|
+
return min(python_values), max(python_values), "binary"
|
|
748
|
+
|
|
749
|
+
return None
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _block_metadata(
|
|
753
|
+
coord_arrays: dict,
|
|
754
|
+
block: Block,
|
|
755
|
+
dims: Iterable[Hashable] | None = None,
|
|
756
|
+
) -> PartitionBounds:
|
|
757
|
+
"""Compute min/max coordinate values for a single partition block.
|
|
758
|
+
|
|
759
|
+
Args:
|
|
760
|
+
coord_arrays: Pre-materialised coordinate arrays keyed by dimension name
|
|
761
|
+
string. Hoist this outside any loop to avoid repeated remote I/O
|
|
762
|
+
for Zarr-backed datasets.
|
|
763
|
+
block: A single block slice dict from block_slices().
|
|
764
|
+
dims: Optional restriction to a subset of dims to compute. Used by
|
|
765
|
+
``read_xarray_table`` to skip unchunked dims whose bounds are
|
|
766
|
+
constant across all partitions and have been precomputed once.
|
|
767
|
+
Defaults to all dims present in ``block``.
|
|
768
|
+
|
|
769
|
+
Returns:
|
|
770
|
+
Dict mapping dimension name to (min_value, max_value, dtype_str).
|
|
771
|
+
Dimensions with an empty slice are omitted; the Rust pruning logic
|
|
772
|
+
treats missing dimensions conservatively (never prunes on them).
|
|
773
|
+
"""
|
|
774
|
+
items = ((d, block[d]) for d in dims) if dims is not None else block.items()
|
|
775
|
+
ranges: PartitionBounds = {}
|
|
776
|
+
for dim, slc in items:
|
|
777
|
+
coord_values = coord_arrays[str(dim)][slc]
|
|
778
|
+
if len(coord_values) == 0:
|
|
779
|
+
continue
|
|
780
|
+
# cftime coordinates are object dtype but carry their own bound
|
|
781
|
+
# encoding, so they must be handled before the string/object skip
|
|
782
|
+
# below (otherwise pruning is silently disabled for them).
|
|
783
|
+
# partition_bounds returns None when the bound overflows int64.
|
|
784
|
+
if cft.is_cftime(coord_values):
|
|
785
|
+
bounds = cft.partition_bounds(coord_values)
|
|
786
|
+
if bounds is not None:
|
|
787
|
+
ranges[str(dim)] = bounds
|
|
788
|
+
continue
|
|
789
|
+
# Use actual min/max rather than first/last so non-monotonic axes are
|
|
790
|
+
# safe. Expected dtype/conversion failures disable pruning only for
|
|
791
|
+
# this dimension; they must not make table registration fail.
|
|
792
|
+
try:
|
|
793
|
+
bounds = _coordinate_partition_bound(coord_values)
|
|
794
|
+
except (OverflowError, TypeError, ValueError):
|
|
795
|
+
bounds = None
|
|
796
|
+
if bounds is not None:
|
|
797
|
+
ranges[str(dim)] = bounds
|
|
798
|
+
return ranges
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def partition_metadata(
|
|
802
|
+
ds: xr.Dataset, blocks: list[Block]
|
|
803
|
+
) -> list[PartitionBounds]:
|
|
804
|
+
"""Compute min/max coordinate values for each partition.
|
|
805
|
+
|
|
806
|
+
This metadata enables filter pushdown: SQL queries with WHERE clauses
|
|
807
|
+
on dimension columns can prune partitions that can't contain matching rows.
|
|
808
|
+
|
|
809
|
+
Args:
|
|
810
|
+
ds: The xarray Dataset containing coordinate values.
|
|
811
|
+
blocks: List of block slices from block_slices().
|
|
812
|
+
|
|
813
|
+
Returns:
|
|
814
|
+
List of dicts mapping dimension name to
|
|
815
|
+
(min_value, max_value, dtype_str) tuples.
|
|
816
|
+
|
|
817
|
+
Values preserve the coordinate's logical type and width. Temporal
|
|
818
|
+
values are signed integer counts in their Arrow unit; strings and
|
|
819
|
+
bytes use lexical bounds. Unsupported or null-tainted dimensions
|
|
820
|
+
are omitted so the Rust layer retains their partitions.
|
|
821
|
+
|
|
822
|
+
Note:
|
|
823
|
+
If a partition has an empty slice for a dimension, that dimension is
|
|
824
|
+
omitted from the partition's metadata. The Rust pruning logic treats
|
|
825
|
+
missing dimensions conservatively (never prunes on them).
|
|
826
|
+
"""
|
|
827
|
+
# Hoist coordinate array reads outside the partition loop.
|
|
828
|
+
# ds.coords[dim].values materializes the full array on every call; doing it
|
|
829
|
+
# N_partitions × N_dims times is wasteful and, for remote Zarr-backed datasets
|
|
830
|
+
# (e.g. ARCO-ERA5 on GCS), may trigger repeated network I/O.
|
|
831
|
+
coord_arrays = {str(dim): ds.coords[dim].values for dim in ds.dims}
|
|
832
|
+
return [_block_metadata(coord_arrays, block) for block in blocks]
|