xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/df.py ADDED
@@ -0,0 +1,832 @@
1
+ import itertools
2
+ import sys
3
+ from collections import defaultdict
4
+ from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping
5
+ from typing import Any
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+ import pyarrow as pa
10
+ import xarray as xr
11
+
12
+ from . import cftime as cft
13
+
14
+
15
+ _CAN_START_THREADS = sys.platform != "emscripten"
16
+ """Whether this interpreter can start OS threads; Pyodide cannot."""
17
+
18
+ _DEFAULT_BLOCK_ROWS = 2**20
19
+ """Rows per partition for a Dataset that records no storage chunks."""
20
+
21
+ Block = dict[Hashable, slice]
22
+ Chunks = dict[str, int] | None
23
+ TableNames = Mapping[tuple[str, ...], str] | None
24
+ """Maps a dimension group's exact dim tuple to the table name it takes."""
25
+
26
+
27
+ # Borrowed from Xarray
28
+ def _get_chunk_slicer(
29
+ dim: Hashable, chunk_index: Mapping, chunk_bounds: Mapping
30
+ ):
31
+ if dim in chunk_index:
32
+ which_chunk = chunk_index[dim]
33
+ return slice(
34
+ chunk_bounds[dim][which_chunk], chunk_bounds[dim][which_chunk + 1]
35
+ )
36
+ return slice(None)
37
+
38
+
39
+ def compute_chunks(
40
+ ds: xr.Dataset, chunks: dict[str, int]
41
+ ) -> dict[Hashable, tuple[int, ...]]:
42
+ """Per-dim chunk-size tuples matching ``ds.chunk(chunks).chunks``.
43
+
44
+ Pure arithmetic replacement for the dask rechunk round-trip; dask's
45
+ ``.chunk()`` eagerly builds a task graph, which dominates
46
+ ``block_slices()`` cost on large datasets.
47
+ """
48
+ existing = dict(ds.chunks) if ds.chunks else {}
49
+ result: dict[Hashable, tuple[int, ...]] = {}
50
+ for dim in ds.dims:
51
+ size = ds.sizes[dim]
52
+ if dim in chunks:
53
+ cs = chunks[dim]
54
+ if cs <= 0 or cs >= size:
55
+ result[dim] = (size,)
56
+ else:
57
+ n_full, rem = divmod(size, cs)
58
+ result[dim] = (cs,) * n_full + ((rem,) if rem else ())
59
+ elif dim in existing:
60
+ result[dim] = tuple(existing[dim])
61
+ else:
62
+ result[dim] = (size,)
63
+ return result
64
+
65
+
66
+ def resolve_chunks(
67
+ ds: xr.Dataset, chunks: Chunks
68
+ ) -> Mapping[Hashable, tuple[int, ...]]:
69
+ """Normalise the user's ``chunks`` argument to per-dim size tuples.
70
+
71
+ Filters out keys for dims this dataset doesn't have (sub-datasets in a
72
+ heterogeneous group need not contain every dimension named in the
73
+ spec), then rechunks arithmetically via ``compute_chunks``. Without a
74
+ spec, a chunked dataset keeps its chunks; an unchunked one opened from
75
+ storage is split along the storage chunks its backend recorded, and is
76
+ one chunk along other dimensions. A dataset that records none (built in
77
+ memory) is split into blocks of about ``_DEFAULT_BLOCK_ROWS`` rows.
78
+
79
+ Returns an empty mapping for scalar datasets; callers should treat that
80
+ as "one block covering everything".
81
+ """
82
+ if chunks is not None:
83
+ chunks = {dim: size for dim, size in chunks.items() if dim in ds.sizes}
84
+ if chunks:
85
+ return compute_chunks(ds, chunks)
86
+ if ds.chunks:
87
+ return {d: tuple(c) for d, c in ds.chunks.items()}
88
+ return compute_chunks(ds, _storage_chunks(ds) or _default_chunks(ds))
89
+
90
+
91
+ def _storage_chunks(ds: xr.Dataset) -> dict[str, int]:
92
+ """Chunk sizes from the backend's ``preferred_chunks`` encoding.
93
+
94
+ Zarr and NetCDF backends record each variable's on-disk chunking there.
95
+ Where variables disagree on a dimension, the first one wins.
96
+ """
97
+ chunks: dict[str, int] = {}
98
+ for var in ds.data_vars.values():
99
+ for dim, size in var.encoding.get("preferred_chunks", {}).items():
100
+ chunks.setdefault(dim, size)
101
+ return chunks
102
+
103
+
104
+ def _default_chunks(ds: xr.Dataset) -> dict[str, int]:
105
+ """Chunk sizes giving blocks of at most about ``_DEFAULT_BLOCK_ROWS`` rows.
106
+
107
+ Inner dimensions stay whole while they fit, so each block is a
108
+ contiguous run of the row-major grid; the outermost dimension that
109
+ does not fit is split, and the dimensions outside it get size one.
110
+ """
111
+ chunks: dict[str, int] = {}
112
+ inner = 1
113
+ for dim in reversed(list(ds.sizes)):
114
+ size = ds.sizes[dim]
115
+ step = max(1, min(size, _DEFAULT_BLOCK_ROWS // inner))
116
+ if step < size:
117
+ chunks[str(dim)] = step
118
+ inner *= step
119
+ return chunks
120
+
121
+
122
+ def _ensure_default_indexes(ds: xr.Dataset) -> xr.Dataset:
123
+ """Attach a default integer index coordinate to every dimension lacking one.
124
+
125
+ xarray allows "dimensions without coordinates"; these are absent from
126
+ ``ds.coords``, so they are dropped from the SQL schema and, once a block is
127
+ sliced out with ``isel``, their position is synthesized *relative to the
128
+ block* (restarting at 0 in every partition). Materialising an explicit
129
+ ``arange`` index up front turns them into ordinary dimension coordinates, so
130
+ they appear as columns and carry their absolute position through chunked
131
+ reads. Datasets whose dimensions already have coordinates are returned
132
+ unchanged.
133
+ """
134
+ missing = {
135
+ dim: np.arange(ds.sizes[dim]) for dim in ds.dims if dim not in ds.coords
136
+ }
137
+ return ds.assign_coords(missing) if missing else ds
138
+
139
+
140
+ def _block_slices_from_resolved(
141
+ ds: xr.Dataset, resolved: Mapping[Hashable, tuple[int, ...]]
142
+ ) -> Iterator[Block]:
143
+ """Emit blocks given pre-resolved per-dim chunk tuples."""
144
+ if not resolved:
145
+ # A dimensionless dataset (e.g. scalar metadata variables) is a
146
+ # single block.
147
+ yield {}
148
+ return
149
+
150
+ chunk_bounds = {
151
+ dim: np.cumsum((0,) + tuple(c)) for dim, c in resolved.items()
152
+ }
153
+ ichunk = {dim: range(len(tuple(c))) for dim, c in resolved.items()}
154
+ ick, icv = zip(*ichunk.items()) # Makes same order of keys and val.
155
+ chunk_idxs = (dict(zip(ick, i)) for i in itertools.product(*icv))
156
+ yield from (
157
+ {
158
+ dim: _get_chunk_slicer(dim, chunk_index, chunk_bounds)
159
+ for dim in ds.dims
160
+ }
161
+ for chunk_index in chunk_idxs
162
+ )
163
+
164
+
165
+ # Adapted from Xarray `map_blocks` implementation.
166
+ def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> Iterator[Block]:
167
+ """Compute block slices for a chunked Dataset."""
168
+ yield from _block_slices_from_resolved(ds, resolve_chunks(ds, chunks))
169
+
170
+
171
+ def explode(ds: xr.Dataset, chunks: Chunks = None) -> Iterator[xr.Dataset]:
172
+ """Explodes a dataset into its chunks."""
173
+ yield from (ds.isel(b) for b in block_slices(ds, chunks=chunks))
174
+
175
+
176
+ def group_vars_by_dims(ds: xr.Dataset) -> dict[tuple[str, ...], list[str]]:
177
+ """Group a Dataset's data variables by their exact dimension tuple.
178
+
179
+ Variables that share dimensions can share a table; each distinct
180
+ dimension tuple becomes its own table when a mixed-dimension Dataset
181
+ is registered::
182
+
183
+ ("time", "lat", "lon"): ["temperature_2m", "wind_speed"],
184
+ ("time", "lat", "lon", "level"): ["pressure", "humidity"]
185
+ """
186
+ groups = defaultdict(list)
187
+ for var_name, var in ds.data_vars.items():
188
+ dims = var.dims
189
+ groups[dims].append(var_name)
190
+ return groups
191
+
192
+
193
+ def default_table_name(dims: tuple[str, ...]) -> str:
194
+ """The table name a dimension group gets when the user names none."""
195
+ # Scalar variables group under empty dims, where "_".join(()) is the
196
+ # empty string; fall back to a valid default table name.
197
+ return "_".join(dims) or "scalar"
198
+
199
+
200
+ def resolve_table_names(
201
+ ds: xr.Dataset,
202
+ table_names: TableNames = None,
203
+ *,
204
+ case_insensitive: bool = False,
205
+ ) -> dict[tuple[str, ...], str]:
206
+ """Name every dimension group of ``ds``, honouring user overrides.
207
+
208
+ ``table_names`` maps a group's exact dimension tuple to the name its
209
+ table should carry; groups it does not mention keep
210
+ [default_table_name][xarray_sql.df.default_table_name]. Keys that
211
+ match no group in ``ds`` are ignored, so one naming map can be reused
212
+ across Datasets that hold different subsets of the same variables.
213
+
214
+ Every engine adapter routes through here, which is what makes
215
+ ``table_names={('time', 'lat', 'lon'): 'surface'}`` mean the same
216
+ thing on DataFusion, DuckDB, and the pyarrow-dataset engines.
217
+
218
+ Args:
219
+ ds: The Dataset whose dimension groups are being named.
220
+ table_names: User overrides; see above.
221
+ case_insensitive: Fold names before comparing them for
222
+ collisions — DuckDB identifiers are case-insensitive even
223
+ quoted, so ``'surface'`` and ``'SURFACE'`` name the same
224
+ table there even though they are distinct strings. Other
225
+ engines compare case-sensitively.
226
+
227
+ Raises:
228
+ ValueError: if two groups would end up with the same name (or,
229
+ with ``case_insensitive=True``, names differing only in
230
+ case), which would silently register one table over the
231
+ other.
232
+ """
233
+ overrides = table_names or {}
234
+ names = {
235
+ dims: overrides.get(dims, default_table_name(dims))
236
+ for dims in group_vars_by_dims(ds)
237
+ }
238
+ taken: dict[str, tuple[tuple[str, ...], str]] = {}
239
+ for dims, name in names.items():
240
+ key = name.casefold() if case_insensitive else name
241
+ if key in taken:
242
+ other_dims, other_name = taken[key]
243
+ detail = (
244
+ f"the same table name {name!r}"
245
+ if other_name == name
246
+ else f"names that collide under DuckDB's case-insensitive "
247
+ f"identifiers: {other_name!r} and {name!r}"
248
+ )
249
+ raise ValueError(
250
+ f"table_names maps two dimension groups to {detail}: "
251
+ f"{other_dims} and {dims}. Give each group a distinct name."
252
+ )
253
+ taken[key] = (dims, name)
254
+ return names
255
+
256
+
257
+ def shared_coord_arrays(ds: xr.Dataset) -> dict[str, np.ndarray]:
258
+ """Materialise ``ds``'s dimension coordinates once, to share.
259
+
260
+ Splitting a Dataset into per-dimension-group tables otherwise reads
261
+ every shared dim coordinate once per table — a network round-trip
262
+ apiece for Zarr-backed parents like ARCO-ERA5.
263
+ """
264
+ return {str(dim): ds.coords[dim].values for dim in ds.dims}
265
+
266
+
267
+ def _block_len(block: Block) -> int:
268
+ return int(np.prod([v.stop - v.start for v in block.values()]))
269
+
270
+
271
+ def from_map_batched(
272
+ func: Callable[..., pd.DataFrame],
273
+ *iterables: tuple[Any, ...],
274
+ args: tuple | None = None,
275
+ schema: pa.Schema = None,
276
+ **kwargs: dict[str, Any],
277
+ ) -> pa.RecordBatchReader:
278
+ """Create a PyArrow RecordBatchReader by mapping a function over iterables.
279
+
280
+ This is equivalent to dask's from_map but returns a PyArrow
281
+ RecordBatchReader that can be used with DataFusion. It iterates over
282
+ RecordBatches which are created via the `func` one-at-a-time.
283
+
284
+ Args:
285
+ func: Function to apply to each element of the iterables. Currently, the
286
+ function must return a Pandas DataFrame.
287
+ *iterables: Iterable objects to map the function over.
288
+ schema: Optional schema needed for the RecordBatchReader.
289
+ args: Additional positional arguments to pass to func.
290
+ **kwargs: Additional keyword arguments to pass to func.
291
+
292
+ Returns:
293
+ A PyArrow RecordBatchReader containing the stream of RecordBatches.
294
+ """
295
+ if args is None:
296
+ args = ()
297
+
298
+ def map_batches():
299
+ for items in zip(*iterables):
300
+ df = func(*items, *args, **kwargs)
301
+ yield pa.RecordBatch.from_pandas(df, schema=schema)
302
+
303
+ return pa.RecordBatchReader.from_batches(schema, map_batches())
304
+
305
+
306
+ def from_map(
307
+ func: Callable,
308
+ *iterables: tuple[Any, ...],
309
+ args: tuple | None = None,
310
+ **kwargs: dict[str, Any],
311
+ ) -> pa.Table:
312
+ """Create a PyArrow Table by mapping a function over iterables.
313
+
314
+ This is equivalent to dask's from_map but returns a PyArrow Table
315
+ that can be used with DataFusion instead of a Dask DataFrame.
316
+
317
+ Args:
318
+ func: Function to apply to each element of the iterables.
319
+ *iterables: Iterable objects to map the function over.
320
+ args: Additional positional arguments to pass to func.
321
+ **kwargs: Additional keyword arguments to pass to func.
322
+
323
+ Returns:
324
+ A PyArrow Table containing the concatenated results.
325
+ """
326
+ if args is None:
327
+ args = ()
328
+
329
+ # Apply the function to each combination of iterable elements
330
+ results = []
331
+ for items in zip(*iterables) if len(iterables) > 1 else iterables[0]:
332
+ if isinstance(items, tuple):
333
+ result = func(*items, *args, **kwargs)
334
+ else:
335
+ result = func(items, *args, **kwargs)
336
+
337
+ # Convert result to PyArrow Table
338
+ if isinstance(result, pd.DataFrame):
339
+ pa_table = pa.Table.from_pandas(result)
340
+ elif isinstance(result, pa.Table):
341
+ pa_table = result
342
+ else:
343
+ # Try to convert to pandas first, then to PyArrow
344
+ try:
345
+ df = pd.DataFrame(result)
346
+ pa_table = pa.Table.from_pandas(df)
347
+ except Exception as e:
348
+ raise ValueError(
349
+ f"Cannot convert function result to PyArrow Table: {e}"
350
+ )
351
+
352
+ results.append(pa_table)
353
+
354
+ # Concatenate all results
355
+ if not results:
356
+ raise ValueError("No results to concatenate")
357
+
358
+ return pa.concat_tables(results)
359
+
360
+
361
+ def pivot(ds: xr.Dataset) -> pd.DataFrame:
362
+ """Converts an xarray Dataset to a pandas DataFrame."""
363
+ return ds.to_dataframe().reset_index() # type: ignore[no-any-return]
364
+
365
+
366
+ def dataset_to_record_batch(
367
+ ds: xr.Dataset, schema: pa.Schema
368
+ ) -> pa.RecordBatch:
369
+ """Convert an xarray Dataset partition to an Arrow RecordBatch.
370
+
371
+ Builds the RecordBatch directly from numpy arrays, bypassing the pandas
372
+ round-trip (to_dataframe → reset_index → from_pandas) used by pivot().
373
+ For large partitions this reduces peak memory from ~5× to ~2× the
374
+ partition size.
375
+
376
+ Dimension coordinates are broadcast to the full partition shape and
377
+ ravelled. np.broadcast_to() is zero-copy; the ravel() forces one copy
378
+ per coordinate (unavoidable, since broadcast arrays are non-contiguous).
379
+ Data variable arrays are ravelled in-place — a zero-copy view when the
380
+ underlying array is already C-contiguous (the common case for numpy-backed
381
+ xarray datasets).
382
+
383
+ Args:
384
+ ds: A partition-sized xarray Dataset (already sliced via isel).
385
+ schema: The Arrow schema for the output, as produced by _parse_schema.
386
+ Column order in the output matches schema field order.
387
+
388
+ Returns:
389
+ A RecordBatch with one column per dimension coordinate and data
390
+ variable, in schema order.
391
+ """
392
+ # Use the data variable's dimension order as canonical so coordinate
393
+ # broadcasts and data variable ravels use the same layout. All data
394
+ # variables are validated to share the same dims tuple.
395
+ if ds.data_vars:
396
+ first_var = next(iter(ds.data_vars.values()))
397
+ dim_names = list(first_var.dims)
398
+ shape = first_var.shape
399
+ else:
400
+ dim_names = list(ds.sizes.keys())
401
+ shape = tuple(ds.sizes[d] for d in dim_names)
402
+
403
+ arrays = []
404
+ for field in schema:
405
+ name = field.name
406
+ if name in ds.coords and name in ds.dims:
407
+ # Broadcast 1-D coordinate to the full N-D partition shape, then ravel.
408
+ axis = dim_names.index(name)
409
+ coord = ds.coords[name].values
410
+ if cft.is_cftime(coord):
411
+ coord = cft.convert_for_field(coord, field)
412
+ reshape = [1] * len(shape)
413
+ reshape[axis] = coord.shape[0]
414
+ arr = np.broadcast_to(coord.reshape(reshape), shape).ravel()
415
+ arrays.append(pa.array(arr, type=field.type))
416
+ else:
417
+ # Data variable: ravel to 1-D (zero-copy for C-contiguous arrays).
418
+ raw = ds[name].values.ravel()
419
+ if cft.is_cftime(ds[name].values):
420
+ raw = cft.convert_for_field(ds[name].values, field)
421
+
422
+ # from_pandas=True maps NaN → Arrow null inside the C++ copy kernel,
423
+ # so SQL aggregates (MAX, MIN, AVG) skip missing values correctly.
424
+ arrays.append(pa.array(raw, type=field.type, from_pandas=True))
425
+
426
+ return pa.RecordBatch.from_arrays(arrays, schema=schema)
427
+
428
+
429
+ DEFAULT_BATCH_SIZE: int = 65_536
430
+ """Default number of rows per emitted Arrow RecordBatch.
431
+
432
+ 64 K rows balances DataFusion pipeline depth against per-batch overhead.
433
+ """
434
+
435
+ _FULL_PIVOT_MAX_ROWS: int = 8_388_608
436
+ """Row cap for the whole-partition coordinate fast path in
437
+ iter_record_batches.
438
+
439
+ Below this, coordinate columns are materialised for the full partition
440
+ with repeat/tile (sequential writes, ~3x faster than per-batch index
441
+ arithmetic) and batches are zero-copy slices; the cost is holding every
442
+ coordinate column of the partition in memory at once (rows x 8 bytes x
443
+ n_dims). Above it — e.g. single-time-step reanalysis partitions with
444
+ tens of millions of rows — the per-batch path keeps peak memory at
445
+ O(batch_size) per coordinate instead.
446
+ """
447
+
448
+
449
+ def _as_single_array(values, type: pa.DataType, *, from_pandas: bool = False):
450
+ """``pa.array`` that always returns a contiguous ``pa.Array``.
451
+
452
+ ``pa.array`` may return a ``ChunkedArray`` instead of an ``Array`` for
453
+ large inputs (observed for numpy fixed-width unicode columns of a few
454
+ million rows — e.g. a string dimension coordinate tiled across a full
455
+ partition). ``RecordBatch.from_arrays`` rejects chunked input, so
456
+ flatten it back to one contiguous array.
457
+ """
458
+ arr = pa.array(values, type=type, from_pandas=from_pandas)
459
+ if isinstance(arr, pa.ChunkedArray):
460
+ arr = arr.combine_chunks()
461
+ return arr
462
+
463
+
464
+ def iter_record_batches(
465
+ ds: xr.Dataset,
466
+ schema: pa.Schema,
467
+ batch_size: int = DEFAULT_BATCH_SIZE,
468
+ ) -> Iterator[pa.RecordBatch]:
469
+ """Yield RecordBatches of at most *batch_size* rows from a partition Dataset.
470
+
471
+ Unlike `dataset_to_record_batch`, which materialises the entire
472
+ partition as one batch, this generator emits smaller batches so that
473
+ DataFusion can begin filtering and aggregating before the full partition
474
+ is loaded. Peak memory per batch is O(batch_size) for coordinate columns
475
+ and O(partition_size) for data-variable columns (which must be loaded in
476
+ full from storage).
477
+
478
+ Coordinate values are computed per batch via strided index arithmetic —
479
+ no broadcast array spanning the whole partition is ever allocated. Data
480
+ variable flat arrays are loaded once (triggering any remote I/O) and then
481
+ sliced as zero-copy views for each batch.
482
+
483
+ Args:
484
+ ds: A partition-sized xarray Dataset (already sliced via isel).
485
+ schema: The Arrow schema for the output, as produced by _parse_schema.
486
+ batch_size: Maximum number of rows per yielded RecordBatch.
487
+
488
+ Yields:
489
+ RecordBatches in schema column order, covering all rows of the
490
+ partition exactly once.
491
+ """
492
+ if ds.data_vars:
493
+ first_var = next(iter(ds.data_vars.values()))
494
+ dim_names = list(first_var.dims)
495
+ shape = first_var.shape
496
+ else:
497
+ dim_names = list(ds.sizes.keys())
498
+ shape = tuple(ds.sizes[d] for d in dim_names)
499
+
500
+ total_rows = int(np.prod(shape))
501
+
502
+ # Preload small 1-D coordinate arrays (negligible memory).
503
+ # Convert cftime objects to numeric values matching the schema type.
504
+ # Projected scans may omit dimension columns from the schema; those
505
+ # dims still shape the iteration but never emit a column.
506
+ coord_values = {}
507
+ schema_names = set(schema.names)
508
+ for name in dim_names:
509
+ # A dim the projection dropped (e.g. time under GROUP BY level) is never
510
+ # read in the batch loop below, which only iterates the schema's fields.
511
+ # Skip it so schema.field(name) is not called for a projected-away name
512
+ # (it raises for cftime coords, which take the convert_for_field path).
513
+ if name not in schema_names:
514
+ continue
515
+ vals = ds.coords[name].values
516
+ if cft.is_cftime(vals):
517
+ coord_values[name] = cft.convert_for_field(vals, schema.field(name))
518
+ else:
519
+ coord_values[name] = vals
520
+
521
+ # C-order stride for each dimension: stride[k] = prod(shape[k+1:]).
522
+ # Flat row index i → coordinate index for dim k: (i // stride[k]) % shape[k].
523
+ strides = [int(np.prod(shape[k + 1 :])) for k in range(len(shape))]
524
+
525
+ # Load data-variable arrays fully (triggers Dask/Zarr compute once).
526
+ # ravel() is a zero-copy view for C-contiguous arrays.
527
+ data_arrays = {}
528
+ for field in schema:
529
+ if field.name not in ds.dims:
530
+ raw = ds[field.name].values
531
+ if cft.is_cftime(raw):
532
+ data_arrays[field.name] = cft.convert_for_field(raw, field)
533
+ else:
534
+ data_arrays[field.name] = raw.ravel()
535
+
536
+ if 0 < total_rows <= _FULL_PIVOT_MAX_ROWS:
537
+ # Fast path: build each coordinate column once for the whole
538
+ # partition. In C order, dim k's flat column is its coord values
539
+ # each repeated prod(shape[k+1:]) times, with that pattern tiled
540
+ # prod(shape[:k]) times — two sequential-write kernels, much
541
+ # faster than per-batch division/modulo plus gather. Batches are
542
+ # then zero-copy slices of the full-partition Arrow arrays.
543
+ full_arrays = []
544
+ for field in schema:
545
+ name = field.name
546
+ if name in ds.coords and name in ds.dims:
547
+ k = dim_names.index(name)
548
+ outer = int(np.prod(shape[:k]))
549
+ col: np.ndarray = np.repeat(coord_values[name], strides[k])
550
+ if outer > 1:
551
+ col = np.tile(col, outer)
552
+ full_arrays.append(_as_single_array(col, field.type))
553
+ else:
554
+ full_arrays.append(
555
+ _as_single_array(
556
+ data_arrays[name], field.type, from_pandas=True
557
+ )
558
+ )
559
+ for row_start in range(0, total_rows, batch_size):
560
+ yield pa.RecordBatch.from_arrays(
561
+ [a.slice(row_start, batch_size) for a in full_arrays],
562
+ schema=schema,
563
+ )
564
+ return
565
+
566
+ for row_start in range(0, total_rows, batch_size):
567
+ row_end = min(row_start + batch_size, total_rows)
568
+ row_idx = np.arange(row_start, row_end)
569
+
570
+ arrays = []
571
+ for field in schema:
572
+ name = field.name
573
+ if name in ds.coords and name in ds.dims:
574
+ k = dim_names.index(name)
575
+ coord_idx = (row_idx // strides[k]) % shape[k]
576
+ arrays.append(
577
+ _as_single_array(coord_values[name][coord_idx], field.type)
578
+ )
579
+ else:
580
+ arrays.append(
581
+ _as_single_array(
582
+ data_arrays[name][row_start:row_end],
583
+ field.type,
584
+ from_pandas=True,
585
+ )
586
+ )
587
+
588
+ yield pa.RecordBatch.from_arrays(arrays, schema=schema)
589
+
590
+
591
+ def _arrow_type_for_object(values: np.ndarray) -> pa.DataType:
592
+ """Infer an Arrow type for a non-cftime object-dtype array.
593
+
594
+ ``pa.from_numpy_dtype`` cannot map numpy object dtype, so let pyarrow infer
595
+ the type from the data instead: strings become ``pa.string()``, bytes
596
+ ``pa.binary()``, and other representable Python scalars their Arrow
597
+ equivalent. An all-null array stays ``pa.null()``, and a column mixing
598
+ incompatible types (e.g. str and int) raises, surfacing a clear error
599
+ rather than a silent coercion. Object-dtype arrays are never Dask/Zarr
600
+ backed, so this triggers no remote I/O.
601
+ """
602
+ return pa.array(np.asarray(values).ravel()).type
603
+
604
+
605
+ def _arrow_type_for_categorical(values: np.ndarray) -> pa.DataType:
606
+ """Return the Arrow value type emitted for a categorical coordinate.
607
+
608
+ xarray preserves a pandas ``CategoricalDtype`` on the coordinate variable,
609
+ but ``.values`` exposes the category labels that are actually written to
610
+ Arrow. Infer from those labels so string categories become UTF-8 and
611
+ numeric categories retain their numeric width.
612
+ """
613
+ values = np.asarray(values)
614
+ if values.dtype == np.dtype("O"):
615
+ return _arrow_type_for_object(values)
616
+ return pa.from_numpy_dtype(values.dtype)
617
+
618
+
619
+ def _parse_schema(ds: xr.Dataset) -> pa.Schema:
620
+ """Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns.
621
+
622
+ Only *dimension coordinates* become dimension columns, so a dimension
623
+ without a coordinate would be dropped. Callers must run the Dataset through
624
+ ``_ensure_default_indexes`` first (the readers do) so every dimension
625
+ has a coordinate and appears as a column.
626
+
627
+ Uses the xarray index type to detect cftime coordinates without
628
+ materializing their data — important for Dask/Zarr-backed datasets
629
+ where .values would trigger eager computation.
630
+
631
+ cftime coordinates are mapped to one of two Arrow types:
632
+
633
+ * **Gregorian-like calendars** (standard, noleap, all_leap, etc.):
634
+ ``pa.timestamp('us')`` so string-based SQL filters work naturally.
635
+ * **Non-Gregorian calendars** (360_day, julian):
636
+ ``pa.int64()`` with ``xarray:units`` / ``xarray:calendar`` metadata
637
+ on the field, preserving lossless CF-convention encoding.
638
+ """
639
+ columns = []
640
+
641
+ for coord_name, coord_var in ds.coords.items():
642
+ # Only include dimension coordinates
643
+ if coord_name in ds.dims:
644
+ if cft.is_cftime_index(ds, coord_name):
645
+ units, calendar = cft.encoding(ds, coord_name)
646
+ columns.append(cft.arrow_field(coord_name, units, calendar))
647
+ elif isinstance(coord_var.dtype, pd.CategoricalDtype):
648
+ columns.append(
649
+ pa.field(
650
+ coord_name,
651
+ _arrow_type_for_categorical(coord_var.values),
652
+ )
653
+ )
654
+ elif coord_var.dtype == np.dtype("O"):
655
+ # Object dtype that isn't cftime (e.g. string station names).
656
+ arrow_type = _arrow_type_for_object(coord_var.values)
657
+ columns.append(pa.field(coord_name, arrow_type))
658
+ else:
659
+ pa_type = pa.from_numpy_dtype(coord_var.dtype)
660
+ columns.append(pa.field(coord_name, pa_type))
661
+
662
+ for var_name, var in ds.data_vars.items():
663
+ # An object-dtype data variable may hold cftime objects (encode it like
664
+ # a cftime coordinate) or strings/other Python scalars (infer the Arrow
665
+ # type from the data). The dtype check keeps the common numeric path off
666
+ # the object branch.
667
+ if var.dtype == np.dtype("O"):
668
+ if cft.is_cftime(var.values):
669
+ # Encode with the same units/calendar as a cftime coordinate.
670
+ cal = var.values.ravel()[0].calendar
671
+ columns.append(
672
+ cft.arrow_field(var_name, cft.DEFAULT_UNITS, cal)
673
+ )
674
+ else:
675
+ arrow_type = _arrow_type_for_object(var.values)
676
+ columns.append(pa.field(var_name, arrow_type))
677
+ else:
678
+ pa_type = pa.from_numpy_dtype(var.dtype)
679
+ columns.append(pa.field(var_name, pa_type))
680
+
681
+ return pa.schema(columns)
682
+
683
+
684
+ # Type alias for partition metadata: maps dimension name to (min, max, dtype_str) values
685
+ PartitionBounds = dict[str, tuple[Any, Any, str]]
686
+
687
+
688
+ _INTEGER_BOUND_TAGS = {
689
+ ("i", 1): "int8",
690
+ ("i", 2): "int16",
691
+ ("i", 4): "int32",
692
+ ("i", 8): "int64",
693
+ ("u", 1): "uint8",
694
+ ("u", 2): "uint16",
695
+ ("u", 4): "uint32",
696
+ ("u", 8): "uint64",
697
+ }
698
+ _FLOAT_BOUND_TAGS = {2: "float16", 4: "float32", 8: "float64"}
699
+ _TEMPORAL_BOUND_UNITS = {"s", "ms", "us", "ns"}
700
+
701
+
702
+ def _coordinate_partition_bound(
703
+ coord_values: np.ndarray,
704
+ ) -> tuple[Any, Any, str] | None:
705
+ """Return an exact, Rust-compatible bound for one coordinate slice.
706
+
707
+ Unsupported, null-tainted, or unordered values return ``None``. Partition
708
+ pruning is optional, so omitting unsafe metadata is always preferable to
709
+ coercing a value or failing dataset registration.
710
+ """
711
+ values = np.asarray(coord_values)
712
+ if values.size == 0:
713
+ return None
714
+
715
+ kind = values.dtype.kind
716
+
717
+ if kind in ("M", "m"):
718
+ if np.isnat(values).any():
719
+ return None
720
+ unit, step = np.datetime_data(values.dtype)
721
+ if unit not in _TEMPORAL_BOUND_UNITS or step != 1:
722
+ return None
723
+ raw = values.astype(np.int64, copy=False)
724
+ prefix = "timestamp" if kind == "M" else "duration"
725
+ return int(raw.min()), int(raw.max()), f"{prefix}_{unit}"
726
+
727
+ if kind == "b":
728
+ return bool(values.min()), bool(values.max()), "bool"
729
+
730
+ if kind in ("i", "u"):
731
+ tag = _INTEGER_BOUND_TAGS.get((kind, values.dtype.itemsize))
732
+ if tag is None:
733
+ return None
734
+ return int(values.min()), int(values.max()), tag
735
+
736
+ if kind == "f":
737
+ tag = _FLOAT_BOUND_TAGS.get(values.dtype.itemsize)
738
+ if tag is None or np.isnan(values).any():
739
+ return None
740
+ return float(values.min()), float(values.max()), tag
741
+
742
+ if kind in ("U", "S", "O"):
743
+ python_values = values.ravel().tolist()
744
+ if all(isinstance(value, str) for value in python_values):
745
+ return min(python_values), max(python_values), "utf8"
746
+ if all(isinstance(value, bytes) for value in python_values):
747
+ return min(python_values), max(python_values), "binary"
748
+
749
+ return None
750
+
751
+
752
+ def _block_metadata(
753
+ coord_arrays: dict,
754
+ block: Block,
755
+ dims: Iterable[Hashable] | None = None,
756
+ ) -> PartitionBounds:
757
+ """Compute min/max coordinate values for a single partition block.
758
+
759
+ Args:
760
+ coord_arrays: Pre-materialised coordinate arrays keyed by dimension name
761
+ string. Hoist this outside any loop to avoid repeated remote I/O
762
+ for Zarr-backed datasets.
763
+ block: A single block slice dict from block_slices().
764
+ dims: Optional restriction to a subset of dims to compute. Used by
765
+ ``read_xarray_table`` to skip unchunked dims whose bounds are
766
+ constant across all partitions and have been precomputed once.
767
+ Defaults to all dims present in ``block``.
768
+
769
+ Returns:
770
+ Dict mapping dimension name to (min_value, max_value, dtype_str).
771
+ Dimensions with an empty slice are omitted; the Rust pruning logic
772
+ treats missing dimensions conservatively (never prunes on them).
773
+ """
774
+ items = ((d, block[d]) for d in dims) if dims is not None else block.items()
775
+ ranges: PartitionBounds = {}
776
+ for dim, slc in items:
777
+ coord_values = coord_arrays[str(dim)][slc]
778
+ if len(coord_values) == 0:
779
+ continue
780
+ # cftime coordinates are object dtype but carry their own bound
781
+ # encoding, so they must be handled before the string/object skip
782
+ # below (otherwise pruning is silently disabled for them).
783
+ # partition_bounds returns None when the bound overflows int64.
784
+ if cft.is_cftime(coord_values):
785
+ bounds = cft.partition_bounds(coord_values)
786
+ if bounds is not None:
787
+ ranges[str(dim)] = bounds
788
+ continue
789
+ # Use actual min/max rather than first/last so non-monotonic axes are
790
+ # safe. Expected dtype/conversion failures disable pruning only for
791
+ # this dimension; they must not make table registration fail.
792
+ try:
793
+ bounds = _coordinate_partition_bound(coord_values)
794
+ except (OverflowError, TypeError, ValueError):
795
+ bounds = None
796
+ if bounds is not None:
797
+ ranges[str(dim)] = bounds
798
+ return ranges
799
+
800
+
801
+ def partition_metadata(
802
+ ds: xr.Dataset, blocks: list[Block]
803
+ ) -> list[PartitionBounds]:
804
+ """Compute min/max coordinate values for each partition.
805
+
806
+ This metadata enables filter pushdown: SQL queries with WHERE clauses
807
+ on dimension columns can prune partitions that can't contain matching rows.
808
+
809
+ Args:
810
+ ds: The xarray Dataset containing coordinate values.
811
+ blocks: List of block slices from block_slices().
812
+
813
+ Returns:
814
+ List of dicts mapping dimension name to
815
+ (min_value, max_value, dtype_str) tuples.
816
+
817
+ Values preserve the coordinate's logical type and width. Temporal
818
+ values are signed integer counts in their Arrow unit; strings and
819
+ bytes use lexical bounds. Unsupported or null-tainted dimensions
820
+ are omitted so the Rust layer retains their partitions.
821
+
822
+ Note:
823
+ If a partition has an empty slice for a dimension, that dimension is
824
+ omitted from the partition's metadata. The Rust pruning logic treats
825
+ missing dimensions conservatively (never prunes on them).
826
+ """
827
+ # Hoist coordinate array reads outside the partition loop.
828
+ # ds.coords[dim].values materializes the full array on every call; doing it
829
+ # N_partitions × N_dims times is wasteful and, for remote Zarr-backed datasets
830
+ # (e.g. ARCO-ERA5 on GCS), may trigger repeated network I/O.
831
+ coord_arrays = {str(dim): ds.coords[dim].values for dim in ds.dims}
832
+ return [_block_metadata(coord_arrays, block) for block in blocks]