xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/ds.py ADDED
@@ -0,0 +1,1286 @@
1
+ """Reconstruct xarray Datasets from SQL query results.
2
+
3
+ The inverse of the forward Dataset-to-table pivot done by
4
+ [xarray_sql.df.pivot][]. Internally defines an [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame]
5
+ wrapper around the DataFusion ``DataFrame`` returned by
6
+ [Context.sql][xarray_sql.sql.Context.sql], with a [XarrayDataFrame.to_dataset][xarray_sql.ds.XarrayDataFrame.to_dataset]
7
+ method that round-trips a query result back to ``xr.Dataset``.
8
+
9
+ Reconstruction is controlled by the ``chunks`` argument to
10
+ [XarrayDataFrame.to_dataset][xarray_sql.ds.XarrayDataFrame.to_dataset] -- the xarray idiom for tuning how a
11
+ result is partitioned -- rather than by reflecting on the query plan:
12
+
13
+ * **Eager** (``chunks=None``, or the default ``"inherit"`` when the
14
+ result keeps no multi-chunk source dimension): the plan executes
15
+ exactly once via ``execute_stream`` and the result is scattered into a
16
+ dense in-memory Dataset. This is the right default for reductions
17
+ (aggregations), whose results are small, and it never re-executes.
18
+ * **Lazy / chunked** (``chunks`` is a mapping, ``"auto"``, or
19
+ ``"inherit"`` over a multi-chunk source dimension): data variables are
20
+ backed by [SQLBackendArray][xarray_sql.ds.SQLBackendArray] wrapped in
21
+ ``xarray.core.indexing.LazilyIndexedArray`` and chunked via xarray's
22
+ configured chunk manager (dask, cubed, ...). Each chunk maps onto the
23
+ source partitions and reads its coordinate range on access by
24
+ translating the indexer into a DataFusion ``filter`` expression, so only
25
+ the requested partitions are materialized as Arrow ``RecordBatch`` es
26
+ and scattered into numpy.
27
+
28
+ ``.compute()`` materializes the whole Dataset in memory.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import warnings
34
+ from collections.abc import Mapping
35
+ from typing import Any, Literal, cast
36
+
37
+ import numpy as np
38
+ import pandas as pd
39
+ import pyarrow as pa
40
+ import pyarrow.compute as pc
41
+ import xarray as xr
42
+
43
+ from .lazyscan import DataFusionHandle, DimSpec, LazyResultHandle
44
+
45
+ Sparsity = Literal["result", "template"]
46
+ """Output coordinate extent for a filtered round-trip.
47
+
48
+ * ``"result"`` keeps only the dim values present in the query result, so
49
+ the output is sparse and equal to whatever rows came back.
50
+ * ``"template"`` reindexes to the registered Dataset's full coord ranges
51
+ and fills absent cells with ``fill_value``.
52
+ """
53
+
54
+
55
+ # ---------------------------------------------------------------------------
56
+ # Private helpers
57
+ # ---------------------------------------------------------------------------
58
+
59
+
60
+ def _result_dims(template: xr.Dataset, columns) -> list[str]:
61
+ """The template's dims that survive into a result's *columns*.
62
+
63
+ In the variables' axis order, not ``template.dims``'s name order. A
64
+ mixed-dimension template has one order per group of variables, so
65
+ the variables the result carries pick theirs: ``AVG(temperature) ...
66
+ GROUP BY level`` keeps ``level`` although the template's first
67
+ variable has none. A result carrying no template variable (only
68
+ aliases such as ``wind``) may keep any template dim.
69
+ """
70
+ columns = set(columns)
71
+ carried = [name for name in template.data_vars if name in columns]
72
+ order: list = []
73
+ for name in carried or list(template.data_vars):
74
+ order.extend(d for d in template[name].dims if d not in order)
75
+ if not order:
76
+ order = list(template.dims)
77
+ return [d for d in order if d in columns]
78
+
79
+
80
+ _TIMEDELTA_PARTS = (
81
+ "weeks",
82
+ "days",
83
+ "hours",
84
+ "minutes",
85
+ "seconds",
86
+ "milliseconds",
87
+ "microseconds",
88
+ "nanoseconds",
89
+ )
90
+
91
+
92
+ def _timedelta(value: Any) -> pd.Timedelta:
93
+ """One duration a database returned as an interval or as text.
94
+
95
+ Arrow's month-day-nano intervals (DuckDB, PostgreSQL) arrive as
96
+ pandas ``DateOffset`` objects; MySQL and Trino return text such as
97
+ ``'21600s'``.
98
+ """
99
+ if value is None:
100
+ return pd.NaT
101
+ if isinstance(value, pd.DateOffset):
102
+ parts = value.kwds
103
+ if parts.get("years") or parts.get("months"):
104
+ raise ValueError(
105
+ f"{value!r} spans calendar months, which have no fixed duration"
106
+ )
107
+ return pd.Timedelta(**{k: parts.get(k, 0) for k in _TIMEDELTA_PARTS})
108
+ return pd.Timedelta(value)
109
+
110
+
111
+ def _as_dtype(coord: xr.DataArray, dtype: np.dtype) -> xr.DataArray:
112
+ """*coord* cast to the template's *dtype*.
113
+
114
+ Databases without a duration type hand timedeltas back as intervals
115
+ or text, which numpy cannot cast; those are parsed first. (Integers
116
+ need nothing: numpy reads them as counts of the template's unit,
117
+ which is how durations are stored where no duration type exists.)
118
+ """
119
+ if dtype.kind == "m" and coord.dtype.kind in "OUS":
120
+ values = np.array(
121
+ [_timedelta(v) for v in coord.values], dtype="timedelta64[ns]"
122
+ )
123
+ return coord.copy(data=values.astype(dtype))
124
+ return coord.astype(dtype)
125
+
126
+
127
+ _NARROWINGS = {("i", "b"), ("u", "b"), ("i", "u"), ("f", "f")}
128
+ """(result kind, template kind) pairs a database may have widened."""
129
+
130
+
131
+ def _restore_dtype(var: xr.DataArray, template: xr.DataArray) -> xr.DataArray:
132
+ """*var* as the *template* variable's dtype, when *var* is that variable.
133
+
134
+ Databases without a type widen it: SQLite stores ``float32`` as
135
+ ``float64`` and ``bool`` as an integer, MySQL ``bool`` as ``int8``, and
136
+ databases without unsigned integers store them as signed ones. A plain
137
+ ``SELECT`` of such a column is narrowed back.
138
+
139
+ Whether the column is the variable is read from its dims, since the
140
+ query itself is not available here: a plain select (filtered or not)
141
+ keeps every dim of the variable, while an aggregate such as
142
+ ``SUM(flag) AS flag`` reduces some away and keeps the result's dtype,
143
+ whatever its values. The narrowing must also be exact, and only
144
+ in-memory values can be checked, so a lazily reconstructed variable
145
+ keeps the result's dtype too.
146
+ """
147
+ dtype = template.dtype
148
+ # ``var.data`` would load a lazy variable; check without reading it.
149
+ if var.dtype == dtype or not var.variable._in_memory:
150
+ return var
151
+ if set(var.dims) != set(template.dims):
152
+ return var
153
+ if (var.dtype.kind, dtype.kind) not in _NARROWINGS:
154
+ return var
155
+ values = var.values
156
+ if dtype.kind == "u" and values.size and values.min() < 0:
157
+ return var # the cast would wrap negative values around
158
+ narrowed = values.astype(dtype)
159
+ if not np.array_equal(
160
+ narrowed.astype(values.dtype), values, equal_nan=True
161
+ ):
162
+ return var
163
+ return var.copy(data=narrowed)
164
+
165
+
166
+ def _apply_template(ds: xr.Dataset, template: xr.Dataset) -> xr.Dataset:
167
+ """Recover metadata that the forward SQL pivot strips.
168
+
169
+ Adds back, where unambiguous:
170
+
171
+ * Data-variable ``attrs`` and ``encoding`` for vars present in
172
+ ``template`` (aggregation aliases like ``air_avg`` get nothing).
173
+ Dtype-bound encoding keys (``dtype``, ``_FillValue``,
174
+ ``missing_value``) are intentionally dropped: SQL may have
175
+ changed the column's dtype (e.g. ``int16`` -> ``float64`` after
176
+ ``AVG`` or a null-introducing filter), and reattaching the
177
+ source's packing would make a later ``ds.to_netcdf()`` write
178
+ corrupt values.
179
+ * Data-variable dtype, where the database widened it and every value
180
+ survives the narrowing exactly (see ``_restore_dtype``).
181
+ * Dim-coordinate dtype, where SQL upcasted (datetime is the
182
+ canonical case).
183
+ * Non-dim coordinates whose dims are all present in ``ds`` (scalar
184
+ coords attach as-is; vector coords use ``.sel``).
185
+ * Dataset-level ``attrs``.
186
+
187
+ Skipped coords are warned about once per call.
188
+ """
189
+ out = ds.copy()
190
+
191
+ # 1. Data-var dtype, attrs, and encoding for vars present in the
192
+ # template. Aggregation aliases absent from template inherit nothing.
193
+ for name in list(out.data_vars):
194
+ if name in template.data_vars:
195
+ out[name] = _restore_dtype(out[name], template[name])
196
+ out[name].attrs = dict(template[name].attrs)
197
+ # Drop dtype-bound encoding keys; SQL may have changed dtype.
198
+ enc = {
199
+ k: v
200
+ for k, v in template[name].encoding.items()
201
+ if k not in {"dtype", "_FillValue", "missing_value"}
202
+ }
203
+ out[name].encoding = enc
204
+
205
+ # 2. Restore dim-coordinate dtype when SQL changed it (e.g. datetime
206
+ # upcast through pyarrow / pandas) and copy the source's dim-coord
207
+ # attrs (``standard_name``, ``long_name``, ``units``, etc.).
208
+ for d in list(out.dims):
209
+ if d in template.coords:
210
+ tdt = template.coords[d].dtype
211
+ if out.coords[d].dtype != tdt:
212
+ try:
213
+ out = out.assign_coords({d: _as_dtype(out.coords[d], tdt)})
214
+ except (ValueError, TypeError):
215
+ pass # incompatible cast; leave as-is
216
+ out[d].attrs = dict(template.coords[d].attrs)
217
+
218
+ # 3. Non-dim coordinates whose dims are all present in the result.
219
+ out_dims = set(out.dims)
220
+ skipped: list[str] = []
221
+ for cname, coord in template.coords.items():
222
+ if cname in template.dims:
223
+ continue # dim coord; already in out
224
+ if not set(coord.dims) <= out_dims:
225
+ continue # spans dims the result lacks
226
+ try:
227
+ if not coord.dims:
228
+ # Scalar coord (e.g. weather_dataset.reference_time).
229
+ out = out.assign_coords({cname: coord})
230
+ else:
231
+ sel = {d: out.coords[d] for d in coord.dims}
232
+ out = out.assign_coords({cname: coord.sel(sel)})
233
+ except (KeyError, ValueError, TypeError):
234
+ skipped.append(cname)
235
+
236
+ # 4. Dataset-level attrs.
237
+ out.attrs = dict(template.attrs)
238
+
239
+ if skipped:
240
+ warnings.warn(
241
+ f"Could not re-attach non-dim coordinates from template: {skipped}",
242
+ stacklevel=3,
243
+ )
244
+ return out
245
+
246
+
247
+ def _axis_numeric(values: np.ndarray) -> np.ndarray:
248
+ """View an axis as float64 for affine position arithmetic."""
249
+ if values.dtype.kind == "M":
250
+ return values.astype("datetime64[ns]").view("int64").astype("float64")
251
+ return values.astype("float64", copy=False)
252
+
253
+
254
+ def _affine_axis(requested: np.ndarray) -> tuple[float, float] | None:
255
+ """``(origin, step)`` when *requested* is uniformly spaced, else None.
256
+
257
+ Uniform spacing must hold exactly enough that ``rint((v - origin) /
258
+ step)`` recovers every index: the deviation of each element from its
259
+ affine prediction is checked against a quarter step. Non-numeric
260
+ axes (strings, cftime objects) never qualify.
261
+ """
262
+ if requested.dtype.kind not in ("i", "u", "f", "M") or len(requested) < 2:
263
+ return None
264
+ numeric = _axis_numeric(requested)
265
+ step = (numeric[-1] - numeric[0]) / (len(numeric) - 1)
266
+ if step == 0 or not np.isfinite(step):
267
+ return None
268
+ predicted = numeric[0] + step * np.arange(len(numeric))
269
+ # Written as a <= comparison so a NaN anywhere in the axis (e.g. a
270
+ # NULL dim value in the result) fails the check and falls back to
271
+ # the non-affine lookup, whose hash strategy matches NaN by value
272
+ # equality (pandas index lookups treat NaN as equal to NaN).
273
+ if not (np.abs(numeric - predicted) <= 0.25 * abs(step)).all():
274
+ return None
275
+ return float(numeric[0]), float(step)
276
+
277
+
278
+ class _CoordLookup:
279
+ """Maps one dimension's coordinate values to their array positions.
280
+
281
+ Built once per dimension per reconstruction, then probed with every
282
+ batch's coordinate column. Three strategies, fastest applicable wins:
283
+
284
+ * **Affine** — the axis is uniformly spaced (the norm for rasters and
285
+ regular time steps, ascending or descending): the position is
286
+ ``rint((value - origin) / step)``, a fused vector op with no per-row
287
+ lookup at all.
288
+ * **Hash** — irregular axes (station networks, arbitrary point sets)
289
+ with unique values: a ``pd.Index`` built once; ``get_indexer``
290
+ probes its persistent hash table per batch, O(1) amortized per
291
+ row instead of a per-row binary search. Building the table once
292
+ here, outside the batch loop, is what makes this fast, and it is
293
+ only correct to reuse because the axis does not change between
294
+ batches. The table holds roughly twice the transient memory of
295
+ the sorted copy the search strategy needs; affine axes build
296
+ neither.
297
+ * **Sorted search** — axes the hash strategy cannot represent:
298
+ duplicate values (a unique-key table has no single position per
299
+ value) or dtypes ``pd.Index`` does not support (e.g. float16):
300
+ ``np.argsort`` once, then ``np.searchsorted`` per batch (O(log n)
301
+ per row). With duplicates, each value resolves to one of the
302
+ positions holding it (which one is unspecified), and rows
303
+ targeting the same cell overwrite in batch order — the scatter's
304
+ last-write-wins semantics.
305
+
306
+ Only the hash strategy can detect a probe value absent from the
307
+ axis; ``positions_for`` then raises ``ValueError`` — a symptom of a
308
+ filtered query whose coordinate discovery missed a value, which
309
+ would otherwise scatter to a wrong cell. The affine strategy rounds
310
+ such a value to the nearest grid position and the sorted strategy
311
+ resolves it to a neighbor, both preserving their historical
312
+ semantics.
313
+ """
314
+
315
+ def __init__(self, requested: np.ndarray) -> None:
316
+ self._affine: tuple[float, float] | None = _affine_axis(requested)
317
+ self._hash_index: pd.Index | None = None
318
+ self._sorted_idx: np.ndarray | None = None
319
+ self._sorted_req: np.ndarray | None = None
320
+ if self._affine is None:
321
+ index: pd.Index | None
322
+ try:
323
+ index = pd.Index(requested)
324
+ except (NotImplementedError, TypeError):
325
+ # Dtypes pandas cannot index (e.g. float16) take the
326
+ # sorted-search strategy, which only needs numpy
327
+ # comparisons.
328
+ index = None
329
+ if index is not None and index.is_unique:
330
+ self._hash_index = index
331
+ else:
332
+ self._sorted_idx = np.argsort(requested)
333
+ self._sorted_req = requested[self._sorted_idx]
334
+
335
+ def positions_for(self, vals: np.ndarray, dim: str) -> np.ndarray:
336
+ """Positions of ``vals`` within the axis, in axis order."""
337
+ if self._affine is not None:
338
+ origin, step = self._affine
339
+ pos = np.rint((_axis_numeric(vals) - origin) / step).astype(np.intp)
340
+ return cast(np.ndarray, pos)
341
+ if self._hash_index is not None:
342
+ pos = np.asarray(self._hash_index.get_indexer(vals))
343
+ if (pos < 0).any():
344
+ missing = vals[pos < 0]
345
+ raise ValueError(
346
+ f"result contains {len(missing)} value(s) for dimension "
347
+ f"{dim!r} not present in its coordinate array "
348
+ f"(first: {missing[0]!r}); the query result does not "
349
+ "match the reconstruction's coordinates."
350
+ )
351
+ return cast(np.ndarray, pos)
352
+ assert self._sorted_req is not None and self._sorted_idx is not None
353
+ pos_in_sorted = np.searchsorted(self._sorted_req, vals)
354
+ return cast(np.ndarray, self._sorted_idx[pos_in_sorted])
355
+
356
+
357
+ def _scatter_batches_to_ndarray(
358
+ batches: list[pa.RecordBatch],
359
+ dimension_columns: list[str],
360
+ requested: dict[str, np.ndarray],
361
+ var_name: str,
362
+ out_shape: tuple[int, ...],
363
+ dtype: np.dtype,
364
+ drop_axes: list[int],
365
+ ) -> np.ndarray:
366
+ """Convert filtered Arrow ``RecordBatch`` rows into a dense N-D numpy array.
367
+
368
+ SQL query results arrive as flat rows; xarray expects N-D arrays.
369
+ This bridges the two: each row carries the dim-coord values that
370
+ identify its cell in the output cube plus the value to write there.
371
+ We look up each row's N-D position within the caller's requested
372
+ coord arrays (see ``_CoordLookup``), then scatter-write the value
373
+ at that index.
374
+
375
+ Missing combinations (sparse results from filtered queries) stay as
376
+ ``NaN`` for floating-point outputs by pre-filling the buffer; integer
377
+ outputs leave them as ``np.empty``-style undefined values.
378
+ """
379
+ # NaN fill for float outputs; default for int/datetime falls through
380
+ # to ``np.empty``-style undefined values (but every output cell is
381
+ # written below for non-sparse cases).
382
+ out = (
383
+ np.full(out_shape, np.nan, dtype=dtype)
384
+ if np.issubdtype(dtype, np.floating)
385
+ else np.empty(out_shape, dtype=dtype)
386
+ )
387
+
388
+ lookups = {d: _CoordLookup(requested[d]) for d in dimension_columns}
389
+
390
+ for batch in batches:
391
+ if batch.num_rows == 0:
392
+ continue
393
+ schema_names = batch.schema.names
394
+ # Build per-dim position arrays for this batch (positions within
395
+ # the caller's requested coord order).
396
+ positions = []
397
+ for d in dimension_columns:
398
+ col_arr = batch.column(schema_names.index(d))
399
+ vals = col_arr.to_numpy(zero_copy_only=False)
400
+ positions.append(lookups[d].positions_for(vals, dim=d))
401
+ value_arr = batch.column(schema_names.index(var_name)).to_numpy(
402
+ zero_copy_only=False
403
+ )
404
+ out[tuple(positions)] = value_arr.astype(dtype, copy=False)
405
+
406
+ if drop_axes:
407
+ out = np.squeeze(out, axis=tuple(drop_axes))
408
+ return cast(np.ndarray, out)
409
+
410
+
411
+ class SQLBackendArray(xr.backends.BackendArray):
412
+ """Read-only lazy N-D array view over a re-executable SQL result.
413
+
414
+ Bridges xarray's lazy-indexing interface
415
+ (``xarray.backends.BackendArray``) to an engine query result,
416
+ so an xarray ``Dataset`` can present a SQL query as if it were a
417
+ materialized N-D array without actually loading any data until the
418
+ caller asks for it. This is the workhorse that lets
419
+ [XarrayDataFrame.to_dataset][xarray_sql.ds.XarrayDataFrame.to_dataset] (and the engine-agnostic
420
+ ``xql.to_dataset(chunks=...)``) return a Dataset cheaply.
421
+
422
+ On each ``__getitem__`` call, the requested xarray indexer is
423
+ translated into per-dimension coordinate windows and a column
424
+ projection, executed through a
425
+ [LazyResultHandle][xarray_sql.lazyscan.LazyResultHandle] (DataFusion, DuckDB,
426
+ or Polars — each renders the windows with its own typed expression
427
+ API). The resulting Arrow ``RecordBatch`` es are scattered into a
428
+ preallocated numpy buffer, so only the requested data is
429
+ materialized.
430
+
431
+ Constraints and caveats:
432
+
433
+ - Read-only: there is no write path; the backend exists to surface
434
+ query results, not to round-trip writes into a SQL store.
435
+ - The underlying engine object may hold non-picklable references
436
+ (DataFusion's ``SessionContext``, a DuckDB connection). The class
437
+ therefore overrides ``__copy__`` and ``__deepcopy__`` to return
438
+ ``self`` -- this is safe because the backend is read-only.
439
+ - ``IndexingSupport.OUTER``: ``BasicIndexer`` and ``OuterIndexer``
440
+ are translated to filter predicates directly; ``VectorizedIndexer``
441
+ paths through xarray's adapter to outer-then-gather and so still
442
+ works, just less efficiently.
443
+
444
+ Raises:
445
+ ValueError, engine exceptions: propagated from the underlying
446
+ filter/project/execute chain if a predicate refers to a
447
+ missing column, the dtype of a literal is incompatible, or
448
+ the execution itself fails.
449
+ ValueError: from the coordinate lookup (``_CoordLookup``) when
450
+ the result contains values for an irregular unique axis
451
+ that are not present in the wrapper's pre-computed coord
452
+ arrays -- usually a symptom of a filtered query whose coord
453
+ discovery missed a value.
454
+
455
+ Constructed by ``_build_lazy_scan``; users should not instantiate
456
+ this class directly.
457
+ """
458
+
459
+ def __init__(
460
+ self,
461
+ handle: LazyResultHandle,
462
+ var_name: str,
463
+ dimension_columns: list[str],
464
+ coord_arrays: dict[str, np.ndarray],
465
+ shape: tuple[int, ...],
466
+ dtype: np.dtype,
467
+ ) -> None:
468
+ self._handle = handle
469
+ self._var_name = var_name
470
+ self._dimension_columns = list(dimension_columns)
471
+ self._coord_arrays = coord_arrays
472
+ # Computed once per dim: whether the whole coordinate array is
473
+ # strictly monotonic, the precondition for translating contiguous
474
+ # positional windows into value ranges (see _dim_spec).
475
+ self._monotonic = {
476
+ d: _strictly_monotonic(coord_arrays[d]) for d in dimension_columns
477
+ }
478
+ self.shape = tuple(shape)
479
+ self.dtype = np.dtype(dtype)
480
+
481
+ def __getitem__(self, key: Any) -> np.ndarray:
482
+ return cast(
483
+ np.ndarray,
484
+ xr.core.indexing.explicit_indexing_adapter(
485
+ key,
486
+ self.shape,
487
+ xr.core.indexing.IndexingSupport.OUTER,
488
+ self._raw_getitem,
489
+ ),
490
+ )
491
+
492
+ def __copy__(self) -> "SQLBackendArray":
493
+ # The backend is read-only; the underlying DataFusion DataFrame
494
+ # holds a non-picklable SessionContext reference, so sharing the
495
+ # same backend across a copy is both safe and necessary.
496
+ return self
497
+
498
+ def __deepcopy__(self, memo: dict) -> "SQLBackendArray":
499
+ return self
500
+
501
+ # ------------------------------------------------------------------
502
+
503
+ def _raw_getitem(self, key: tuple) -> np.ndarray:
504
+ """Materialize the indexed region described by *key* via the engine.
505
+
506
+ ``key`` is a tuple of ``int``/``slice``/1-D integer-array, one per
507
+ dim, in ``_dimension_columns`` order.
508
+ """
509
+ requested: dict[str, np.ndarray] = {}
510
+ # Per-dim windows for the engine. Dims whose indexer covers the
511
+ # full extent are omitted entirely so the engine doesn't have to
512
+ # evaluate a tautology.
513
+ specs: dict[str, DimSpec] = {}
514
+ drop_axes: list[int] = []
515
+ for axis, (dim, k) in enumerate(
516
+ zip(self._dimension_columns, key, strict=True)
517
+ ):
518
+ coord = self._coord_arrays[dim]
519
+ contiguous = False
520
+ if isinstance(k, slice):
521
+ start = 0 if k.start is None else k.start
522
+ stop = len(coord) if k.stop is None else k.stop
523
+ step = 1 if k.step is None else k.step
524
+ requested[dim] = np.asarray(coord[start:stop:step])
525
+ contiguous = step == 1
526
+ if start == 0 and stop >= len(coord) and step == 1:
527
+ continue
528
+ elif isinstance(k, (int, np.integer)):
529
+ requested[dim] = np.asarray([coord[int(k)]])
530
+ drop_axes.append(axis)
531
+ else:
532
+ arr = np.asarray(k)
533
+ requested[dim] = np.asarray(coord[arr])
534
+ if (
535
+ len(arr) == len(coord)
536
+ and (arr == np.arange(len(coord))).all()
537
+ ):
538
+ continue
539
+ contiguous = len(arr) > 1 and bool((np.diff(arr) == 1).all())
540
+ specs[dim] = _dim_spec(
541
+ requested[dim], contiguous, self._monotonic[dim]
542
+ )
543
+
544
+ out_shape = tuple(len(requested[d]) for d in self._dimension_columns)
545
+ if any(n == 0 for n in out_shape):
546
+ empty = np.empty(out_shape, dtype=self.dtype)
547
+ squeezed = (
548
+ np.squeeze(empty, axis=tuple(drop_axes)) if drop_axes else empty
549
+ )
550
+ return cast(np.ndarray, squeezed)
551
+
552
+ batches = self._handle.fetch(
553
+ specs, self._dimension_columns + [self._var_name]
554
+ )
555
+ return _scatter_batches_to_ndarray(
556
+ batches=batches,
557
+ dimension_columns=self._dimension_columns,
558
+ requested=requested,
559
+ var_name=self._var_name,
560
+ out_shape=out_shape,
561
+ dtype=self.dtype,
562
+ drop_axes=drop_axes,
563
+ )
564
+
565
+
566
+ def _strictly_monotonic(coord: np.ndarray) -> bool:
567
+ """Whether ``coord`` is strictly increasing or strictly decreasing.
568
+
569
+ Strict monotonicity of the whole coordinate array is the
570
+ precondition for translating a contiguous positional window into a
571
+ value range: with duplicated or unsorted values, ``[min, max]`` of a
572
+ window admits coordinate values at positions outside the window.
573
+ NaN/NaT (whose comparisons are all false) and non-comparable object
574
+ arrays report ``False``, which safely falls back to value lists.
575
+ """
576
+ if len(coord) < 2:
577
+ return True
578
+ head, tail = coord[:-1], coord[1:]
579
+ try:
580
+ return bool((tail > head).all() or (tail < head).all())
581
+ except TypeError:
582
+ return False
583
+
584
+
585
+ def _dim_spec(
586
+ vals: np.ndarray, contiguous: bool, coord_monotonic: bool
587
+ ) -> DimSpec:
588
+ """The engine window for one dim's requested coordinate values.
589
+
590
+ A contiguous run of positions over a strictly monotonic coordinate
591
+ array is exactly the value range ``[min, max]`` — a two-literal
592
+ predicate engines can push into range pruning. Monotonicity must
593
+ hold for the *entire* coordinate array (``coord_monotonic``), not
594
+ just the requested window: template coords are used verbatim, and
595
+ over a non-monotonic array a window's ``[min, max]`` admits values
596
+ at positions outside the window, which the scatter would then write
597
+ to wrong cells. Anything else (stepped slices, fancy indexers,
598
+ non-monotonic or duplicated coords) must be an explicit value list:
599
+ a range would admit rows the scatter did not request.
600
+ """
601
+ if contiguous and coord_monotonic and len(vals) > 1:
602
+ return ("range", vals.min(), vals.max())
603
+ return ("values", vals, None)
604
+
605
+
606
+ def _c_order_grid(
607
+ dim_cols: dict[str, np.ndarray],
608
+ coord_arrays: dict[str, np.ndarray],
609
+ dimension_columns: list[str],
610
+ total_rows: int,
611
+ ) -> bool:
612
+ """Whether the result rows form the complete grid in C order.
613
+
614
+ True iff the row count is exactly the coordinate product and every
615
+ dimension column is its coordinates repeated/tiled in C order — the
616
+ shape any unfiltered or bbox-windowed scan produces. When it holds,
617
+ data variables are dense row-major arrays already and can be
618
+ reshaped instead of scatter-written (one memcpy versus a per-row
619
+ position lookup and write; see ``_CoordLookup``).
620
+ """
621
+ shape = tuple(len(coord_arrays[d]) for d in dimension_columns)
622
+ if total_rows != int(np.prod(shape)) or total_rows == 0:
623
+ return False
624
+ for k, d in enumerate(dimension_columns):
625
+ inner = int(np.prod(shape[k + 1 :]))
626
+ outer = int(np.prod(shape[:k]))
627
+ view = dim_cols[d].reshape(outer, shape[k], inner)
628
+ if not (view == coord_arrays[d][None, :, None]).all():
629
+ return False
630
+ return True
631
+
632
+
633
+ def _dataset_from_batches(
634
+ batches: list[pa.RecordBatch],
635
+ dimension_columns: list[str],
636
+ field_names: list[str],
637
+ field_types: dict[str, Any],
638
+ ) -> xr.Dataset:
639
+ """Build a dense in-memory Dataset from Arrow ``RecordBatch`` es.
640
+
641
+ The engine-agnostic core of the eager round-trip: derives the
642
+ coordinates and every data variable from a single already-executed
643
+ result, whichever engine produced it. ``field_types`` values only
644
+ need a ``to_pandas_dtype()`` method (both ``pyarrow.DataType`` and
645
+ DataFusion's Arrow type wrappers qualify).
646
+
647
+ Complete grid-ordered results (unfiltered scans, bbox windows) are
648
+ reshaped directly; anything else — sparse results from filtered
649
+ queries, engine-reordered rows — falls back to the positional
650
+ scatter, which handles arbitrary row order.
651
+ """
652
+ dim_cols: dict[str, np.ndarray] = {}
653
+ coord_arrays: dict[str, np.ndarray] = {}
654
+ for d in dimension_columns:
655
+ if not batches:
656
+ dim_cols[d] = np.asarray([])
657
+ coord_arrays[d] = np.asarray([])
658
+ continue
659
+ vals = np.concatenate(
660
+ [
661
+ b.column(b.schema.names.index(d)).to_numpy(zero_copy_only=False)
662
+ for b in batches
663
+ ]
664
+ )
665
+ dim_cols[d] = vals
666
+ # Preserve the order coordinate values first appear in the result so an
667
+ # ORDER BY direction (e.g. ``ORDER BY level DESC``) carries through to
668
+ # the Dataset dimension instead of being force-sorted ascending.
669
+ # pd.unique keeps first-appearance order; the scatter below argsorts
670
+ # internally, so arbitrarily-ordered coordinates are placed correctly.
671
+ coord_arrays[d] = np.asarray(pd.unique(vals))
672
+ shape = tuple(len(coord_arrays[d]) for d in dimension_columns)
673
+ total_rows = sum(b.num_rows for b in batches)
674
+
675
+ grid_ordered = _c_order_grid(
676
+ dim_cols, coord_arrays, dimension_columns, total_rows
677
+ )
678
+
679
+ data_vars: dict[str, xr.Variable] = {}
680
+ for name in field_names:
681
+ if name in dimension_columns:
682
+ continue
683
+ np_dtype = np.dtype(field_types[name].to_pandas_dtype())
684
+ if grid_ordered:
685
+ flat = np.concatenate(
686
+ [
687
+ b.column(b.schema.names.index(name)).to_numpy(
688
+ zero_copy_only=False
689
+ )
690
+ for b in batches
691
+ ]
692
+ )
693
+ dense = flat.astype(np_dtype, copy=False).reshape(shape)
694
+ else:
695
+ dense = _scatter_batches_to_ndarray(
696
+ batches=batches,
697
+ dimension_columns=dimension_columns,
698
+ requested=coord_arrays,
699
+ var_name=name,
700
+ out_shape=shape,
701
+ dtype=np_dtype,
702
+ drop_axes=[],
703
+ )
704
+ data_vars[name] = xr.Variable(dimension_columns, dense)
705
+
706
+ coords_arg = {d: coord_arrays[d] for d in dimension_columns}
707
+ return xr.Dataset(data_vars=data_vars, coords=coords_arg)
708
+
709
+
710
+ _PURE_SCAN_NODES = {"Projection", "Sort", "TableScan", "SubqueryAlias"}
711
+
712
+
713
+ def _unfiltered_scan_table(inner_df: Any) -> str | None:
714
+ """Return the scanned table name iff the query is a pure unfiltered scan.
715
+
716
+ A pure scan only contains ``Projection``, ``Sort``, ``TableScan``,
717
+ ``SubqueryAlias`` nodes and exactly one ``TableScan``. Anything else
718
+ (``Filter``, ``Aggregate``, ``Join``, ``Union``, ``Limit``, multi-table
719
+ joins, ...) returns ``None`` so the caller falls back to per-dim
720
+ discovery. The returned name is the registered table the caller can
721
+ look up to source coord arrays from.
722
+ """
723
+ try:
724
+ lp = inner_df.logical_plan()
725
+ except Exception:
726
+ return None
727
+ table_name: str | None = None
728
+ stack = [lp]
729
+ while stack:
730
+ node = stack.pop()
731
+ try:
732
+ variant = node.to_variant()
733
+ except Exception:
734
+ return None
735
+ cls = type(variant).__name__
736
+ if cls not in _PURE_SCAN_NODES:
737
+ return None
738
+ if cls == "TableScan":
739
+ try:
740
+ this = variant.table_name()
741
+ except (AttributeError, TypeError):
742
+ return None
743
+ if not isinstance(this, str):
744
+ return None
745
+ if table_name is not None and table_name != this:
746
+ return None # multi-table scan; not a single source
747
+ table_name = this
748
+ stack.extend(node.inputs())
749
+ return table_name
750
+
751
+
752
+ def _maybe_template_coords(
753
+ templates: dict[str, xr.Dataset] | None,
754
+ dimension_columns: list[str],
755
+ inner_df: Any,
756
+ ) -> dict[str, np.ndarray] | None:
757
+ """Use the scanned table's registered coord arrays directly when safe.
758
+
759
+ Returns coord arrays sourced from the registered Dataset for the
760
+ scanned table iff the query is an unfiltered scan over that single
761
+ table and the registered Dataset carries all requested dims. Returns
762
+ ``None`` otherwise so the caller falls back to per-dim discovery.
763
+ Skipping discovery avoids one full plan execution per dim and
764
+ preserves the source's coordinate order.
765
+
766
+ Coord values come from the **scanned** registered Dataset, not from
767
+ any user-supplied ``template=`` (which is for metadata recovery
768
+ only). That keeps the fast path correct when a user with multiple
769
+ registered Datasets passes a metadata template that differs from
770
+ the query's source.
771
+ """
772
+ if not templates:
773
+ return None
774
+ table = _unfiltered_scan_table(inner_df)
775
+ if table is None or table not in templates:
776
+ return None
777
+ source = templates[table]
778
+ if not all(d in source.coords for d in dimension_columns):
779
+ return None
780
+ return {d: np.asarray(source.coords[d].values) for d in dimension_columns}
781
+
782
+
783
+ _ONE_PASS_BYTES = 64 * 2**20
784
+ """Size up to which a lazy result that must discover its coordinates is
785
+ read in a single pass instead."""
786
+
787
+
788
+ def _read_or_discover(
789
+ handle: LazyResultHandle, columns: list[str], dimension_columns: list[str]
790
+ ) -> tuple[list[pa.RecordBatch] | None, dict[str, np.ndarray] | None]:
791
+ """Execute the query once: the whole result if small, else its coordinates.
792
+
793
+ A lazy result must know each dimension's coordinates before any data
794
+ is read. One streamed pass serves both cases. A result that fits in
795
+ ``_ONE_PASS_BYTES`` -- an aggregation, which no window filter can push
796
+ below, or a narrow selection -- is returned whole, to be built in
797
+ memory. Past the budget, the pass keeps going but keeps only each
798
+ dimension's distinct values, so discovery executes the query once
799
+ rather than once per dimension. Returns ``(None, None)`` for handles
800
+ that cannot stream, which then discover one dimension at a time.
801
+ """
802
+ stream = getattr(handle, "stream", None)
803
+ if stream is None:
804
+ return None, None
805
+ batches: list[pa.RecordBatch] = []
806
+ total = 0
807
+ distinct: dict[str, pa.Array] | None = None
808
+ for batch in stream(columns):
809
+ if distinct is None:
810
+ total += batch.nbytes
811
+ batches.append(batch)
812
+ if total <= _ONE_PASS_BYTES:
813
+ continue
814
+ # Over budget: from here on, keep only distinct coordinates.
815
+ distinct = {}
816
+ pending, batches = batches, []
817
+ else:
818
+ pending = [batch]
819
+ for b in pending:
820
+ for d in dimension_columns:
821
+ values = pc.unique(b.column(b.schema.names.index(d)))
822
+ if d in distinct:
823
+ values = pc.unique(pa.concat_arrays([distinct[d], values]))
824
+ distinct[d] = values
825
+ if distinct is None:
826
+ return batches, None
827
+ return None, {
828
+ # Sorted, as from ``distinct``, so positional slices map onto
829
+ # contiguous value ranges.
830
+ d: np.sort(distinct[d].to_numpy(zero_copy_only=False))
831
+ for d in dimension_columns
832
+ }
833
+
834
+
835
+ def _build_lazy_scan(
836
+ handle: LazyResultHandle,
837
+ dimension_columns: list[str],
838
+ field_names: list[str],
839
+ field_types: dict[str, Any],
840
+ coord_arrays: dict[str, np.ndarray] | None = None,
841
+ ) -> xr.Dataset:
842
+ """Build a lazy Dataset whose data vars are [SQLBackendArray][xarray_sql.ds.SQLBackendArray].
843
+
844
+ Each data variable stays lazy: every access reads its coordinate range
845
+ via a pushdown filter, either directly or, once wrapped by
846
+ ``Dataset.chunk``, one chunk at a time. Coordinates come either from the
847
+ caller (the scanned table's registered Dataset for unfiltered DataFusion
848
+ scans -- see ``_maybe_template_coords`` -- or an explicitly trusted
849
+ template) or from per-dim distinct queries through the handle; over a
850
+ registered pushdown table the engine projects to that single coordinate
851
+ column, so discovery reads coordinate values only (no data-variable I/O).
852
+ """
853
+ if coord_arrays is None:
854
+ coord_arrays = {}
855
+ for d in dimension_columns:
856
+ # ``distinct`` returns engine order; sort ascending so
857
+ # positional slices map onto contiguous value ranges.
858
+ coord_arrays[d] = np.sort(handle.distinct(d))
859
+ shape = tuple(len(coord_arrays[d]) for d in dimension_columns)
860
+
861
+ data_vars: dict[str, xr.Variable] = {}
862
+ for name in field_names:
863
+ if name in dimension_columns:
864
+ continue
865
+ np_dtype = field_types[name].to_pandas_dtype()
866
+ backend = SQLBackendArray(
867
+ handle=handle,
868
+ var_name=name,
869
+ dimension_columns=dimension_columns,
870
+ coord_arrays=coord_arrays,
871
+ shape=shape,
872
+ dtype=np_dtype,
873
+ )
874
+ lazy = xr.core.indexing.LazilyIndexedArray(backend)
875
+ data_vars[name] = xr.Variable(dimension_columns, lazy)
876
+
877
+ coords_arg = {d: coord_arrays[d] for d in dimension_columns}
878
+ return xr.Dataset(data_vars=data_vars, coords=coords_arg)
879
+
880
+
881
+ def _auto_chunk_target_bytes() -> int:
882
+ """Byte target for ``chunks="auto"`` (the chunk manager's, else 128 MiB)."""
883
+ try:
884
+ import dask
885
+ from dask.utils import parse_bytes
886
+
887
+ return int(parse_bytes(dask.config.get("array.chunk-size")))
888
+ except Exception:
889
+ return 128 * 1024 * 1024
890
+
891
+
892
+ def _auto_chunks(
893
+ template: xr.Dataset | None,
894
+ dimension_columns: list[str],
895
+ field_types: dict[str, Any],
896
+ ) -> dict[str, int] | None:
897
+ """Resolve ``chunks="auto"`` to a source-partition-aligned chunk spec.
898
+
899
+ Sizes chunks to roughly the chunk manager's byte target (dask's
900
+ ``array.chunk-size``, default 128 MiB) but snaps boundaries to whole source
901
+ partitions, so every chunk is a union of source partitions -- no chunk splits
902
+ a partition (which would make adjacent chunks re-read it). This is what makes
903
+ ``"auto"`` useful for finely partitioned sources (e.g. ERA5
904
+ ``chunks={"time": 1}``): it coarsens many tiny partitions into memory-sized,
905
+ aligned chunks. Returns ``None`` when there is no resolvable source grid to
906
+ align to, so the caller falls back to the chunk manager's own ``"auto"``.
907
+ """
908
+ if template is None:
909
+ return None
910
+ part = template.chunksizes # dim -> tuple of source chunk lengths
911
+ chunked_dims = [
912
+ d for d in dimension_columns if d in part and len(part[d]) > 1
913
+ ]
914
+ if not chunked_dims:
915
+ return None
916
+
917
+ itemsizes = [
918
+ np.dtype(t.to_pandas_dtype()).itemsize
919
+ for name, t in field_types.items()
920
+ if name not in dimension_columns
921
+ ]
922
+ itemsize = max(itemsizes) if itemsizes else 8
923
+
924
+ # Bytes in one source-partition block: the nominal source chunk length per
925
+ # dimension (``part[d][0]``) multiplied across all dims, times itemsize.
926
+ block_bytes = itemsize
927
+ for d in dimension_columns:
928
+ if d in part:
929
+ block_bytes *= int(part[d][0])
930
+ # Number of source partitions to merge per chunk to approach the target.
931
+ merge = max(1, _auto_chunk_target_bytes() // max(block_bytes, 1))
932
+
933
+ # Absorb the coarsening into the most finely partitioned dimension; the rest
934
+ # keep their source chunk length. xarray caps an oversize chunk at the dim
935
+ # length, so an over-large merge simply yields a single chunk on that dim.
936
+ primary = max(chunked_dims, key=lambda d: len(part[d]))
937
+ return {
938
+ d: int(part[d][0]) * (merge if d == primary else 1)
939
+ for d in chunked_dims
940
+ }
941
+
942
+
943
+ def _result_to_xarray(
944
+ inner_df: Any,
945
+ dimension_columns: list[str],
946
+ template: xr.Dataset | None,
947
+ sparsity: Sparsity,
948
+ fill_value: Any,
949
+ chunks: Mapping[str, int] | str | None,
950
+ templates: dict[str, xr.Dataset] | None = None,
951
+ ) -> xr.Dataset:
952
+ """Reconstruct an ``xr.Dataset`` from a SQL result.
953
+
954
+ ``chunks`` (already resolved by ``XarrayDataFrame._resolve_chunks``)
955
+ selects the execution strategy:
956
+
957
+ * ``None`` -> lazy, unchunked: the data variables are lazily indexed
958
+ [SQLBackendArray][xarray_sql.ds.SQLBackendArray]s, so each access
959
+ re-executes the query narrowed to the selection and projected to
960
+ the variable read. No chunk manager is involved. A result whose
961
+ coordinates must be discovered is first streamed in one pass, which
962
+ keeps it in memory if small (``_read_or_discover``), so an
963
+ aggregation executes once.
964
+ * a mapping (or ``"auto"``) -> lazy/chunked: build [SQLBackendArray][xarray_sql.ds.SQLBackendArray]
965
+ data variables (``_build_lazy_scan``) and wrap them with
966
+ ``Dataset.chunk`` so each chunk reads its coordinate range via filter
967
+ pushdown. The chunk grid maps onto the source partitions. Chunking goes
968
+ through xarray's configured chunk manager (dask, cubed, ...), so no
969
+ chunked-array backend is imported directly here.
970
+ """
971
+ if sparsity not in ("result", "template"):
972
+ raise ValueError(
973
+ f"sparsity must be 'result' or 'template', got {sparsity!r}"
974
+ )
975
+ if sparsity == "template" and template is None:
976
+ raise ValueError(
977
+ "sparsity='template' requires template= to be supplied"
978
+ )
979
+
980
+ schema = inner_df.schema()
981
+ field_names = [f.name for f in schema]
982
+ field_types = {f.name: f.type for f in schema}
983
+
984
+ handle = DataFusionHandle(inner_df)
985
+ coord_arrays = _maybe_template_coords(
986
+ templates, dimension_columns, inner_df
987
+ )
988
+ batches = None
989
+ if chunks is None and coord_arrays is None:
990
+ batches, coord_arrays = _read_or_discover(
991
+ handle, field_names, dimension_columns
992
+ )
993
+ if batches is not None:
994
+ ds = _dataset_from_batches(
995
+ batches, dimension_columns, field_names, field_types
996
+ )
997
+ else:
998
+ ds = _build_lazy_scan(
999
+ handle,
1000
+ dimension_columns,
1001
+ field_names,
1002
+ field_types,
1003
+ coord_arrays=coord_arrays,
1004
+ )
1005
+ return _finish_dataset(
1006
+ ds,
1007
+ dimension_columns,
1008
+ template,
1009
+ sparsity,
1010
+ fill_value,
1011
+ chunks,
1012
+ field_types,
1013
+ )
1014
+
1015
+
1016
+ def _finish_dataset(
1017
+ ds: xr.Dataset,
1018
+ dimension_columns: list[str],
1019
+ template: xr.Dataset | None,
1020
+ sparsity: Sparsity,
1021
+ fill_value: Any,
1022
+ chunks: Mapping[str, int] | str | None,
1023
+ field_types: dict[str, Any],
1024
+ ) -> xr.Dataset:
1025
+ """Shared reconstruction tail: sparsity, template metadata, chunking."""
1026
+ if sparsity == "template":
1027
+ assert template is not None
1028
+ indexers = {
1029
+ d: template.coords[d].values
1030
+ for d in dimension_columns
1031
+ if d in template.coords and d in template.dims
1032
+ }
1033
+ if indexers:
1034
+ ds = ds.reindex(indexers, fill_value=fill_value)
1035
+
1036
+ if template is not None:
1037
+ ds = _apply_template(ds, template)
1038
+
1039
+ if chunks is not None:
1040
+ if chunks == "auto":
1041
+ # Snap the byte-budgeted "auto" sizing to source partition
1042
+ # boundaries; fall back to the chunk manager's own "auto" when there
1043
+ # is no source grid to align to.
1044
+ chunks = (
1045
+ _auto_chunks(template, dimension_columns, field_types) or "auto"
1046
+ )
1047
+ # Wrap the lazy data variables in the configured chunk manager (dask by
1048
+ # default). Each chunk reads its coordinate range via pushdown on access.
1049
+ ds = ds.chunk(chunks)
1050
+ return ds
1051
+
1052
+
1053
+ # ---------------------------------------------------------------------------
1054
+ # Public wrapper
1055
+ # ---------------------------------------------------------------------------
1056
+
1057
+
1058
+ class XarrayDataFrame:
1059
+ """Wrapper around a DataFusion ``DataFrame`` with xarray-aware helpers.
1060
+
1061
+ Returned by [xarray_sql.Context.sql][]. Forwards every
1062
+ attribute it does not define itself to the wrapped DataFrame, so
1063
+ ``.collect()``, ``.schema()``, ``.show()``, ``.count()`` all work
1064
+ unchanged.
1065
+
1066
+ Carries a private snapshot of the context's registered Datasets so
1067
+ ``to_dataset`` can default ``dims`` and recover metadata
1068
+ dropped by the forward pivot.
1069
+
1070
+ Users should not construct this class directly; let
1071
+ [Context.sql][xarray_sql.sql.Context.sql] produce it.
1072
+ """
1073
+
1074
+ def __init__(
1075
+ self,
1076
+ inner: Any,
1077
+ templates: dict[str, xr.Dataset] | None = None,
1078
+ ) -> None:
1079
+ """Construct a wrapper.
1080
+
1081
+ Args:
1082
+ inner: The underlying ``datafusion.DataFrame`` returned by
1083
+ [Context.sql][xarray_sql.sql.Context.sql].
1084
+ templates: Snapshot of the registered Datasets on the producing
1085
+ context, keyed by the SQL identifier each was registered
1086
+ under. Used by ``to_dataset`` to recover metadata that
1087
+ the forward pivot strips. ``None`` means no metadata
1088
+ recovery is possible from registrations alone; callers may
1089
+ still pass ``template=`` to ``to_dataset`` explicitly.
1090
+ """
1091
+ object.__setattr__(self, "_inner", inner)
1092
+ object.__setattr__(self, "_templates", dict(templates or {}))
1093
+
1094
+ def to_pandas(self) -> pd.DataFrame:
1095
+ """Materialize the result as a ``pd.DataFrame`` (DataFusion API)."""
1096
+ return self._inner.to_pandas()
1097
+
1098
+ def to_dataset(
1099
+ self,
1100
+ dims: list[str] | None = None,
1101
+ template: xr.Dataset | str | None = None,
1102
+ sparsity: Sparsity = "result",
1103
+ fill_value: Any = np.nan,
1104
+ chunks: Mapping[str, int] | str | None = "inherit",
1105
+ ) -> xr.Dataset:
1106
+ """Convert the result to an ``xr.Dataset``.
1107
+
1108
+ Args:
1109
+ dims: Result columns to use as Dataset dimensions. When
1110
+ ``None``, defaults to a registered Dataset's dimensions that
1111
+ survive into the result columns, so an aggregation that drops
1112
+ dims (e.g. ``GROUP BY time`` over a ``(time, lat, lon)`` grid)
1113
+ round-trips on the remaining dim. Raises when no dimension
1114
+ survives, or when several registered Datasets imply different
1115
+ dims (pass ``dims`` explicitly then).
1116
+ template: Source to recover metadata (attrs, encoding, non-dim
1117
+ coordinates, dim-coord dtype) from. Either an ``xr.Dataset``
1118
+ used directly, or the name of a registered table (e.g.
1119
+ ``"era5.surface"``) whose Dataset is looked up. When ``None``
1120
+ and exactly one Dataset is registered, that one is used.
1121
+ sparsity: ``"result"`` (default) keeps only dim values
1122
+ present in the result. ``"template"`` reindexes to the
1123
+ template's full coord ranges, filling absent cells with
1124
+ ``fill_value``; requires a template.
1125
+ fill_value: Used when ``sparsity="template"`` reindexes
1126
+ to a wider extent. Defaults to ``np.nan``.
1127
+ chunks: Output chunking, controlling laziness (an xarray idiom).
1128
+
1129
+ * ``"inherit"`` (default): reuse the source Dataset's chunk
1130
+ sizes, but only for dimensions that were genuinely split into
1131
+ multiple chunks in the input -- so the output chunk grid maps
1132
+ onto the source partitions. A reduction that drops the chunked
1133
+ dimension (e.g. a global aggregation) inherits nothing, and
1134
+ neither does a result with no resolvable source Dataset;
1135
+ both are returned as with ``None``.
1136
+ * ``None``: lazy without a chunk manager, as with
1137
+ ``xr.open_dataset(chunks=None)``. Each access re-executes
1138
+ the query narrowed to the selection and projected to the
1139
+ variable read, so ``.load()`` executes it once per
1140
+ variable. Dask is not required. The context must stay
1141
+ usable while the result is read.
1142
+ * a mapping (e.g. ``{"time": 100}``): chunk explicitly. Each
1143
+ chunk reads its coordinate range lazily via filter pushdown on
1144
+ access, through xarray's configured chunk manager (dask,
1145
+ cubed, ...).
1146
+ * ``"auto"``: size chunks to the chunk manager's byte target but
1147
+ snap boundaries to whole source partitions, so each chunk is a
1148
+ union of source partitions. Useful for finely partitioned
1149
+ sources (e.g. ERA5 ``chunks={"time": 1}``), coarsening many
1150
+ tiny partitions into memory-sized, aligned chunks.
1151
+
1152
+ Returns:
1153
+ An ``xr.Dataset`` with ``dims`` as dimensions and the
1154
+ remaining result columns as data variables.
1155
+
1156
+ Raises:
1157
+ ValueError: ``dims`` cannot be inferred, names a missing
1158
+ column, or the result has duplicate dim tuples;
1159
+ ``template`` names an unknown registered table; or
1160
+ ``sparsity="template"`` is requested without a
1161
+ resolvable template.
1162
+ """
1163
+ if not isinstance(template, xr.Dataset):
1164
+ # ``template`` is a registered-table name or None; look it up.
1165
+ template = self._resolve_template(template)
1166
+ if dims is None:
1167
+ dims = self._infer_dimension_columns(preferred_template=template)
1168
+ resolved_chunks = self._resolve_chunks(chunks, template, dims)
1169
+ return _result_to_xarray(
1170
+ inner_df=self._inner,
1171
+ dimension_columns=dims,
1172
+ template=template,
1173
+ sparsity=sparsity,
1174
+ fill_value=fill_value,
1175
+ chunks=resolved_chunks,
1176
+ templates=self._templates,
1177
+ )
1178
+
1179
+ # ------------------------------------------------------------------
1180
+ # Internals
1181
+ # ------------------------------------------------------------------
1182
+
1183
+ @staticmethod
1184
+ def _resolve_chunks(
1185
+ chunks: Mapping[str, int] | str | None,
1186
+ template: xr.Dataset | None,
1187
+ dimension_columns: list[str],
1188
+ ) -> Mapping[str, int] | str | None:
1189
+ """Resolve the ``chunks`` argument to a concrete spec or ``None``.
1190
+
1191
+ ``None`` selects the lazy, unchunked path; anything else selects the
1192
+ chunked path. ``"inherit"`` reuses the source Dataset's chunk sizes --
1193
+ but only for dimensions actually split into more than one chunk in the
1194
+ input (a single full chunk is not "chunked"), so reductions that drop
1195
+ the chunked dimension resolve to ``None`` automatically. Mappings
1196
+ pass through unchanged; ``"auto"`` passes through here and is snapped to
1197
+ source partition boundaries later (see ``_auto_chunks``).
1198
+ """
1199
+ if chunks is None:
1200
+ return None
1201
+ if chunks == "inherit":
1202
+ if template is None:
1203
+ return None
1204
+ sizes = template.chunksizes # dim -> tuple of chunk lengths
1205
+ inherited = {
1206
+ d: sizes[d][0]
1207
+ for d in dimension_columns
1208
+ if d in sizes and len(sizes[d]) > 1
1209
+ }
1210
+ return inherited or None
1211
+ return chunks
1212
+
1213
+ def _resolve_template(self, name: str | None) -> xr.Dataset | None:
1214
+ """Pick a template Dataset for metadata recovery by registered name.
1215
+
1216
+ Priority:
1217
+ 1. The named registered table (``name``).
1218
+ 2. If exactly one Dataset is registered on the context, use it.
1219
+ 3. None.
1220
+ """
1221
+ templates = self._templates
1222
+ if name is not None:
1223
+ if name not in templates:
1224
+ raise ValueError(
1225
+ f"template={name!r} is not a registered table on this "
1226
+ f"context. Registered: {list(templates)}"
1227
+ )
1228
+ return templates[name]
1229
+ if len(templates) == 1:
1230
+ return next(iter(templates.values()))
1231
+ return None
1232
+
1233
+ def _infer_dimension_columns(
1234
+ self, preferred_template: xr.Dataset | None = None
1235
+ ) -> list[str]:
1236
+ """Pick a default ``dimension_columns`` from the registry, or raise.
1237
+
1238
+ A registered Dataset's dims that survive into the result columns
1239
+ become the dimensions, so aggregations that drop dims (e.g.
1240
+ ``GROUP BY time`` over a ``(time, lat, lon)`` grid) round-trip on the
1241
+ surviving dim(s). Uses the data variable's dim order (via
1242
+ ``_result_dims``) so the original axis order is preserved.
1243
+ """
1244
+ result_cols = set(self._result_columns())
1245
+
1246
+ def surviving(template: xr.Dataset) -> list[str]:
1247
+ # Template dims still present in the result, in var axis order.
1248
+ return _result_dims(template, result_cols)
1249
+
1250
+ if preferred_template is not None:
1251
+ preferred = surviving(preferred_template)
1252
+ if preferred:
1253
+ return preferred
1254
+ if not self._templates:
1255
+ raise ValueError(
1256
+ "dims cannot be inferred (no registered "
1257
+ "Dataset on this result); pass dims=[...] "
1258
+ "explicitly."
1259
+ )
1260
+ candidates = {tuple(surviving(t)) for t in self._templates.values()}
1261
+ candidates.discard(()) # templates with no surviving dim
1262
+ if len(candidates) == 1:
1263
+ return list(next(iter(candidates)))
1264
+ if not candidates:
1265
+ raise ValueError(
1266
+ "dims cannot be inferred: no registered Dataset "
1267
+ "dimension survives in the result columns. Pass "
1268
+ "dims=[...] explicitly."
1269
+ )
1270
+ raise ValueError(
1271
+ "dims cannot be inferred unambiguously: multiple "
1272
+ "registered Datasets are compatible with the result. Pass "
1273
+ "dims=[...] explicitly."
1274
+ )
1275
+
1276
+ def _result_columns(self) -> list[str]:
1277
+ """Return the result's column names without materializing rows."""
1278
+ return [field.name for field in self._inner.schema()]
1279
+
1280
+ def __getattr__(self, name: str) -> Any:
1281
+ # Runs only when ``name`` is not found via normal lookup, so this
1282
+ # safely forwards anything we have not overridden.
1283
+ return getattr(self._inner, name)
1284
+
1285
+ def __repr__(self) -> str:
1286
+ return repr(self._inner)