xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
xarray_sql/ds.py
ADDED
|
@@ -0,0 +1,1286 @@
|
|
|
1
|
+
"""Reconstruct xarray Datasets from SQL query results.
|
|
2
|
+
|
|
3
|
+
The inverse of the forward Dataset-to-table pivot done by
|
|
4
|
+
[xarray_sql.df.pivot][]. Internally defines an [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame]
|
|
5
|
+
wrapper around the DataFusion ``DataFrame`` returned by
|
|
6
|
+
[Context.sql][xarray_sql.sql.Context.sql], with a [XarrayDataFrame.to_dataset][xarray_sql.ds.XarrayDataFrame.to_dataset]
|
|
7
|
+
method that round-trips a query result back to ``xr.Dataset``.
|
|
8
|
+
|
|
9
|
+
Reconstruction is controlled by the ``chunks`` argument to
|
|
10
|
+
[XarrayDataFrame.to_dataset][xarray_sql.ds.XarrayDataFrame.to_dataset] -- the xarray idiom for tuning how a
|
|
11
|
+
result is partitioned -- rather than by reflecting on the query plan:
|
|
12
|
+
|
|
13
|
+
* **Eager** (``chunks=None``, or the default ``"inherit"`` when the
|
|
14
|
+
result keeps no multi-chunk source dimension): the plan executes
|
|
15
|
+
exactly once via ``execute_stream`` and the result is scattered into a
|
|
16
|
+
dense in-memory Dataset. This is the right default for reductions
|
|
17
|
+
(aggregations), whose results are small, and it never re-executes.
|
|
18
|
+
* **Lazy / chunked** (``chunks`` is a mapping, ``"auto"``, or
|
|
19
|
+
``"inherit"`` over a multi-chunk source dimension): data variables are
|
|
20
|
+
backed by [SQLBackendArray][xarray_sql.ds.SQLBackendArray] wrapped in
|
|
21
|
+
``xarray.core.indexing.LazilyIndexedArray`` and chunked via xarray's
|
|
22
|
+
configured chunk manager (dask, cubed, ...). Each chunk maps onto the
|
|
23
|
+
source partitions and reads its coordinate range on access by
|
|
24
|
+
translating the indexer into a DataFusion ``filter`` expression, so only
|
|
25
|
+
the requested partitions are materialized as Arrow ``RecordBatch`` es
|
|
26
|
+
and scattered into numpy.
|
|
27
|
+
|
|
28
|
+
``.compute()`` materializes the whole Dataset in memory.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import warnings
|
|
34
|
+
from collections.abc import Mapping
|
|
35
|
+
from typing import Any, Literal, cast
|
|
36
|
+
|
|
37
|
+
import numpy as np
|
|
38
|
+
import pandas as pd
|
|
39
|
+
import pyarrow as pa
|
|
40
|
+
import pyarrow.compute as pc
|
|
41
|
+
import xarray as xr
|
|
42
|
+
|
|
43
|
+
from .lazyscan import DataFusionHandle, DimSpec, LazyResultHandle
|
|
44
|
+
|
|
45
|
+
Sparsity = Literal["result", "template"]
|
|
46
|
+
"""Output coordinate extent for a filtered round-trip.
|
|
47
|
+
|
|
48
|
+
* ``"result"`` keeps only the dim values present in the query result, so
|
|
49
|
+
the output is sparse and equal to whatever rows came back.
|
|
50
|
+
* ``"template"`` reindexes to the registered Dataset's full coord ranges
|
|
51
|
+
and fills absent cells with ``fill_value``.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ---------------------------------------------------------------------------
|
|
56
|
+
# Private helpers
|
|
57
|
+
# ---------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _result_dims(template: xr.Dataset, columns) -> list[str]:
|
|
61
|
+
"""The template's dims that survive into a result's *columns*.
|
|
62
|
+
|
|
63
|
+
In the variables' axis order, not ``template.dims``'s name order. A
|
|
64
|
+
mixed-dimension template has one order per group of variables, so
|
|
65
|
+
the variables the result carries pick theirs: ``AVG(temperature) ...
|
|
66
|
+
GROUP BY level`` keeps ``level`` although the template's first
|
|
67
|
+
variable has none. A result carrying no template variable (only
|
|
68
|
+
aliases such as ``wind``) may keep any template dim.
|
|
69
|
+
"""
|
|
70
|
+
columns = set(columns)
|
|
71
|
+
carried = [name for name in template.data_vars if name in columns]
|
|
72
|
+
order: list = []
|
|
73
|
+
for name in carried or list(template.data_vars):
|
|
74
|
+
order.extend(d for d in template[name].dims if d not in order)
|
|
75
|
+
if not order:
|
|
76
|
+
order = list(template.dims)
|
|
77
|
+
return [d for d in order if d in columns]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_TIMEDELTA_PARTS = (
|
|
81
|
+
"weeks",
|
|
82
|
+
"days",
|
|
83
|
+
"hours",
|
|
84
|
+
"minutes",
|
|
85
|
+
"seconds",
|
|
86
|
+
"milliseconds",
|
|
87
|
+
"microseconds",
|
|
88
|
+
"nanoseconds",
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _timedelta(value: Any) -> pd.Timedelta:
|
|
93
|
+
"""One duration a database returned as an interval or as text.
|
|
94
|
+
|
|
95
|
+
Arrow's month-day-nano intervals (DuckDB, PostgreSQL) arrive as
|
|
96
|
+
pandas ``DateOffset`` objects; MySQL and Trino return text such as
|
|
97
|
+
``'21600s'``.
|
|
98
|
+
"""
|
|
99
|
+
if value is None:
|
|
100
|
+
return pd.NaT
|
|
101
|
+
if isinstance(value, pd.DateOffset):
|
|
102
|
+
parts = value.kwds
|
|
103
|
+
if parts.get("years") or parts.get("months"):
|
|
104
|
+
raise ValueError(
|
|
105
|
+
f"{value!r} spans calendar months, which have no fixed duration"
|
|
106
|
+
)
|
|
107
|
+
return pd.Timedelta(**{k: parts.get(k, 0) for k in _TIMEDELTA_PARTS})
|
|
108
|
+
return pd.Timedelta(value)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _as_dtype(coord: xr.DataArray, dtype: np.dtype) -> xr.DataArray:
|
|
112
|
+
"""*coord* cast to the template's *dtype*.
|
|
113
|
+
|
|
114
|
+
Databases without a duration type hand timedeltas back as intervals
|
|
115
|
+
or text, which numpy cannot cast; those are parsed first. (Integers
|
|
116
|
+
need nothing: numpy reads them as counts of the template's unit,
|
|
117
|
+
which is how durations are stored where no duration type exists.)
|
|
118
|
+
"""
|
|
119
|
+
if dtype.kind == "m" and coord.dtype.kind in "OUS":
|
|
120
|
+
values = np.array(
|
|
121
|
+
[_timedelta(v) for v in coord.values], dtype="timedelta64[ns]"
|
|
122
|
+
)
|
|
123
|
+
return coord.copy(data=values.astype(dtype))
|
|
124
|
+
return coord.astype(dtype)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
_NARROWINGS = {("i", "b"), ("u", "b"), ("i", "u"), ("f", "f")}
|
|
128
|
+
"""(result kind, template kind) pairs a database may have widened."""
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _restore_dtype(var: xr.DataArray, template: xr.DataArray) -> xr.DataArray:
|
|
132
|
+
"""*var* as the *template* variable's dtype, when *var* is that variable.
|
|
133
|
+
|
|
134
|
+
Databases without a type widen it: SQLite stores ``float32`` as
|
|
135
|
+
``float64`` and ``bool`` as an integer, MySQL ``bool`` as ``int8``, and
|
|
136
|
+
databases without unsigned integers store them as signed ones. A plain
|
|
137
|
+
``SELECT`` of such a column is narrowed back.
|
|
138
|
+
|
|
139
|
+
Whether the column is the variable is read from its dims, since the
|
|
140
|
+
query itself is not available here: a plain select (filtered or not)
|
|
141
|
+
keeps every dim of the variable, while an aggregate such as
|
|
142
|
+
``SUM(flag) AS flag`` reduces some away and keeps the result's dtype,
|
|
143
|
+
whatever its values. The narrowing must also be exact, and only
|
|
144
|
+
in-memory values can be checked, so a lazily reconstructed variable
|
|
145
|
+
keeps the result's dtype too.
|
|
146
|
+
"""
|
|
147
|
+
dtype = template.dtype
|
|
148
|
+
# ``var.data`` would load a lazy variable; check without reading it.
|
|
149
|
+
if var.dtype == dtype or not var.variable._in_memory:
|
|
150
|
+
return var
|
|
151
|
+
if set(var.dims) != set(template.dims):
|
|
152
|
+
return var
|
|
153
|
+
if (var.dtype.kind, dtype.kind) not in _NARROWINGS:
|
|
154
|
+
return var
|
|
155
|
+
values = var.values
|
|
156
|
+
if dtype.kind == "u" and values.size and values.min() < 0:
|
|
157
|
+
return var # the cast would wrap negative values around
|
|
158
|
+
narrowed = values.astype(dtype)
|
|
159
|
+
if not np.array_equal(
|
|
160
|
+
narrowed.astype(values.dtype), values, equal_nan=True
|
|
161
|
+
):
|
|
162
|
+
return var
|
|
163
|
+
return var.copy(data=narrowed)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _apply_template(ds: xr.Dataset, template: xr.Dataset) -> xr.Dataset:
|
|
167
|
+
"""Recover metadata that the forward SQL pivot strips.
|
|
168
|
+
|
|
169
|
+
Adds back, where unambiguous:
|
|
170
|
+
|
|
171
|
+
* Data-variable ``attrs`` and ``encoding`` for vars present in
|
|
172
|
+
``template`` (aggregation aliases like ``air_avg`` get nothing).
|
|
173
|
+
Dtype-bound encoding keys (``dtype``, ``_FillValue``,
|
|
174
|
+
``missing_value``) are intentionally dropped: SQL may have
|
|
175
|
+
changed the column's dtype (e.g. ``int16`` -> ``float64`` after
|
|
176
|
+
``AVG`` or a null-introducing filter), and reattaching the
|
|
177
|
+
source's packing would make a later ``ds.to_netcdf()`` write
|
|
178
|
+
corrupt values.
|
|
179
|
+
* Data-variable dtype, where the database widened it and every value
|
|
180
|
+
survives the narrowing exactly (see ``_restore_dtype``).
|
|
181
|
+
* Dim-coordinate dtype, where SQL upcasted (datetime is the
|
|
182
|
+
canonical case).
|
|
183
|
+
* Non-dim coordinates whose dims are all present in ``ds`` (scalar
|
|
184
|
+
coords attach as-is; vector coords use ``.sel``).
|
|
185
|
+
* Dataset-level ``attrs``.
|
|
186
|
+
|
|
187
|
+
Skipped coords are warned about once per call.
|
|
188
|
+
"""
|
|
189
|
+
out = ds.copy()
|
|
190
|
+
|
|
191
|
+
# 1. Data-var dtype, attrs, and encoding for vars present in the
|
|
192
|
+
# template. Aggregation aliases absent from template inherit nothing.
|
|
193
|
+
for name in list(out.data_vars):
|
|
194
|
+
if name in template.data_vars:
|
|
195
|
+
out[name] = _restore_dtype(out[name], template[name])
|
|
196
|
+
out[name].attrs = dict(template[name].attrs)
|
|
197
|
+
# Drop dtype-bound encoding keys; SQL may have changed dtype.
|
|
198
|
+
enc = {
|
|
199
|
+
k: v
|
|
200
|
+
for k, v in template[name].encoding.items()
|
|
201
|
+
if k not in {"dtype", "_FillValue", "missing_value"}
|
|
202
|
+
}
|
|
203
|
+
out[name].encoding = enc
|
|
204
|
+
|
|
205
|
+
# 2. Restore dim-coordinate dtype when SQL changed it (e.g. datetime
|
|
206
|
+
# upcast through pyarrow / pandas) and copy the source's dim-coord
|
|
207
|
+
# attrs (``standard_name``, ``long_name``, ``units``, etc.).
|
|
208
|
+
for d in list(out.dims):
|
|
209
|
+
if d in template.coords:
|
|
210
|
+
tdt = template.coords[d].dtype
|
|
211
|
+
if out.coords[d].dtype != tdt:
|
|
212
|
+
try:
|
|
213
|
+
out = out.assign_coords({d: _as_dtype(out.coords[d], tdt)})
|
|
214
|
+
except (ValueError, TypeError):
|
|
215
|
+
pass # incompatible cast; leave as-is
|
|
216
|
+
out[d].attrs = dict(template.coords[d].attrs)
|
|
217
|
+
|
|
218
|
+
# 3. Non-dim coordinates whose dims are all present in the result.
|
|
219
|
+
out_dims = set(out.dims)
|
|
220
|
+
skipped: list[str] = []
|
|
221
|
+
for cname, coord in template.coords.items():
|
|
222
|
+
if cname in template.dims:
|
|
223
|
+
continue # dim coord; already in out
|
|
224
|
+
if not set(coord.dims) <= out_dims:
|
|
225
|
+
continue # spans dims the result lacks
|
|
226
|
+
try:
|
|
227
|
+
if not coord.dims:
|
|
228
|
+
# Scalar coord (e.g. weather_dataset.reference_time).
|
|
229
|
+
out = out.assign_coords({cname: coord})
|
|
230
|
+
else:
|
|
231
|
+
sel = {d: out.coords[d] for d in coord.dims}
|
|
232
|
+
out = out.assign_coords({cname: coord.sel(sel)})
|
|
233
|
+
except (KeyError, ValueError, TypeError):
|
|
234
|
+
skipped.append(cname)
|
|
235
|
+
|
|
236
|
+
# 4. Dataset-level attrs.
|
|
237
|
+
out.attrs = dict(template.attrs)
|
|
238
|
+
|
|
239
|
+
if skipped:
|
|
240
|
+
warnings.warn(
|
|
241
|
+
f"Could not re-attach non-dim coordinates from template: {skipped}",
|
|
242
|
+
stacklevel=3,
|
|
243
|
+
)
|
|
244
|
+
return out
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _axis_numeric(values: np.ndarray) -> np.ndarray:
|
|
248
|
+
"""View an axis as float64 for affine position arithmetic."""
|
|
249
|
+
if values.dtype.kind == "M":
|
|
250
|
+
return values.astype("datetime64[ns]").view("int64").astype("float64")
|
|
251
|
+
return values.astype("float64", copy=False)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _affine_axis(requested: np.ndarray) -> tuple[float, float] | None:
|
|
255
|
+
"""``(origin, step)`` when *requested* is uniformly spaced, else None.
|
|
256
|
+
|
|
257
|
+
Uniform spacing must hold exactly enough that ``rint((v - origin) /
|
|
258
|
+
step)`` recovers every index: the deviation of each element from its
|
|
259
|
+
affine prediction is checked against a quarter step. Non-numeric
|
|
260
|
+
axes (strings, cftime objects) never qualify.
|
|
261
|
+
"""
|
|
262
|
+
if requested.dtype.kind not in ("i", "u", "f", "M") or len(requested) < 2:
|
|
263
|
+
return None
|
|
264
|
+
numeric = _axis_numeric(requested)
|
|
265
|
+
step = (numeric[-1] - numeric[0]) / (len(numeric) - 1)
|
|
266
|
+
if step == 0 or not np.isfinite(step):
|
|
267
|
+
return None
|
|
268
|
+
predicted = numeric[0] + step * np.arange(len(numeric))
|
|
269
|
+
# Written as a <= comparison so a NaN anywhere in the axis (e.g. a
|
|
270
|
+
# NULL dim value in the result) fails the check and falls back to
|
|
271
|
+
# the non-affine lookup, whose hash strategy matches NaN by value
|
|
272
|
+
# equality (pandas index lookups treat NaN as equal to NaN).
|
|
273
|
+
if not (np.abs(numeric - predicted) <= 0.25 * abs(step)).all():
|
|
274
|
+
return None
|
|
275
|
+
return float(numeric[0]), float(step)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
class _CoordLookup:
|
|
279
|
+
"""Maps one dimension's coordinate values to their array positions.
|
|
280
|
+
|
|
281
|
+
Built once per dimension per reconstruction, then probed with every
|
|
282
|
+
batch's coordinate column. Three strategies, fastest applicable wins:
|
|
283
|
+
|
|
284
|
+
* **Affine** — the axis is uniformly spaced (the norm for rasters and
|
|
285
|
+
regular time steps, ascending or descending): the position is
|
|
286
|
+
``rint((value - origin) / step)``, a fused vector op with no per-row
|
|
287
|
+
lookup at all.
|
|
288
|
+
* **Hash** — irregular axes (station networks, arbitrary point sets)
|
|
289
|
+
with unique values: a ``pd.Index`` built once; ``get_indexer``
|
|
290
|
+
probes its persistent hash table per batch, O(1) amortized per
|
|
291
|
+
row instead of a per-row binary search. Building the table once
|
|
292
|
+
here, outside the batch loop, is what makes this fast, and it is
|
|
293
|
+
only correct to reuse because the axis does not change between
|
|
294
|
+
batches. The table holds roughly twice the transient memory of
|
|
295
|
+
the sorted copy the search strategy needs; affine axes build
|
|
296
|
+
neither.
|
|
297
|
+
* **Sorted search** — axes the hash strategy cannot represent:
|
|
298
|
+
duplicate values (a unique-key table has no single position per
|
|
299
|
+
value) or dtypes ``pd.Index`` does not support (e.g. float16):
|
|
300
|
+
``np.argsort`` once, then ``np.searchsorted`` per batch (O(log n)
|
|
301
|
+
per row). With duplicates, each value resolves to one of the
|
|
302
|
+
positions holding it (which one is unspecified), and rows
|
|
303
|
+
targeting the same cell overwrite in batch order — the scatter's
|
|
304
|
+
last-write-wins semantics.
|
|
305
|
+
|
|
306
|
+
Only the hash strategy can detect a probe value absent from the
|
|
307
|
+
axis; ``positions_for`` then raises ``ValueError`` — a symptom of a
|
|
308
|
+
filtered query whose coordinate discovery missed a value, which
|
|
309
|
+
would otherwise scatter to a wrong cell. The affine strategy rounds
|
|
310
|
+
such a value to the nearest grid position and the sorted strategy
|
|
311
|
+
resolves it to a neighbor, both preserving their historical
|
|
312
|
+
semantics.
|
|
313
|
+
"""
|
|
314
|
+
|
|
315
|
+
def __init__(self, requested: np.ndarray) -> None:
|
|
316
|
+
self._affine: tuple[float, float] | None = _affine_axis(requested)
|
|
317
|
+
self._hash_index: pd.Index | None = None
|
|
318
|
+
self._sorted_idx: np.ndarray | None = None
|
|
319
|
+
self._sorted_req: np.ndarray | None = None
|
|
320
|
+
if self._affine is None:
|
|
321
|
+
index: pd.Index | None
|
|
322
|
+
try:
|
|
323
|
+
index = pd.Index(requested)
|
|
324
|
+
except (NotImplementedError, TypeError):
|
|
325
|
+
# Dtypes pandas cannot index (e.g. float16) take the
|
|
326
|
+
# sorted-search strategy, which only needs numpy
|
|
327
|
+
# comparisons.
|
|
328
|
+
index = None
|
|
329
|
+
if index is not None and index.is_unique:
|
|
330
|
+
self._hash_index = index
|
|
331
|
+
else:
|
|
332
|
+
self._sorted_idx = np.argsort(requested)
|
|
333
|
+
self._sorted_req = requested[self._sorted_idx]
|
|
334
|
+
|
|
335
|
+
def positions_for(self, vals: np.ndarray, dim: str) -> np.ndarray:
|
|
336
|
+
"""Positions of ``vals`` within the axis, in axis order."""
|
|
337
|
+
if self._affine is not None:
|
|
338
|
+
origin, step = self._affine
|
|
339
|
+
pos = np.rint((_axis_numeric(vals) - origin) / step).astype(np.intp)
|
|
340
|
+
return cast(np.ndarray, pos)
|
|
341
|
+
if self._hash_index is not None:
|
|
342
|
+
pos = np.asarray(self._hash_index.get_indexer(vals))
|
|
343
|
+
if (pos < 0).any():
|
|
344
|
+
missing = vals[pos < 0]
|
|
345
|
+
raise ValueError(
|
|
346
|
+
f"result contains {len(missing)} value(s) for dimension "
|
|
347
|
+
f"{dim!r} not present in its coordinate array "
|
|
348
|
+
f"(first: {missing[0]!r}); the query result does not "
|
|
349
|
+
"match the reconstruction's coordinates."
|
|
350
|
+
)
|
|
351
|
+
return cast(np.ndarray, pos)
|
|
352
|
+
assert self._sorted_req is not None and self._sorted_idx is not None
|
|
353
|
+
pos_in_sorted = np.searchsorted(self._sorted_req, vals)
|
|
354
|
+
return cast(np.ndarray, self._sorted_idx[pos_in_sorted])
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _scatter_batches_to_ndarray(
|
|
358
|
+
batches: list[pa.RecordBatch],
|
|
359
|
+
dimension_columns: list[str],
|
|
360
|
+
requested: dict[str, np.ndarray],
|
|
361
|
+
var_name: str,
|
|
362
|
+
out_shape: tuple[int, ...],
|
|
363
|
+
dtype: np.dtype,
|
|
364
|
+
drop_axes: list[int],
|
|
365
|
+
) -> np.ndarray:
|
|
366
|
+
"""Convert filtered Arrow ``RecordBatch`` rows into a dense N-D numpy array.
|
|
367
|
+
|
|
368
|
+
SQL query results arrive as flat rows; xarray expects N-D arrays.
|
|
369
|
+
This bridges the two: each row carries the dim-coord values that
|
|
370
|
+
identify its cell in the output cube plus the value to write there.
|
|
371
|
+
We look up each row's N-D position within the caller's requested
|
|
372
|
+
coord arrays (see ``_CoordLookup``), then scatter-write the value
|
|
373
|
+
at that index.
|
|
374
|
+
|
|
375
|
+
Missing combinations (sparse results from filtered queries) stay as
|
|
376
|
+
``NaN`` for floating-point outputs by pre-filling the buffer; integer
|
|
377
|
+
outputs leave them as ``np.empty``-style undefined values.
|
|
378
|
+
"""
|
|
379
|
+
# NaN fill for float outputs; default for int/datetime falls through
|
|
380
|
+
# to ``np.empty``-style undefined values (but every output cell is
|
|
381
|
+
# written below for non-sparse cases).
|
|
382
|
+
out = (
|
|
383
|
+
np.full(out_shape, np.nan, dtype=dtype)
|
|
384
|
+
if np.issubdtype(dtype, np.floating)
|
|
385
|
+
else np.empty(out_shape, dtype=dtype)
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
lookups = {d: _CoordLookup(requested[d]) for d in dimension_columns}
|
|
389
|
+
|
|
390
|
+
for batch in batches:
|
|
391
|
+
if batch.num_rows == 0:
|
|
392
|
+
continue
|
|
393
|
+
schema_names = batch.schema.names
|
|
394
|
+
# Build per-dim position arrays for this batch (positions within
|
|
395
|
+
# the caller's requested coord order).
|
|
396
|
+
positions = []
|
|
397
|
+
for d in dimension_columns:
|
|
398
|
+
col_arr = batch.column(schema_names.index(d))
|
|
399
|
+
vals = col_arr.to_numpy(zero_copy_only=False)
|
|
400
|
+
positions.append(lookups[d].positions_for(vals, dim=d))
|
|
401
|
+
value_arr = batch.column(schema_names.index(var_name)).to_numpy(
|
|
402
|
+
zero_copy_only=False
|
|
403
|
+
)
|
|
404
|
+
out[tuple(positions)] = value_arr.astype(dtype, copy=False)
|
|
405
|
+
|
|
406
|
+
if drop_axes:
|
|
407
|
+
out = np.squeeze(out, axis=tuple(drop_axes))
|
|
408
|
+
return cast(np.ndarray, out)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
class SQLBackendArray(xr.backends.BackendArray):
|
|
412
|
+
"""Read-only lazy N-D array view over a re-executable SQL result.
|
|
413
|
+
|
|
414
|
+
Bridges xarray's lazy-indexing interface
|
|
415
|
+
(``xarray.backends.BackendArray``) to an engine query result,
|
|
416
|
+
so an xarray ``Dataset`` can present a SQL query as if it were a
|
|
417
|
+
materialized N-D array without actually loading any data until the
|
|
418
|
+
caller asks for it. This is the workhorse that lets
|
|
419
|
+
[XarrayDataFrame.to_dataset][xarray_sql.ds.XarrayDataFrame.to_dataset] (and the engine-agnostic
|
|
420
|
+
``xql.to_dataset(chunks=...)``) return a Dataset cheaply.
|
|
421
|
+
|
|
422
|
+
On each ``__getitem__`` call, the requested xarray indexer is
|
|
423
|
+
translated into per-dimension coordinate windows and a column
|
|
424
|
+
projection, executed through a
|
|
425
|
+
[LazyResultHandle][xarray_sql.lazyscan.LazyResultHandle] (DataFusion, DuckDB,
|
|
426
|
+
or Polars — each renders the windows with its own typed expression
|
|
427
|
+
API). The resulting Arrow ``RecordBatch`` es are scattered into a
|
|
428
|
+
preallocated numpy buffer, so only the requested data is
|
|
429
|
+
materialized.
|
|
430
|
+
|
|
431
|
+
Constraints and caveats:
|
|
432
|
+
|
|
433
|
+
- Read-only: there is no write path; the backend exists to surface
|
|
434
|
+
query results, not to round-trip writes into a SQL store.
|
|
435
|
+
- The underlying engine object may hold non-picklable references
|
|
436
|
+
(DataFusion's ``SessionContext``, a DuckDB connection). The class
|
|
437
|
+
therefore overrides ``__copy__`` and ``__deepcopy__`` to return
|
|
438
|
+
``self`` -- this is safe because the backend is read-only.
|
|
439
|
+
- ``IndexingSupport.OUTER``: ``BasicIndexer`` and ``OuterIndexer``
|
|
440
|
+
are translated to filter predicates directly; ``VectorizedIndexer``
|
|
441
|
+
paths through xarray's adapter to outer-then-gather and so still
|
|
442
|
+
works, just less efficiently.
|
|
443
|
+
|
|
444
|
+
Raises:
|
|
445
|
+
ValueError, engine exceptions: propagated from the underlying
|
|
446
|
+
filter/project/execute chain if a predicate refers to a
|
|
447
|
+
missing column, the dtype of a literal is incompatible, or
|
|
448
|
+
the execution itself fails.
|
|
449
|
+
ValueError: from the coordinate lookup (``_CoordLookup``) when
|
|
450
|
+
the result contains values for an irregular unique axis
|
|
451
|
+
that are not present in the wrapper's pre-computed coord
|
|
452
|
+
arrays -- usually a symptom of a filtered query whose coord
|
|
453
|
+
discovery missed a value.
|
|
454
|
+
|
|
455
|
+
Constructed by ``_build_lazy_scan``; users should not instantiate
|
|
456
|
+
this class directly.
|
|
457
|
+
"""
|
|
458
|
+
|
|
459
|
+
def __init__(
|
|
460
|
+
self,
|
|
461
|
+
handle: LazyResultHandle,
|
|
462
|
+
var_name: str,
|
|
463
|
+
dimension_columns: list[str],
|
|
464
|
+
coord_arrays: dict[str, np.ndarray],
|
|
465
|
+
shape: tuple[int, ...],
|
|
466
|
+
dtype: np.dtype,
|
|
467
|
+
) -> None:
|
|
468
|
+
self._handle = handle
|
|
469
|
+
self._var_name = var_name
|
|
470
|
+
self._dimension_columns = list(dimension_columns)
|
|
471
|
+
self._coord_arrays = coord_arrays
|
|
472
|
+
# Computed once per dim: whether the whole coordinate array is
|
|
473
|
+
# strictly monotonic, the precondition for translating contiguous
|
|
474
|
+
# positional windows into value ranges (see _dim_spec).
|
|
475
|
+
self._monotonic = {
|
|
476
|
+
d: _strictly_monotonic(coord_arrays[d]) for d in dimension_columns
|
|
477
|
+
}
|
|
478
|
+
self.shape = tuple(shape)
|
|
479
|
+
self.dtype = np.dtype(dtype)
|
|
480
|
+
|
|
481
|
+
def __getitem__(self, key: Any) -> np.ndarray:
|
|
482
|
+
return cast(
|
|
483
|
+
np.ndarray,
|
|
484
|
+
xr.core.indexing.explicit_indexing_adapter(
|
|
485
|
+
key,
|
|
486
|
+
self.shape,
|
|
487
|
+
xr.core.indexing.IndexingSupport.OUTER,
|
|
488
|
+
self._raw_getitem,
|
|
489
|
+
),
|
|
490
|
+
)
|
|
491
|
+
|
|
492
|
+
def __copy__(self) -> "SQLBackendArray":
|
|
493
|
+
# The backend is read-only; the underlying DataFusion DataFrame
|
|
494
|
+
# holds a non-picklable SessionContext reference, so sharing the
|
|
495
|
+
# same backend across a copy is both safe and necessary.
|
|
496
|
+
return self
|
|
497
|
+
|
|
498
|
+
def __deepcopy__(self, memo: dict) -> "SQLBackendArray":
|
|
499
|
+
return self
|
|
500
|
+
|
|
501
|
+
# ------------------------------------------------------------------
|
|
502
|
+
|
|
503
|
+
def _raw_getitem(self, key: tuple) -> np.ndarray:
|
|
504
|
+
"""Materialize the indexed region described by *key* via the engine.
|
|
505
|
+
|
|
506
|
+
``key`` is a tuple of ``int``/``slice``/1-D integer-array, one per
|
|
507
|
+
dim, in ``_dimension_columns`` order.
|
|
508
|
+
"""
|
|
509
|
+
requested: dict[str, np.ndarray] = {}
|
|
510
|
+
# Per-dim windows for the engine. Dims whose indexer covers the
|
|
511
|
+
# full extent are omitted entirely so the engine doesn't have to
|
|
512
|
+
# evaluate a tautology.
|
|
513
|
+
specs: dict[str, DimSpec] = {}
|
|
514
|
+
drop_axes: list[int] = []
|
|
515
|
+
for axis, (dim, k) in enumerate(
|
|
516
|
+
zip(self._dimension_columns, key, strict=True)
|
|
517
|
+
):
|
|
518
|
+
coord = self._coord_arrays[dim]
|
|
519
|
+
contiguous = False
|
|
520
|
+
if isinstance(k, slice):
|
|
521
|
+
start = 0 if k.start is None else k.start
|
|
522
|
+
stop = len(coord) if k.stop is None else k.stop
|
|
523
|
+
step = 1 if k.step is None else k.step
|
|
524
|
+
requested[dim] = np.asarray(coord[start:stop:step])
|
|
525
|
+
contiguous = step == 1
|
|
526
|
+
if start == 0 and stop >= len(coord) and step == 1:
|
|
527
|
+
continue
|
|
528
|
+
elif isinstance(k, (int, np.integer)):
|
|
529
|
+
requested[dim] = np.asarray([coord[int(k)]])
|
|
530
|
+
drop_axes.append(axis)
|
|
531
|
+
else:
|
|
532
|
+
arr = np.asarray(k)
|
|
533
|
+
requested[dim] = np.asarray(coord[arr])
|
|
534
|
+
if (
|
|
535
|
+
len(arr) == len(coord)
|
|
536
|
+
and (arr == np.arange(len(coord))).all()
|
|
537
|
+
):
|
|
538
|
+
continue
|
|
539
|
+
contiguous = len(arr) > 1 and bool((np.diff(arr) == 1).all())
|
|
540
|
+
specs[dim] = _dim_spec(
|
|
541
|
+
requested[dim], contiguous, self._monotonic[dim]
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
out_shape = tuple(len(requested[d]) for d in self._dimension_columns)
|
|
545
|
+
if any(n == 0 for n in out_shape):
|
|
546
|
+
empty = np.empty(out_shape, dtype=self.dtype)
|
|
547
|
+
squeezed = (
|
|
548
|
+
np.squeeze(empty, axis=tuple(drop_axes)) if drop_axes else empty
|
|
549
|
+
)
|
|
550
|
+
return cast(np.ndarray, squeezed)
|
|
551
|
+
|
|
552
|
+
batches = self._handle.fetch(
|
|
553
|
+
specs, self._dimension_columns + [self._var_name]
|
|
554
|
+
)
|
|
555
|
+
return _scatter_batches_to_ndarray(
|
|
556
|
+
batches=batches,
|
|
557
|
+
dimension_columns=self._dimension_columns,
|
|
558
|
+
requested=requested,
|
|
559
|
+
var_name=self._var_name,
|
|
560
|
+
out_shape=out_shape,
|
|
561
|
+
dtype=self.dtype,
|
|
562
|
+
drop_axes=drop_axes,
|
|
563
|
+
)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _strictly_monotonic(coord: np.ndarray) -> bool:
|
|
567
|
+
"""Whether ``coord`` is strictly increasing or strictly decreasing.
|
|
568
|
+
|
|
569
|
+
Strict monotonicity of the whole coordinate array is the
|
|
570
|
+
precondition for translating a contiguous positional window into a
|
|
571
|
+
value range: with duplicated or unsorted values, ``[min, max]`` of a
|
|
572
|
+
window admits coordinate values at positions outside the window.
|
|
573
|
+
NaN/NaT (whose comparisons are all false) and non-comparable object
|
|
574
|
+
arrays report ``False``, which safely falls back to value lists.
|
|
575
|
+
"""
|
|
576
|
+
if len(coord) < 2:
|
|
577
|
+
return True
|
|
578
|
+
head, tail = coord[:-1], coord[1:]
|
|
579
|
+
try:
|
|
580
|
+
return bool((tail > head).all() or (tail < head).all())
|
|
581
|
+
except TypeError:
|
|
582
|
+
return False
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def _dim_spec(
|
|
586
|
+
vals: np.ndarray, contiguous: bool, coord_monotonic: bool
|
|
587
|
+
) -> DimSpec:
|
|
588
|
+
"""The engine window for one dim's requested coordinate values.
|
|
589
|
+
|
|
590
|
+
A contiguous run of positions over a strictly monotonic coordinate
|
|
591
|
+
array is exactly the value range ``[min, max]`` — a two-literal
|
|
592
|
+
predicate engines can push into range pruning. Monotonicity must
|
|
593
|
+
hold for the *entire* coordinate array (``coord_monotonic``), not
|
|
594
|
+
just the requested window: template coords are used verbatim, and
|
|
595
|
+
over a non-monotonic array a window's ``[min, max]`` admits values
|
|
596
|
+
at positions outside the window, which the scatter would then write
|
|
597
|
+
to wrong cells. Anything else (stepped slices, fancy indexers,
|
|
598
|
+
non-monotonic or duplicated coords) must be an explicit value list:
|
|
599
|
+
a range would admit rows the scatter did not request.
|
|
600
|
+
"""
|
|
601
|
+
if contiguous and coord_monotonic and len(vals) > 1:
|
|
602
|
+
return ("range", vals.min(), vals.max())
|
|
603
|
+
return ("values", vals, None)
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def _c_order_grid(
|
|
607
|
+
dim_cols: dict[str, np.ndarray],
|
|
608
|
+
coord_arrays: dict[str, np.ndarray],
|
|
609
|
+
dimension_columns: list[str],
|
|
610
|
+
total_rows: int,
|
|
611
|
+
) -> bool:
|
|
612
|
+
"""Whether the result rows form the complete grid in C order.
|
|
613
|
+
|
|
614
|
+
True iff the row count is exactly the coordinate product and every
|
|
615
|
+
dimension column is its coordinates repeated/tiled in C order — the
|
|
616
|
+
shape any unfiltered or bbox-windowed scan produces. When it holds,
|
|
617
|
+
data variables are dense row-major arrays already and can be
|
|
618
|
+
reshaped instead of scatter-written (one memcpy versus a per-row
|
|
619
|
+
position lookup and write; see ``_CoordLookup``).
|
|
620
|
+
"""
|
|
621
|
+
shape = tuple(len(coord_arrays[d]) for d in dimension_columns)
|
|
622
|
+
if total_rows != int(np.prod(shape)) or total_rows == 0:
|
|
623
|
+
return False
|
|
624
|
+
for k, d in enumerate(dimension_columns):
|
|
625
|
+
inner = int(np.prod(shape[k + 1 :]))
|
|
626
|
+
outer = int(np.prod(shape[:k]))
|
|
627
|
+
view = dim_cols[d].reshape(outer, shape[k], inner)
|
|
628
|
+
if not (view == coord_arrays[d][None, :, None]).all():
|
|
629
|
+
return False
|
|
630
|
+
return True
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _dataset_from_batches(
|
|
634
|
+
batches: list[pa.RecordBatch],
|
|
635
|
+
dimension_columns: list[str],
|
|
636
|
+
field_names: list[str],
|
|
637
|
+
field_types: dict[str, Any],
|
|
638
|
+
) -> xr.Dataset:
|
|
639
|
+
"""Build a dense in-memory Dataset from Arrow ``RecordBatch`` es.
|
|
640
|
+
|
|
641
|
+
The engine-agnostic core of the eager round-trip: derives the
|
|
642
|
+
coordinates and every data variable from a single already-executed
|
|
643
|
+
result, whichever engine produced it. ``field_types`` values only
|
|
644
|
+
need a ``to_pandas_dtype()`` method (both ``pyarrow.DataType`` and
|
|
645
|
+
DataFusion's Arrow type wrappers qualify).
|
|
646
|
+
|
|
647
|
+
Complete grid-ordered results (unfiltered scans, bbox windows) are
|
|
648
|
+
reshaped directly; anything else — sparse results from filtered
|
|
649
|
+
queries, engine-reordered rows — falls back to the positional
|
|
650
|
+
scatter, which handles arbitrary row order.
|
|
651
|
+
"""
|
|
652
|
+
dim_cols: dict[str, np.ndarray] = {}
|
|
653
|
+
coord_arrays: dict[str, np.ndarray] = {}
|
|
654
|
+
for d in dimension_columns:
|
|
655
|
+
if not batches:
|
|
656
|
+
dim_cols[d] = np.asarray([])
|
|
657
|
+
coord_arrays[d] = np.asarray([])
|
|
658
|
+
continue
|
|
659
|
+
vals = np.concatenate(
|
|
660
|
+
[
|
|
661
|
+
b.column(b.schema.names.index(d)).to_numpy(zero_copy_only=False)
|
|
662
|
+
for b in batches
|
|
663
|
+
]
|
|
664
|
+
)
|
|
665
|
+
dim_cols[d] = vals
|
|
666
|
+
# Preserve the order coordinate values first appear in the result so an
|
|
667
|
+
# ORDER BY direction (e.g. ``ORDER BY level DESC``) carries through to
|
|
668
|
+
# the Dataset dimension instead of being force-sorted ascending.
|
|
669
|
+
# pd.unique keeps first-appearance order; the scatter below argsorts
|
|
670
|
+
# internally, so arbitrarily-ordered coordinates are placed correctly.
|
|
671
|
+
coord_arrays[d] = np.asarray(pd.unique(vals))
|
|
672
|
+
shape = tuple(len(coord_arrays[d]) for d in dimension_columns)
|
|
673
|
+
total_rows = sum(b.num_rows for b in batches)
|
|
674
|
+
|
|
675
|
+
grid_ordered = _c_order_grid(
|
|
676
|
+
dim_cols, coord_arrays, dimension_columns, total_rows
|
|
677
|
+
)
|
|
678
|
+
|
|
679
|
+
data_vars: dict[str, xr.Variable] = {}
|
|
680
|
+
for name in field_names:
|
|
681
|
+
if name in dimension_columns:
|
|
682
|
+
continue
|
|
683
|
+
np_dtype = np.dtype(field_types[name].to_pandas_dtype())
|
|
684
|
+
if grid_ordered:
|
|
685
|
+
flat = np.concatenate(
|
|
686
|
+
[
|
|
687
|
+
b.column(b.schema.names.index(name)).to_numpy(
|
|
688
|
+
zero_copy_only=False
|
|
689
|
+
)
|
|
690
|
+
for b in batches
|
|
691
|
+
]
|
|
692
|
+
)
|
|
693
|
+
dense = flat.astype(np_dtype, copy=False).reshape(shape)
|
|
694
|
+
else:
|
|
695
|
+
dense = _scatter_batches_to_ndarray(
|
|
696
|
+
batches=batches,
|
|
697
|
+
dimension_columns=dimension_columns,
|
|
698
|
+
requested=coord_arrays,
|
|
699
|
+
var_name=name,
|
|
700
|
+
out_shape=shape,
|
|
701
|
+
dtype=np_dtype,
|
|
702
|
+
drop_axes=[],
|
|
703
|
+
)
|
|
704
|
+
data_vars[name] = xr.Variable(dimension_columns, dense)
|
|
705
|
+
|
|
706
|
+
coords_arg = {d: coord_arrays[d] for d in dimension_columns}
|
|
707
|
+
return xr.Dataset(data_vars=data_vars, coords=coords_arg)
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
_PURE_SCAN_NODES = {"Projection", "Sort", "TableScan", "SubqueryAlias"}
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def _unfiltered_scan_table(inner_df: Any) -> str | None:
|
|
714
|
+
"""Return the scanned table name iff the query is a pure unfiltered scan.
|
|
715
|
+
|
|
716
|
+
A pure scan only contains ``Projection``, ``Sort``, ``TableScan``,
|
|
717
|
+
``SubqueryAlias`` nodes and exactly one ``TableScan``. Anything else
|
|
718
|
+
(``Filter``, ``Aggregate``, ``Join``, ``Union``, ``Limit``, multi-table
|
|
719
|
+
joins, ...) returns ``None`` so the caller falls back to per-dim
|
|
720
|
+
discovery. The returned name is the registered table the caller can
|
|
721
|
+
look up to source coord arrays from.
|
|
722
|
+
"""
|
|
723
|
+
try:
|
|
724
|
+
lp = inner_df.logical_plan()
|
|
725
|
+
except Exception:
|
|
726
|
+
return None
|
|
727
|
+
table_name: str | None = None
|
|
728
|
+
stack = [lp]
|
|
729
|
+
while stack:
|
|
730
|
+
node = stack.pop()
|
|
731
|
+
try:
|
|
732
|
+
variant = node.to_variant()
|
|
733
|
+
except Exception:
|
|
734
|
+
return None
|
|
735
|
+
cls = type(variant).__name__
|
|
736
|
+
if cls not in _PURE_SCAN_NODES:
|
|
737
|
+
return None
|
|
738
|
+
if cls == "TableScan":
|
|
739
|
+
try:
|
|
740
|
+
this = variant.table_name()
|
|
741
|
+
except (AttributeError, TypeError):
|
|
742
|
+
return None
|
|
743
|
+
if not isinstance(this, str):
|
|
744
|
+
return None
|
|
745
|
+
if table_name is not None and table_name != this:
|
|
746
|
+
return None # multi-table scan; not a single source
|
|
747
|
+
table_name = this
|
|
748
|
+
stack.extend(node.inputs())
|
|
749
|
+
return table_name
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _maybe_template_coords(
|
|
753
|
+
templates: dict[str, xr.Dataset] | None,
|
|
754
|
+
dimension_columns: list[str],
|
|
755
|
+
inner_df: Any,
|
|
756
|
+
) -> dict[str, np.ndarray] | None:
|
|
757
|
+
"""Use the scanned table's registered coord arrays directly when safe.
|
|
758
|
+
|
|
759
|
+
Returns coord arrays sourced from the registered Dataset for the
|
|
760
|
+
scanned table iff the query is an unfiltered scan over that single
|
|
761
|
+
table and the registered Dataset carries all requested dims. Returns
|
|
762
|
+
``None`` otherwise so the caller falls back to per-dim discovery.
|
|
763
|
+
Skipping discovery avoids one full plan execution per dim and
|
|
764
|
+
preserves the source's coordinate order.
|
|
765
|
+
|
|
766
|
+
Coord values come from the **scanned** registered Dataset, not from
|
|
767
|
+
any user-supplied ``template=`` (which is for metadata recovery
|
|
768
|
+
only). That keeps the fast path correct when a user with multiple
|
|
769
|
+
registered Datasets passes a metadata template that differs from
|
|
770
|
+
the query's source.
|
|
771
|
+
"""
|
|
772
|
+
if not templates:
|
|
773
|
+
return None
|
|
774
|
+
table = _unfiltered_scan_table(inner_df)
|
|
775
|
+
if table is None or table not in templates:
|
|
776
|
+
return None
|
|
777
|
+
source = templates[table]
|
|
778
|
+
if not all(d in source.coords for d in dimension_columns):
|
|
779
|
+
return None
|
|
780
|
+
return {d: np.asarray(source.coords[d].values) for d in dimension_columns}
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
_ONE_PASS_BYTES = 64 * 2**20
|
|
784
|
+
"""Size up to which a lazy result that must discover its coordinates is
|
|
785
|
+
read in a single pass instead."""
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def _read_or_discover(
|
|
789
|
+
handle: LazyResultHandle, columns: list[str], dimension_columns: list[str]
|
|
790
|
+
) -> tuple[list[pa.RecordBatch] | None, dict[str, np.ndarray] | None]:
|
|
791
|
+
"""Execute the query once: the whole result if small, else its coordinates.
|
|
792
|
+
|
|
793
|
+
A lazy result must know each dimension's coordinates before any data
|
|
794
|
+
is read. One streamed pass serves both cases. A result that fits in
|
|
795
|
+
``_ONE_PASS_BYTES`` -- an aggregation, which no window filter can push
|
|
796
|
+
below, or a narrow selection -- is returned whole, to be built in
|
|
797
|
+
memory. Past the budget, the pass keeps going but keeps only each
|
|
798
|
+
dimension's distinct values, so discovery executes the query once
|
|
799
|
+
rather than once per dimension. Returns ``(None, None)`` for handles
|
|
800
|
+
that cannot stream, which then discover one dimension at a time.
|
|
801
|
+
"""
|
|
802
|
+
stream = getattr(handle, "stream", None)
|
|
803
|
+
if stream is None:
|
|
804
|
+
return None, None
|
|
805
|
+
batches: list[pa.RecordBatch] = []
|
|
806
|
+
total = 0
|
|
807
|
+
distinct: dict[str, pa.Array] | None = None
|
|
808
|
+
for batch in stream(columns):
|
|
809
|
+
if distinct is None:
|
|
810
|
+
total += batch.nbytes
|
|
811
|
+
batches.append(batch)
|
|
812
|
+
if total <= _ONE_PASS_BYTES:
|
|
813
|
+
continue
|
|
814
|
+
# Over budget: from here on, keep only distinct coordinates.
|
|
815
|
+
distinct = {}
|
|
816
|
+
pending, batches = batches, []
|
|
817
|
+
else:
|
|
818
|
+
pending = [batch]
|
|
819
|
+
for b in pending:
|
|
820
|
+
for d in dimension_columns:
|
|
821
|
+
values = pc.unique(b.column(b.schema.names.index(d)))
|
|
822
|
+
if d in distinct:
|
|
823
|
+
values = pc.unique(pa.concat_arrays([distinct[d], values]))
|
|
824
|
+
distinct[d] = values
|
|
825
|
+
if distinct is None:
|
|
826
|
+
return batches, None
|
|
827
|
+
return None, {
|
|
828
|
+
# Sorted, as from ``distinct``, so positional slices map onto
|
|
829
|
+
# contiguous value ranges.
|
|
830
|
+
d: np.sort(distinct[d].to_numpy(zero_copy_only=False))
|
|
831
|
+
for d in dimension_columns
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def _build_lazy_scan(
|
|
836
|
+
handle: LazyResultHandle,
|
|
837
|
+
dimension_columns: list[str],
|
|
838
|
+
field_names: list[str],
|
|
839
|
+
field_types: dict[str, Any],
|
|
840
|
+
coord_arrays: dict[str, np.ndarray] | None = None,
|
|
841
|
+
) -> xr.Dataset:
|
|
842
|
+
"""Build a lazy Dataset whose data vars are [SQLBackendArray][xarray_sql.ds.SQLBackendArray].
|
|
843
|
+
|
|
844
|
+
Each data variable stays lazy: every access reads its coordinate range
|
|
845
|
+
via a pushdown filter, either directly or, once wrapped by
|
|
846
|
+
``Dataset.chunk``, one chunk at a time. Coordinates come either from the
|
|
847
|
+
caller (the scanned table's registered Dataset for unfiltered DataFusion
|
|
848
|
+
scans -- see ``_maybe_template_coords`` -- or an explicitly trusted
|
|
849
|
+
template) or from per-dim distinct queries through the handle; over a
|
|
850
|
+
registered pushdown table the engine projects to that single coordinate
|
|
851
|
+
column, so discovery reads coordinate values only (no data-variable I/O).
|
|
852
|
+
"""
|
|
853
|
+
if coord_arrays is None:
|
|
854
|
+
coord_arrays = {}
|
|
855
|
+
for d in dimension_columns:
|
|
856
|
+
# ``distinct`` returns engine order; sort ascending so
|
|
857
|
+
# positional slices map onto contiguous value ranges.
|
|
858
|
+
coord_arrays[d] = np.sort(handle.distinct(d))
|
|
859
|
+
shape = tuple(len(coord_arrays[d]) for d in dimension_columns)
|
|
860
|
+
|
|
861
|
+
data_vars: dict[str, xr.Variable] = {}
|
|
862
|
+
for name in field_names:
|
|
863
|
+
if name in dimension_columns:
|
|
864
|
+
continue
|
|
865
|
+
np_dtype = field_types[name].to_pandas_dtype()
|
|
866
|
+
backend = SQLBackendArray(
|
|
867
|
+
handle=handle,
|
|
868
|
+
var_name=name,
|
|
869
|
+
dimension_columns=dimension_columns,
|
|
870
|
+
coord_arrays=coord_arrays,
|
|
871
|
+
shape=shape,
|
|
872
|
+
dtype=np_dtype,
|
|
873
|
+
)
|
|
874
|
+
lazy = xr.core.indexing.LazilyIndexedArray(backend)
|
|
875
|
+
data_vars[name] = xr.Variable(dimension_columns, lazy)
|
|
876
|
+
|
|
877
|
+
coords_arg = {d: coord_arrays[d] for d in dimension_columns}
|
|
878
|
+
return xr.Dataset(data_vars=data_vars, coords=coords_arg)
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _auto_chunk_target_bytes() -> int:
|
|
882
|
+
"""Byte target for ``chunks="auto"`` (the chunk manager's, else 128 MiB)."""
|
|
883
|
+
try:
|
|
884
|
+
import dask
|
|
885
|
+
from dask.utils import parse_bytes
|
|
886
|
+
|
|
887
|
+
return int(parse_bytes(dask.config.get("array.chunk-size")))
|
|
888
|
+
except Exception:
|
|
889
|
+
return 128 * 1024 * 1024
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def _auto_chunks(
|
|
893
|
+
template: xr.Dataset | None,
|
|
894
|
+
dimension_columns: list[str],
|
|
895
|
+
field_types: dict[str, Any],
|
|
896
|
+
) -> dict[str, int] | None:
|
|
897
|
+
"""Resolve ``chunks="auto"`` to a source-partition-aligned chunk spec.
|
|
898
|
+
|
|
899
|
+
Sizes chunks to roughly the chunk manager's byte target (dask's
|
|
900
|
+
``array.chunk-size``, default 128 MiB) but snaps boundaries to whole source
|
|
901
|
+
partitions, so every chunk is a union of source partitions -- no chunk splits
|
|
902
|
+
a partition (which would make adjacent chunks re-read it). This is what makes
|
|
903
|
+
``"auto"`` useful for finely partitioned sources (e.g. ERA5
|
|
904
|
+
``chunks={"time": 1}``): it coarsens many tiny partitions into memory-sized,
|
|
905
|
+
aligned chunks. Returns ``None`` when there is no resolvable source grid to
|
|
906
|
+
align to, so the caller falls back to the chunk manager's own ``"auto"``.
|
|
907
|
+
"""
|
|
908
|
+
if template is None:
|
|
909
|
+
return None
|
|
910
|
+
part = template.chunksizes # dim -> tuple of source chunk lengths
|
|
911
|
+
chunked_dims = [
|
|
912
|
+
d for d in dimension_columns if d in part and len(part[d]) > 1
|
|
913
|
+
]
|
|
914
|
+
if not chunked_dims:
|
|
915
|
+
return None
|
|
916
|
+
|
|
917
|
+
itemsizes = [
|
|
918
|
+
np.dtype(t.to_pandas_dtype()).itemsize
|
|
919
|
+
for name, t in field_types.items()
|
|
920
|
+
if name not in dimension_columns
|
|
921
|
+
]
|
|
922
|
+
itemsize = max(itemsizes) if itemsizes else 8
|
|
923
|
+
|
|
924
|
+
# Bytes in one source-partition block: the nominal source chunk length per
|
|
925
|
+
# dimension (``part[d][0]``) multiplied across all dims, times itemsize.
|
|
926
|
+
block_bytes = itemsize
|
|
927
|
+
for d in dimension_columns:
|
|
928
|
+
if d in part:
|
|
929
|
+
block_bytes *= int(part[d][0])
|
|
930
|
+
# Number of source partitions to merge per chunk to approach the target.
|
|
931
|
+
merge = max(1, _auto_chunk_target_bytes() // max(block_bytes, 1))
|
|
932
|
+
|
|
933
|
+
# Absorb the coarsening into the most finely partitioned dimension; the rest
|
|
934
|
+
# keep their source chunk length. xarray caps an oversize chunk at the dim
|
|
935
|
+
# length, so an over-large merge simply yields a single chunk on that dim.
|
|
936
|
+
primary = max(chunked_dims, key=lambda d: len(part[d]))
|
|
937
|
+
return {
|
|
938
|
+
d: int(part[d][0]) * (merge if d == primary else 1)
|
|
939
|
+
for d in chunked_dims
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def _result_to_xarray(
|
|
944
|
+
inner_df: Any,
|
|
945
|
+
dimension_columns: list[str],
|
|
946
|
+
template: xr.Dataset | None,
|
|
947
|
+
sparsity: Sparsity,
|
|
948
|
+
fill_value: Any,
|
|
949
|
+
chunks: Mapping[str, int] | str | None,
|
|
950
|
+
templates: dict[str, xr.Dataset] | None = None,
|
|
951
|
+
) -> xr.Dataset:
|
|
952
|
+
"""Reconstruct an ``xr.Dataset`` from a SQL result.
|
|
953
|
+
|
|
954
|
+
``chunks`` (already resolved by ``XarrayDataFrame._resolve_chunks``)
|
|
955
|
+
selects the execution strategy:
|
|
956
|
+
|
|
957
|
+
* ``None`` -> lazy, unchunked: the data variables are lazily indexed
|
|
958
|
+
[SQLBackendArray][xarray_sql.ds.SQLBackendArray]s, so each access
|
|
959
|
+
re-executes the query narrowed to the selection and projected to
|
|
960
|
+
the variable read. No chunk manager is involved. A result whose
|
|
961
|
+
coordinates must be discovered is first streamed in one pass, which
|
|
962
|
+
keeps it in memory if small (``_read_or_discover``), so an
|
|
963
|
+
aggregation executes once.
|
|
964
|
+
* a mapping (or ``"auto"``) -> lazy/chunked: build [SQLBackendArray][xarray_sql.ds.SQLBackendArray]
|
|
965
|
+
data variables (``_build_lazy_scan``) and wrap them with
|
|
966
|
+
``Dataset.chunk`` so each chunk reads its coordinate range via filter
|
|
967
|
+
pushdown. The chunk grid maps onto the source partitions. Chunking goes
|
|
968
|
+
through xarray's configured chunk manager (dask, cubed, ...), so no
|
|
969
|
+
chunked-array backend is imported directly here.
|
|
970
|
+
"""
|
|
971
|
+
if sparsity not in ("result", "template"):
|
|
972
|
+
raise ValueError(
|
|
973
|
+
f"sparsity must be 'result' or 'template', got {sparsity!r}"
|
|
974
|
+
)
|
|
975
|
+
if sparsity == "template" and template is None:
|
|
976
|
+
raise ValueError(
|
|
977
|
+
"sparsity='template' requires template= to be supplied"
|
|
978
|
+
)
|
|
979
|
+
|
|
980
|
+
schema = inner_df.schema()
|
|
981
|
+
field_names = [f.name for f in schema]
|
|
982
|
+
field_types = {f.name: f.type for f in schema}
|
|
983
|
+
|
|
984
|
+
handle = DataFusionHandle(inner_df)
|
|
985
|
+
coord_arrays = _maybe_template_coords(
|
|
986
|
+
templates, dimension_columns, inner_df
|
|
987
|
+
)
|
|
988
|
+
batches = None
|
|
989
|
+
if chunks is None and coord_arrays is None:
|
|
990
|
+
batches, coord_arrays = _read_or_discover(
|
|
991
|
+
handle, field_names, dimension_columns
|
|
992
|
+
)
|
|
993
|
+
if batches is not None:
|
|
994
|
+
ds = _dataset_from_batches(
|
|
995
|
+
batches, dimension_columns, field_names, field_types
|
|
996
|
+
)
|
|
997
|
+
else:
|
|
998
|
+
ds = _build_lazy_scan(
|
|
999
|
+
handle,
|
|
1000
|
+
dimension_columns,
|
|
1001
|
+
field_names,
|
|
1002
|
+
field_types,
|
|
1003
|
+
coord_arrays=coord_arrays,
|
|
1004
|
+
)
|
|
1005
|
+
return _finish_dataset(
|
|
1006
|
+
ds,
|
|
1007
|
+
dimension_columns,
|
|
1008
|
+
template,
|
|
1009
|
+
sparsity,
|
|
1010
|
+
fill_value,
|
|
1011
|
+
chunks,
|
|
1012
|
+
field_types,
|
|
1013
|
+
)
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def _finish_dataset(
|
|
1017
|
+
ds: xr.Dataset,
|
|
1018
|
+
dimension_columns: list[str],
|
|
1019
|
+
template: xr.Dataset | None,
|
|
1020
|
+
sparsity: Sparsity,
|
|
1021
|
+
fill_value: Any,
|
|
1022
|
+
chunks: Mapping[str, int] | str | None,
|
|
1023
|
+
field_types: dict[str, Any],
|
|
1024
|
+
) -> xr.Dataset:
|
|
1025
|
+
"""Shared reconstruction tail: sparsity, template metadata, chunking."""
|
|
1026
|
+
if sparsity == "template":
|
|
1027
|
+
assert template is not None
|
|
1028
|
+
indexers = {
|
|
1029
|
+
d: template.coords[d].values
|
|
1030
|
+
for d in dimension_columns
|
|
1031
|
+
if d in template.coords and d in template.dims
|
|
1032
|
+
}
|
|
1033
|
+
if indexers:
|
|
1034
|
+
ds = ds.reindex(indexers, fill_value=fill_value)
|
|
1035
|
+
|
|
1036
|
+
if template is not None:
|
|
1037
|
+
ds = _apply_template(ds, template)
|
|
1038
|
+
|
|
1039
|
+
if chunks is not None:
|
|
1040
|
+
if chunks == "auto":
|
|
1041
|
+
# Snap the byte-budgeted "auto" sizing to source partition
|
|
1042
|
+
# boundaries; fall back to the chunk manager's own "auto" when there
|
|
1043
|
+
# is no source grid to align to.
|
|
1044
|
+
chunks = (
|
|
1045
|
+
_auto_chunks(template, dimension_columns, field_types) or "auto"
|
|
1046
|
+
)
|
|
1047
|
+
# Wrap the lazy data variables in the configured chunk manager (dask by
|
|
1048
|
+
# default). Each chunk reads its coordinate range via pushdown on access.
|
|
1049
|
+
ds = ds.chunk(chunks)
|
|
1050
|
+
return ds
|
|
1051
|
+
|
|
1052
|
+
|
|
1053
|
+
# ---------------------------------------------------------------------------
|
|
1054
|
+
# Public wrapper
|
|
1055
|
+
# ---------------------------------------------------------------------------
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
class XarrayDataFrame:
|
|
1059
|
+
"""Wrapper around a DataFusion ``DataFrame`` with xarray-aware helpers.
|
|
1060
|
+
|
|
1061
|
+
Returned by [xarray_sql.Context.sql][]. Forwards every
|
|
1062
|
+
attribute it does not define itself to the wrapped DataFrame, so
|
|
1063
|
+
``.collect()``, ``.schema()``, ``.show()``, ``.count()`` all work
|
|
1064
|
+
unchanged.
|
|
1065
|
+
|
|
1066
|
+
Carries a private snapshot of the context's registered Datasets so
|
|
1067
|
+
``to_dataset`` can default ``dims`` and recover metadata
|
|
1068
|
+
dropped by the forward pivot.
|
|
1069
|
+
|
|
1070
|
+
Users should not construct this class directly; let
|
|
1071
|
+
[Context.sql][xarray_sql.sql.Context.sql] produce it.
|
|
1072
|
+
"""
|
|
1073
|
+
|
|
1074
|
+
def __init__(
|
|
1075
|
+
self,
|
|
1076
|
+
inner: Any,
|
|
1077
|
+
templates: dict[str, xr.Dataset] | None = None,
|
|
1078
|
+
) -> None:
|
|
1079
|
+
"""Construct a wrapper.
|
|
1080
|
+
|
|
1081
|
+
Args:
|
|
1082
|
+
inner: The underlying ``datafusion.DataFrame`` returned by
|
|
1083
|
+
[Context.sql][xarray_sql.sql.Context.sql].
|
|
1084
|
+
templates: Snapshot of the registered Datasets on the producing
|
|
1085
|
+
context, keyed by the SQL identifier each was registered
|
|
1086
|
+
under. Used by ``to_dataset`` to recover metadata that
|
|
1087
|
+
the forward pivot strips. ``None`` means no metadata
|
|
1088
|
+
recovery is possible from registrations alone; callers may
|
|
1089
|
+
still pass ``template=`` to ``to_dataset`` explicitly.
|
|
1090
|
+
"""
|
|
1091
|
+
object.__setattr__(self, "_inner", inner)
|
|
1092
|
+
object.__setattr__(self, "_templates", dict(templates or {}))
|
|
1093
|
+
|
|
1094
|
+
def to_pandas(self) -> pd.DataFrame:
|
|
1095
|
+
"""Materialize the result as a ``pd.DataFrame`` (DataFusion API)."""
|
|
1096
|
+
return self._inner.to_pandas()
|
|
1097
|
+
|
|
1098
|
+
def to_dataset(
|
|
1099
|
+
self,
|
|
1100
|
+
dims: list[str] | None = None,
|
|
1101
|
+
template: xr.Dataset | str | None = None,
|
|
1102
|
+
sparsity: Sparsity = "result",
|
|
1103
|
+
fill_value: Any = np.nan,
|
|
1104
|
+
chunks: Mapping[str, int] | str | None = "inherit",
|
|
1105
|
+
) -> xr.Dataset:
|
|
1106
|
+
"""Convert the result to an ``xr.Dataset``.
|
|
1107
|
+
|
|
1108
|
+
Args:
|
|
1109
|
+
dims: Result columns to use as Dataset dimensions. When
|
|
1110
|
+
``None``, defaults to a registered Dataset's dimensions that
|
|
1111
|
+
survive into the result columns, so an aggregation that drops
|
|
1112
|
+
dims (e.g. ``GROUP BY time`` over a ``(time, lat, lon)`` grid)
|
|
1113
|
+
round-trips on the remaining dim. Raises when no dimension
|
|
1114
|
+
survives, or when several registered Datasets imply different
|
|
1115
|
+
dims (pass ``dims`` explicitly then).
|
|
1116
|
+
template: Source to recover metadata (attrs, encoding, non-dim
|
|
1117
|
+
coordinates, dim-coord dtype) from. Either an ``xr.Dataset``
|
|
1118
|
+
used directly, or the name of a registered table (e.g.
|
|
1119
|
+
``"era5.surface"``) whose Dataset is looked up. When ``None``
|
|
1120
|
+
and exactly one Dataset is registered, that one is used.
|
|
1121
|
+
sparsity: ``"result"`` (default) keeps only dim values
|
|
1122
|
+
present in the result. ``"template"`` reindexes to the
|
|
1123
|
+
template's full coord ranges, filling absent cells with
|
|
1124
|
+
``fill_value``; requires a template.
|
|
1125
|
+
fill_value: Used when ``sparsity="template"`` reindexes
|
|
1126
|
+
to a wider extent. Defaults to ``np.nan``.
|
|
1127
|
+
chunks: Output chunking, controlling laziness (an xarray idiom).
|
|
1128
|
+
|
|
1129
|
+
* ``"inherit"`` (default): reuse the source Dataset's chunk
|
|
1130
|
+
sizes, but only for dimensions that were genuinely split into
|
|
1131
|
+
multiple chunks in the input -- so the output chunk grid maps
|
|
1132
|
+
onto the source partitions. A reduction that drops the chunked
|
|
1133
|
+
dimension (e.g. a global aggregation) inherits nothing, and
|
|
1134
|
+
neither does a result with no resolvable source Dataset;
|
|
1135
|
+
both are returned as with ``None``.
|
|
1136
|
+
* ``None``: lazy without a chunk manager, as with
|
|
1137
|
+
``xr.open_dataset(chunks=None)``. Each access re-executes
|
|
1138
|
+
the query narrowed to the selection and projected to the
|
|
1139
|
+
variable read, so ``.load()`` executes it once per
|
|
1140
|
+
variable. Dask is not required. The context must stay
|
|
1141
|
+
usable while the result is read.
|
|
1142
|
+
* a mapping (e.g. ``{"time": 100}``): chunk explicitly. Each
|
|
1143
|
+
chunk reads its coordinate range lazily via filter pushdown on
|
|
1144
|
+
access, through xarray's configured chunk manager (dask,
|
|
1145
|
+
cubed, ...).
|
|
1146
|
+
* ``"auto"``: size chunks to the chunk manager's byte target but
|
|
1147
|
+
snap boundaries to whole source partitions, so each chunk is a
|
|
1148
|
+
union of source partitions. Useful for finely partitioned
|
|
1149
|
+
sources (e.g. ERA5 ``chunks={"time": 1}``), coarsening many
|
|
1150
|
+
tiny partitions into memory-sized, aligned chunks.
|
|
1151
|
+
|
|
1152
|
+
Returns:
|
|
1153
|
+
An ``xr.Dataset`` with ``dims`` as dimensions and the
|
|
1154
|
+
remaining result columns as data variables.
|
|
1155
|
+
|
|
1156
|
+
Raises:
|
|
1157
|
+
ValueError: ``dims`` cannot be inferred, names a missing
|
|
1158
|
+
column, or the result has duplicate dim tuples;
|
|
1159
|
+
``template`` names an unknown registered table; or
|
|
1160
|
+
``sparsity="template"`` is requested without a
|
|
1161
|
+
resolvable template.
|
|
1162
|
+
"""
|
|
1163
|
+
if not isinstance(template, xr.Dataset):
|
|
1164
|
+
# ``template`` is a registered-table name or None; look it up.
|
|
1165
|
+
template = self._resolve_template(template)
|
|
1166
|
+
if dims is None:
|
|
1167
|
+
dims = self._infer_dimension_columns(preferred_template=template)
|
|
1168
|
+
resolved_chunks = self._resolve_chunks(chunks, template, dims)
|
|
1169
|
+
return _result_to_xarray(
|
|
1170
|
+
inner_df=self._inner,
|
|
1171
|
+
dimension_columns=dims,
|
|
1172
|
+
template=template,
|
|
1173
|
+
sparsity=sparsity,
|
|
1174
|
+
fill_value=fill_value,
|
|
1175
|
+
chunks=resolved_chunks,
|
|
1176
|
+
templates=self._templates,
|
|
1177
|
+
)
|
|
1178
|
+
|
|
1179
|
+
# ------------------------------------------------------------------
|
|
1180
|
+
# Internals
|
|
1181
|
+
# ------------------------------------------------------------------
|
|
1182
|
+
|
|
1183
|
+
@staticmethod
|
|
1184
|
+
def _resolve_chunks(
|
|
1185
|
+
chunks: Mapping[str, int] | str | None,
|
|
1186
|
+
template: xr.Dataset | None,
|
|
1187
|
+
dimension_columns: list[str],
|
|
1188
|
+
) -> Mapping[str, int] | str | None:
|
|
1189
|
+
"""Resolve the ``chunks`` argument to a concrete spec or ``None``.
|
|
1190
|
+
|
|
1191
|
+
``None`` selects the lazy, unchunked path; anything else selects the
|
|
1192
|
+
chunked path. ``"inherit"`` reuses the source Dataset's chunk sizes --
|
|
1193
|
+
but only for dimensions actually split into more than one chunk in the
|
|
1194
|
+
input (a single full chunk is not "chunked"), so reductions that drop
|
|
1195
|
+
the chunked dimension resolve to ``None`` automatically. Mappings
|
|
1196
|
+
pass through unchanged; ``"auto"`` passes through here and is snapped to
|
|
1197
|
+
source partition boundaries later (see ``_auto_chunks``).
|
|
1198
|
+
"""
|
|
1199
|
+
if chunks is None:
|
|
1200
|
+
return None
|
|
1201
|
+
if chunks == "inherit":
|
|
1202
|
+
if template is None:
|
|
1203
|
+
return None
|
|
1204
|
+
sizes = template.chunksizes # dim -> tuple of chunk lengths
|
|
1205
|
+
inherited = {
|
|
1206
|
+
d: sizes[d][0]
|
|
1207
|
+
for d in dimension_columns
|
|
1208
|
+
if d in sizes and len(sizes[d]) > 1
|
|
1209
|
+
}
|
|
1210
|
+
return inherited or None
|
|
1211
|
+
return chunks
|
|
1212
|
+
|
|
1213
|
+
def _resolve_template(self, name: str | None) -> xr.Dataset | None:
|
|
1214
|
+
"""Pick a template Dataset for metadata recovery by registered name.
|
|
1215
|
+
|
|
1216
|
+
Priority:
|
|
1217
|
+
1. The named registered table (``name``).
|
|
1218
|
+
2. If exactly one Dataset is registered on the context, use it.
|
|
1219
|
+
3. None.
|
|
1220
|
+
"""
|
|
1221
|
+
templates = self._templates
|
|
1222
|
+
if name is not None:
|
|
1223
|
+
if name not in templates:
|
|
1224
|
+
raise ValueError(
|
|
1225
|
+
f"template={name!r} is not a registered table on this "
|
|
1226
|
+
f"context. Registered: {list(templates)}"
|
|
1227
|
+
)
|
|
1228
|
+
return templates[name]
|
|
1229
|
+
if len(templates) == 1:
|
|
1230
|
+
return next(iter(templates.values()))
|
|
1231
|
+
return None
|
|
1232
|
+
|
|
1233
|
+
def _infer_dimension_columns(
|
|
1234
|
+
self, preferred_template: xr.Dataset | None = None
|
|
1235
|
+
) -> list[str]:
|
|
1236
|
+
"""Pick a default ``dimension_columns`` from the registry, or raise.
|
|
1237
|
+
|
|
1238
|
+
A registered Dataset's dims that survive into the result columns
|
|
1239
|
+
become the dimensions, so aggregations that drop dims (e.g.
|
|
1240
|
+
``GROUP BY time`` over a ``(time, lat, lon)`` grid) round-trip on the
|
|
1241
|
+
surviving dim(s). Uses the data variable's dim order (via
|
|
1242
|
+
``_result_dims``) so the original axis order is preserved.
|
|
1243
|
+
"""
|
|
1244
|
+
result_cols = set(self._result_columns())
|
|
1245
|
+
|
|
1246
|
+
def surviving(template: xr.Dataset) -> list[str]:
|
|
1247
|
+
# Template dims still present in the result, in var axis order.
|
|
1248
|
+
return _result_dims(template, result_cols)
|
|
1249
|
+
|
|
1250
|
+
if preferred_template is not None:
|
|
1251
|
+
preferred = surviving(preferred_template)
|
|
1252
|
+
if preferred:
|
|
1253
|
+
return preferred
|
|
1254
|
+
if not self._templates:
|
|
1255
|
+
raise ValueError(
|
|
1256
|
+
"dims cannot be inferred (no registered "
|
|
1257
|
+
"Dataset on this result); pass dims=[...] "
|
|
1258
|
+
"explicitly."
|
|
1259
|
+
)
|
|
1260
|
+
candidates = {tuple(surviving(t)) for t in self._templates.values()}
|
|
1261
|
+
candidates.discard(()) # templates with no surviving dim
|
|
1262
|
+
if len(candidates) == 1:
|
|
1263
|
+
return list(next(iter(candidates)))
|
|
1264
|
+
if not candidates:
|
|
1265
|
+
raise ValueError(
|
|
1266
|
+
"dims cannot be inferred: no registered Dataset "
|
|
1267
|
+
"dimension survives in the result columns. Pass "
|
|
1268
|
+
"dims=[...] explicitly."
|
|
1269
|
+
)
|
|
1270
|
+
raise ValueError(
|
|
1271
|
+
"dims cannot be inferred unambiguously: multiple "
|
|
1272
|
+
"registered Datasets are compatible with the result. Pass "
|
|
1273
|
+
"dims=[...] explicitly."
|
|
1274
|
+
)
|
|
1275
|
+
|
|
1276
|
+
def _result_columns(self) -> list[str]:
|
|
1277
|
+
"""Return the result's column names without materializing rows."""
|
|
1278
|
+
return [field.name for field in self._inner.schema()]
|
|
1279
|
+
|
|
1280
|
+
def __getattr__(self, name: str) -> Any:
|
|
1281
|
+
# Runs only when ``name`` is not found via normal lookup, so this
|
|
1282
|
+
# safely forwards anything we have not overridden.
|
|
1283
|
+
return getattr(self._inner, name)
|
|
1284
|
+
|
|
1285
|
+
def __repr__(self) -> str:
|
|
1286
|
+
return repr(self._inner)
|