xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
|
@@ -0,0 +1,1239 @@
|
|
|
1
|
+
"""Engine-neutral pyarrow views of lazy xarray Datasets.
|
|
2
|
+
|
|
3
|
+
Two ways to hand a lazy ``xarray.Dataset`` to an Arrow-speaking query
|
|
4
|
+
engine:
|
|
5
|
+
|
|
6
|
+
* [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset] — a real ``pyarrow.dataset.Dataset``
|
|
7
|
+
subclass (the pattern Lance uses for ``LanceDataset``). Consumers of
|
|
8
|
+
the pyarrow dataset protocol — DuckDB via ``con.register``, Polars via
|
|
9
|
+
``pl.scan_pyarrow_dataset``, or pyarrow itself — call
|
|
10
|
+
[scanner][xarray_sql.backends.pyarrow.XarrayPushdownDataset.scanner] with the columns a query needs
|
|
11
|
+
and the predicate it pushed down, so the scan loads only the needed
|
|
12
|
+
data variables from only the chunks whose coordinate ranges can
|
|
13
|
+
satisfy the predicate. Construct one with [arrow_dataset][xarray_sql.backends.pyarrow.arrow_dataset].
|
|
14
|
+
* [XarrayArrowStream][xarray_sql.backends.pyarrow.XarrayArrowStream] — a re-scannable Arrow C-stream
|
|
15
|
+
(PyCapsule) view. No source-level pushdown, but works with any
|
|
16
|
+
PyCapsule consumer; the dependency-light fallback.
|
|
17
|
+
|
|
18
|
+
Correctness contract shared by all consumers of the pushdown dataset:
|
|
19
|
+
engines may delete the filter conjuncts they push down (DuckDB does),
|
|
20
|
+
so the returned scanner applies the expression exactly via
|
|
21
|
+
``pyarrow.dataset.Scanner``; chunk pruning is only ever an optimization
|
|
22
|
+
on top.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import itertools
|
|
28
|
+
import math
|
|
29
|
+
import re
|
|
30
|
+
import threading
|
|
31
|
+
import weakref
|
|
32
|
+
from collections import deque
|
|
33
|
+
from collections.abc import Callable, Iterator
|
|
34
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
35
|
+
from typing import Any
|
|
36
|
+
|
|
37
|
+
import numpy as np
|
|
38
|
+
import pyarrow as pa
|
|
39
|
+
import pyarrow.compute as pc
|
|
40
|
+
import pyarrow.dataset as pads
|
|
41
|
+
import pyarrow.fs as pafs
|
|
42
|
+
import xarray as xr
|
|
43
|
+
|
|
44
|
+
from ..df import (
|
|
45
|
+
_CAN_START_THREADS,
|
|
46
|
+
Block,
|
|
47
|
+
Chunks,
|
|
48
|
+
DEFAULT_BATCH_SIZE,
|
|
49
|
+
TableNames,
|
|
50
|
+
_ensure_default_indexes,
|
|
51
|
+
_parse_schema,
|
|
52
|
+
group_vars_by_dims,
|
|
53
|
+
iter_record_batches,
|
|
54
|
+
resolve_chunks,
|
|
55
|
+
resolve_table_names,
|
|
56
|
+
shared_coord_arrays,
|
|
57
|
+
)
|
|
58
|
+
from ..geometry import GEOMETRY_COLUMN, build_geometry, geometry_field
|
|
59
|
+
from ..reader import XarrayRecordBatchReader
|
|
60
|
+
|
|
61
|
+
DEFAULT_PREFETCH = 4 if _CAN_START_THREADS else 1
|
|
62
|
+
"""Chunk loads kept in flight ahead of the consumer during a scan.
|
|
63
|
+
|
|
64
|
+
Where threads are unavailable (Pyodide), chunks load one at a time."""
|
|
65
|
+
|
|
66
|
+
_SHADOW_FANOUT = 1024
|
|
67
|
+
"""Maximum fragments per shadow level.
|
|
68
|
+
|
|
69
|
+
A dimension with more chunks than this gets a two-level shadow: a coarse
|
|
70
|
+
level of at most this many buckets, refined per surviving bucket. This
|
|
71
|
+
bounds shadow construction cost for finely partitioned datasets (e.g.
|
|
72
|
+
hundreds of thousands of single-step time chunks) at registration and
|
|
73
|
+
query time alike.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
_REFINE_MAX_FRACTION = 0.25
|
|
77
|
+
"""Skip fine-level pruning when the coarse pass kept more buckets.
|
|
78
|
+
|
|
79
|
+
Refinement builds one sub-shadow per surviving bucket; when a predicate
|
|
80
|
+
matches most of the axis that cost cannot pay for itself, so the scan
|
|
81
|
+
falls back to the (sound) coarse answer.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
_STRICT_LEVEL_BUDGET = 4096
|
|
85
|
+
"""Maximum guarantee fragments per level of the strictness analysis.
|
|
86
|
+
|
|
87
|
+
Strictness classifies bucket-products of surviving chunks recursively:
|
|
88
|
+
whole buckets prove (or prune) at once, and only mixed cells refine.
|
|
89
|
+
Each level builds at most this many fragments, so grids of millions of
|
|
90
|
+
chunks resolve in a handful of vectorized passes.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
_STRICT_MAX_DEPTH = 6
|
|
94
|
+
"""Recursion bound for the strictness analysis (a backstop: realistic
|
|
95
|
+
grids terminate in 2-3 levels)."""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _guarantee_shadow(
|
|
99
|
+
guarantees: list[tuple[str, pc.Expression]], schema: pa.Schema
|
|
100
|
+
) -> pads.FileSystemDataset:
|
|
101
|
+
"""A path-only dataset whose fragments carry the given guarantees.
|
|
102
|
+
|
|
103
|
+
Each ``(path, guarantee)`` pair becomes one fragment: the path is a
|
|
104
|
+
label (never opened) and the guarantee is its
|
|
105
|
+
``partition_expression``, so ``get_fragments(filter=...)`` delegates
|
|
106
|
+
satisfiability to Arrow's guarantee simplification.
|
|
107
|
+
"""
|
|
108
|
+
fmt = pads.IpcFileFormat()
|
|
109
|
+
fs = pafs.LocalFileSystem()
|
|
110
|
+
fragments = [
|
|
111
|
+
fmt.make_fragment(path, fs, partition_expression=guarantee)
|
|
112
|
+
for path, guarantee in guarantees
|
|
113
|
+
]
|
|
114
|
+
return pads.FileSystemDataset(fragments, schema, fmt, fs)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class _DimShadow:
|
|
118
|
+
"""Chunk-pruning index for one dimension of the source grid.
|
|
119
|
+
|
|
120
|
+
Fragment ``i`` of a shadow ``FileSystemDataset`` carries the
|
|
121
|
+
guarantee ``dim ∈ [min, max]`` of chunk-span ``i`` as its
|
|
122
|
+
``partition_expression``; ``get_fragments(filter=...)`` then lets
|
|
123
|
+
Arrow's guarantee simplification decide which spans can satisfy a
|
|
124
|
+
predicate — sound for every predicate shape, conservative on columns
|
|
125
|
+
the guarantee does not mention, and the fragments' paths are never
|
|
126
|
+
opened.
|
|
127
|
+
|
|
128
|
+
Axes with more than ``_SHADOW_FANOUT`` chunks use two levels: a
|
|
129
|
+
coarse shadow over buckets of consecutive chunks, plus per-bucket
|
|
130
|
+
fine shadows built lazily for the buckets a query keeps.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
def __init__(
|
|
134
|
+
self,
|
|
135
|
+
name: str,
|
|
136
|
+
schema: pa.Schema,
|
|
137
|
+
coord: np.ndarray,
|
|
138
|
+
bounds: np.ndarray,
|
|
139
|
+
):
|
|
140
|
+
self._name = name
|
|
141
|
+
# The full table schema, not just this dimension's field: the
|
|
142
|
+
# pushed predicate may reference any column, and get_fragments
|
|
143
|
+
# must be able to bind all of them (guarantees stay per-dim;
|
|
144
|
+
# unmentioned columns are conservatively unconstrained).
|
|
145
|
+
self._schema = schema
|
|
146
|
+
self._field_type = schema.field(name).type
|
|
147
|
+
self._coord = coord
|
|
148
|
+
self._bounds = bounds
|
|
149
|
+
self._n = len(bounds) - 1
|
|
150
|
+
self._step = max(1, math.ceil(self._n / _SHADOW_FANOUT))
|
|
151
|
+
self._n_buckets = math.ceil(self._n / self._step)
|
|
152
|
+
self._coarse = self._build(
|
|
153
|
+
[
|
|
154
|
+
(b * self._step, min((b + 1) * self._step, self._n))
|
|
155
|
+
for b in range(self._n_buckets)
|
|
156
|
+
]
|
|
157
|
+
)
|
|
158
|
+
self._fine: dict[int, pads.FileSystemDataset] = {}
|
|
159
|
+
|
|
160
|
+
def _build(self, spans: list[tuple[int, int]]) -> pads.FileSystemDataset:
|
|
161
|
+
"""A shadow whose fragment ``i`` guarantees chunk-span ``spans[i]``."""
|
|
162
|
+
guarantees: list[tuple[str, pc.Expression]] = []
|
|
163
|
+
for i, (lo_chunk, hi_chunk) in enumerate(spans):
|
|
164
|
+
vals = self._coord[self._bounds[lo_chunk] : self._bounds[hi_chunk]]
|
|
165
|
+
if (vals.dtype.kind == "f" and np.isnan(vals).any()) or (
|
|
166
|
+
vals.dtype.kind == "M" and np.isnat(vals).any()
|
|
167
|
+
):
|
|
168
|
+
# NaN/NaT poisons min/max into a (dim >= NaN) guarantee
|
|
169
|
+
# that Arrow simplifies every predicate against as false,
|
|
170
|
+
# silently pruning rows. An always-true guarantee keeps
|
|
171
|
+
# the span unprunable instead.
|
|
172
|
+
guarantee = pc.scalar(True)
|
|
173
|
+
else:
|
|
174
|
+
# min/max (not first/last) so descending axes like
|
|
175
|
+
# latitude 90→-90 carry correct ranges.
|
|
176
|
+
lo = pa.scalar(vals.min(), type=self._field_type)
|
|
177
|
+
hi = pa.scalar(vals.max(), type=self._field_type)
|
|
178
|
+
guarantee = (pc.field(self._name) >= lo) & (
|
|
179
|
+
pc.field(self._name) <= hi
|
|
180
|
+
)
|
|
181
|
+
guarantees.append((str(i), guarantee))
|
|
182
|
+
return _guarantee_shadow(guarantees, self._schema)
|
|
183
|
+
|
|
184
|
+
@staticmethod
|
|
185
|
+
def _kept_indices(
|
|
186
|
+
shadow: pads.FileSystemDataset, filter: pc.Expression
|
|
187
|
+
) -> list[int]:
|
|
188
|
+
return sorted(
|
|
189
|
+
int(frag.path) for frag in shadow.get_fragments(filter=filter)
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def kept(self, filter: pc.Expression) -> list[int] | None:
|
|
193
|
+
"""Chunk indices that can satisfy ``filter``; ``None`` means all."""
|
|
194
|
+
try:
|
|
195
|
+
buckets = self._kept_indices(self._coarse, filter)
|
|
196
|
+
if self._step == 1:
|
|
197
|
+
return buckets if len(buckets) < self._n else None
|
|
198
|
+
if len(buckets) > _REFINE_MAX_FRACTION * self._n_buckets:
|
|
199
|
+
# Refining most of the axis costs more than it saves;
|
|
200
|
+
# answer with the coarse buckets, which is still sound.
|
|
201
|
+
return (
|
|
202
|
+
None
|
|
203
|
+
if len(buckets) == self._n_buckets
|
|
204
|
+
else [
|
|
205
|
+
i
|
|
206
|
+
for b in buckets
|
|
207
|
+
for i in range(
|
|
208
|
+
b * self._step,
|
|
209
|
+
min((b + 1) * self._step, self._n),
|
|
210
|
+
)
|
|
211
|
+
]
|
|
212
|
+
)
|
|
213
|
+
kept: list[int] = []
|
|
214
|
+
for b in buckets:
|
|
215
|
+
fine = self._fine.get(b)
|
|
216
|
+
if fine is None:
|
|
217
|
+
start = b * self._step
|
|
218
|
+
stop = min((b + 1) * self._step, self._n)
|
|
219
|
+
fine = self._build([(i, i + 1) for i in range(start, stop)])
|
|
220
|
+
self._fine[b] = fine
|
|
221
|
+
start = b * self._step
|
|
222
|
+
kept.extend(start + i for i in self._kept_indices(fine, filter))
|
|
223
|
+
return kept
|
|
224
|
+
except (pa.ArrowInvalid, pa.ArrowNotImplementedError, TypeError):
|
|
225
|
+
return None # conservative: scan every chunk of this dim
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
class XarrayArrowStream:
|
|
229
|
+
"""A re-scannable Arrow C-stream view over a lazy xarray Dataset.
|
|
230
|
+
|
|
231
|
+
Arrow PyCapsule consumers (DuckDB among them) call
|
|
232
|
+
``__arrow_c_stream__`` once per scan. Each call constructs a fresh
|
|
233
|
+
[XarrayRecordBatchReader][xarray_sql.reader.XarrayRecordBatchReader] over the same
|
|
234
|
+
lazy Dataset, so — unlike registering a ``pyarrow.RecordBatchReader``
|
|
235
|
+
directly, which is exhausted after one query — the same registered
|
|
236
|
+
table supports any number of queries, and data is only read while a
|
|
237
|
+
query is executing.
|
|
238
|
+
|
|
239
|
+
The PyCapsule scan path gets no source-level pushdown (the producer
|
|
240
|
+
never sees the query's columns or filters), so
|
|
241
|
+
[XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset] is the default registration object;
|
|
242
|
+
this class remains as the dependency-light fallback.
|
|
243
|
+
"""
|
|
244
|
+
|
|
245
|
+
def __init__(
|
|
246
|
+
self,
|
|
247
|
+
ds: xr.Dataset,
|
|
248
|
+
chunks: Chunks = None,
|
|
249
|
+
*,
|
|
250
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
251
|
+
_iteration_callback: (
|
|
252
|
+
Callable[[Block, list[str] | None], None] | None
|
|
253
|
+
) = None,
|
|
254
|
+
):
|
|
255
|
+
# Validate eagerly (same checks XarrayRecordBatchReader runs) so
|
|
256
|
+
# registration fails fast instead of erroring mid-query.
|
|
257
|
+
probe = XarrayRecordBatchReader(ds, chunks, batch_size=batch_size)
|
|
258
|
+
self._ds = ds
|
|
259
|
+
self._chunks = chunks
|
|
260
|
+
self._batch_size = batch_size
|
|
261
|
+
self._schema = probe.schema
|
|
262
|
+
self._iteration_callback = _iteration_callback
|
|
263
|
+
|
|
264
|
+
def __arrow_c_stream__(
|
|
265
|
+
self, requested_schema: object | None = None
|
|
266
|
+
) -> object:
|
|
267
|
+
reader = XarrayRecordBatchReader(
|
|
268
|
+
self._ds,
|
|
269
|
+
self._chunks,
|
|
270
|
+
batch_size=self._batch_size,
|
|
271
|
+
_iteration_callback=self._iteration_callback,
|
|
272
|
+
)
|
|
273
|
+
return reader.__arrow_c_stream__(requested_schema)
|
|
274
|
+
|
|
275
|
+
def __arrow_c_schema__(self) -> object:
|
|
276
|
+
return self._schema.__arrow_c_schema__()
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class XarrayPushdownDataset(pads.Dataset):
|
|
280
|
+
"""A pushdown-capable ``pyarrow.dataset.Dataset`` view of a Dataset.
|
|
281
|
+
|
|
282
|
+
Consumers that speak the pyarrow dataset protocol (DuckDB, Polars,
|
|
283
|
+
...) call [scanner][xarray_sql.backends.pyarrow.XarrayPushdownDataset.scanner] with the columns a query needs and the
|
|
284
|
+
predicate it pushed down; the scan then loads only the needed data
|
|
285
|
+
variables from only the chunks whose coordinate ranges can satisfy
|
|
286
|
+
the predicate.
|
|
287
|
+
|
|
288
|
+
The base class is never initialized (there is no C++ dataset behind
|
|
289
|
+
this object — the same construction Lance uses for ``LanceDataset``);
|
|
290
|
+
every entry point consumers touch is overridden in Python, and the
|
|
291
|
+
few inherited members that would read uninitialized native state are
|
|
292
|
+
stubbed out.
|
|
293
|
+
|
|
294
|
+
References:
|
|
295
|
+
Lance's ``LanceDataset``, a ``pyarrow.dataset.Dataset`` subclass
|
|
296
|
+
built the same way: https://github.com/lancedb/lance
|
|
297
|
+
(``python/python/lance/dataset.py``).
|
|
298
|
+
"""
|
|
299
|
+
|
|
300
|
+
def __init__(
|
|
301
|
+
self,
|
|
302
|
+
ds: xr.Dataset,
|
|
303
|
+
chunks: Chunks = None,
|
|
304
|
+
*,
|
|
305
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
306
|
+
prefetch: int = DEFAULT_PREFETCH,
|
|
307
|
+
prefetch_bytes: int | None = None,
|
|
308
|
+
coalesce_rows: int | None = None,
|
|
309
|
+
geometry: tuple[str, str] | None = None,
|
|
310
|
+
geometry_encoding: str = "wkb",
|
|
311
|
+
geometry_crs: str | None = "OGC:CRS84",
|
|
312
|
+
coord_arrays: dict[str, np.ndarray] | None = None,
|
|
313
|
+
_iteration_callback: (
|
|
314
|
+
Callable[[Block, list[str] | None], None] | None
|
|
315
|
+
) = None,
|
|
316
|
+
):
|
|
317
|
+
# Deliberately no super().__init__() — see class docstring.
|
|
318
|
+
ds = _ensure_default_indexes(ds)
|
|
319
|
+
if ds.data_vars:
|
|
320
|
+
fst = next(iter(ds.values())).dims
|
|
321
|
+
if not all(da.dims == fst for da in ds.values()):
|
|
322
|
+
raise ValueError(
|
|
323
|
+
"All dimensions must be equal. "
|
|
324
|
+
"Please filter data_vars in the Dataset."
|
|
325
|
+
)
|
|
326
|
+
self._ds = ds
|
|
327
|
+
self._schema = _parse_schema(ds)
|
|
328
|
+
self._geometry = tuple(geometry) if geometry else None
|
|
329
|
+
self._geometry_encoding = geometry_encoding
|
|
330
|
+
if self._geometry:
|
|
331
|
+
if GEOMETRY_COLUMN in self._schema.names:
|
|
332
|
+
raise ValueError(
|
|
333
|
+
f"geometry= would shadow an existing column named "
|
|
334
|
+
f"{GEOMETRY_COLUMN!r}."
|
|
335
|
+
)
|
|
336
|
+
missing = [d for d in self._geometry if d not in self._schema.names]
|
|
337
|
+
if missing:
|
|
338
|
+
raise ValueError(
|
|
339
|
+
f"geometry dims {missing} are not columns of the "
|
|
340
|
+
f"table; available: {self._schema.names}"
|
|
341
|
+
)
|
|
342
|
+
self._schema = self._schema.append(
|
|
343
|
+
geometry_field(geometry_encoding, geometry_crs)
|
|
344
|
+
)
|
|
345
|
+
self._resolved = resolve_chunks(ds, chunks)
|
|
346
|
+
self._chunk_bounds = {
|
|
347
|
+
d: np.cumsum((0, *sizes)) for d, sizes in self._resolved.items()
|
|
348
|
+
}
|
|
349
|
+
# Reuse pre-materialised coordinate arrays where the caller has
|
|
350
|
+
# them (e.g. shared across the tables of a dim-group split); each
|
|
351
|
+
# missing dim costs one read, a network round-trip for Zarr.
|
|
352
|
+
self._coord_arrays = dict(coord_arrays or {})
|
|
353
|
+
for d in ds.dims:
|
|
354
|
+
if str(d) not in self._coord_arrays:
|
|
355
|
+
self._coord_arrays[str(d)] = ds.coords[d].values
|
|
356
|
+
if batch_size <= 0:
|
|
357
|
+
# A zero size would never advance the zero-column scan's
|
|
358
|
+
# row loop; fail here rather than mid-scan.
|
|
359
|
+
raise ValueError(f"batch_size must be positive, got {batch_size}")
|
|
360
|
+
self._batch_size = batch_size
|
|
361
|
+
self._prefetch = prefetch
|
|
362
|
+
self._prefetch_bytes = prefetch_bytes
|
|
363
|
+
self._coalesce_rows = coalesce_rows
|
|
364
|
+
self._iteration_callback = _iteration_callback
|
|
365
|
+
self._shadows: dict[str, _DimShadow] | None = None
|
|
366
|
+
self._span_cache: dict[
|
|
367
|
+
str, tuple[np.ndarray, np.ndarray, np.ndarray]
|
|
368
|
+
] = {}
|
|
369
|
+
# One long-lived pool shared by every scan, its threads spawned
|
|
370
|
+
# NOW — never from inside an engine's scan callback. Creating a
|
|
371
|
+
# pool (and its OS threads) per scan deadlocks when the scan is
|
|
372
|
+
# driven from an engine executing under another thread pool
|
|
373
|
+
# (dask computing chunks of a lazy round-trip): thread startup
|
|
374
|
+
# and concurrent.futures' global shutdown lock interleave with
|
|
375
|
+
# the engine's callback needing the GIL. Scans that stop early
|
|
376
|
+
# cancel their queued loads instead of tearing the pool down.
|
|
377
|
+
# Like any thread state, the pool does not survive fork(); use
|
|
378
|
+
# from forked workers (e.g. PyTorch DataLoader) is tracked in
|
|
379
|
+
# https://github.com/xqlsystems/xarray-sql/issues/145.
|
|
380
|
+
self._pool: ThreadPoolExecutor | None = None
|
|
381
|
+
if self._prefetch > 1:
|
|
382
|
+
self._pool = ThreadPoolExecutor(max_workers=self._prefetch)
|
|
383
|
+
# Each pre-spawn task parks on the barrier, so no thread can
|
|
384
|
+
# take a second task and the executor is forced to start all
|
|
385
|
+
# ``prefetch`` OS threads before __init__ returns (submitting
|
|
386
|
+
# plain no-ops lets one idle thread absorb several of them,
|
|
387
|
+
# leaving the rest to spawn later inside an engine's scan
|
|
388
|
+
# callback — the deadlock this pre-spawn exists to prevent).
|
|
389
|
+
barrier = threading.Barrier(self._prefetch + 1)
|
|
390
|
+
spawn = [
|
|
391
|
+
self._pool.submit(barrier.wait) for _ in range(self._prefetch)
|
|
392
|
+
]
|
|
393
|
+
barrier.wait()
|
|
394
|
+
for f in spawn:
|
|
395
|
+
f.result()
|
|
396
|
+
# Stop the pool's threads when the dataset dies; live scans
|
|
397
|
+
# keep the dataset alive through their generator closures,
|
|
398
|
+
# so nothing in flight is cut short. The callback is bound
|
|
399
|
+
# to the executor, not the dataset, so the finalizer holds
|
|
400
|
+
# no reference cycle back to self.
|
|
401
|
+
weakref.finalize(
|
|
402
|
+
self, self._pool.shutdown, wait=False, cancel_futures=True
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
# ------------------------------------------------------------------
|
|
406
|
+
# The consumer-facing surface
|
|
407
|
+
# ------------------------------------------------------------------
|
|
408
|
+
|
|
409
|
+
@property
|
|
410
|
+
def schema(self) -> pa.Schema:
|
|
411
|
+
return self._schema
|
|
412
|
+
|
|
413
|
+
def scanner(
|
|
414
|
+
self,
|
|
415
|
+
columns: list[str] | None = None,
|
|
416
|
+
filter: pc.Expression | None = None,
|
|
417
|
+
batch_size: int | None = None,
|
|
418
|
+
**kwargs: Any,
|
|
419
|
+
) -> pads.Scanner:
|
|
420
|
+
"""Build a scanner for the requested columns and predicate.
|
|
421
|
+
|
|
422
|
+
``filter`` is applied exactly by the returned scanner (DuckDB
|
|
423
|
+
deletes the conjuncts it pushes down and trusts the source to
|
|
424
|
+
enforce them); chunk pruning and column selection only reduce
|
|
425
|
+
how much data is read to get there. ``batch_size`` caps rows per
|
|
426
|
+
emitted batch (Polars passes it through ``to_batches``). Extra
|
|
427
|
+
keyword arguments from other pyarrow-dataset consumers are
|
|
428
|
+
accepted and ignored.
|
|
429
|
+
"""
|
|
430
|
+
kept = None if filter is None else self._prune(filter)
|
|
431
|
+
blocks = (
|
|
432
|
+
self._coalesced_blocks(kept)
|
|
433
|
+
if self._coalesce_rows
|
|
434
|
+
else self._blocks(kept)
|
|
435
|
+
)
|
|
436
|
+
return self._scanner_for_blocks(blocks, columns, filter, batch_size)
|
|
437
|
+
|
|
438
|
+
def _scanner_for_blocks(
|
|
439
|
+
self,
|
|
440
|
+
blocks: Iterator[Block] | list[Block],
|
|
441
|
+
columns: list[str] | None,
|
|
442
|
+
filter: pc.Expression | None,
|
|
443
|
+
batch_size: int | None = None,
|
|
444
|
+
) -> pads.Scanner:
|
|
445
|
+
"""A scanner over the given blocks; shared by dataset and fragments."""
|
|
446
|
+
# ``None`` means every column (the pyarrow convention); an
|
|
447
|
+
# explicitly empty list is a real projection ("no payload"), not
|
|
448
|
+
# a request for the full schema.
|
|
449
|
+
proj = list(self._schema.names) if columns is None else list(columns)
|
|
450
|
+
scan_names = self._scan_columns(proj, filter)
|
|
451
|
+
scan_schema = pa.schema([self._schema.field(n) for n in scan_names])
|
|
452
|
+
size = batch_size or self._batch_size
|
|
453
|
+
if self._geometry and GEOMETRY_COLUMN in scan_names:
|
|
454
|
+
batches = self._batches_with_geometry(scan_schema, blocks, size)
|
|
455
|
+
else:
|
|
456
|
+
batches = self._batch_generator(scan_schema, blocks, size)
|
|
457
|
+
return pads.Scanner.from_batches(
|
|
458
|
+
batches, schema=scan_schema, columns=proj, filter=filter
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
def _batches_with_geometry(
|
|
462
|
+
self,
|
|
463
|
+
scan_schema: pa.Schema,
|
|
464
|
+
blocks: Iterator[Block] | list[Block],
|
|
465
|
+
batch_size: int,
|
|
466
|
+
) -> Iterator[pa.RecordBatch]:
|
|
467
|
+
"""Emit ``scan_schema`` batches, synthesizing the geometry column.
|
|
468
|
+
|
|
469
|
+
The pivot never materializes geometry: batches are produced with
|
|
470
|
+
the coordinate dims the geometry derives from, and the geometry
|
|
471
|
+
column is built per batch from those columns (for the native
|
|
472
|
+
encoding the point struct's children *are* the coordinate
|
|
473
|
+
arrays — a schema annotation, not a copy).
|
|
474
|
+
"""
|
|
475
|
+
assert self._geometry is not None
|
|
476
|
+
x_dim, y_dim = self._geometry
|
|
477
|
+
base_names = [n for n in scan_schema.names if n != GEOMETRY_COLUMN]
|
|
478
|
+
for d in (x_dim, y_dim):
|
|
479
|
+
if d not in base_names:
|
|
480
|
+
base_names.append(d)
|
|
481
|
+
base_schema = pa.schema([self._schema.field(n) for n in base_names])
|
|
482
|
+
for batch in self._batch_generator(base_schema, blocks, batch_size):
|
|
483
|
+
geom = build_geometry(
|
|
484
|
+
self._geometry_encoding,
|
|
485
|
+
batch.column(base_names.index(x_dim)),
|
|
486
|
+
batch.column(base_names.index(y_dim)),
|
|
487
|
+
)
|
|
488
|
+
arrays = [
|
|
489
|
+
geom
|
|
490
|
+
if name == GEOMETRY_COLUMN
|
|
491
|
+
else batch.column(base_names.index(name))
|
|
492
|
+
for name in scan_schema.names
|
|
493
|
+
]
|
|
494
|
+
yield pa.RecordBatch.from_arrays(arrays, schema=scan_schema)
|
|
495
|
+
|
|
496
|
+
def _batch_generator(
|
|
497
|
+
self,
|
|
498
|
+
scan_schema: pa.Schema,
|
|
499
|
+
blocks: Iterator[Block] | list[Block],
|
|
500
|
+
batch_size: int,
|
|
501
|
+
) -> Iterator[pa.RecordBatch]:
|
|
502
|
+
names = list(scan_schema.names)
|
|
503
|
+
data_vars = [n for n in names if n in self._ds.data_vars]
|
|
504
|
+
# Select only the needed variables before slicing so unrequested
|
|
505
|
+
# variables are never loaded (dimension coords come via coords).
|
|
506
|
+
base = (
|
|
507
|
+
self._ds[data_vars]
|
|
508
|
+
if data_vars
|
|
509
|
+
else self._ds.drop_vars(list(self._ds.data_vars))
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
def load(block: Block) -> list[pa.RecordBatch]:
|
|
513
|
+
if self._iteration_callback is not None:
|
|
514
|
+
self._iteration_callback(block, names)
|
|
515
|
+
if not names:
|
|
516
|
+
# Zero-column projection: row counts are chunk
|
|
517
|
+
# arithmetic; no coordinate or variable data is read.
|
|
518
|
+
out = []
|
|
519
|
+
rows = self._block_rows(block)
|
|
520
|
+
while rows > 0:
|
|
521
|
+
n = min(rows, batch_size)
|
|
522
|
+
out.append(
|
|
523
|
+
pa.table({"_": np.empty(n, np.int8)})
|
|
524
|
+
.select([])
|
|
525
|
+
.to_batches()[0]
|
|
526
|
+
)
|
|
527
|
+
rows -= n
|
|
528
|
+
return out
|
|
529
|
+
return list(
|
|
530
|
+
iter_record_batches(base.isel(block), scan_schema, batch_size)
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
# Estimated pivoted bytes per row: gates admission when a
|
|
534
|
+
# byte budget is set, so peak memory tracks bytes in flight
|
|
535
|
+
# rather than block count (blocks vary in size under
|
|
536
|
+
# coalesce_rows).
|
|
537
|
+
row_width = 0
|
|
538
|
+
for field in scan_schema:
|
|
539
|
+
try:
|
|
540
|
+
row_width += np.dtype(field.type.to_pandas_dtype()).itemsize
|
|
541
|
+
except (TypeError, NotImplementedError):
|
|
542
|
+
row_width += 8
|
|
543
|
+
|
|
544
|
+
def generate() -> Iterator[pa.RecordBatch]:
|
|
545
|
+
block_iter = iter(blocks)
|
|
546
|
+
first = next(block_iter, None)
|
|
547
|
+
if first is None:
|
|
548
|
+
return
|
|
549
|
+
second = next(block_iter, None)
|
|
550
|
+
if self._pool is None or second is None:
|
|
551
|
+
# Single-block scans (a lazy round-trip window that maps
|
|
552
|
+
# onto one source chunk) skip the pool entirely.
|
|
553
|
+
yield from load(first)
|
|
554
|
+
if second is not None:
|
|
555
|
+
yield from load(second)
|
|
556
|
+
for block in block_iter:
|
|
557
|
+
yield from load(block)
|
|
558
|
+
return
|
|
559
|
+
pool = self._pool
|
|
560
|
+
budget = self._prefetch_bytes
|
|
561
|
+
pending: deque = deque()
|
|
562
|
+
inflight = 0
|
|
563
|
+
|
|
564
|
+
def submit(block: Block) -> None:
|
|
565
|
+
nonlocal inflight
|
|
566
|
+
estimate = self._block_rows(block) * row_width
|
|
567
|
+
pending.append((pool.submit(load, block), estimate))
|
|
568
|
+
inflight += estimate
|
|
569
|
+
|
|
570
|
+
def drain_one() -> Iterator[pa.RecordBatch]:
|
|
571
|
+
nonlocal inflight
|
|
572
|
+
future, estimate = pending.popleft()
|
|
573
|
+
inflight -= estimate
|
|
574
|
+
yield from future.result()
|
|
575
|
+
|
|
576
|
+
try:
|
|
577
|
+
submit(first)
|
|
578
|
+
submit(second)
|
|
579
|
+
for block in block_iter:
|
|
580
|
+
submit(block)
|
|
581
|
+
while len(pending) > 1 and (
|
|
582
|
+
len(pending) >= self._prefetch
|
|
583
|
+
or (budget is not None and inflight > budget)
|
|
584
|
+
):
|
|
585
|
+
yield from drain_one()
|
|
586
|
+
while pending:
|
|
587
|
+
yield from drain_one()
|
|
588
|
+
finally:
|
|
589
|
+
# Consumer may stop early (e.g. LIMIT): drop queued work
|
|
590
|
+
# without waiting for in-flight loads. The pool itself is
|
|
591
|
+
# shared across scans and stays up.
|
|
592
|
+
for future, _ in pending:
|
|
593
|
+
future.cancel()
|
|
594
|
+
|
|
595
|
+
return generate()
|
|
596
|
+
|
|
597
|
+
def get_fragments(
|
|
598
|
+
self, filter: pc.Expression | None = None
|
|
599
|
+
) -> list["_XarrayFragment"]:
|
|
600
|
+
"""One fragment per chunk of the source grid, pruned by ``filter``.
|
|
601
|
+
|
|
602
|
+
This is how DataFusion consumes the dataset
|
|
603
|
+
(``SessionContext.register_dataset`` plans one partition per
|
|
604
|
+
fragment and scans them in parallel), and enables the Dask
|
|
605
|
+
pattern ``from_map(lambda f: f.to_table().to_pandas(),
|
|
606
|
+
ds.get_fragments())``.
|
|
607
|
+
"""
|
|
608
|
+
kept = None if filter is None else self._prune(filter)
|
|
609
|
+
return [_XarrayFragment(self, block) for block in self._blocks(kept)]
|
|
610
|
+
|
|
611
|
+
def count_rows(
|
|
612
|
+
self, filter: pc.Expression | None = None, **kwargs: Any
|
|
613
|
+
) -> int:
|
|
614
|
+
"""Count rows, reading as little data as possible.
|
|
615
|
+
|
|
616
|
+
Without a filter the count is pure chunk arithmetic — no I/O at
|
|
617
|
+
all. With a filter, chunks are split three ways: pruned chunks
|
|
618
|
+
contribute nothing, chunks whose coordinate ranges *prove* the
|
|
619
|
+
filter true contribute their exact size arithmetically, and only
|
|
620
|
+
the undecided boundary chunks are scanned (reading just the
|
|
621
|
+
columns the filter references).
|
|
622
|
+
"""
|
|
623
|
+
if not self._ds.sizes:
|
|
624
|
+
return int(self.scanner(columns=[], filter=filter).count_rows())
|
|
625
|
+
if filter is None:
|
|
626
|
+
return int(np.prod([self._ds.sizes[d] for d in self._ds.dims]))
|
|
627
|
+
kept = self._prune(filter)
|
|
628
|
+
proven, boundary = self._strict_partition(kept, filter)
|
|
629
|
+
return proven + int(
|
|
630
|
+
self._scanner_for_blocks(boundary, [], filter).count_rows()
|
|
631
|
+
)
|
|
632
|
+
|
|
633
|
+
# Inherited convenience methods (to_table, head, to_batches, take)
|
|
634
|
+
# route through scanner() and keep working; the members below would
|
|
635
|
+
# touch the uninitialized native dataset.
|
|
636
|
+
|
|
637
|
+
@property
|
|
638
|
+
def partition_expression(self) -> pc.Expression:
|
|
639
|
+
# The dataset-level guarantee: trivially true. The base class
|
|
640
|
+
# getter reads native state this object does not have.
|
|
641
|
+
return pc.scalar(True)
|
|
642
|
+
|
|
643
|
+
def filter(self, expression: pc.Expression):
|
|
644
|
+
# A lazily-composed filter view is implementable; tracked in
|
|
645
|
+
# https://github.com/xqlsystems/xarray-sql/issues/239.
|
|
646
|
+
raise NotImplementedError(
|
|
647
|
+
"Use scanner(filter=...) or the engine's WHERE clause."
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
def replace_schema(self, schema: pa.Schema):
|
|
651
|
+
raise NotImplementedError
|
|
652
|
+
|
|
653
|
+
# Guarded delegation for sort_by/join/join_asof (engine-level SQL
|
|
654
|
+
# joins already work through scanner()) is tracked in
|
|
655
|
+
# https://github.com/xqlsystems/xarray-sql/issues/240.
|
|
656
|
+
|
|
657
|
+
def sort_by(self, sorting, **kwargs):
|
|
658
|
+
raise NotImplementedError
|
|
659
|
+
|
|
660
|
+
def join(self, *args, **kwargs):
|
|
661
|
+
raise NotImplementedError
|
|
662
|
+
|
|
663
|
+
def join_asof(self, *args, **kwargs):
|
|
664
|
+
raise NotImplementedError
|
|
665
|
+
|
|
666
|
+
def __reduce__(self):
|
|
667
|
+
raise TypeError("XarrayPushdownDataset is not picklable.")
|
|
668
|
+
|
|
669
|
+
# ------------------------------------------------------------------
|
|
670
|
+
# Projection: which columns must be read
|
|
671
|
+
# ------------------------------------------------------------------
|
|
672
|
+
|
|
673
|
+
def _scan_columns(
|
|
674
|
+
self, proj: list[str], filter: pc.Expression | None
|
|
675
|
+
) -> list[str]:
|
|
676
|
+
"""Columns to read: the projection plus any the filter references.
|
|
677
|
+
|
|
678
|
+
The consumer's column list need not include filter-only columns
|
|
679
|
+
(DuckDB drops pushed conjuncts from its plan and has no upstream
|
|
680
|
+
use for them). Rather than parsing the expression, probe it
|
|
681
|
+
against an empty table and grow the column set from the "no match
|
|
682
|
+
for field" errors until it evaluates; on anything unexpected fall
|
|
683
|
+
back to scanning every column, which is always correct.
|
|
684
|
+
"""
|
|
685
|
+
if filter is None:
|
|
686
|
+
return proj
|
|
687
|
+
wanted = set(proj)
|
|
688
|
+
for _ in range(len(self._schema.names) + 1):
|
|
689
|
+
probe = pa.table(
|
|
690
|
+
{
|
|
691
|
+
n: pa.array([], type=self._schema.field(n).type)
|
|
692
|
+
for n in self._schema.names
|
|
693
|
+
if n in wanted
|
|
694
|
+
}
|
|
695
|
+
)
|
|
696
|
+
try:
|
|
697
|
+
probe.filter(filter)
|
|
698
|
+
except pa.lib.ArrowInvalid as exc:
|
|
699
|
+
match = re.search(r"FieldRef\.Name\((.*?)\)", str(exc))
|
|
700
|
+
name = match.group(1) if match else None
|
|
701
|
+
if (
|
|
702
|
+
name is not None
|
|
703
|
+
and name in set(self._schema.names) - wanted
|
|
704
|
+
):
|
|
705
|
+
wanted.add(name)
|
|
706
|
+
continue
|
|
707
|
+
return list(self._schema.names)
|
|
708
|
+
else:
|
|
709
|
+
return [n for n in self._schema.names if n in wanted]
|
|
710
|
+
return list(self._schema.names)
|
|
711
|
+
|
|
712
|
+
# ------------------------------------------------------------------
|
|
713
|
+
# Pruning: which chunks can satisfy the predicate
|
|
714
|
+
# ------------------------------------------------------------------
|
|
715
|
+
|
|
716
|
+
def _dim_shadows(self) -> dict[str, _DimShadow]:
|
|
717
|
+
"""One pruning index per prunable dimension, built lazily.
|
|
718
|
+
|
|
719
|
+
Keeping one shadow per dimension (Σ n_d fragments) instead of one
|
|
720
|
+
per chunk (Π n_d) is what keeps this cheap for finely partitioned
|
|
721
|
+
datasets, and is sound: a chunk is dropped only when the full
|
|
722
|
+
predicate is provably false given that single dimension's range.
|
|
723
|
+
"""
|
|
724
|
+
if self._shadows is not None:
|
|
725
|
+
return self._shadows
|
|
726
|
+
shadows: dict[str, _DimShadow] = {}
|
|
727
|
+
for dim in self._resolved:
|
|
728
|
+
name = str(dim)
|
|
729
|
+
if name not in self._schema.names:
|
|
730
|
+
continue
|
|
731
|
+
coord = self._coord_arrays[name]
|
|
732
|
+
if coord.dtype.kind not in ("i", "u", "f", "M"):
|
|
733
|
+
continue # strings/objects/cftime: never prune this dim
|
|
734
|
+
try:
|
|
735
|
+
shadows[name] = _DimShadow(
|
|
736
|
+
name, self._schema, coord, self._chunk_bounds[dim]
|
|
737
|
+
)
|
|
738
|
+
except (pa.ArrowInvalid, pa.ArrowNotImplementedError, TypeError):
|
|
739
|
+
continue # conservative: no pruning on this dim
|
|
740
|
+
self._shadows = shadows
|
|
741
|
+
return shadows
|
|
742
|
+
|
|
743
|
+
def _prune(self, filter: pc.Expression) -> dict[str, list[int]]:
|
|
744
|
+
"""Per-dimension chunk indices that can satisfy ``filter``.
|
|
745
|
+
|
|
746
|
+
Satisfiability is delegated to Arrow's guarantee simplification
|
|
747
|
+
(see ``_DimShadow``) — no expression decoding here, and
|
|
748
|
+
predicates on columns a shadow knows nothing about are
|
|
749
|
+
conservatively kept. Dimensions without a shadow, or where every
|
|
750
|
+
chunk survives, are absent from the result.
|
|
751
|
+
"""
|
|
752
|
+
kept: dict[str, list[int]] = {}
|
|
753
|
+
for name, shadow in self._dim_shadows().items():
|
|
754
|
+
indices = shadow.kept(filter)
|
|
755
|
+
if indices is not None:
|
|
756
|
+
kept[name] = indices
|
|
757
|
+
return kept
|
|
758
|
+
|
|
759
|
+
# ------------------------------------------------------------------
|
|
760
|
+
# Strictness: which surviving chunks satisfy the filter entirely
|
|
761
|
+
# ------------------------------------------------------------------
|
|
762
|
+
|
|
763
|
+
def _chunk_spans(
|
|
764
|
+
self, name: str
|
|
765
|
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
766
|
+
"""Cached vectorized per-chunk ``(lo, hi, poisoned)`` for a dim."""
|
|
767
|
+
cached = self._span_cache.get(name)
|
|
768
|
+
if cached is not None:
|
|
769
|
+
return cached
|
|
770
|
+
coord = self._coord_arrays[name]
|
|
771
|
+
starts = self._chunk_bounds[name][:-1]
|
|
772
|
+
lo = np.minimum.reduceat(coord, starts)
|
|
773
|
+
hi = np.maximum.reduceat(coord, starts)
|
|
774
|
+
if coord.dtype.kind == "f":
|
|
775
|
+
bad_values = np.isnan(coord)
|
|
776
|
+
elif coord.dtype.kind == "M":
|
|
777
|
+
bad_values = np.isnat(coord)
|
|
778
|
+
else:
|
|
779
|
+
bad_values = np.zeros(len(coord), dtype=bool)
|
|
780
|
+
bad = np.bitwise_or.reduceat(bad_values, starts)
|
|
781
|
+
self._span_cache[name] = (lo, hi, bad)
|
|
782
|
+
return self._span_cache[name]
|
|
783
|
+
|
|
784
|
+
def _strict_partition(
|
|
785
|
+
self,
|
|
786
|
+
kept: dict[str, list[int]] | None,
|
|
787
|
+
filter: pc.Expression,
|
|
788
|
+
) -> tuple[int, list[Block] | Iterator[Block]]:
|
|
789
|
+
"""``(rows proven to satisfy filter, boundary blocks to scan)``.
|
|
790
|
+
|
|
791
|
+
A cell of the surviving chunk grid with conjunctive coordinate
|
|
792
|
+
guarantee ``G`` satisfies ``filter`` everywhere iff
|
|
793
|
+
``G ∧ ¬filter`` is unsatisfiable, and can be *dropped* entirely
|
|
794
|
+
iff ``G ∧ filter`` is — both decided by Arrow's guarantee
|
|
795
|
+
simplification. Cells are classified hierarchically: each level
|
|
796
|
+
buckets the surviving indices into at most
|
|
797
|
+
``_STRICT_LEVEL_BUDGET`` products, proves/prunes whole buckets
|
|
798
|
+
at once (the prune side refines the per-dimension pruning with
|
|
799
|
+
cross-dimension information), and recurses only into mixed
|
|
800
|
+
cells — so million-chunk axes resolve in two or three levels.
|
|
801
|
+
Anything undecidable (NaN spans, non-numeric dims, expression
|
|
802
|
+
shapes the simplifier rejects) conservatively lands in the
|
|
803
|
+
boundary set, which the caller scans exactly.
|
|
804
|
+
"""
|
|
805
|
+
dims = list(self._resolved.keys())
|
|
806
|
+
lists = {d: list(self._surviving(kept, d)) for d in dims}
|
|
807
|
+
if not dims or any(not v for v in lists.values()):
|
|
808
|
+
return 0, []
|
|
809
|
+
lens = {d: np.diff(self._chunk_bounds[d]) for d in dims}
|
|
810
|
+
outer = self._outer_rows()
|
|
811
|
+
usable = {
|
|
812
|
+
d: str(d) in self._schema.names
|
|
813
|
+
and self._coord_arrays[str(d)].dtype.kind in ("i", "u", "f", "M")
|
|
814
|
+
for d in dims
|
|
815
|
+
}
|
|
816
|
+
proven = 0
|
|
817
|
+
boundary: list[Block] = []
|
|
818
|
+
|
|
819
|
+
def bucket_guarantee(
|
|
820
|
+
d: Any, indices: np.ndarray
|
|
821
|
+
) -> pc.Expression | None:
|
|
822
|
+
"""Conjunctive [min, max] guarantee over one dimension's index
|
|
823
|
+
bucket, or None when unprovable (NaN spans, non-numeric dims)."""
|
|
824
|
+
if not usable[d]:
|
|
825
|
+
return None
|
|
826
|
+
lo, hi, bad = self._chunk_spans(str(d))
|
|
827
|
+
if bad[indices].any():
|
|
828
|
+
return None
|
|
829
|
+
field_type = self._schema.field(str(d)).type
|
|
830
|
+
return (
|
|
831
|
+
pc.field(str(d)) >= pa.scalar(lo[indices].min(), field_type)
|
|
832
|
+
) & (pc.field(str(d)) <= pa.scalar(hi[indices].max(), field_type))
|
|
833
|
+
|
|
834
|
+
def rows_of(cell: dict) -> int:
|
|
835
|
+
"""Rows spanned by ``cell``: the product of its chunks' lengths
|
|
836
|
+
per dimension, times the rows of dimensions outside the grid."""
|
|
837
|
+
rows = outer
|
|
838
|
+
for d in dims:
|
|
839
|
+
rows *= int(lens[d][np.asarray(cell[d])].sum())
|
|
840
|
+
return rows
|
|
841
|
+
|
|
842
|
+
def classify(cell: dict, depth: int) -> None:
|
|
843
|
+
"""Prove, prune, or split one cell of the surviving chunk grid.
|
|
844
|
+
|
|
845
|
+
A cell is a hyper-rectangle of chunk indices per dimension.
|
|
846
|
+
Its indices are bucketed into at most
|
|
847
|
+
``_STRICT_LEVEL_BUDGET`` products; Arrow's guarantee
|
|
848
|
+
simplification then decides each bucket-product wholesale:
|
|
849
|
+
proven (every row satisfies ``filter``, counted into
|
|
850
|
+
``proven`` without reading), pruned (provably empty,
|
|
851
|
+
dropped), or mixed (recurse). Undecidable single-chunk
|
|
852
|
+
cells land in ``boundary`` for exact scanning.
|
|
853
|
+
"""
|
|
854
|
+
nonlocal proven
|
|
855
|
+
ks = {d: len(cell[d]) for d in dims}
|
|
856
|
+
while int(np.prod(list(ks.values()))) > _STRICT_LEVEL_BUDGET:
|
|
857
|
+
widest = max(ks, key=lambda d: ks[d])
|
|
858
|
+
if ks[widest] == 1:
|
|
859
|
+
break
|
|
860
|
+
ks[widest] = max(1, ks[widest] // 2)
|
|
861
|
+
buckets = {
|
|
862
|
+
d: np.array_split(np.asarray(cell[d]), ks[d]) for d in dims
|
|
863
|
+
}
|
|
864
|
+
combos = list(itertools.product(*(range(ks[d]) for d in dims)))
|
|
865
|
+
guarantees: list[pc.Expression | None] = []
|
|
866
|
+
for combo in combos:
|
|
867
|
+
g: pc.Expression | None = None
|
|
868
|
+
complete = True
|
|
869
|
+
for d, b in zip(dims, combo):
|
|
870
|
+
gd = bucket_guarantee(d, buckets[d][b])
|
|
871
|
+
if gd is None:
|
|
872
|
+
if usable[d]:
|
|
873
|
+
complete = False # NaN span: never provable
|
|
874
|
+
continue
|
|
875
|
+
g = gd if g is None else g & gd
|
|
876
|
+
guarantees.append(g if (g is not None and complete) else None)
|
|
877
|
+
decidable = [i for i, g in enumerate(guarantees) if g is not None]
|
|
878
|
+
satisfiable = set(decidable)
|
|
879
|
+
unstrict = set(decidable)
|
|
880
|
+
if decidable:
|
|
881
|
+
shadow = _guarantee_shadow(
|
|
882
|
+
[(str(i), guarantees[i]) for i in decidable], self._schema
|
|
883
|
+
)
|
|
884
|
+
satisfiable = {
|
|
885
|
+
int(f.path) for f in shadow.get_fragments(filter=filter)
|
|
886
|
+
}
|
|
887
|
+
unstrict = {
|
|
888
|
+
int(f.path) for f in shadow.get_fragments(filter=~filter)
|
|
889
|
+
}
|
|
890
|
+
for i, combo in enumerate(combos):
|
|
891
|
+
subcell = {d: list(buckets[d][b]) for d, b in zip(dims, combo)}
|
|
892
|
+
if guarantees[i] is not None:
|
|
893
|
+
if i not in satisfiable:
|
|
894
|
+
continue # provably empty: cross-dim refinement
|
|
895
|
+
if i not in unstrict:
|
|
896
|
+
proven += rows_of(subcell)
|
|
897
|
+
continue
|
|
898
|
+
if all(len(v) == 1 for v in subcell.values()):
|
|
899
|
+
boundary.append(
|
|
900
|
+
self._block_for_combo(
|
|
901
|
+
tuple(v[0] for v in subcell.values())
|
|
902
|
+
)
|
|
903
|
+
)
|
|
904
|
+
elif depth < _STRICT_MAX_DEPTH:
|
|
905
|
+
classify(subcell, depth + 1)
|
|
906
|
+
else:
|
|
907
|
+
# Depth backstop; unreachable for realistic grids.
|
|
908
|
+
for c in itertools.product(*subcell.values()):
|
|
909
|
+
boundary.append(self._block_for_combo(c))
|
|
910
|
+
|
|
911
|
+
try:
|
|
912
|
+
classify(lists, 0)
|
|
913
|
+
except (pa.ArrowInvalid, pa.ArrowNotImplementedError, TypeError):
|
|
914
|
+
return 0, self._blocks(kept) # scan every survivor, exactly
|
|
915
|
+
return proven, boundary
|
|
916
|
+
|
|
917
|
+
def _block_rows(self, block: Block) -> int:
|
|
918
|
+
rows = 1
|
|
919
|
+
for d, sl in block.items():
|
|
920
|
+
size = self._ds.sizes[d]
|
|
921
|
+
start, stop, _ = sl.indices(size)
|
|
922
|
+
rows *= stop - start
|
|
923
|
+
return rows
|
|
924
|
+
|
|
925
|
+
# ------------------------------------------------------------------
|
|
926
|
+
# Scan: load surviving chunks, prefetching ahead of the consumer
|
|
927
|
+
# ------------------------------------------------------------------
|
|
928
|
+
|
|
929
|
+
def _surviving(
|
|
930
|
+
self, kept: dict[str, list[int]] | None, dim: Any
|
|
931
|
+
) -> list[int] | range:
|
|
932
|
+
"""One dim's surviving chunk indices; every chunk when unpruned."""
|
|
933
|
+
return (kept or {}).get(str(dim), range(len(self._resolved[dim])))
|
|
934
|
+
|
|
935
|
+
def _outer_rows(self) -> int:
|
|
936
|
+
"""Rows contributed per grid cell by dims outside the chunk grid.
|
|
937
|
+
|
|
938
|
+
Unresolved dims span their full extent in every block, so they
|
|
939
|
+
multiply every block's row count uniformly.
|
|
940
|
+
"""
|
|
941
|
+
rows = 1
|
|
942
|
+
for d in self._ds.dims:
|
|
943
|
+
if d not in self._resolved:
|
|
944
|
+
rows *= self._ds.sizes[d]
|
|
945
|
+
return rows
|
|
946
|
+
|
|
947
|
+
def _combos(
|
|
948
|
+
self, kept: dict[str, list[int]] | None
|
|
949
|
+
) -> Iterator[tuple[int, ...]]:
|
|
950
|
+
"""Surviving chunk-index combinations, in grid order."""
|
|
951
|
+
dims = list(self._resolved.keys())
|
|
952
|
+
if not dims:
|
|
953
|
+
return
|
|
954
|
+
yield from itertools.product(*(self._surviving(kept, d) for d in dims))
|
|
955
|
+
|
|
956
|
+
def _block_for_combo(self, combo: tuple[int, ...]) -> Block:
|
|
957
|
+
block: Block = {d: slice(None) for d in self._ds.dims}
|
|
958
|
+
for d, i in zip(self._resolved.keys(), combo):
|
|
959
|
+
bounds = self._chunk_bounds[d]
|
|
960
|
+
block[d] = slice(int(bounds[i]), int(bounds[i + 1]))
|
|
961
|
+
return block
|
|
962
|
+
|
|
963
|
+
def _blocks(self, kept: dict[str, list[int]] | None) -> Iterator[Block]:
|
|
964
|
+
"""Yield isel-able block slices for the surviving chunk grid."""
|
|
965
|
+
if not self._resolved:
|
|
966
|
+
yield {}
|
|
967
|
+
return
|
|
968
|
+
for combo in self._combos(kept):
|
|
969
|
+
yield self._block_for_combo(combo)
|
|
970
|
+
|
|
971
|
+
def _coalesced_blocks(
|
|
972
|
+
self, kept: dict[str, list[int]] | None
|
|
973
|
+
) -> Iterator[Block]:
|
|
974
|
+
"""Blocks with runs of consecutive chunks merged along one dim.
|
|
975
|
+
|
|
976
|
+
Runs are merged along the most finely chunked dimension while
|
|
977
|
+
the merged block stays under ``coalesce_rows`` rows. One merged
|
|
978
|
+
block is one ``isel`` — on Zarr sources its member chunks are
|
|
979
|
+
fetched by the store's own concurrent batch read instead of one
|
|
980
|
+
request per chunk through the prefetch pool.
|
|
981
|
+
"""
|
|
982
|
+
if not self._resolved:
|
|
983
|
+
yield {}
|
|
984
|
+
return
|
|
985
|
+
dims = list(self._resolved.keys())
|
|
986
|
+
merge_dim = max(dims, key=lambda d: len(self._resolved[d]))
|
|
987
|
+
others = [d for d in dims if d != merge_dim]
|
|
988
|
+
ranges = {d: list(self._surviving(kept, d)) for d in dims}
|
|
989
|
+
merge_bounds = self._chunk_bounds[merge_dim]
|
|
990
|
+
outer_rows = self._outer_rows()
|
|
991
|
+
|
|
992
|
+
def flush(prefix: tuple[int, ...], run: list[int]) -> Block:
|
|
993
|
+
block: Block = {d: slice(None) for d in self._ds.dims}
|
|
994
|
+
for d, i in zip(others, prefix):
|
|
995
|
+
bounds = self._chunk_bounds[d]
|
|
996
|
+
block[d] = slice(int(bounds[i]), int(bounds[i + 1]))
|
|
997
|
+
block[merge_dim] = slice(
|
|
998
|
+
int(merge_bounds[run[0]]), int(merge_bounds[run[-1] + 1])
|
|
999
|
+
)
|
|
1000
|
+
return block
|
|
1001
|
+
|
|
1002
|
+
coalesce_rows = self._coalesce_rows
|
|
1003
|
+
assert coalesce_rows is not None # only reached with coalescing on
|
|
1004
|
+
for prefix in itertools.product(*(ranges[d] for d in others)):
|
|
1005
|
+
per_row = outer_rows
|
|
1006
|
+
for d, i in zip(others, prefix):
|
|
1007
|
+
bounds = self._chunk_bounds[d]
|
|
1008
|
+
per_row *= int(bounds[i + 1] - bounds[i])
|
|
1009
|
+
run: list[int] = []
|
|
1010
|
+
run_rows = 0
|
|
1011
|
+
for i in ranges[merge_dim]:
|
|
1012
|
+
rows = int(merge_bounds[i + 1] - merge_bounds[i]) * per_row
|
|
1013
|
+
if run and (
|
|
1014
|
+
i != run[-1] + 1 or run_rows + rows > coalesce_rows
|
|
1015
|
+
):
|
|
1016
|
+
yield flush(prefix, run)
|
|
1017
|
+
run, run_rows = [], 0
|
|
1018
|
+
run.append(i)
|
|
1019
|
+
run_rows += rows
|
|
1020
|
+
if run:
|
|
1021
|
+
yield flush(prefix, run)
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
class _XarrayFragment:
|
|
1025
|
+
"""One chunk of the source grid, presented as a dataset fragment.
|
|
1026
|
+
|
|
1027
|
+
Fragment consumers (DataFusion's ``DatasetExec`` plans one partition
|
|
1028
|
+
per fragment; Dask maps over them) call [scanner][xarray_sql.backends.pyarrow.XarrayPushdownDataset.scanner] with the
|
|
1029
|
+
columns and predicate for this piece; the pushed filter is applied
|
|
1030
|
+
row-exactly, same as the parent dataset's scanner.
|
|
1031
|
+
"""
|
|
1032
|
+
|
|
1033
|
+
def __init__(self, dataset: XarrayPushdownDataset, block: Block):
|
|
1034
|
+
self._dataset = dataset
|
|
1035
|
+
self._block = block
|
|
1036
|
+
|
|
1037
|
+
@property
|
|
1038
|
+
def physical_schema(self) -> pa.Schema:
|
|
1039
|
+
return self._dataset.schema
|
|
1040
|
+
|
|
1041
|
+
def scanner(
|
|
1042
|
+
self,
|
|
1043
|
+
schema: pa.Schema | None = None,
|
|
1044
|
+
columns: list[str] | None = None,
|
|
1045
|
+
filter: pc.Expression | None = None,
|
|
1046
|
+
batch_size: int | None = None,
|
|
1047
|
+
**kwargs: Any,
|
|
1048
|
+
) -> pads.Scanner:
|
|
1049
|
+
return self._dataset._scanner_for_blocks(
|
|
1050
|
+
[self._block], columns, filter, batch_size
|
|
1051
|
+
)
|
|
1052
|
+
|
|
1053
|
+
def to_batches(self, **kwargs: Any) -> Iterator[pa.RecordBatch]:
|
|
1054
|
+
return iter(self.scanner(**kwargs).to_batches())
|
|
1055
|
+
|
|
1056
|
+
def to_table(self, **kwargs: Any) -> pa.Table:
|
|
1057
|
+
return self.scanner(**kwargs).to_table()
|
|
1058
|
+
|
|
1059
|
+
def count_rows(self, **kwargs: Any) -> int:
|
|
1060
|
+
return int(self.scanner(**kwargs).count_rows())
|
|
1061
|
+
|
|
1062
|
+
def __dask_tokenize__(self) -> tuple:
|
|
1063
|
+
# Dask hashes from_map inputs; the parent dataset is unpicklable,
|
|
1064
|
+
# so provide a deterministic token from the fragment's identity.
|
|
1065
|
+
return (
|
|
1066
|
+
"xarray_sql._XarrayFragment",
|
|
1067
|
+
repr(self._block),
|
|
1068
|
+
self._dataset.schema.to_string(),
|
|
1069
|
+
)
|
|
1070
|
+
|
|
1071
|
+
def __repr__(self) -> str:
|
|
1072
|
+
return f"_XarrayFragment({self._block!r})"
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
def arrow_dataset(
|
|
1076
|
+
ds: xr.Dataset,
|
|
1077
|
+
chunks: Chunks = None,
|
|
1078
|
+
*,
|
|
1079
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
1080
|
+
prefetch: int = DEFAULT_PREFETCH,
|
|
1081
|
+
prefetch_bytes: int | None = None,
|
|
1082
|
+
coalesce_rows: int | None = None,
|
|
1083
|
+
geometry: tuple[str, str] | None = None,
|
|
1084
|
+
geometry_encoding: str = "wkb",
|
|
1085
|
+
geometry_crs: str | None = "OGC:CRS84",
|
|
1086
|
+
) -> XarrayPushdownDataset:
|
|
1087
|
+
"""A pushdown-capable ``pyarrow.dataset.Dataset`` view of ``ds``.
|
|
1088
|
+
|
|
1089
|
+
The returned object works anywhere a pyarrow dataset does, keeping
|
|
1090
|
+
projection pushdown and coordinate-range chunk pruning::
|
|
1091
|
+
|
|
1092
|
+
import polars as pl
|
|
1093
|
+
lf = pl.scan_pyarrow_dataset(xql.arrow_dataset(ds))
|
|
1094
|
+
|
|
1095
|
+
import duckdb
|
|
1096
|
+
duckdb.connect().register("t", xql.arrow_dataset(ds))
|
|
1097
|
+
|
|
1098
|
+
xql.arrow_dataset(ds).to_table(columns=["t2m"], filter=...)
|
|
1099
|
+
|
|
1100
|
+
Args:
|
|
1101
|
+
ds: An xarray Dataset. All data variables must share the same
|
|
1102
|
+
dimensions (select a variable subset first otherwise).
|
|
1103
|
+
chunks: Xarray-like chunks specification controlling partition
|
|
1104
|
+
granularity. Defaults to the Dataset's existing chunks, or for
|
|
1105
|
+
an unchunked Dataset its storage chunks.
|
|
1106
|
+
batch_size: Maximum rows per emitted Arrow RecordBatch.
|
|
1107
|
+
prefetch: Chunk loads kept in flight ahead of the consumer
|
|
1108
|
+
(memory scales with ``prefetch`` x pivoted chunk size).
|
|
1109
|
+
prefetch_bytes: Optional cap on *estimated pivoted bytes* in
|
|
1110
|
+
flight; admission then tracks bytes rather than block count,
|
|
1111
|
+
which keeps peak memory steady when ``coalesce_rows`` makes
|
|
1112
|
+
blocks large or uneven. ``prefetch`` still bounds
|
|
1113
|
+
concurrency (thread count).
|
|
1114
|
+
coalesce_rows: When set, merge runs of consecutive surviving
|
|
1115
|
+
chunks along the most finely chunked dimension into single
|
|
1116
|
+
reads of at most this many rows. Fewer, larger source
|
|
1117
|
+
requests — the win on remote stores, where each merged read
|
|
1118
|
+
fetches its member chunks through the store's own concurrent
|
|
1119
|
+
batching. Memory scales with ``prefetch`` x the *merged*
|
|
1120
|
+
block size, so size accordingly (e.g. ``8_000_000``).
|
|
1121
|
+
geometry: ``(x_dim, y_dim)`` coordinate dims to derive a
|
|
1122
|
+
``geometry`` point column from (see
|
|
1123
|
+
[xarray_sql.geometry][]). With the default ``"wkb"``
|
|
1124
|
+
encoding, DuckDB (spatial loaded) sees a native ``GEOMETRY``
|
|
1125
|
+
column, so ``ST_Within(geometry, ...)`` works directly.
|
|
1126
|
+
geometry_encoding: ``"wkb"`` (default; DuckDB-consumable) or
|
|
1127
|
+
``"point"`` (GeoArrow native separated coordinates — the
|
|
1128
|
+
struct children are the coordinate arrays; for GeoPandas,
|
|
1129
|
+
lonboard, geoarrow-rs consumers).
|
|
1130
|
+
geometry_crs: CRS tag carried in the extension metadata.
|
|
1131
|
+
Defaults to ``OGC:CRS84`` (plain longitude/latitude); pass
|
|
1132
|
+
``None`` to omit, or an authority code / PROJJSON string.
|
|
1133
|
+
|
|
1134
|
+
Returns:
|
|
1135
|
+
An [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset].
|
|
1136
|
+
"""
|
|
1137
|
+
return XarrayPushdownDataset(
|
|
1138
|
+
ds,
|
|
1139
|
+
chunks,
|
|
1140
|
+
batch_size=batch_size,
|
|
1141
|
+
prefetch=prefetch,
|
|
1142
|
+
prefetch_bytes=prefetch_bytes,
|
|
1143
|
+
coalesce_rows=coalesce_rows,
|
|
1144
|
+
geometry=geometry,
|
|
1145
|
+
geometry_encoding=geometry_encoding,
|
|
1146
|
+
geometry_crs=geometry_crs,
|
|
1147
|
+
)
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def arrow_datasets(
|
|
1151
|
+
ds: xr.Dataset,
|
|
1152
|
+
name: str | None = None,
|
|
1153
|
+
*,
|
|
1154
|
+
chunks: Chunks = None,
|
|
1155
|
+
table_names: TableNames = None,
|
|
1156
|
+
**kwargs: Any,
|
|
1157
|
+
) -> dict[str, XarrayPushdownDataset]:
|
|
1158
|
+
"""Named pushdown datasets, one per dimension group of ``ds``.
|
|
1159
|
+
|
|
1160
|
+
[arrow_dataset][xarray_sql.backends.pyarrow.arrow_dataset] needs a
|
|
1161
|
+
Dataset whose data variables all share one set of dimensions. This
|
|
1162
|
+
applies the same split the DataFusion and DuckDB adapters do — one
|
|
1163
|
+
table per dimension group — and hands back the tables *named*, ready
|
|
1164
|
+
to register on an engine that has no connection object to dispatch
|
|
1165
|
+
on::
|
|
1166
|
+
|
|
1167
|
+
tables = xql.arrow_datasets(ds, 'era5', table_names={
|
|
1168
|
+
('time', 'latitude', 'longitude'): 'surface',
|
|
1169
|
+
('time', 'level', 'latitude', 'longitude'): 'atmosphere',
|
|
1170
|
+
})
|
|
1171
|
+
|
|
1172
|
+
ctx = pl.SQLContext()
|
|
1173
|
+
for table, dataset in tables.items(): # 'era5_surface', ...
|
|
1174
|
+
ctx.register(table, pl.scan_pyarrow_dataset(dataset))
|
|
1175
|
+
|
|
1176
|
+
ctx.execute('SELECT AVG("2m_temperature") FROM era5_surface')
|
|
1177
|
+
|
|
1178
|
+
Dimension coordinates are read once and shared across the returned
|
|
1179
|
+
tables, which is a network round-trip saved per dimension per group
|
|
1180
|
+
on Zarr-backed stores.
|
|
1181
|
+
|
|
1182
|
+
Args:
|
|
1183
|
+
ds: An xarray Dataset, with variables on any mix of dimensions.
|
|
1184
|
+
name: Prefix for the table names, as in ``era5_surface``. Omit it
|
|
1185
|
+
to get the bare group names (``surface``).
|
|
1186
|
+
chunks: Xarray-like chunks specification controlling partition
|
|
1187
|
+
granularity. Keys naming dimensions a group does not have are
|
|
1188
|
+
ignored for that group. Defaults to the Dataset's existing
|
|
1189
|
+
chunks, or for an unchunked Dataset its storage chunks.
|
|
1190
|
+
table_names: Maps a dimension group's exact dim tuple to the name
|
|
1191
|
+
its table takes, e.g.
|
|
1192
|
+
``{('time', 'latitude', 'longitude'): 'surface'}``. Groups
|
|
1193
|
+
left unnamed take their dimensions joined by underscores
|
|
1194
|
+
(``time_latitude_longitude``); the group holding scalar
|
|
1195
|
+
variables, if any, takes ``scalar``.
|
|
1196
|
+
**kwargs: Forwarded to
|
|
1197
|
+
[XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset]
|
|
1198
|
+
(``batch_size``, ``prefetch``, ...), applied to every
|
|
1199
|
+
returned table. ``geometry=(x_dim, y_dim)`` only reaches the
|
|
1200
|
+
groups whose dimensions include both — a mixed-dimension
|
|
1201
|
+
Dataset otherwise has groups the geometry dims are not
|
|
1202
|
+
columns of; those groups come back without a geometry
|
|
1203
|
+
column rather than raising.
|
|
1204
|
+
|
|
1205
|
+
Returns:
|
|
1206
|
+
Table name to
|
|
1207
|
+
[XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset].
|
|
1208
|
+
A Dataset with a single dimension group yields one entry, keyed
|
|
1209
|
+
by ``name`` when given.
|
|
1210
|
+
"""
|
|
1211
|
+
groups = group_vars_by_dims(ds)
|
|
1212
|
+
names = resolve_table_names(ds, table_names)
|
|
1213
|
+
|
|
1214
|
+
def table_name(dims: tuple[str, ...]) -> str:
|
|
1215
|
+
group = names[dims]
|
|
1216
|
+
return f"{name}_{group}" if name else group
|
|
1217
|
+
|
|
1218
|
+
if len(groups) <= 1:
|
|
1219
|
+
# One group is one table, named `name` — there is no group to
|
|
1220
|
+
# tell apart. Without a `name`, it takes the group's own. Built
|
|
1221
|
+
# directly (not via arrow_dataset(), whose signature is a fixed
|
|
1222
|
+
# subset) so every kwarg XarrayPushdownDataset accepts works
|
|
1223
|
+
# here exactly as it does in the multi-group branch below.
|
|
1224
|
+
only = name if name else next(iter(names.values()), "scalar")
|
|
1225
|
+
return {only: XarrayPushdownDataset(ds, chunks, **kwargs)}
|
|
1226
|
+
|
|
1227
|
+
geometry = kwargs.pop("geometry", None)
|
|
1228
|
+
coord_arrays = shared_coord_arrays(ds)
|
|
1229
|
+
tables = {}
|
|
1230
|
+
for dims, var_names in groups.items():
|
|
1231
|
+
group_kwargs = dict(kwargs)
|
|
1232
|
+
if geometry is not None:
|
|
1233
|
+
group_kwargs["geometry"] = (
|
|
1234
|
+
geometry if set(geometry).issubset(dims) else None
|
|
1235
|
+
)
|
|
1236
|
+
tables[table_name(dims)] = XarrayPushdownDataset(
|
|
1237
|
+
ds[var_names], chunks, coord_arrays=coord_arrays, **group_kwargs
|
|
1238
|
+
)
|
|
1239
|
+
return tables
|