xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1239 @@
1
+ """Engine-neutral pyarrow views of lazy xarray Datasets.
2
+
3
+ Two ways to hand a lazy ``xarray.Dataset`` to an Arrow-speaking query
4
+ engine:
5
+
6
+ * [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset] — a real ``pyarrow.dataset.Dataset``
7
+ subclass (the pattern Lance uses for ``LanceDataset``). Consumers of
8
+ the pyarrow dataset protocol — DuckDB via ``con.register``, Polars via
9
+ ``pl.scan_pyarrow_dataset``, or pyarrow itself — call
10
+ [scanner][xarray_sql.backends.pyarrow.XarrayPushdownDataset.scanner] with the columns a query needs
11
+ and the predicate it pushed down, so the scan loads only the needed
12
+ data variables from only the chunks whose coordinate ranges can
13
+ satisfy the predicate. Construct one with [arrow_dataset][xarray_sql.backends.pyarrow.arrow_dataset].
14
+ * [XarrayArrowStream][xarray_sql.backends.pyarrow.XarrayArrowStream] — a re-scannable Arrow C-stream
15
+ (PyCapsule) view. No source-level pushdown, but works with any
16
+ PyCapsule consumer; the dependency-light fallback.
17
+
18
+ Correctness contract shared by all consumers of the pushdown dataset:
19
+ engines may delete the filter conjuncts they push down (DuckDB does),
20
+ so the returned scanner applies the expression exactly via
21
+ ``pyarrow.dataset.Scanner``; chunk pruning is only ever an optimization
22
+ on top.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import itertools
28
+ import math
29
+ import re
30
+ import threading
31
+ import weakref
32
+ from collections import deque
33
+ from collections.abc import Callable, Iterator
34
+ from concurrent.futures import ThreadPoolExecutor
35
+ from typing import Any
36
+
37
+ import numpy as np
38
+ import pyarrow as pa
39
+ import pyarrow.compute as pc
40
+ import pyarrow.dataset as pads
41
+ import pyarrow.fs as pafs
42
+ import xarray as xr
43
+
44
+ from ..df import (
45
+ _CAN_START_THREADS,
46
+ Block,
47
+ Chunks,
48
+ DEFAULT_BATCH_SIZE,
49
+ TableNames,
50
+ _ensure_default_indexes,
51
+ _parse_schema,
52
+ group_vars_by_dims,
53
+ iter_record_batches,
54
+ resolve_chunks,
55
+ resolve_table_names,
56
+ shared_coord_arrays,
57
+ )
58
+ from ..geometry import GEOMETRY_COLUMN, build_geometry, geometry_field
59
+ from ..reader import XarrayRecordBatchReader
60
+
61
+ DEFAULT_PREFETCH = 4 if _CAN_START_THREADS else 1
62
+ """Chunk loads kept in flight ahead of the consumer during a scan.
63
+
64
+ Where threads are unavailable (Pyodide), chunks load one at a time."""
65
+
66
+ _SHADOW_FANOUT = 1024
67
+ """Maximum fragments per shadow level.
68
+
69
+ A dimension with more chunks than this gets a two-level shadow: a coarse
70
+ level of at most this many buckets, refined per surviving bucket. This
71
+ bounds shadow construction cost for finely partitioned datasets (e.g.
72
+ hundreds of thousands of single-step time chunks) at registration and
73
+ query time alike.
74
+ """
75
+
76
+ _REFINE_MAX_FRACTION = 0.25
77
+ """Skip fine-level pruning when the coarse pass kept more buckets.
78
+
79
+ Refinement builds one sub-shadow per surviving bucket; when a predicate
80
+ matches most of the axis that cost cannot pay for itself, so the scan
81
+ falls back to the (sound) coarse answer.
82
+ """
83
+
84
+ _STRICT_LEVEL_BUDGET = 4096
85
+ """Maximum guarantee fragments per level of the strictness analysis.
86
+
87
+ Strictness classifies bucket-products of surviving chunks recursively:
88
+ whole buckets prove (or prune) at once, and only mixed cells refine.
89
+ Each level builds at most this many fragments, so grids of millions of
90
+ chunks resolve in a handful of vectorized passes.
91
+ """
92
+
93
+ _STRICT_MAX_DEPTH = 6
94
+ """Recursion bound for the strictness analysis (a backstop: realistic
95
+ grids terminate in 2-3 levels)."""
96
+
97
+
98
+ def _guarantee_shadow(
99
+ guarantees: list[tuple[str, pc.Expression]], schema: pa.Schema
100
+ ) -> pads.FileSystemDataset:
101
+ """A path-only dataset whose fragments carry the given guarantees.
102
+
103
+ Each ``(path, guarantee)`` pair becomes one fragment: the path is a
104
+ label (never opened) and the guarantee is its
105
+ ``partition_expression``, so ``get_fragments(filter=...)`` delegates
106
+ satisfiability to Arrow's guarantee simplification.
107
+ """
108
+ fmt = pads.IpcFileFormat()
109
+ fs = pafs.LocalFileSystem()
110
+ fragments = [
111
+ fmt.make_fragment(path, fs, partition_expression=guarantee)
112
+ for path, guarantee in guarantees
113
+ ]
114
+ return pads.FileSystemDataset(fragments, schema, fmt, fs)
115
+
116
+
117
+ class _DimShadow:
118
+ """Chunk-pruning index for one dimension of the source grid.
119
+
120
+ Fragment ``i`` of a shadow ``FileSystemDataset`` carries the
121
+ guarantee ``dim ∈ [min, max]`` of chunk-span ``i`` as its
122
+ ``partition_expression``; ``get_fragments(filter=...)`` then lets
123
+ Arrow's guarantee simplification decide which spans can satisfy a
124
+ predicate — sound for every predicate shape, conservative on columns
125
+ the guarantee does not mention, and the fragments' paths are never
126
+ opened.
127
+
128
+ Axes with more than ``_SHADOW_FANOUT`` chunks use two levels: a
129
+ coarse shadow over buckets of consecutive chunks, plus per-bucket
130
+ fine shadows built lazily for the buckets a query keeps.
131
+ """
132
+
133
+ def __init__(
134
+ self,
135
+ name: str,
136
+ schema: pa.Schema,
137
+ coord: np.ndarray,
138
+ bounds: np.ndarray,
139
+ ):
140
+ self._name = name
141
+ # The full table schema, not just this dimension's field: the
142
+ # pushed predicate may reference any column, and get_fragments
143
+ # must be able to bind all of them (guarantees stay per-dim;
144
+ # unmentioned columns are conservatively unconstrained).
145
+ self._schema = schema
146
+ self._field_type = schema.field(name).type
147
+ self._coord = coord
148
+ self._bounds = bounds
149
+ self._n = len(bounds) - 1
150
+ self._step = max(1, math.ceil(self._n / _SHADOW_FANOUT))
151
+ self._n_buckets = math.ceil(self._n / self._step)
152
+ self._coarse = self._build(
153
+ [
154
+ (b * self._step, min((b + 1) * self._step, self._n))
155
+ for b in range(self._n_buckets)
156
+ ]
157
+ )
158
+ self._fine: dict[int, pads.FileSystemDataset] = {}
159
+
160
+ def _build(self, spans: list[tuple[int, int]]) -> pads.FileSystemDataset:
161
+ """A shadow whose fragment ``i`` guarantees chunk-span ``spans[i]``."""
162
+ guarantees: list[tuple[str, pc.Expression]] = []
163
+ for i, (lo_chunk, hi_chunk) in enumerate(spans):
164
+ vals = self._coord[self._bounds[lo_chunk] : self._bounds[hi_chunk]]
165
+ if (vals.dtype.kind == "f" and np.isnan(vals).any()) or (
166
+ vals.dtype.kind == "M" and np.isnat(vals).any()
167
+ ):
168
+ # NaN/NaT poisons min/max into a (dim >= NaN) guarantee
169
+ # that Arrow simplifies every predicate against as false,
170
+ # silently pruning rows. An always-true guarantee keeps
171
+ # the span unprunable instead.
172
+ guarantee = pc.scalar(True)
173
+ else:
174
+ # min/max (not first/last) so descending axes like
175
+ # latitude 90→-90 carry correct ranges.
176
+ lo = pa.scalar(vals.min(), type=self._field_type)
177
+ hi = pa.scalar(vals.max(), type=self._field_type)
178
+ guarantee = (pc.field(self._name) >= lo) & (
179
+ pc.field(self._name) <= hi
180
+ )
181
+ guarantees.append((str(i), guarantee))
182
+ return _guarantee_shadow(guarantees, self._schema)
183
+
184
+ @staticmethod
185
+ def _kept_indices(
186
+ shadow: pads.FileSystemDataset, filter: pc.Expression
187
+ ) -> list[int]:
188
+ return sorted(
189
+ int(frag.path) for frag in shadow.get_fragments(filter=filter)
190
+ )
191
+
192
+ def kept(self, filter: pc.Expression) -> list[int] | None:
193
+ """Chunk indices that can satisfy ``filter``; ``None`` means all."""
194
+ try:
195
+ buckets = self._kept_indices(self._coarse, filter)
196
+ if self._step == 1:
197
+ return buckets if len(buckets) < self._n else None
198
+ if len(buckets) > _REFINE_MAX_FRACTION * self._n_buckets:
199
+ # Refining most of the axis costs more than it saves;
200
+ # answer with the coarse buckets, which is still sound.
201
+ return (
202
+ None
203
+ if len(buckets) == self._n_buckets
204
+ else [
205
+ i
206
+ for b in buckets
207
+ for i in range(
208
+ b * self._step,
209
+ min((b + 1) * self._step, self._n),
210
+ )
211
+ ]
212
+ )
213
+ kept: list[int] = []
214
+ for b in buckets:
215
+ fine = self._fine.get(b)
216
+ if fine is None:
217
+ start = b * self._step
218
+ stop = min((b + 1) * self._step, self._n)
219
+ fine = self._build([(i, i + 1) for i in range(start, stop)])
220
+ self._fine[b] = fine
221
+ start = b * self._step
222
+ kept.extend(start + i for i in self._kept_indices(fine, filter))
223
+ return kept
224
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError, TypeError):
225
+ return None # conservative: scan every chunk of this dim
226
+
227
+
228
+ class XarrayArrowStream:
229
+ """A re-scannable Arrow C-stream view over a lazy xarray Dataset.
230
+
231
+ Arrow PyCapsule consumers (DuckDB among them) call
232
+ ``__arrow_c_stream__`` once per scan. Each call constructs a fresh
233
+ [XarrayRecordBatchReader][xarray_sql.reader.XarrayRecordBatchReader] over the same
234
+ lazy Dataset, so — unlike registering a ``pyarrow.RecordBatchReader``
235
+ directly, which is exhausted after one query — the same registered
236
+ table supports any number of queries, and data is only read while a
237
+ query is executing.
238
+
239
+ The PyCapsule scan path gets no source-level pushdown (the producer
240
+ never sees the query's columns or filters), so
241
+ [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset] is the default registration object;
242
+ this class remains as the dependency-light fallback.
243
+ """
244
+
245
+ def __init__(
246
+ self,
247
+ ds: xr.Dataset,
248
+ chunks: Chunks = None,
249
+ *,
250
+ batch_size: int = DEFAULT_BATCH_SIZE,
251
+ _iteration_callback: (
252
+ Callable[[Block, list[str] | None], None] | None
253
+ ) = None,
254
+ ):
255
+ # Validate eagerly (same checks XarrayRecordBatchReader runs) so
256
+ # registration fails fast instead of erroring mid-query.
257
+ probe = XarrayRecordBatchReader(ds, chunks, batch_size=batch_size)
258
+ self._ds = ds
259
+ self._chunks = chunks
260
+ self._batch_size = batch_size
261
+ self._schema = probe.schema
262
+ self._iteration_callback = _iteration_callback
263
+
264
+ def __arrow_c_stream__(
265
+ self, requested_schema: object | None = None
266
+ ) -> object:
267
+ reader = XarrayRecordBatchReader(
268
+ self._ds,
269
+ self._chunks,
270
+ batch_size=self._batch_size,
271
+ _iteration_callback=self._iteration_callback,
272
+ )
273
+ return reader.__arrow_c_stream__(requested_schema)
274
+
275
+ def __arrow_c_schema__(self) -> object:
276
+ return self._schema.__arrow_c_schema__()
277
+
278
+
279
+ class XarrayPushdownDataset(pads.Dataset):
280
+ """A pushdown-capable ``pyarrow.dataset.Dataset`` view of a Dataset.
281
+
282
+ Consumers that speak the pyarrow dataset protocol (DuckDB, Polars,
283
+ ...) call [scanner][xarray_sql.backends.pyarrow.XarrayPushdownDataset.scanner] with the columns a query needs and the
284
+ predicate it pushed down; the scan then loads only the needed data
285
+ variables from only the chunks whose coordinate ranges can satisfy
286
+ the predicate.
287
+
288
+ The base class is never initialized (there is no C++ dataset behind
289
+ this object — the same construction Lance uses for ``LanceDataset``);
290
+ every entry point consumers touch is overridden in Python, and the
291
+ few inherited members that would read uninitialized native state are
292
+ stubbed out.
293
+
294
+ References:
295
+ Lance's ``LanceDataset``, a ``pyarrow.dataset.Dataset`` subclass
296
+ built the same way: https://github.com/lancedb/lance
297
+ (``python/python/lance/dataset.py``).
298
+ """
299
+
300
+ def __init__(
301
+ self,
302
+ ds: xr.Dataset,
303
+ chunks: Chunks = None,
304
+ *,
305
+ batch_size: int = DEFAULT_BATCH_SIZE,
306
+ prefetch: int = DEFAULT_PREFETCH,
307
+ prefetch_bytes: int | None = None,
308
+ coalesce_rows: int | None = None,
309
+ geometry: tuple[str, str] | None = None,
310
+ geometry_encoding: str = "wkb",
311
+ geometry_crs: str | None = "OGC:CRS84",
312
+ coord_arrays: dict[str, np.ndarray] | None = None,
313
+ _iteration_callback: (
314
+ Callable[[Block, list[str] | None], None] | None
315
+ ) = None,
316
+ ):
317
+ # Deliberately no super().__init__() — see class docstring.
318
+ ds = _ensure_default_indexes(ds)
319
+ if ds.data_vars:
320
+ fst = next(iter(ds.values())).dims
321
+ if not all(da.dims == fst for da in ds.values()):
322
+ raise ValueError(
323
+ "All dimensions must be equal. "
324
+ "Please filter data_vars in the Dataset."
325
+ )
326
+ self._ds = ds
327
+ self._schema = _parse_schema(ds)
328
+ self._geometry = tuple(geometry) if geometry else None
329
+ self._geometry_encoding = geometry_encoding
330
+ if self._geometry:
331
+ if GEOMETRY_COLUMN in self._schema.names:
332
+ raise ValueError(
333
+ f"geometry= would shadow an existing column named "
334
+ f"{GEOMETRY_COLUMN!r}."
335
+ )
336
+ missing = [d for d in self._geometry if d not in self._schema.names]
337
+ if missing:
338
+ raise ValueError(
339
+ f"geometry dims {missing} are not columns of the "
340
+ f"table; available: {self._schema.names}"
341
+ )
342
+ self._schema = self._schema.append(
343
+ geometry_field(geometry_encoding, geometry_crs)
344
+ )
345
+ self._resolved = resolve_chunks(ds, chunks)
346
+ self._chunk_bounds = {
347
+ d: np.cumsum((0, *sizes)) for d, sizes in self._resolved.items()
348
+ }
349
+ # Reuse pre-materialised coordinate arrays where the caller has
350
+ # them (e.g. shared across the tables of a dim-group split); each
351
+ # missing dim costs one read, a network round-trip for Zarr.
352
+ self._coord_arrays = dict(coord_arrays or {})
353
+ for d in ds.dims:
354
+ if str(d) not in self._coord_arrays:
355
+ self._coord_arrays[str(d)] = ds.coords[d].values
356
+ if batch_size <= 0:
357
+ # A zero size would never advance the zero-column scan's
358
+ # row loop; fail here rather than mid-scan.
359
+ raise ValueError(f"batch_size must be positive, got {batch_size}")
360
+ self._batch_size = batch_size
361
+ self._prefetch = prefetch
362
+ self._prefetch_bytes = prefetch_bytes
363
+ self._coalesce_rows = coalesce_rows
364
+ self._iteration_callback = _iteration_callback
365
+ self._shadows: dict[str, _DimShadow] | None = None
366
+ self._span_cache: dict[
367
+ str, tuple[np.ndarray, np.ndarray, np.ndarray]
368
+ ] = {}
369
+ # One long-lived pool shared by every scan, its threads spawned
370
+ # NOW — never from inside an engine's scan callback. Creating a
371
+ # pool (and its OS threads) per scan deadlocks when the scan is
372
+ # driven from an engine executing under another thread pool
373
+ # (dask computing chunks of a lazy round-trip): thread startup
374
+ # and concurrent.futures' global shutdown lock interleave with
375
+ # the engine's callback needing the GIL. Scans that stop early
376
+ # cancel their queued loads instead of tearing the pool down.
377
+ # Like any thread state, the pool does not survive fork(); use
378
+ # from forked workers (e.g. PyTorch DataLoader) is tracked in
379
+ # https://github.com/xqlsystems/xarray-sql/issues/145.
380
+ self._pool: ThreadPoolExecutor | None = None
381
+ if self._prefetch > 1:
382
+ self._pool = ThreadPoolExecutor(max_workers=self._prefetch)
383
+ # Each pre-spawn task parks on the barrier, so no thread can
384
+ # take a second task and the executor is forced to start all
385
+ # ``prefetch`` OS threads before __init__ returns (submitting
386
+ # plain no-ops lets one idle thread absorb several of them,
387
+ # leaving the rest to spawn later inside an engine's scan
388
+ # callback — the deadlock this pre-spawn exists to prevent).
389
+ barrier = threading.Barrier(self._prefetch + 1)
390
+ spawn = [
391
+ self._pool.submit(barrier.wait) for _ in range(self._prefetch)
392
+ ]
393
+ barrier.wait()
394
+ for f in spawn:
395
+ f.result()
396
+ # Stop the pool's threads when the dataset dies; live scans
397
+ # keep the dataset alive through their generator closures,
398
+ # so nothing in flight is cut short. The callback is bound
399
+ # to the executor, not the dataset, so the finalizer holds
400
+ # no reference cycle back to self.
401
+ weakref.finalize(
402
+ self, self._pool.shutdown, wait=False, cancel_futures=True
403
+ )
404
+
405
+ # ------------------------------------------------------------------
406
+ # The consumer-facing surface
407
+ # ------------------------------------------------------------------
408
+
409
+ @property
410
+ def schema(self) -> pa.Schema:
411
+ return self._schema
412
+
413
+ def scanner(
414
+ self,
415
+ columns: list[str] | None = None,
416
+ filter: pc.Expression | None = None,
417
+ batch_size: int | None = None,
418
+ **kwargs: Any,
419
+ ) -> pads.Scanner:
420
+ """Build a scanner for the requested columns and predicate.
421
+
422
+ ``filter`` is applied exactly by the returned scanner (DuckDB
423
+ deletes the conjuncts it pushes down and trusts the source to
424
+ enforce them); chunk pruning and column selection only reduce
425
+ how much data is read to get there. ``batch_size`` caps rows per
426
+ emitted batch (Polars passes it through ``to_batches``). Extra
427
+ keyword arguments from other pyarrow-dataset consumers are
428
+ accepted and ignored.
429
+ """
430
+ kept = None if filter is None else self._prune(filter)
431
+ blocks = (
432
+ self._coalesced_blocks(kept)
433
+ if self._coalesce_rows
434
+ else self._blocks(kept)
435
+ )
436
+ return self._scanner_for_blocks(blocks, columns, filter, batch_size)
437
+
438
+ def _scanner_for_blocks(
439
+ self,
440
+ blocks: Iterator[Block] | list[Block],
441
+ columns: list[str] | None,
442
+ filter: pc.Expression | None,
443
+ batch_size: int | None = None,
444
+ ) -> pads.Scanner:
445
+ """A scanner over the given blocks; shared by dataset and fragments."""
446
+ # ``None`` means every column (the pyarrow convention); an
447
+ # explicitly empty list is a real projection ("no payload"), not
448
+ # a request for the full schema.
449
+ proj = list(self._schema.names) if columns is None else list(columns)
450
+ scan_names = self._scan_columns(proj, filter)
451
+ scan_schema = pa.schema([self._schema.field(n) for n in scan_names])
452
+ size = batch_size or self._batch_size
453
+ if self._geometry and GEOMETRY_COLUMN in scan_names:
454
+ batches = self._batches_with_geometry(scan_schema, blocks, size)
455
+ else:
456
+ batches = self._batch_generator(scan_schema, blocks, size)
457
+ return pads.Scanner.from_batches(
458
+ batches, schema=scan_schema, columns=proj, filter=filter
459
+ )
460
+
461
+ def _batches_with_geometry(
462
+ self,
463
+ scan_schema: pa.Schema,
464
+ blocks: Iterator[Block] | list[Block],
465
+ batch_size: int,
466
+ ) -> Iterator[pa.RecordBatch]:
467
+ """Emit ``scan_schema`` batches, synthesizing the geometry column.
468
+
469
+ The pivot never materializes geometry: batches are produced with
470
+ the coordinate dims the geometry derives from, and the geometry
471
+ column is built per batch from those columns (for the native
472
+ encoding the point struct's children *are* the coordinate
473
+ arrays — a schema annotation, not a copy).
474
+ """
475
+ assert self._geometry is not None
476
+ x_dim, y_dim = self._geometry
477
+ base_names = [n for n in scan_schema.names if n != GEOMETRY_COLUMN]
478
+ for d in (x_dim, y_dim):
479
+ if d not in base_names:
480
+ base_names.append(d)
481
+ base_schema = pa.schema([self._schema.field(n) for n in base_names])
482
+ for batch in self._batch_generator(base_schema, blocks, batch_size):
483
+ geom = build_geometry(
484
+ self._geometry_encoding,
485
+ batch.column(base_names.index(x_dim)),
486
+ batch.column(base_names.index(y_dim)),
487
+ )
488
+ arrays = [
489
+ geom
490
+ if name == GEOMETRY_COLUMN
491
+ else batch.column(base_names.index(name))
492
+ for name in scan_schema.names
493
+ ]
494
+ yield pa.RecordBatch.from_arrays(arrays, schema=scan_schema)
495
+
496
+ def _batch_generator(
497
+ self,
498
+ scan_schema: pa.Schema,
499
+ blocks: Iterator[Block] | list[Block],
500
+ batch_size: int,
501
+ ) -> Iterator[pa.RecordBatch]:
502
+ names = list(scan_schema.names)
503
+ data_vars = [n for n in names if n in self._ds.data_vars]
504
+ # Select only the needed variables before slicing so unrequested
505
+ # variables are never loaded (dimension coords come via coords).
506
+ base = (
507
+ self._ds[data_vars]
508
+ if data_vars
509
+ else self._ds.drop_vars(list(self._ds.data_vars))
510
+ )
511
+
512
+ def load(block: Block) -> list[pa.RecordBatch]:
513
+ if self._iteration_callback is not None:
514
+ self._iteration_callback(block, names)
515
+ if not names:
516
+ # Zero-column projection: row counts are chunk
517
+ # arithmetic; no coordinate or variable data is read.
518
+ out = []
519
+ rows = self._block_rows(block)
520
+ while rows > 0:
521
+ n = min(rows, batch_size)
522
+ out.append(
523
+ pa.table({"_": np.empty(n, np.int8)})
524
+ .select([])
525
+ .to_batches()[0]
526
+ )
527
+ rows -= n
528
+ return out
529
+ return list(
530
+ iter_record_batches(base.isel(block), scan_schema, batch_size)
531
+ )
532
+
533
+ # Estimated pivoted bytes per row: gates admission when a
534
+ # byte budget is set, so peak memory tracks bytes in flight
535
+ # rather than block count (blocks vary in size under
536
+ # coalesce_rows).
537
+ row_width = 0
538
+ for field in scan_schema:
539
+ try:
540
+ row_width += np.dtype(field.type.to_pandas_dtype()).itemsize
541
+ except (TypeError, NotImplementedError):
542
+ row_width += 8
543
+
544
+ def generate() -> Iterator[pa.RecordBatch]:
545
+ block_iter = iter(blocks)
546
+ first = next(block_iter, None)
547
+ if first is None:
548
+ return
549
+ second = next(block_iter, None)
550
+ if self._pool is None or second is None:
551
+ # Single-block scans (a lazy round-trip window that maps
552
+ # onto one source chunk) skip the pool entirely.
553
+ yield from load(first)
554
+ if second is not None:
555
+ yield from load(second)
556
+ for block in block_iter:
557
+ yield from load(block)
558
+ return
559
+ pool = self._pool
560
+ budget = self._prefetch_bytes
561
+ pending: deque = deque()
562
+ inflight = 0
563
+
564
+ def submit(block: Block) -> None:
565
+ nonlocal inflight
566
+ estimate = self._block_rows(block) * row_width
567
+ pending.append((pool.submit(load, block), estimate))
568
+ inflight += estimate
569
+
570
+ def drain_one() -> Iterator[pa.RecordBatch]:
571
+ nonlocal inflight
572
+ future, estimate = pending.popleft()
573
+ inflight -= estimate
574
+ yield from future.result()
575
+
576
+ try:
577
+ submit(first)
578
+ submit(second)
579
+ for block in block_iter:
580
+ submit(block)
581
+ while len(pending) > 1 and (
582
+ len(pending) >= self._prefetch
583
+ or (budget is not None and inflight > budget)
584
+ ):
585
+ yield from drain_one()
586
+ while pending:
587
+ yield from drain_one()
588
+ finally:
589
+ # Consumer may stop early (e.g. LIMIT): drop queued work
590
+ # without waiting for in-flight loads. The pool itself is
591
+ # shared across scans and stays up.
592
+ for future, _ in pending:
593
+ future.cancel()
594
+
595
+ return generate()
596
+
597
+ def get_fragments(
598
+ self, filter: pc.Expression | None = None
599
+ ) -> list["_XarrayFragment"]:
600
+ """One fragment per chunk of the source grid, pruned by ``filter``.
601
+
602
+ This is how DataFusion consumes the dataset
603
+ (``SessionContext.register_dataset`` plans one partition per
604
+ fragment and scans them in parallel), and enables the Dask
605
+ pattern ``from_map(lambda f: f.to_table().to_pandas(),
606
+ ds.get_fragments())``.
607
+ """
608
+ kept = None if filter is None else self._prune(filter)
609
+ return [_XarrayFragment(self, block) for block in self._blocks(kept)]
610
+
611
+ def count_rows(
612
+ self, filter: pc.Expression | None = None, **kwargs: Any
613
+ ) -> int:
614
+ """Count rows, reading as little data as possible.
615
+
616
+ Without a filter the count is pure chunk arithmetic — no I/O at
617
+ all. With a filter, chunks are split three ways: pruned chunks
618
+ contribute nothing, chunks whose coordinate ranges *prove* the
619
+ filter true contribute their exact size arithmetically, and only
620
+ the undecided boundary chunks are scanned (reading just the
621
+ columns the filter references).
622
+ """
623
+ if not self._ds.sizes:
624
+ return int(self.scanner(columns=[], filter=filter).count_rows())
625
+ if filter is None:
626
+ return int(np.prod([self._ds.sizes[d] for d in self._ds.dims]))
627
+ kept = self._prune(filter)
628
+ proven, boundary = self._strict_partition(kept, filter)
629
+ return proven + int(
630
+ self._scanner_for_blocks(boundary, [], filter).count_rows()
631
+ )
632
+
633
+ # Inherited convenience methods (to_table, head, to_batches, take)
634
+ # route through scanner() and keep working; the members below would
635
+ # touch the uninitialized native dataset.
636
+
637
+ @property
638
+ def partition_expression(self) -> pc.Expression:
639
+ # The dataset-level guarantee: trivially true. The base class
640
+ # getter reads native state this object does not have.
641
+ return pc.scalar(True)
642
+
643
+ def filter(self, expression: pc.Expression):
644
+ # A lazily-composed filter view is implementable; tracked in
645
+ # https://github.com/xqlsystems/xarray-sql/issues/239.
646
+ raise NotImplementedError(
647
+ "Use scanner(filter=...) or the engine's WHERE clause."
648
+ )
649
+
650
+ def replace_schema(self, schema: pa.Schema):
651
+ raise NotImplementedError
652
+
653
+ # Guarded delegation for sort_by/join/join_asof (engine-level SQL
654
+ # joins already work through scanner()) is tracked in
655
+ # https://github.com/xqlsystems/xarray-sql/issues/240.
656
+
657
+ def sort_by(self, sorting, **kwargs):
658
+ raise NotImplementedError
659
+
660
+ def join(self, *args, **kwargs):
661
+ raise NotImplementedError
662
+
663
+ def join_asof(self, *args, **kwargs):
664
+ raise NotImplementedError
665
+
666
+ def __reduce__(self):
667
+ raise TypeError("XarrayPushdownDataset is not picklable.")
668
+
669
+ # ------------------------------------------------------------------
670
+ # Projection: which columns must be read
671
+ # ------------------------------------------------------------------
672
+
673
+ def _scan_columns(
674
+ self, proj: list[str], filter: pc.Expression | None
675
+ ) -> list[str]:
676
+ """Columns to read: the projection plus any the filter references.
677
+
678
+ The consumer's column list need not include filter-only columns
679
+ (DuckDB drops pushed conjuncts from its plan and has no upstream
680
+ use for them). Rather than parsing the expression, probe it
681
+ against an empty table and grow the column set from the "no match
682
+ for field" errors until it evaluates; on anything unexpected fall
683
+ back to scanning every column, which is always correct.
684
+ """
685
+ if filter is None:
686
+ return proj
687
+ wanted = set(proj)
688
+ for _ in range(len(self._schema.names) + 1):
689
+ probe = pa.table(
690
+ {
691
+ n: pa.array([], type=self._schema.field(n).type)
692
+ for n in self._schema.names
693
+ if n in wanted
694
+ }
695
+ )
696
+ try:
697
+ probe.filter(filter)
698
+ except pa.lib.ArrowInvalid as exc:
699
+ match = re.search(r"FieldRef\.Name\((.*?)\)", str(exc))
700
+ name = match.group(1) if match else None
701
+ if (
702
+ name is not None
703
+ and name in set(self._schema.names) - wanted
704
+ ):
705
+ wanted.add(name)
706
+ continue
707
+ return list(self._schema.names)
708
+ else:
709
+ return [n for n in self._schema.names if n in wanted]
710
+ return list(self._schema.names)
711
+
712
+ # ------------------------------------------------------------------
713
+ # Pruning: which chunks can satisfy the predicate
714
+ # ------------------------------------------------------------------
715
+
716
+ def _dim_shadows(self) -> dict[str, _DimShadow]:
717
+ """One pruning index per prunable dimension, built lazily.
718
+
719
+ Keeping one shadow per dimension (Σ n_d fragments) instead of one
720
+ per chunk (Π n_d) is what keeps this cheap for finely partitioned
721
+ datasets, and is sound: a chunk is dropped only when the full
722
+ predicate is provably false given that single dimension's range.
723
+ """
724
+ if self._shadows is not None:
725
+ return self._shadows
726
+ shadows: dict[str, _DimShadow] = {}
727
+ for dim in self._resolved:
728
+ name = str(dim)
729
+ if name not in self._schema.names:
730
+ continue
731
+ coord = self._coord_arrays[name]
732
+ if coord.dtype.kind not in ("i", "u", "f", "M"):
733
+ continue # strings/objects/cftime: never prune this dim
734
+ try:
735
+ shadows[name] = _DimShadow(
736
+ name, self._schema, coord, self._chunk_bounds[dim]
737
+ )
738
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError, TypeError):
739
+ continue # conservative: no pruning on this dim
740
+ self._shadows = shadows
741
+ return shadows
742
+
743
+ def _prune(self, filter: pc.Expression) -> dict[str, list[int]]:
744
+ """Per-dimension chunk indices that can satisfy ``filter``.
745
+
746
+ Satisfiability is delegated to Arrow's guarantee simplification
747
+ (see ``_DimShadow``) — no expression decoding here, and
748
+ predicates on columns a shadow knows nothing about are
749
+ conservatively kept. Dimensions without a shadow, or where every
750
+ chunk survives, are absent from the result.
751
+ """
752
+ kept: dict[str, list[int]] = {}
753
+ for name, shadow in self._dim_shadows().items():
754
+ indices = shadow.kept(filter)
755
+ if indices is not None:
756
+ kept[name] = indices
757
+ return kept
758
+
759
+ # ------------------------------------------------------------------
760
+ # Strictness: which surviving chunks satisfy the filter entirely
761
+ # ------------------------------------------------------------------
762
+
763
+ def _chunk_spans(
764
+ self, name: str
765
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
766
+ """Cached vectorized per-chunk ``(lo, hi, poisoned)`` for a dim."""
767
+ cached = self._span_cache.get(name)
768
+ if cached is not None:
769
+ return cached
770
+ coord = self._coord_arrays[name]
771
+ starts = self._chunk_bounds[name][:-1]
772
+ lo = np.minimum.reduceat(coord, starts)
773
+ hi = np.maximum.reduceat(coord, starts)
774
+ if coord.dtype.kind == "f":
775
+ bad_values = np.isnan(coord)
776
+ elif coord.dtype.kind == "M":
777
+ bad_values = np.isnat(coord)
778
+ else:
779
+ bad_values = np.zeros(len(coord), dtype=bool)
780
+ bad = np.bitwise_or.reduceat(bad_values, starts)
781
+ self._span_cache[name] = (lo, hi, bad)
782
+ return self._span_cache[name]
783
+
784
+ def _strict_partition(
785
+ self,
786
+ kept: dict[str, list[int]] | None,
787
+ filter: pc.Expression,
788
+ ) -> tuple[int, list[Block] | Iterator[Block]]:
789
+ """``(rows proven to satisfy filter, boundary blocks to scan)``.
790
+
791
+ A cell of the surviving chunk grid with conjunctive coordinate
792
+ guarantee ``G`` satisfies ``filter`` everywhere iff
793
+ ``G ∧ ¬filter`` is unsatisfiable, and can be *dropped* entirely
794
+ iff ``G ∧ filter`` is — both decided by Arrow's guarantee
795
+ simplification. Cells are classified hierarchically: each level
796
+ buckets the surviving indices into at most
797
+ ``_STRICT_LEVEL_BUDGET`` products, proves/prunes whole buckets
798
+ at once (the prune side refines the per-dimension pruning with
799
+ cross-dimension information), and recurses only into mixed
800
+ cells — so million-chunk axes resolve in two or three levels.
801
+ Anything undecidable (NaN spans, non-numeric dims, expression
802
+ shapes the simplifier rejects) conservatively lands in the
803
+ boundary set, which the caller scans exactly.
804
+ """
805
+ dims = list(self._resolved.keys())
806
+ lists = {d: list(self._surviving(kept, d)) for d in dims}
807
+ if not dims or any(not v for v in lists.values()):
808
+ return 0, []
809
+ lens = {d: np.diff(self._chunk_bounds[d]) for d in dims}
810
+ outer = self._outer_rows()
811
+ usable = {
812
+ d: str(d) in self._schema.names
813
+ and self._coord_arrays[str(d)].dtype.kind in ("i", "u", "f", "M")
814
+ for d in dims
815
+ }
816
+ proven = 0
817
+ boundary: list[Block] = []
818
+
819
+ def bucket_guarantee(
820
+ d: Any, indices: np.ndarray
821
+ ) -> pc.Expression | None:
822
+ """Conjunctive [min, max] guarantee over one dimension's index
823
+ bucket, or None when unprovable (NaN spans, non-numeric dims)."""
824
+ if not usable[d]:
825
+ return None
826
+ lo, hi, bad = self._chunk_spans(str(d))
827
+ if bad[indices].any():
828
+ return None
829
+ field_type = self._schema.field(str(d)).type
830
+ return (
831
+ pc.field(str(d)) >= pa.scalar(lo[indices].min(), field_type)
832
+ ) & (pc.field(str(d)) <= pa.scalar(hi[indices].max(), field_type))
833
+
834
+ def rows_of(cell: dict) -> int:
835
+ """Rows spanned by ``cell``: the product of its chunks' lengths
836
+ per dimension, times the rows of dimensions outside the grid."""
837
+ rows = outer
838
+ for d in dims:
839
+ rows *= int(lens[d][np.asarray(cell[d])].sum())
840
+ return rows
841
+
842
+ def classify(cell: dict, depth: int) -> None:
843
+ """Prove, prune, or split one cell of the surviving chunk grid.
844
+
845
+ A cell is a hyper-rectangle of chunk indices per dimension.
846
+ Its indices are bucketed into at most
847
+ ``_STRICT_LEVEL_BUDGET`` products; Arrow's guarantee
848
+ simplification then decides each bucket-product wholesale:
849
+ proven (every row satisfies ``filter``, counted into
850
+ ``proven`` without reading), pruned (provably empty,
851
+ dropped), or mixed (recurse). Undecidable single-chunk
852
+ cells land in ``boundary`` for exact scanning.
853
+ """
854
+ nonlocal proven
855
+ ks = {d: len(cell[d]) for d in dims}
856
+ while int(np.prod(list(ks.values()))) > _STRICT_LEVEL_BUDGET:
857
+ widest = max(ks, key=lambda d: ks[d])
858
+ if ks[widest] == 1:
859
+ break
860
+ ks[widest] = max(1, ks[widest] // 2)
861
+ buckets = {
862
+ d: np.array_split(np.asarray(cell[d]), ks[d]) for d in dims
863
+ }
864
+ combos = list(itertools.product(*(range(ks[d]) for d in dims)))
865
+ guarantees: list[pc.Expression | None] = []
866
+ for combo in combos:
867
+ g: pc.Expression | None = None
868
+ complete = True
869
+ for d, b in zip(dims, combo):
870
+ gd = bucket_guarantee(d, buckets[d][b])
871
+ if gd is None:
872
+ if usable[d]:
873
+ complete = False # NaN span: never provable
874
+ continue
875
+ g = gd if g is None else g & gd
876
+ guarantees.append(g if (g is not None and complete) else None)
877
+ decidable = [i for i, g in enumerate(guarantees) if g is not None]
878
+ satisfiable = set(decidable)
879
+ unstrict = set(decidable)
880
+ if decidable:
881
+ shadow = _guarantee_shadow(
882
+ [(str(i), guarantees[i]) for i in decidable], self._schema
883
+ )
884
+ satisfiable = {
885
+ int(f.path) for f in shadow.get_fragments(filter=filter)
886
+ }
887
+ unstrict = {
888
+ int(f.path) for f in shadow.get_fragments(filter=~filter)
889
+ }
890
+ for i, combo in enumerate(combos):
891
+ subcell = {d: list(buckets[d][b]) for d, b in zip(dims, combo)}
892
+ if guarantees[i] is not None:
893
+ if i not in satisfiable:
894
+ continue # provably empty: cross-dim refinement
895
+ if i not in unstrict:
896
+ proven += rows_of(subcell)
897
+ continue
898
+ if all(len(v) == 1 for v in subcell.values()):
899
+ boundary.append(
900
+ self._block_for_combo(
901
+ tuple(v[0] for v in subcell.values())
902
+ )
903
+ )
904
+ elif depth < _STRICT_MAX_DEPTH:
905
+ classify(subcell, depth + 1)
906
+ else:
907
+ # Depth backstop; unreachable for realistic grids.
908
+ for c in itertools.product(*subcell.values()):
909
+ boundary.append(self._block_for_combo(c))
910
+
911
+ try:
912
+ classify(lists, 0)
913
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError, TypeError):
914
+ return 0, self._blocks(kept) # scan every survivor, exactly
915
+ return proven, boundary
916
+
917
+ def _block_rows(self, block: Block) -> int:
918
+ rows = 1
919
+ for d, sl in block.items():
920
+ size = self._ds.sizes[d]
921
+ start, stop, _ = sl.indices(size)
922
+ rows *= stop - start
923
+ return rows
924
+
925
+ # ------------------------------------------------------------------
926
+ # Scan: load surviving chunks, prefetching ahead of the consumer
927
+ # ------------------------------------------------------------------
928
+
929
+ def _surviving(
930
+ self, kept: dict[str, list[int]] | None, dim: Any
931
+ ) -> list[int] | range:
932
+ """One dim's surviving chunk indices; every chunk when unpruned."""
933
+ return (kept or {}).get(str(dim), range(len(self._resolved[dim])))
934
+
935
+ def _outer_rows(self) -> int:
936
+ """Rows contributed per grid cell by dims outside the chunk grid.
937
+
938
+ Unresolved dims span their full extent in every block, so they
939
+ multiply every block's row count uniformly.
940
+ """
941
+ rows = 1
942
+ for d in self._ds.dims:
943
+ if d not in self._resolved:
944
+ rows *= self._ds.sizes[d]
945
+ return rows
946
+
947
+ def _combos(
948
+ self, kept: dict[str, list[int]] | None
949
+ ) -> Iterator[tuple[int, ...]]:
950
+ """Surviving chunk-index combinations, in grid order."""
951
+ dims = list(self._resolved.keys())
952
+ if not dims:
953
+ return
954
+ yield from itertools.product(*(self._surviving(kept, d) for d in dims))
955
+
956
+ def _block_for_combo(self, combo: tuple[int, ...]) -> Block:
957
+ block: Block = {d: slice(None) for d in self._ds.dims}
958
+ for d, i in zip(self._resolved.keys(), combo):
959
+ bounds = self._chunk_bounds[d]
960
+ block[d] = slice(int(bounds[i]), int(bounds[i + 1]))
961
+ return block
962
+
963
+ def _blocks(self, kept: dict[str, list[int]] | None) -> Iterator[Block]:
964
+ """Yield isel-able block slices for the surviving chunk grid."""
965
+ if not self._resolved:
966
+ yield {}
967
+ return
968
+ for combo in self._combos(kept):
969
+ yield self._block_for_combo(combo)
970
+
971
+ def _coalesced_blocks(
972
+ self, kept: dict[str, list[int]] | None
973
+ ) -> Iterator[Block]:
974
+ """Blocks with runs of consecutive chunks merged along one dim.
975
+
976
+ Runs are merged along the most finely chunked dimension while
977
+ the merged block stays under ``coalesce_rows`` rows. One merged
978
+ block is one ``isel`` — on Zarr sources its member chunks are
979
+ fetched by the store's own concurrent batch read instead of one
980
+ request per chunk through the prefetch pool.
981
+ """
982
+ if not self._resolved:
983
+ yield {}
984
+ return
985
+ dims = list(self._resolved.keys())
986
+ merge_dim = max(dims, key=lambda d: len(self._resolved[d]))
987
+ others = [d for d in dims if d != merge_dim]
988
+ ranges = {d: list(self._surviving(kept, d)) for d in dims}
989
+ merge_bounds = self._chunk_bounds[merge_dim]
990
+ outer_rows = self._outer_rows()
991
+
992
+ def flush(prefix: tuple[int, ...], run: list[int]) -> Block:
993
+ block: Block = {d: slice(None) for d in self._ds.dims}
994
+ for d, i in zip(others, prefix):
995
+ bounds = self._chunk_bounds[d]
996
+ block[d] = slice(int(bounds[i]), int(bounds[i + 1]))
997
+ block[merge_dim] = slice(
998
+ int(merge_bounds[run[0]]), int(merge_bounds[run[-1] + 1])
999
+ )
1000
+ return block
1001
+
1002
+ coalesce_rows = self._coalesce_rows
1003
+ assert coalesce_rows is not None # only reached with coalescing on
1004
+ for prefix in itertools.product(*(ranges[d] for d in others)):
1005
+ per_row = outer_rows
1006
+ for d, i in zip(others, prefix):
1007
+ bounds = self._chunk_bounds[d]
1008
+ per_row *= int(bounds[i + 1] - bounds[i])
1009
+ run: list[int] = []
1010
+ run_rows = 0
1011
+ for i in ranges[merge_dim]:
1012
+ rows = int(merge_bounds[i + 1] - merge_bounds[i]) * per_row
1013
+ if run and (
1014
+ i != run[-1] + 1 or run_rows + rows > coalesce_rows
1015
+ ):
1016
+ yield flush(prefix, run)
1017
+ run, run_rows = [], 0
1018
+ run.append(i)
1019
+ run_rows += rows
1020
+ if run:
1021
+ yield flush(prefix, run)
1022
+
1023
+
1024
+ class _XarrayFragment:
1025
+ """One chunk of the source grid, presented as a dataset fragment.
1026
+
1027
+ Fragment consumers (DataFusion's ``DatasetExec`` plans one partition
1028
+ per fragment; Dask maps over them) call [scanner][xarray_sql.backends.pyarrow.XarrayPushdownDataset.scanner] with the
1029
+ columns and predicate for this piece; the pushed filter is applied
1030
+ row-exactly, same as the parent dataset's scanner.
1031
+ """
1032
+
1033
+ def __init__(self, dataset: XarrayPushdownDataset, block: Block):
1034
+ self._dataset = dataset
1035
+ self._block = block
1036
+
1037
+ @property
1038
+ def physical_schema(self) -> pa.Schema:
1039
+ return self._dataset.schema
1040
+
1041
+ def scanner(
1042
+ self,
1043
+ schema: pa.Schema | None = None,
1044
+ columns: list[str] | None = None,
1045
+ filter: pc.Expression | None = None,
1046
+ batch_size: int | None = None,
1047
+ **kwargs: Any,
1048
+ ) -> pads.Scanner:
1049
+ return self._dataset._scanner_for_blocks(
1050
+ [self._block], columns, filter, batch_size
1051
+ )
1052
+
1053
+ def to_batches(self, **kwargs: Any) -> Iterator[pa.RecordBatch]:
1054
+ return iter(self.scanner(**kwargs).to_batches())
1055
+
1056
+ def to_table(self, **kwargs: Any) -> pa.Table:
1057
+ return self.scanner(**kwargs).to_table()
1058
+
1059
+ def count_rows(self, **kwargs: Any) -> int:
1060
+ return int(self.scanner(**kwargs).count_rows())
1061
+
1062
+ def __dask_tokenize__(self) -> tuple:
1063
+ # Dask hashes from_map inputs; the parent dataset is unpicklable,
1064
+ # so provide a deterministic token from the fragment's identity.
1065
+ return (
1066
+ "xarray_sql._XarrayFragment",
1067
+ repr(self._block),
1068
+ self._dataset.schema.to_string(),
1069
+ )
1070
+
1071
+ def __repr__(self) -> str:
1072
+ return f"_XarrayFragment({self._block!r})"
1073
+
1074
+
1075
+ def arrow_dataset(
1076
+ ds: xr.Dataset,
1077
+ chunks: Chunks = None,
1078
+ *,
1079
+ batch_size: int = DEFAULT_BATCH_SIZE,
1080
+ prefetch: int = DEFAULT_PREFETCH,
1081
+ prefetch_bytes: int | None = None,
1082
+ coalesce_rows: int | None = None,
1083
+ geometry: tuple[str, str] | None = None,
1084
+ geometry_encoding: str = "wkb",
1085
+ geometry_crs: str | None = "OGC:CRS84",
1086
+ ) -> XarrayPushdownDataset:
1087
+ """A pushdown-capable ``pyarrow.dataset.Dataset`` view of ``ds``.
1088
+
1089
+ The returned object works anywhere a pyarrow dataset does, keeping
1090
+ projection pushdown and coordinate-range chunk pruning::
1091
+
1092
+ import polars as pl
1093
+ lf = pl.scan_pyarrow_dataset(xql.arrow_dataset(ds))
1094
+
1095
+ import duckdb
1096
+ duckdb.connect().register("t", xql.arrow_dataset(ds))
1097
+
1098
+ xql.arrow_dataset(ds).to_table(columns=["t2m"], filter=...)
1099
+
1100
+ Args:
1101
+ ds: An xarray Dataset. All data variables must share the same
1102
+ dimensions (select a variable subset first otherwise).
1103
+ chunks: Xarray-like chunks specification controlling partition
1104
+ granularity. Defaults to the Dataset's existing chunks, or for
1105
+ an unchunked Dataset its storage chunks.
1106
+ batch_size: Maximum rows per emitted Arrow RecordBatch.
1107
+ prefetch: Chunk loads kept in flight ahead of the consumer
1108
+ (memory scales with ``prefetch`` x pivoted chunk size).
1109
+ prefetch_bytes: Optional cap on *estimated pivoted bytes* in
1110
+ flight; admission then tracks bytes rather than block count,
1111
+ which keeps peak memory steady when ``coalesce_rows`` makes
1112
+ blocks large or uneven. ``prefetch`` still bounds
1113
+ concurrency (thread count).
1114
+ coalesce_rows: When set, merge runs of consecutive surviving
1115
+ chunks along the most finely chunked dimension into single
1116
+ reads of at most this many rows. Fewer, larger source
1117
+ requests — the win on remote stores, where each merged read
1118
+ fetches its member chunks through the store's own concurrent
1119
+ batching. Memory scales with ``prefetch`` x the *merged*
1120
+ block size, so size accordingly (e.g. ``8_000_000``).
1121
+ geometry: ``(x_dim, y_dim)`` coordinate dims to derive a
1122
+ ``geometry`` point column from (see
1123
+ [xarray_sql.geometry][]). With the default ``"wkb"``
1124
+ encoding, DuckDB (spatial loaded) sees a native ``GEOMETRY``
1125
+ column, so ``ST_Within(geometry, ...)`` works directly.
1126
+ geometry_encoding: ``"wkb"`` (default; DuckDB-consumable) or
1127
+ ``"point"`` (GeoArrow native separated coordinates — the
1128
+ struct children are the coordinate arrays; for GeoPandas,
1129
+ lonboard, geoarrow-rs consumers).
1130
+ geometry_crs: CRS tag carried in the extension metadata.
1131
+ Defaults to ``OGC:CRS84`` (plain longitude/latitude); pass
1132
+ ``None`` to omit, or an authority code / PROJJSON string.
1133
+
1134
+ Returns:
1135
+ An [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset].
1136
+ """
1137
+ return XarrayPushdownDataset(
1138
+ ds,
1139
+ chunks,
1140
+ batch_size=batch_size,
1141
+ prefetch=prefetch,
1142
+ prefetch_bytes=prefetch_bytes,
1143
+ coalesce_rows=coalesce_rows,
1144
+ geometry=geometry,
1145
+ geometry_encoding=geometry_encoding,
1146
+ geometry_crs=geometry_crs,
1147
+ )
1148
+
1149
+
1150
+ def arrow_datasets(
1151
+ ds: xr.Dataset,
1152
+ name: str | None = None,
1153
+ *,
1154
+ chunks: Chunks = None,
1155
+ table_names: TableNames = None,
1156
+ **kwargs: Any,
1157
+ ) -> dict[str, XarrayPushdownDataset]:
1158
+ """Named pushdown datasets, one per dimension group of ``ds``.
1159
+
1160
+ [arrow_dataset][xarray_sql.backends.pyarrow.arrow_dataset] needs a
1161
+ Dataset whose data variables all share one set of dimensions. This
1162
+ applies the same split the DataFusion and DuckDB adapters do — one
1163
+ table per dimension group — and hands back the tables *named*, ready
1164
+ to register on an engine that has no connection object to dispatch
1165
+ on::
1166
+
1167
+ tables = xql.arrow_datasets(ds, 'era5', table_names={
1168
+ ('time', 'latitude', 'longitude'): 'surface',
1169
+ ('time', 'level', 'latitude', 'longitude'): 'atmosphere',
1170
+ })
1171
+
1172
+ ctx = pl.SQLContext()
1173
+ for table, dataset in tables.items(): # 'era5_surface', ...
1174
+ ctx.register(table, pl.scan_pyarrow_dataset(dataset))
1175
+
1176
+ ctx.execute('SELECT AVG("2m_temperature") FROM era5_surface')
1177
+
1178
+ Dimension coordinates are read once and shared across the returned
1179
+ tables, which is a network round-trip saved per dimension per group
1180
+ on Zarr-backed stores.
1181
+
1182
+ Args:
1183
+ ds: An xarray Dataset, with variables on any mix of dimensions.
1184
+ name: Prefix for the table names, as in ``era5_surface``. Omit it
1185
+ to get the bare group names (``surface``).
1186
+ chunks: Xarray-like chunks specification controlling partition
1187
+ granularity. Keys naming dimensions a group does not have are
1188
+ ignored for that group. Defaults to the Dataset's existing
1189
+ chunks, or for an unchunked Dataset its storage chunks.
1190
+ table_names: Maps a dimension group's exact dim tuple to the name
1191
+ its table takes, e.g.
1192
+ ``{('time', 'latitude', 'longitude'): 'surface'}``. Groups
1193
+ left unnamed take their dimensions joined by underscores
1194
+ (``time_latitude_longitude``); the group holding scalar
1195
+ variables, if any, takes ``scalar``.
1196
+ **kwargs: Forwarded to
1197
+ [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset]
1198
+ (``batch_size``, ``prefetch``, ...), applied to every
1199
+ returned table. ``geometry=(x_dim, y_dim)`` only reaches the
1200
+ groups whose dimensions include both — a mixed-dimension
1201
+ Dataset otherwise has groups the geometry dims are not
1202
+ columns of; those groups come back without a geometry
1203
+ column rather than raising.
1204
+
1205
+ Returns:
1206
+ Table name to
1207
+ [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset].
1208
+ A Dataset with a single dimension group yields one entry, keyed
1209
+ by ``name`` when given.
1210
+ """
1211
+ groups = group_vars_by_dims(ds)
1212
+ names = resolve_table_names(ds, table_names)
1213
+
1214
+ def table_name(dims: tuple[str, ...]) -> str:
1215
+ group = names[dims]
1216
+ return f"{name}_{group}" if name else group
1217
+
1218
+ if len(groups) <= 1:
1219
+ # One group is one table, named `name` — there is no group to
1220
+ # tell apart. Without a `name`, it takes the group's own. Built
1221
+ # directly (not via arrow_dataset(), whose signature is a fixed
1222
+ # subset) so every kwarg XarrayPushdownDataset accepts works
1223
+ # here exactly as it does in the multi-group branch below.
1224
+ only = name if name else next(iter(names.values()), "scalar")
1225
+ return {only: XarrayPushdownDataset(ds, chunks, **kwargs)}
1226
+
1227
+ geometry = kwargs.pop("geometry", None)
1228
+ coord_arrays = shared_coord_arrays(ds)
1229
+ tables = {}
1230
+ for dims, var_names in groups.items():
1231
+ group_kwargs = dict(kwargs)
1232
+ if geometry is not None:
1233
+ group_kwargs["geometry"] = (
1234
+ geometry if set(geometry).issubset(dims) else None
1235
+ )
1236
+ tables[table_name(dims)] = XarrayPushdownDataset(
1237
+ ds[var_names], chunks, coord_arrays=coord_arrays, **group_kwargs
1238
+ )
1239
+ return tables