xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
xarray_sql/proj.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""PROJ-backed CRS transforms for SQL — the optional geo extension.
|
|
2
|
+
|
|
3
|
+
Geospatial SQL dialects expose coordinate reference system (CRS)
|
|
4
|
+
transforms as a scalar function — PostGIS and DuckDB-spatial both call it
|
|
5
|
+
``ST_Transform`` — because a CRS transform is row-independent: each
|
|
6
|
+
point's new coordinate depends only on its own old coordinate. This
|
|
7
|
+
module brings the same capability to xarray-sql as a vectorized scalar
|
|
8
|
+
UDF over Arrow arrays::
|
|
9
|
+
|
|
10
|
+
SELECT x, y,
|
|
11
|
+
reproject(x, y, 'EPSG:32610', 'EPSG:4326')['x'] AS lon,
|
|
12
|
+
reproject(x, y, 'EPSG:32610', 'EPSG:4326')['y'] AS lat
|
|
13
|
+
FROM grid
|
|
14
|
+
|
|
15
|
+
The CRS pair is part of the *query*, not baked in at registration time,
|
|
16
|
+
so one registered UDF serves any transform — and, because the arguments
|
|
17
|
+
are ordinary SQL expressions, the CRS may even vary per row (e.g. a
|
|
18
|
+
``CASE`` expression selecting the UTM zone from the longitude).
|
|
19
|
+
|
|
20
|
+
Design notes:
|
|
21
|
+
|
|
22
|
+
* **Both output coordinates come from one call**, returned as an Arrow
|
|
23
|
+
struct ``{x, y}`` (in ``always_xy`` order: easting/longitude first).
|
|
24
|
+
Splitting the transform into two scalar UDFs would run PROJ twice per
|
|
25
|
+
row and, worse, evaluate the two projections concurrently on separate
|
|
26
|
+
expression trees.
|
|
27
|
+
* **All pyproj work runs on a dedicated pool of Python threads.**
|
|
28
|
+
DataFusion's runtime workers are not Python-created threads, and
|
|
29
|
+
pyproj (< 3.8, see pyproj#1541) leaves a dangling ``PJ_CONTEXT``
|
|
30
|
+
behind when their ephemeral Python thread states are torn down, so
|
|
31
|
+
calling pyproj in place segfaults — the UDF hands each batch to the
|
|
32
|
+
pool instead. Pool threads are long-lived, so each caches one
|
|
33
|
+
transformer per CRS pair (transformers must not be shared across
|
|
34
|
+
threads), amortizing the expensive construction — PROJ database
|
|
35
|
+
lookups and candidate-operation selection — across record batches.
|
|
36
|
+
Concurrent partitions still transform in parallel across the pool.
|
|
37
|
+
* Any CRS spelling ``pyproj.CRS`` accepts works: authority codes
|
|
38
|
+
(``EPSG:4326``), WKT, PROJ strings (``+proj=utm +zone=10``), etc.
|
|
39
|
+
An unknown CRS raises ``pyproj.exceptions.CRSError`` and fails the
|
|
40
|
+
query loudly rather than returning wrong coordinates.
|
|
41
|
+
* Non-finite or NULL input coordinates yield NaN output (PROJ itself
|
|
42
|
+
would return ``inf``); NULL CRS arguments yield NaN as well.
|
|
43
|
+
|
|
44
|
+
Requires ``pyproj`` (``pip install xarray-sql[geo]``). When pyproj is
|
|
45
|
+
installed, [xarray_sql.Context][] registers ``reproject()``
|
|
46
|
+
automatically; [register][xarray_sql.proj.register] is the explicit hook for plain
|
|
47
|
+
DataFusion ``SessionContext`` objects or custom UDF names.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
from __future__ import annotations
|
|
51
|
+
|
|
52
|
+
import os
|
|
53
|
+
import threading
|
|
54
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
55
|
+
|
|
56
|
+
import numpy as np
|
|
57
|
+
import pyarrow as pa
|
|
58
|
+
import pyarrow.compute as pc
|
|
59
|
+
import pyproj
|
|
60
|
+
from datafusion import udf
|
|
61
|
+
|
|
62
|
+
__all__ = ["register"]
|
|
63
|
+
|
|
64
|
+
RETURN_TYPE = pa.struct([("x", pa.float64()), ("y", pa.float64())])
|
|
65
|
+
"""Arrow type returned by ``reproject()``: destination coordinates in
|
|
66
|
+
``always_xy`` order — ``x`` is easting/longitude, ``y`` is
|
|
67
|
+
northing/latitude."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# ---------------------------------------------------------------------------
|
|
71
|
+
# The PROJ worker pool
|
|
72
|
+
# ---------------------------------------------------------------------------
|
|
73
|
+
#
|
|
74
|
+
# DataFusion evaluates UDFs on its tokio runtime's worker threads, which
|
|
75
|
+
# are not created by Python: a Python thread state is created and
|
|
76
|
+
# destroyed around every UDF call. pyproj keeps its per-thread PJ_CONTEXT
|
|
77
|
+
# in CPython thread-specific storage but does not clear that pointer when
|
|
78
|
+
# the context dies with the ephemeral thread state (fixed by pyproj#1541,
|
|
79
|
+
# unreleased as of 3.7.2), so the next call on the same OS thread
|
|
80
|
+
# dereferences a dangling context and segfaults inside ``proj_create``.
|
|
81
|
+
# Python-owned threads keep their thread state — and thus their contexts —
|
|
82
|
+
# alive for the thread's lifetime, so the UDF never calls pyproj in place:
|
|
83
|
+
# every batch is handed to a small pool of Python-owned worker threads.
|
|
84
|
+
# The pool stays worthwhile on fixed pyproj too: ephemeral thread states
|
|
85
|
+
# would rebuild context and transformer per batch (0.07–12 ms measured)
|
|
86
|
+
# versus ~10 µs for the pool round-trip. pyproj releases the GIL during
|
|
87
|
+
# the transform loop, so concurrent partitions still run in parallel
|
|
88
|
+
# across the pool.
|
|
89
|
+
|
|
90
|
+
_local = threading.local()
|
|
91
|
+
_pool_lock = threading.Lock()
|
|
92
|
+
_pool: ThreadPoolExecutor | None = None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _proj_pool() -> ThreadPoolExecutor:
|
|
96
|
+
"""Return the process-wide pool that runs all pyproj work."""
|
|
97
|
+
global _pool
|
|
98
|
+
if _pool is None:
|
|
99
|
+
with _pool_lock:
|
|
100
|
+
if _pool is None:
|
|
101
|
+
_pool = ThreadPoolExecutor(
|
|
102
|
+
max_workers=os.cpu_count() or 4,
|
|
103
|
+
thread_name_prefix="xarray-sql-proj",
|
|
104
|
+
)
|
|
105
|
+
return _pool
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _transformer(src_crs: str, dst_crs: str) -> pyproj.Transformer:
|
|
109
|
+
"""Return a cached ``Transformer`` owned by the calling pool thread.
|
|
110
|
+
|
|
111
|
+
PROJ transformers are not safe to share across threads, so each pool
|
|
112
|
+
thread keeps its own transformer per ``(src, dst)`` pair; the cache
|
|
113
|
+
also amortizes construction (expensive PROJ database lookups) across
|
|
114
|
+
record batches. ``always_xy=True`` fixes the argument order to
|
|
115
|
+
(easting/longitude, northing/latitude) regardless of the CRS's
|
|
116
|
+
declared axis order.
|
|
117
|
+
"""
|
|
118
|
+
cache = getattr(_local, "transformers", None)
|
|
119
|
+
if cache is None:
|
|
120
|
+
cache = _local.transformers = {}
|
|
121
|
+
key = (src_crs, dst_crs)
|
|
122
|
+
transformer = cache.get(key)
|
|
123
|
+
if transformer is None:
|
|
124
|
+
transformer = cache[key] = pyproj.Transformer.from_crs(
|
|
125
|
+
src_crs, dst_crs, always_xy=True
|
|
126
|
+
)
|
|
127
|
+
return transformer
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _transform_chunk(
|
|
131
|
+
src_crs: str, dst_crs: str, xs: np.ndarray, ys: np.ndarray
|
|
132
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
133
|
+
"""Transform one coordinate chunk; runs on a PROJ pool thread."""
|
|
134
|
+
tx, ty = _transformer(src_crs, dst_crs).transform(xs, ys)
|
|
135
|
+
return tx, ty
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
# The UDF
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _reproject(
|
|
144
|
+
x: pa.Array, y: pa.Array, src_crs: pa.Array, dst_crs: pa.Array
|
|
145
|
+
) -> pa.Array:
|
|
146
|
+
"""Vectorized ``reproject`` kernel over one Arrow record batch.
|
|
147
|
+
|
|
148
|
+
DataFusion broadcasts scalar arguments (the usual literal CRS
|
|
149
|
+
strings) to full-length arrays before calling in, so all four
|
|
150
|
+
arguments arrive with one value per row. The common case — one CRS
|
|
151
|
+
pair for the whole batch — never touches the strings row by row:
|
|
152
|
+
uniqueness is established with a vectorized Arrow kernel and the
|
|
153
|
+
batch becomes a single PROJ call. (Materializing the CRS columns
|
|
154
|
+
as Python strings costs two object allocations per row, which at
|
|
155
|
+
billions of rows dwarfs the transform itself.) Only when the CRS
|
|
156
|
+
genuinely varies within the batch are rows grouped by pair and
|
|
157
|
+
transformed per group.
|
|
158
|
+
"""
|
|
159
|
+
# Zero-copy read-only views when the batch has no nulls; with nulls,
|
|
160
|
+
# pyarrow must materialize the validity bitmap as NaN (the NULL -> NaN
|
|
161
|
+
# contract). pyproj copies into its own writable buffer either way --
|
|
162
|
+
# PROJ mutates buffers in place -- so this is the minimal-copy path.
|
|
163
|
+
xs = np.asarray(x.to_numpy(zero_copy_only=False), dtype="float64")
|
|
164
|
+
ys = np.asarray(y.to_numpy(zero_copy_only=False), dtype="float64")
|
|
165
|
+
|
|
166
|
+
out_x = np.full(xs.shape, np.nan)
|
|
167
|
+
out_y = np.full(ys.shape, np.nan)
|
|
168
|
+
valid = np.isfinite(xs) & np.isfinite(ys)
|
|
169
|
+
src_unique = pc.unique(src_crs)
|
|
170
|
+
dst_unique = pc.unique(dst_crs)
|
|
171
|
+
|
|
172
|
+
if len(src_unique) == 1 and len(dst_unique) == 1:
|
|
173
|
+
groups = [(src_unique[0].as_py(), dst_unique[0].as_py(), valid)]
|
|
174
|
+
else:
|
|
175
|
+
pairs = list(zip(src_crs.to_pylist(), dst_crs.to_pylist()))
|
|
176
|
+
groups = [
|
|
177
|
+
(
|
|
178
|
+
src,
|
|
179
|
+
dst,
|
|
180
|
+
valid
|
|
181
|
+
& np.fromiter(
|
|
182
|
+
(p == (src, dst) for p in pairs),
|
|
183
|
+
dtype=bool,
|
|
184
|
+
count=len(pairs),
|
|
185
|
+
),
|
|
186
|
+
)
|
|
187
|
+
for src, dst in set(pairs)
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
for src, dst, mask in groups:
|
|
191
|
+
if src is None or dst is None or not mask.any():
|
|
192
|
+
continue
|
|
193
|
+
tx, ty = (
|
|
194
|
+
_proj_pool()
|
|
195
|
+
.submit(_transform_chunk, src, dst, xs[mask], ys[mask])
|
|
196
|
+
.result()
|
|
197
|
+
)
|
|
198
|
+
out_x[mask] = tx
|
|
199
|
+
out_y[mask] = ty
|
|
200
|
+
|
|
201
|
+
# PROJ signals out-of-domain points with inf; normalize to NaN so
|
|
202
|
+
# the result round-trips to xarray like any other missing value.
|
|
203
|
+
invalid = ~(np.isfinite(out_x) & np.isfinite(out_y))
|
|
204
|
+
out_x[invalid] = np.nan
|
|
205
|
+
out_y[invalid] = np.nan
|
|
206
|
+
|
|
207
|
+
return pa.StructArray.from_arrays(
|
|
208
|
+
[pa.array(out_x), pa.array(out_y)], names=["x", "y"]
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def register(ctx, name: str = "reproject") -> None:
|
|
213
|
+
"""Register the ``reproject(x, y, src_crs, dst_crs)`` scalar UDF.
|
|
214
|
+
|
|
215
|
+
Works on any DataFusion ``SessionContext`` (``Context``
|
|
216
|
+
registers it automatically when pyproj is installed). The UDF
|
|
217
|
+
returns a ``{x, y}`` struct of destination coordinates, so a query
|
|
218
|
+
selects components with subscripts::
|
|
219
|
+
|
|
220
|
+
SELECT reproject(x, y, 'EPSG:32610', 'EPSG:4326')['x'] AS lon
|
|
221
|
+
FROM grid
|
|
222
|
+
|
|
223
|
+
Args:
|
|
224
|
+
ctx: The DataFusion session context to register the UDF on.
|
|
225
|
+
name: SQL name for the function (default ``"reproject"``).
|
|
226
|
+
"""
|
|
227
|
+
ctx.register_udf(
|
|
228
|
+
udf(
|
|
229
|
+
_reproject,
|
|
230
|
+
[pa.float64(), pa.float64(), pa.utf8(), pa.utf8()],
|
|
231
|
+
RETURN_TYPE,
|
|
232
|
+
"immutable",
|
|
233
|
+
name,
|
|
234
|
+
)
|
|
235
|
+
)
|
xarray_sql/reader.py
ADDED
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""Lazy Arrow stream reader for xarray Datasets.
|
|
2
|
+
|
|
3
|
+
This module provides XarrayRecordBatchReader, which implements the Arrow
|
|
4
|
+
PyCapsule Interface (__arrow_c_stream__) to enable zero-copy, lazy streaming
|
|
5
|
+
of xarray data to DataFusion and other Arrow consumers.
|
|
6
|
+
|
|
7
|
+
The implementation delegates to PyArrow's RecordBatchReader for the
|
|
8
|
+
actual stream implementation, wrapping xarray block iteration in a generator.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Callable, Iterator
|
|
14
|
+
from typing import TYPE_CHECKING
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pyarrow as pa
|
|
18
|
+
import xarray as xr
|
|
19
|
+
|
|
20
|
+
from .df import (
|
|
21
|
+
Block,
|
|
22
|
+
Chunks,
|
|
23
|
+
DEFAULT_BATCH_SIZE,
|
|
24
|
+
_block_len,
|
|
25
|
+
_block_metadata,
|
|
26
|
+
_block_slices_from_resolved,
|
|
27
|
+
_ensure_default_indexes,
|
|
28
|
+
_parse_schema,
|
|
29
|
+
block_slices,
|
|
30
|
+
iter_record_batches,
|
|
31
|
+
resolve_chunks,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from ._native import LazyArrowStreamTable
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class XarrayRecordBatchReader:
|
|
39
|
+
"""A lazy Arrow stream reader for xarray Datasets.
|
|
40
|
+
|
|
41
|
+
Implements the Arrow PyCapsule Interface (__arrow_c_stream__) to enable
|
|
42
|
+
zero-copy, lazy streaming of xarray data to DataFusion and other Arrow
|
|
43
|
+
consumers.
|
|
44
|
+
|
|
45
|
+
The key property is that xarray blocks are only converted to Arrow
|
|
46
|
+
RecordBatches when the consumer calls get_next (e.g., during DataFusion's
|
|
47
|
+
collect()), NOT when the reader is created or registered.
|
|
48
|
+
|
|
49
|
+
Attributes:
|
|
50
|
+
schema: The Arrow schema for the stream.
|
|
51
|
+
|
|
52
|
+
Example:
|
|
53
|
+
>>> import xarray as xr
|
|
54
|
+
>>> from xarray_sql import XarrayRecordBatchReader
|
|
55
|
+
>>> ds = xr.tutorial.open_dataset('air_temperature')
|
|
56
|
+
>>> reader = XarrayRecordBatchReader(ds, chunks={'time': 240})
|
|
57
|
+
>>> # At this point, NO data has been read from xarray
|
|
58
|
+
>>> # Data is only read when consumed:
|
|
59
|
+
>>> import pyarrow as pa
|
|
60
|
+
>>> pa_reader = pa.RecordBatchReader.from_stream(reader)
|
|
61
|
+
>>> for batch in pa_reader:
|
|
62
|
+
... print(batch.num_rows) # Data read here
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __init__(
|
|
66
|
+
self,
|
|
67
|
+
ds: xr.Dataset,
|
|
68
|
+
chunks: Chunks = None,
|
|
69
|
+
*,
|
|
70
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
71
|
+
_iteration_callback: (
|
|
72
|
+
Callable[[Block, list[str] | None], None] | None
|
|
73
|
+
) = None,
|
|
74
|
+
):
|
|
75
|
+
"""Initialize the lazy reader.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
ds: An xarray Dataset. All data_vars must share the same dimensions.
|
|
79
|
+
chunks: Xarray-like chunks specification. If not provided, uses
|
|
80
|
+
the Dataset's existing chunks, or for an unchunked Dataset
|
|
81
|
+
its storage chunks.
|
|
82
|
+
batch_size: Maximum rows per emitted Arrow RecordBatch. Smaller
|
|
83
|
+
values let DataFusion start processing earlier at the cost of
|
|
84
|
+
more Python→Arrow conversion calls.
|
|
85
|
+
_iteration_callback: Internal callback for testing. Called with
|
|
86
|
+
each block dict just before it's converted to Arrow. This
|
|
87
|
+
allows tests to track when iteration actually occurs.
|
|
88
|
+
"""
|
|
89
|
+
self._ds = ds
|
|
90
|
+
self._chunks = chunks
|
|
91
|
+
self._batch_size = batch_size
|
|
92
|
+
self._schema = _parse_schema(ds)
|
|
93
|
+
self._iteration_callback = _iteration_callback
|
|
94
|
+
self._consumed = False
|
|
95
|
+
|
|
96
|
+
# Validate dimensions
|
|
97
|
+
fst = next(iter(ds.values())).dims
|
|
98
|
+
if not all(da.dims == fst for da in ds.values()):
|
|
99
|
+
raise ValueError(
|
|
100
|
+
"All dimensions must be equal. Please filter data_vars in the Dataset."
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def schema(self) -> pa.Schema:
|
|
105
|
+
"""The Arrow schema for this stream."""
|
|
106
|
+
return self._schema
|
|
107
|
+
|
|
108
|
+
def _generate_batches(self) -> Iterator[pa.RecordBatch]:
|
|
109
|
+
"""Generate RecordBatches lazily from xarray blocks.
|
|
110
|
+
|
|
111
|
+
This generator is only consumed when the Arrow stream's get_next
|
|
112
|
+
is called, ensuring true lazy evaluation. Each xarray block is
|
|
113
|
+
emitted as one or more RecordBatches of at most self._batch_size rows.
|
|
114
|
+
"""
|
|
115
|
+
for block in block_slices(self._ds, self._chunks):
|
|
116
|
+
# Call the iteration callback if provided (for testing).
|
|
117
|
+
# XarrayRecordBatchReader has no projection concept, so always passes None.
|
|
118
|
+
if self._iteration_callback is not None:
|
|
119
|
+
self._iteration_callback(block, None)
|
|
120
|
+
|
|
121
|
+
yield from iter_record_batches(
|
|
122
|
+
self._ds.isel(block), self._schema, self._batch_size
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
def __arrow_c_stream__(
|
|
126
|
+
self, requested_schema: object | None = None
|
|
127
|
+
) -> object:
|
|
128
|
+
"""Export as Arrow C Stream via PyCapsule.
|
|
129
|
+
|
|
130
|
+
This method is called by Arrow consumers (like DataFusion) to get
|
|
131
|
+
a C-level stream interface. The actual data iteration only begins
|
|
132
|
+
when the consumer calls get_next on the stream.
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
requested_schema: Optional schema for type casting. Currently
|
|
136
|
+
passed through to PyArrow's implementation.
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
PyCapsule containing ArrowArrayStream pointer with name
|
|
140
|
+
"arrow_array_stream".
|
|
141
|
+
|
|
142
|
+
Raises:
|
|
143
|
+
RuntimeError: If the stream has already been consumed.
|
|
144
|
+
"""
|
|
145
|
+
if self._consumed:
|
|
146
|
+
raise RuntimeError(
|
|
147
|
+
"Stream already consumed. XarrayRecordBatchReader can only "
|
|
148
|
+
"be iterated once. Create a new reader for additional iterations."
|
|
149
|
+
)
|
|
150
|
+
self._consumed = True
|
|
151
|
+
|
|
152
|
+
# Create a PyArrow RecordBatchReader from our generator
|
|
153
|
+
# The generator is NOT consumed here - only when get_next is called
|
|
154
|
+
reader = pa.RecordBatchReader.from_batches(
|
|
155
|
+
self._schema, self._generate_batches()
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# Delegate to PyArrow's __arrow_c_stream__ implementation
|
|
159
|
+
return reader.__arrow_c_stream__(requested_schema)
|
|
160
|
+
|
|
161
|
+
def __arrow_c_schema__(
|
|
162
|
+
self, requested_schema: object | None = None
|
|
163
|
+
) -> object:
|
|
164
|
+
"""Export the schema as Arrow C Schema via PyCapsule.
|
|
165
|
+
|
|
166
|
+
This allows consumers to inspect the schema without consuming the stream.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
requested_schema: Optional schema for negotiation (unused).
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
PyCapsule containing ArrowSchema pointer.
|
|
173
|
+
"""
|
|
174
|
+
return self._schema.__arrow_c_schema__()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
|
|
178
|
+
"""Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
ds: An Xarray Dataset. All `data_vars` must share the same dimensions.
|
|
182
|
+
chunks: Xarray-like chunks. If not provided, will default to the
|
|
183
|
+
Dataset's chunks, or for an unchunked Dataset its storage chunks.
|
|
184
|
+
The product of the chunk sizes becomes the
|
|
185
|
+
standard length of each dataframe partition.
|
|
186
|
+
|
|
187
|
+
Returns:
|
|
188
|
+
A PyArrow RecordBatchReader, which is a table representation of the input
|
|
189
|
+
Dataset.
|
|
190
|
+
"""
|
|
191
|
+
ds = _ensure_default_indexes(ds)
|
|
192
|
+
reader = XarrayRecordBatchReader(ds, chunks=chunks)
|
|
193
|
+
return pa.RecordBatchReader.from_stream(reader)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def read_xarray_table(
|
|
197
|
+
ds: xr.Dataset,
|
|
198
|
+
chunks: Chunks = None,
|
|
199
|
+
*,
|
|
200
|
+
batch_size: int = DEFAULT_BATCH_SIZE,
|
|
201
|
+
coord_arrays: dict[str, np.ndarray] | None = None,
|
|
202
|
+
_iteration_callback: (
|
|
203
|
+
Callable[[Block, list[str] | None], None] | None
|
|
204
|
+
) = None,
|
|
205
|
+
) -> "LazyArrowStreamTable":
|
|
206
|
+
"""Create a lazy DataFusion table from an xarray Dataset.
|
|
207
|
+
|
|
208
|
+
This is the simplest way to register xarray data with DataFusion.
|
|
209
|
+
Data is only read when queries are executed, not during registration.
|
|
210
|
+
The table can be queried multiple times.
|
|
211
|
+
|
|
212
|
+
Each chunk becomes a separate partition, enabling DataFusion's parallel
|
|
213
|
+
execution across multiple cores.
|
|
214
|
+
|
|
215
|
+
Note:
|
|
216
|
+
SQL queries with WHERE clauses on dimension columns (time, lat, lon, etc.)
|
|
217
|
+
automatically prune partitions that can't contain matching rows — this is
|
|
218
|
+
called *filter pushdown*. For example:
|
|
219
|
+
|
|
220
|
+
# This query will skip loading partitions with time < '2020-02-01'
|
|
221
|
+
result = ctx.sql('SELECT * FROM air WHERE time > "2020-02-01"').collect()
|
|
222
|
+
|
|
223
|
+
Supported operators: `=`, `<`, `>`, `<=`, `>=`, `BETWEEN`, `IN`, `AND`, `OR`.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
ds: An xarray Dataset. All data_vars must share the same dimensions.
|
|
227
|
+
chunks: Xarray-like chunks specification. If not provided, uses
|
|
228
|
+
the Dataset's existing chunks, or for an unchunked Dataset its
|
|
229
|
+
storage chunks.
|
|
230
|
+
batch_size: Maximum rows per Arrow RecordBatch emitted per partition.
|
|
231
|
+
Smaller values let DataFusion start processing earlier; the default
|
|
232
|
+
(65 536) works well for most datasets.
|
|
233
|
+
coord_arrays: Pre-materialised coordinate arrays keyed by dim-name
|
|
234
|
+
string. Hand in to share a single read across multiple tables
|
|
235
|
+
built from the same parent Dataset (e.g. surface + atmosphere
|
|
236
|
+
from ARCO-ERA5); the dim coords are otherwise read once per
|
|
237
|
+
``read_xarray_table`` call, which is a network round-trip for
|
|
238
|
+
Zarr-backed datasets.
|
|
239
|
+
_iteration_callback: Internal callback for testing. Called with
|
|
240
|
+
each block dict just before it's converted to Arrow.
|
|
241
|
+
|
|
242
|
+
Returns:
|
|
243
|
+
A LazyArrowStreamTable ready for registration with DataFusion.
|
|
244
|
+
|
|
245
|
+
Example:
|
|
246
|
+
>>> from datafusion import SessionContext
|
|
247
|
+
>>> import xarray as xr
|
|
248
|
+
>>> from xarray_sql import read_xarray_table
|
|
249
|
+
>>>
|
|
250
|
+
>>> ds = xr.tutorial.open_dataset('air_temperature')
|
|
251
|
+
>>> table = read_xarray_table(ds, chunks={'time': 240})
|
|
252
|
+
>>>
|
|
253
|
+
>>> ctx = SessionContext()
|
|
254
|
+
>>> ctx.register_table('air', table)
|
|
255
|
+
>>>
|
|
256
|
+
>>> # Data is only read here, during query execution
|
|
257
|
+
>>> # Filters on 'time' will prune partitions automatically!
|
|
258
|
+
>>> result = ctx.sql('SELECT AVG(air) FROM air').collect()
|
|
259
|
+
"""
|
|
260
|
+
from ._native import LazyArrowStreamTable
|
|
261
|
+
|
|
262
|
+
ds = _ensure_default_indexes(ds)
|
|
263
|
+
schema = _parse_schema(ds)
|
|
264
|
+
|
|
265
|
+
# Hoist coordinate reads once; avoids N_partitions remote I/O calls for
|
|
266
|
+
# Zarr-backed datasets (e.g. ARCO-ERA5 on GCS). When the caller supplies
|
|
267
|
+
# pre-materialised arrays (e.g. shared across surface + atmosphere
|
|
268
|
+
# tables), reuse them and skip the extra read.
|
|
269
|
+
if coord_arrays is None:
|
|
270
|
+
coord_arrays = {str(dim): ds.coords[dim].values for dim in ds.dims}
|
|
271
|
+
|
|
272
|
+
# Determine which column names are data variables (not dimension coordinates).
|
|
273
|
+
# Used by the factory to skip loading unrequested variables.
|
|
274
|
+
data_var_names = set(ds.data_vars.keys())
|
|
275
|
+
|
|
276
|
+
def make_partition_factory(
|
|
277
|
+
block: Block,
|
|
278
|
+
) -> Callable[[list[str] | None], pa.RecordBatchReader]:
|
|
279
|
+
def make_stream(
|
|
280
|
+
projection_names: list[str] | None,
|
|
281
|
+
) -> pa.RecordBatchReader:
|
|
282
|
+
if _iteration_callback is not None:
|
|
283
|
+
_iteration_callback(block, projection_names)
|
|
284
|
+
|
|
285
|
+
if projection_names is not None:
|
|
286
|
+
# Restrict to the data variables mentioned in the projection.
|
|
287
|
+
# Dimension coordinates come along automatically via coords.
|
|
288
|
+
data_vars_needed = [
|
|
289
|
+
c for c in projection_names if c in data_var_names
|
|
290
|
+
]
|
|
291
|
+
if data_vars_needed:
|
|
292
|
+
ds_block = ds[data_vars_needed].isel(block)
|
|
293
|
+
else:
|
|
294
|
+
# Only dimension coords requested — drop all data vars to avoid
|
|
295
|
+
# loading them unnecessarily (e.g. for queries like SELECT lat, lon).
|
|
296
|
+
ds_block = ds.drop_vars(list(ds.data_vars)).isel(block)
|
|
297
|
+
batch_schema = pa.schema(
|
|
298
|
+
[schema.field(name) for name in projection_names]
|
|
299
|
+
)
|
|
300
|
+
else:
|
|
301
|
+
ds_block = ds.isel(block)
|
|
302
|
+
batch_schema = schema
|
|
303
|
+
|
|
304
|
+
return pa.RecordBatchReader.from_batches(
|
|
305
|
+
batch_schema,
|
|
306
|
+
iter_record_batches(ds_block, batch_schema, batch_size),
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
return make_stream
|
|
310
|
+
|
|
311
|
+
# Separate dims whose chunk bounds vary across partitions from those
|
|
312
|
+
# whose bounds are constant (one chunk spanning the whole axis). For the
|
|
313
|
+
# latter we compute min/max once instead of re-scanning the full coord
|
|
314
|
+
# array on every partition — dominant cost when registering hundreds of
|
|
315
|
+
# thousands of single-time-step partitions on a 4-D dataset like ERA5.
|
|
316
|
+
resolved = resolve_chunks(ds, chunks)
|
|
317
|
+
varying_dims = [d for d, tup in resolved.items() if len(tup) > 1]
|
|
318
|
+
static_dims = [d for d in ds.dims if d not in varying_dims]
|
|
319
|
+
static_block: Block = {d: slice(None) for d in static_dims}
|
|
320
|
+
static_ranges = _block_metadata(
|
|
321
|
+
coord_arrays, static_block, dims=static_dims
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
def partition_pairs():
|
|
325
|
+
"""Lazily yield (factory, metadata, num_rows) for each partition.
|
|
326
|
+
|
|
327
|
+
Consuming this generator one item at a time means Python never holds
|
|
328
|
+
all N block dicts, metadata dicts, and factory closures simultaneously.
|
|
329
|
+
Peak Python memory during registration is O(1) per partition instead
|
|
330
|
+
of O(N_partitions).
|
|
331
|
+
"""
|
|
332
|
+
for block in _block_slices_from_resolved(ds, resolved):
|
|
333
|
+
dynamic = _block_metadata(coord_arrays, block, dims=varying_dims)
|
|
334
|
+
yield (
|
|
335
|
+
make_partition_factory(block),
|
|
336
|
+
{**static_ranges, **dynamic},
|
|
337
|
+
# Exact row count for this partition (product of the chunk's
|
|
338
|
+
# per-dimension sizes), so the scan can report exact
|
|
339
|
+
# Statistics::num_rows to the optimizer.
|
|
340
|
+
_block_len(block),
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
return LazyArrowStreamTable(partition_pairs(), schema)
|