xarray-sql 0.2.0__cp311-cp311-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +10 -0
- xarray_sql/_native.cp311-win_amd64.pyd +4 -0
- xarray_sql/core.py +48 -0
- xarray_sql/df.py +172 -0
- xarray_sql/df_test.py +341 -0
- xarray_sql/reader.py +253 -0
- xarray_sql/reader_test.py +944 -0
- xarray_sql/sql.py +18 -0
- xarray_sql/sql_test.py +194 -0
- xarray_sql-0.2.0.dist-info/METADATA +224 -0
- xarray_sql-0.2.0.dist-info/RECORD +13 -0
- xarray_sql-0.2.0.dist-info/WHEEL +4 -0
- xarray_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
xarray_sql/reader.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Lazy Arrow stream reader for xarray Datasets.
|
|
2
|
+
|
|
3
|
+
This module provides XarrayRecordBatchReader, which implements the Arrow
|
|
4
|
+
PyCapsule Interface (__arrow_c_stream__) to enable zero-copy, lazy streaming
|
|
5
|
+
of xarray data to DataFusion and other Arrow consumers.
|
|
6
|
+
|
|
7
|
+
The implementation delegates to PyArrow's RecordBatchReader for the
|
|
8
|
+
actual stream implementation, wrapping xarray block iteration in a generator.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import typing as t
|
|
14
|
+
|
|
15
|
+
import pyarrow as pa
|
|
16
|
+
import xarray as xr
|
|
17
|
+
|
|
18
|
+
from .df import Block, Chunks, block_slices, pivot, _parse_schema
|
|
19
|
+
|
|
20
|
+
if t.TYPE_CHECKING:
|
|
21
|
+
from ._native import LazyArrowStreamTable
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class XarrayRecordBatchReader:
|
|
25
|
+
"""A lazy Arrow stream reader for xarray Datasets.
|
|
26
|
+
|
|
27
|
+
Implements the Arrow PyCapsule Interface (__arrow_c_stream__) to enable
|
|
28
|
+
zero-copy, lazy streaming of xarray data to DataFusion and other Arrow
|
|
29
|
+
consumers.
|
|
30
|
+
|
|
31
|
+
The key property is that xarray blocks are only converted to Arrow
|
|
32
|
+
RecordBatches when the consumer calls get_next (e.g., during DataFusion's
|
|
33
|
+
collect()), NOT when the reader is created or registered.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
schema: The Arrow schema for the stream.
|
|
37
|
+
|
|
38
|
+
Example:
|
|
39
|
+
>>> import xarray as xr
|
|
40
|
+
>>> from xarray_sql import XarrayRecordBatchReader
|
|
41
|
+
>>> ds = xr.tutorial.open_dataset('air_temperature')
|
|
42
|
+
>>> reader = XarrayRecordBatchReader(ds, chunks={'time': 240})
|
|
43
|
+
>>> # At this point, NO data has been read from xarray
|
|
44
|
+
>>> # Data is only read when consumed:
|
|
45
|
+
>>> import pyarrow as pa
|
|
46
|
+
>>> pa_reader = pa.RecordBatchReader.from_stream(reader)
|
|
47
|
+
>>> for batch in pa_reader:
|
|
48
|
+
... print(batch.num_rows) # Data read here
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
ds: xr.Dataset,
|
|
54
|
+
chunks: Chunks = None,
|
|
55
|
+
*,
|
|
56
|
+
_iteration_callback: t.Optional[t.Callable[[Block], None]] = None,
|
|
57
|
+
):
|
|
58
|
+
"""Initialize the lazy reader.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
ds: An xarray Dataset. All data_vars must share the same dimensions.
|
|
62
|
+
chunks: Xarray-like chunks specification. If not provided, uses
|
|
63
|
+
the Dataset's existing chunks.
|
|
64
|
+
_iteration_callback: Internal callback for testing. Called with
|
|
65
|
+
each block dict just before it's converted to Arrow. This
|
|
66
|
+
allows tests to track when iteration actually occurs.
|
|
67
|
+
"""
|
|
68
|
+
self._ds = ds
|
|
69
|
+
self._chunks = chunks
|
|
70
|
+
self._schema = _parse_schema(ds)
|
|
71
|
+
self._iteration_callback = _iteration_callback
|
|
72
|
+
self._consumed = False
|
|
73
|
+
|
|
74
|
+
# Validate dimensions
|
|
75
|
+
fst = next(iter(ds.values())).dims
|
|
76
|
+
if not all(da.dims == fst for da in ds.values()):
|
|
77
|
+
raise ValueError(
|
|
78
|
+
"All dimensions must be equal. Please filter data_vars in the Dataset."
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def schema(self) -> pa.Schema:
|
|
83
|
+
"""The Arrow schema for this stream."""
|
|
84
|
+
return self._schema
|
|
85
|
+
|
|
86
|
+
def _generate_batches(self) -> t.Iterator[pa.RecordBatch]:
|
|
87
|
+
"""Generate RecordBatches lazily from xarray blocks.
|
|
88
|
+
|
|
89
|
+
This generator is only consumed when the Arrow stream's get_next
|
|
90
|
+
is called, ensuring true lazy evaluation.
|
|
91
|
+
"""
|
|
92
|
+
for block in block_slices(self._ds, self._chunks):
|
|
93
|
+
# Call the iteration callback if provided (for testing)
|
|
94
|
+
if self._iteration_callback is not None:
|
|
95
|
+
self._iteration_callback(block)
|
|
96
|
+
|
|
97
|
+
# Convert this block to a RecordBatch
|
|
98
|
+
df = pivot(self._ds.isel(block))
|
|
99
|
+
yield pa.RecordBatch.from_pandas(df, schema=self._schema)
|
|
100
|
+
|
|
101
|
+
def __arrow_c_stream__(
|
|
102
|
+
self, requested_schema: t.Optional[object] = None
|
|
103
|
+
) -> object:
|
|
104
|
+
"""Export as Arrow C Stream via PyCapsule.
|
|
105
|
+
|
|
106
|
+
This method is called by Arrow consumers (like DataFusion) to get
|
|
107
|
+
a C-level stream interface. The actual data iteration only begins
|
|
108
|
+
when the consumer calls get_next on the stream.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
requested_schema: Optional schema for type casting. Currently
|
|
112
|
+
passed through to PyArrow's implementation.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
PyCapsule containing ArrowArrayStream pointer with name
|
|
116
|
+
"arrow_array_stream".
|
|
117
|
+
|
|
118
|
+
Raises:
|
|
119
|
+
RuntimeError: If the stream has already been consumed.
|
|
120
|
+
"""
|
|
121
|
+
if self._consumed:
|
|
122
|
+
raise RuntimeError(
|
|
123
|
+
"Stream already consumed. XarrayRecordBatchReader can only "
|
|
124
|
+
"be iterated once. Create a new reader for additional iterations."
|
|
125
|
+
)
|
|
126
|
+
self._consumed = True
|
|
127
|
+
|
|
128
|
+
# Create a PyArrow RecordBatchReader from our generator
|
|
129
|
+
# The generator is NOT consumed here - only when get_next is called
|
|
130
|
+
reader = pa.RecordBatchReader.from_batches(
|
|
131
|
+
self._schema, self._generate_batches()
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
# Delegate to PyArrow's __arrow_c_stream__ implementation
|
|
135
|
+
return reader.__arrow_c_stream__(requested_schema)
|
|
136
|
+
|
|
137
|
+
def __arrow_c_schema__(
|
|
138
|
+
self, requested_schema: t.Optional[object] = None
|
|
139
|
+
) -> object:
|
|
140
|
+
"""Export the schema as Arrow C Schema via PyCapsule.
|
|
141
|
+
|
|
142
|
+
This allows consumers to inspect the schema without consuming the stream.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
requested_schema: Optional schema for negotiation (unused).
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
PyCapsule containing ArrowSchema pointer.
|
|
149
|
+
"""
|
|
150
|
+
return self._schema.__arrow_c_schema__()
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
|
|
154
|
+
"""Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
|
|
155
|
+
|
|
156
|
+
Args:
|
|
157
|
+
ds: An Xarray Dataset. All `data_vars` must share the same dimensions.
|
|
158
|
+
chunks: Xarray-like chunks. If not provided, will default to the Dataset's
|
|
159
|
+
chunks. The product of the chunk sizes becomes the standard length of each
|
|
160
|
+
dataframe partition.
|
|
161
|
+
|
|
162
|
+
Returns:
|
|
163
|
+
A PyArrow RecordBatchReader, which is a table representation of the input
|
|
164
|
+
Dataset.
|
|
165
|
+
"""
|
|
166
|
+
reader = XarrayRecordBatchReader(ds, chunks=chunks)
|
|
167
|
+
return pa.RecordBatchReader.from_stream(reader)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def read_xarray_table(
|
|
171
|
+
ds: xr.Dataset,
|
|
172
|
+
chunks: Chunks = None,
|
|
173
|
+
*,
|
|
174
|
+
_iteration_callback: t.Optional[t.Callable[[Block], None]] = None,
|
|
175
|
+
) -> "LazyArrowStreamTable":
|
|
176
|
+
"""Create a lazy DataFusion table from an xarray Dataset.
|
|
177
|
+
|
|
178
|
+
This is the simplest way to register xarray data with DataFusion.
|
|
179
|
+
Data is only read when queries are executed, not during registration.
|
|
180
|
+
The table can be queried multiple times.
|
|
181
|
+
|
|
182
|
+
Each chunk becomes a separate partition, enabling DataFusion's parallel
|
|
183
|
+
execution across multiple cores.
|
|
184
|
+
|
|
185
|
+
Note:
|
|
186
|
+
Due to a bug in DataFusion v51.0.0's collect() method, use
|
|
187
|
+
`to_arrow_table()` instead of `collect()` for aggregation queries
|
|
188
|
+
to ensure complete results::
|
|
189
|
+
|
|
190
|
+
# Correct - use to_arrow_table()
|
|
191
|
+
result = ctx.sql('SELECT lat, AVG(temp) FROM t GROUP BY lat').to_arrow_table()
|
|
192
|
+
|
|
193
|
+
# May return partial results with collect()
|
|
194
|
+
result = ctx.sql('SELECT lat, AVG(temp) FROM t GROUP BY lat').collect()
|
|
195
|
+
|
|
196
|
+
This should be fixed when we upgrade datafusion-python to 52 (#107).
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
ds: An xarray Dataset. All data_vars must share the same dimensions.
|
|
200
|
+
chunks: Xarray-like chunks specification. If not provided, uses
|
|
201
|
+
the Dataset's existing chunks.
|
|
202
|
+
_iteration_callback: Internal callback for testing. Called with
|
|
203
|
+
each block dict just before it's converted to Arrow.
|
|
204
|
+
|
|
205
|
+
Returns:
|
|
206
|
+
A LazyArrowStreamTable ready for registration with DataFusion.
|
|
207
|
+
|
|
208
|
+
Example:
|
|
209
|
+
>>> from datafusion import SessionContext
|
|
210
|
+
>>> import xarray as xr
|
|
211
|
+
>>> from xarray_sql import read_xarray_table
|
|
212
|
+
>>>
|
|
213
|
+
>>> ds = xr.tutorial.open_dataset('air_temperature')
|
|
214
|
+
>>> table = read_xarray_table(ds, chunks={'time': 240})
|
|
215
|
+
>>>
|
|
216
|
+
>>> ctx = SessionContext()
|
|
217
|
+
>>> ctx.register_table('air', table)
|
|
218
|
+
>>>
|
|
219
|
+
>>> # Data is only read here, during query execution
|
|
220
|
+
>>> result = ctx.sql('SELECT AVG(air) FROM air').to_arrow_table()
|
|
221
|
+
>>> # Can query again - each query creates a fresh stream
|
|
222
|
+
>>> result2 = ctx.sql('SELECT * FROM air LIMIT 10').to_arrow_table()
|
|
223
|
+
"""
|
|
224
|
+
from ._native import LazyArrowStreamTable
|
|
225
|
+
|
|
226
|
+
# Get schema from dataset without creating a stream
|
|
227
|
+
schema = _parse_schema(ds)
|
|
228
|
+
|
|
229
|
+
blocks = block_slices(ds, chunks)
|
|
230
|
+
|
|
231
|
+
# Create a factory function for each block (partition)
|
|
232
|
+
# Each factory produces a RecordBatchReader for its specific chunk
|
|
233
|
+
def make_partition_factory(
|
|
234
|
+
block: Block,
|
|
235
|
+
) -> t.Callable[[], pa.RecordBatchReader]:
|
|
236
|
+
"""Create a factory function for a specific block/chunk."""
|
|
237
|
+
|
|
238
|
+
def make_stream() -> pa.RecordBatchReader:
|
|
239
|
+
# Call the iteration callback if provided (for testing)
|
|
240
|
+
if _iteration_callback is not None:
|
|
241
|
+
_iteration_callback(block)
|
|
242
|
+
|
|
243
|
+
# Extract just this block from the dataset and convert to Arrow
|
|
244
|
+
df = pivot(ds.isel(block))
|
|
245
|
+
batch = pa.RecordBatch.from_pandas(df, schema=schema)
|
|
246
|
+
return pa.RecordBatchReader.from_batches(schema, [batch])
|
|
247
|
+
|
|
248
|
+
return make_stream
|
|
249
|
+
|
|
250
|
+
# Create one factory per block
|
|
251
|
+
factories = [make_partition_factory(block) for block in blocks]
|
|
252
|
+
|
|
253
|
+
return LazyArrowStreamTable(factories, schema)
|