xarray-sql 0.2.0__cp311-cp311-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/reader.py ADDED
@@ -0,0 +1,253 @@
1
+ """Lazy Arrow stream reader for xarray Datasets.
2
+
3
+ This module provides XarrayRecordBatchReader, which implements the Arrow
4
+ PyCapsule Interface (__arrow_c_stream__) to enable zero-copy, lazy streaming
5
+ of xarray data to DataFusion and other Arrow consumers.
6
+
7
+ The implementation delegates to PyArrow's RecordBatchReader for the
8
+ actual stream implementation, wrapping xarray block iteration in a generator.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import typing as t
14
+
15
+ import pyarrow as pa
16
+ import xarray as xr
17
+
18
+ from .df import Block, Chunks, block_slices, pivot, _parse_schema
19
+
20
+ if t.TYPE_CHECKING:
21
+ from ._native import LazyArrowStreamTable
22
+
23
+
24
+ class XarrayRecordBatchReader:
25
+ """A lazy Arrow stream reader for xarray Datasets.
26
+
27
+ Implements the Arrow PyCapsule Interface (__arrow_c_stream__) to enable
28
+ zero-copy, lazy streaming of xarray data to DataFusion and other Arrow
29
+ consumers.
30
+
31
+ The key property is that xarray blocks are only converted to Arrow
32
+ RecordBatches when the consumer calls get_next (e.g., during DataFusion's
33
+ collect()), NOT when the reader is created or registered.
34
+
35
+ Attributes:
36
+ schema: The Arrow schema for the stream.
37
+
38
+ Example:
39
+ >>> import xarray as xr
40
+ >>> from xarray_sql import XarrayRecordBatchReader
41
+ >>> ds = xr.tutorial.open_dataset('air_temperature')
42
+ >>> reader = XarrayRecordBatchReader(ds, chunks={'time': 240})
43
+ >>> # At this point, NO data has been read from xarray
44
+ >>> # Data is only read when consumed:
45
+ >>> import pyarrow as pa
46
+ >>> pa_reader = pa.RecordBatchReader.from_stream(reader)
47
+ >>> for batch in pa_reader:
48
+ ... print(batch.num_rows) # Data read here
49
+ """
50
+
51
+ def __init__(
52
+ self,
53
+ ds: xr.Dataset,
54
+ chunks: Chunks = None,
55
+ *,
56
+ _iteration_callback: t.Optional[t.Callable[[Block], None]] = None,
57
+ ):
58
+ """Initialize the lazy reader.
59
+
60
+ Args:
61
+ ds: An xarray Dataset. All data_vars must share the same dimensions.
62
+ chunks: Xarray-like chunks specification. If not provided, uses
63
+ the Dataset's existing chunks.
64
+ _iteration_callback: Internal callback for testing. Called with
65
+ each block dict just before it's converted to Arrow. This
66
+ allows tests to track when iteration actually occurs.
67
+ """
68
+ self._ds = ds
69
+ self._chunks = chunks
70
+ self._schema = _parse_schema(ds)
71
+ self._iteration_callback = _iteration_callback
72
+ self._consumed = False
73
+
74
+ # Validate dimensions
75
+ fst = next(iter(ds.values())).dims
76
+ if not all(da.dims == fst for da in ds.values()):
77
+ raise ValueError(
78
+ "All dimensions must be equal. Please filter data_vars in the Dataset."
79
+ )
80
+
81
+ @property
82
+ def schema(self) -> pa.Schema:
83
+ """The Arrow schema for this stream."""
84
+ return self._schema
85
+
86
+ def _generate_batches(self) -> t.Iterator[pa.RecordBatch]:
87
+ """Generate RecordBatches lazily from xarray blocks.
88
+
89
+ This generator is only consumed when the Arrow stream's get_next
90
+ is called, ensuring true lazy evaluation.
91
+ """
92
+ for block in block_slices(self._ds, self._chunks):
93
+ # Call the iteration callback if provided (for testing)
94
+ if self._iteration_callback is not None:
95
+ self._iteration_callback(block)
96
+
97
+ # Convert this block to a RecordBatch
98
+ df = pivot(self._ds.isel(block))
99
+ yield pa.RecordBatch.from_pandas(df, schema=self._schema)
100
+
101
+ def __arrow_c_stream__(
102
+ self, requested_schema: t.Optional[object] = None
103
+ ) -> object:
104
+ """Export as Arrow C Stream via PyCapsule.
105
+
106
+ This method is called by Arrow consumers (like DataFusion) to get
107
+ a C-level stream interface. The actual data iteration only begins
108
+ when the consumer calls get_next on the stream.
109
+
110
+ Args:
111
+ requested_schema: Optional schema for type casting. Currently
112
+ passed through to PyArrow's implementation.
113
+
114
+ Returns:
115
+ PyCapsule containing ArrowArrayStream pointer with name
116
+ "arrow_array_stream".
117
+
118
+ Raises:
119
+ RuntimeError: If the stream has already been consumed.
120
+ """
121
+ if self._consumed:
122
+ raise RuntimeError(
123
+ "Stream already consumed. XarrayRecordBatchReader can only "
124
+ "be iterated once. Create a new reader for additional iterations."
125
+ )
126
+ self._consumed = True
127
+
128
+ # Create a PyArrow RecordBatchReader from our generator
129
+ # The generator is NOT consumed here - only when get_next is called
130
+ reader = pa.RecordBatchReader.from_batches(
131
+ self._schema, self._generate_batches()
132
+ )
133
+
134
+ # Delegate to PyArrow's __arrow_c_stream__ implementation
135
+ return reader.__arrow_c_stream__(requested_schema)
136
+
137
+ def __arrow_c_schema__(
138
+ self, requested_schema: t.Optional[object] = None
139
+ ) -> object:
140
+ """Export the schema as Arrow C Schema via PyCapsule.
141
+
142
+ This allows consumers to inspect the schema without consuming the stream.
143
+
144
+ Args:
145
+ requested_schema: Optional schema for negotiation (unused).
146
+
147
+ Returns:
148
+ PyCapsule containing ArrowSchema pointer.
149
+ """
150
+ return self._schema.__arrow_c_schema__()
151
+
152
+
153
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
154
+ """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
155
+
156
+ Args:
157
+ ds: An Xarray Dataset. All `data_vars` must share the same dimensions.
158
+ chunks: Xarray-like chunks. If not provided, will default to the Dataset's
159
+ chunks. The product of the chunk sizes becomes the standard length of each
160
+ dataframe partition.
161
+
162
+ Returns:
163
+ A PyArrow RecordBatchReader, which is a table representation of the input
164
+ Dataset.
165
+ """
166
+ reader = XarrayRecordBatchReader(ds, chunks=chunks)
167
+ return pa.RecordBatchReader.from_stream(reader)
168
+
169
+
170
+ def read_xarray_table(
171
+ ds: xr.Dataset,
172
+ chunks: Chunks = None,
173
+ *,
174
+ _iteration_callback: t.Optional[t.Callable[[Block], None]] = None,
175
+ ) -> "LazyArrowStreamTable":
176
+ """Create a lazy DataFusion table from an xarray Dataset.
177
+
178
+ This is the simplest way to register xarray data with DataFusion.
179
+ Data is only read when queries are executed, not during registration.
180
+ The table can be queried multiple times.
181
+
182
+ Each chunk becomes a separate partition, enabling DataFusion's parallel
183
+ execution across multiple cores.
184
+
185
+ Note:
186
+ Due to a bug in DataFusion v51.0.0's collect() method, use
187
+ `to_arrow_table()` instead of `collect()` for aggregation queries
188
+ to ensure complete results::
189
+
190
+ # Correct - use to_arrow_table()
191
+ result = ctx.sql('SELECT lat, AVG(temp) FROM t GROUP BY lat').to_arrow_table()
192
+
193
+ # May return partial results with collect()
194
+ result = ctx.sql('SELECT lat, AVG(temp) FROM t GROUP BY lat').collect()
195
+
196
+ This should be fixed when we upgrade datafusion-python to 52 (#107).
197
+
198
+ Args:
199
+ ds: An xarray Dataset. All data_vars must share the same dimensions.
200
+ chunks: Xarray-like chunks specification. If not provided, uses
201
+ the Dataset's existing chunks.
202
+ _iteration_callback: Internal callback for testing. Called with
203
+ each block dict just before it's converted to Arrow.
204
+
205
+ Returns:
206
+ A LazyArrowStreamTable ready for registration with DataFusion.
207
+
208
+ Example:
209
+ >>> from datafusion import SessionContext
210
+ >>> import xarray as xr
211
+ >>> from xarray_sql import read_xarray_table
212
+ >>>
213
+ >>> ds = xr.tutorial.open_dataset('air_temperature')
214
+ >>> table = read_xarray_table(ds, chunks={'time': 240})
215
+ >>>
216
+ >>> ctx = SessionContext()
217
+ >>> ctx.register_table('air', table)
218
+ >>>
219
+ >>> # Data is only read here, during query execution
220
+ >>> result = ctx.sql('SELECT AVG(air) FROM air').to_arrow_table()
221
+ >>> # Can query again - each query creates a fresh stream
222
+ >>> result2 = ctx.sql('SELECT * FROM air LIMIT 10').to_arrow_table()
223
+ """
224
+ from ._native import LazyArrowStreamTable
225
+
226
+ # Get schema from dataset without creating a stream
227
+ schema = _parse_schema(ds)
228
+
229
+ blocks = block_slices(ds, chunks)
230
+
231
+ # Create a factory function for each block (partition)
232
+ # Each factory produces a RecordBatchReader for its specific chunk
233
+ def make_partition_factory(
234
+ block: Block,
235
+ ) -> t.Callable[[], pa.RecordBatchReader]:
236
+ """Create a factory function for a specific block/chunk."""
237
+
238
+ def make_stream() -> pa.RecordBatchReader:
239
+ # Call the iteration callback if provided (for testing)
240
+ if _iteration_callback is not None:
241
+ _iteration_callback(block)
242
+
243
+ # Extract just this block from the dataset and convert to Arrow
244
+ df = pivot(ds.isel(block))
245
+ batch = pa.RecordBatch.from_pandas(df, schema=schema)
246
+ return pa.RecordBatchReader.from_batches(schema, [batch])
247
+
248
+ return make_stream
249
+
250
+ # Create one factory per block
251
+ factories = [make_partition_factory(block) for block in blocks]
252
+
253
+ return LazyArrowStreamTable(factories, schema)