xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/geometry.py ADDED
@@ -0,0 +1,131 @@
1
+ """GeoArrow point-geometry columns derived from coordinate dimensions.
2
+
3
+ A regular grid's pivot already materializes per-row x/y coordinate
4
+ columns; a point-geometry column is those same values under a GeoArrow
5
+ extension annotation. Two encodings:
6
+
7
+ * ``"wkb"`` (default) — 21-byte WKB points under the ``geoarrow.wkb``
8
+ extension name. DuckDB (>= 1.2, spatial loaded) ingests the column as
9
+ a native ``GEOMETRY`` with the CRS attached, so ``ST_Within(geometry,
10
+ ...)`` works with no ``ST_Point(x, y)`` construction in user SQL.
11
+ * ``"point"`` — GeoArrow native points with *separated* coordinates
12
+ (``struct<x: double, y: double>`` under ``geoarrow.point``): the
13
+ child arrays are the coordinate columns themselves, no per-row
14
+ parsing for consumers that execute on native layouts (GeoPandas 1.x,
15
+ geoarrow-rs, lonboard, SedonaDB). DuckDB does not consume this
16
+ encoding.
17
+
18
+ The CRS rides in the extension metadata (GeoArrow 0.2 allows
19
+ authority:code strings alongside PROJJSON). ``OGC:CRS84`` is the
20
+ correct tag for plain longitude/latitude grids.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ from typing import Any
27
+
28
+ import numpy as np
29
+ import pyarrow as pa
30
+
31
+ GEOMETRY_COLUMN = "geometry"
32
+
33
+ _ENCODINGS = ("wkb", "point")
34
+
35
+
36
+ def geometry_field(encoding: str, crs: str | None) -> pa.Field:
37
+ """The schema field for the derived geometry column."""
38
+ if encoding not in _ENCODINGS:
39
+ raise ValueError(
40
+ f"geometry_encoding must be one of {_ENCODINGS}, got {encoding!r}"
41
+ )
42
+ metadata = {
43
+ b"ARROW:extension:name": f"geoarrow.{encoding}".encode(),
44
+ }
45
+ if crs is not None:
46
+ metadata[b"ARROW:extension:metadata"] = json.dumps(
47
+ {"crs": crs}
48
+ ).encode()
49
+ storage = (
50
+ pa.binary()
51
+ if encoding == "wkb"
52
+ else pa.struct([("x", pa.float64()), ("y", pa.float64())])
53
+ )
54
+ return pa.field(GEOMETRY_COLUMN, storage, metadata=metadata)
55
+
56
+
57
+ def build_geometry(encoding: str, x: pa.Array, y: pa.Array) -> pa.Array:
58
+ """Point geometries for one batch's x/y coordinate columns."""
59
+ if encoding == "point":
60
+ return pa.StructArray.from_arrays(
61
+ [x.cast(pa.float64()), y.cast(pa.float64())], ["x", "y"]
62
+ )
63
+ return _wkb_points(
64
+ np.ascontiguousarray(x.to_numpy(zero_copy_only=False), "<f8"),
65
+ np.ascontiguousarray(y.to_numpy(zero_copy_only=False), "<f8"),
66
+ )
67
+
68
+
69
+ def _wkb_points(x: np.ndarray, y: np.ndarray) -> pa.Array:
70
+ """Vectorized 21-byte little-endian WKB point encoding."""
71
+ n = len(x)
72
+ # ``pa.binary()`` carries int32 offsets, which the final offset
73
+ # (n * 21) overflows past ~102M points; the buffers would build
74
+ # silently corrupt. Unreachable through the pivot (batch_size caps
75
+ # rows per batch well below this), so guard rather than widen the
76
+ # storage to large_binary, which DuckDB's ingestion expects not to
77
+ # see.
78
+ if n * 21 > np.iinfo(np.int32).max:
79
+ raise ValueError(
80
+ f"cannot WKB-encode {n:,} points in a single batch: "
81
+ "pa.binary() offsets are int32 and n * 21 bytes would "
82
+ "overflow them. Use a smaller batch_size."
83
+ )
84
+ buf: np.ndarray = np.empty((n, 21), dtype=np.uint8)
85
+ buf[:, 0] = 1 # little-endian byte order mark
86
+ buf[:, 1:5] = np.array([1, 0, 0, 0], dtype=np.uint8) # WKB type 1: Point
87
+ buf[:, 5:13] = x.view(np.uint8).reshape(n, 8)
88
+ buf[:, 13:21] = y.view(np.uint8).reshape(n, 8)
89
+ offsets = pa.py_buffer(
90
+ np.arange(0, (n + 1) * 21, 21, dtype=np.int32).tobytes()
91
+ )
92
+ return pa.Array.from_buffers(
93
+ pa.binary(), n, [None, offsets, pa.py_buffer(buf.tobytes())]
94
+ )
95
+
96
+
97
+ def bbox_conjuncts(
98
+ bounds: Any, x: str = "x", y: str = "y", pad: float = 0.0
99
+ ) -> str:
100
+ """SQL bbox conjuncts for a geometry's envelope — the pruning half.
101
+
102
+ Engines do not push ``ST_*`` functions into the scan, so a
103
+ geometry-only predicate reads every chunk; pairing it with range
104
+ conjuncts on the coordinate columns restores pruning. This helper
105
+ renders those conjuncts from a geometry's envelope::
106
+
107
+ poly = shapely.from_wkt("POLYGON (...)")
108
+ sql = (
109
+ f"SELECT avg(risk) FROM eri "
110
+ f"WHERE {xql.bbox_conjuncts(poly, x='x', y='y')} "
111
+ f"AND ST_Within(geometry, ST_GeomFromText('{poly.wkt}'))"
112
+ )
113
+
114
+ Args:
115
+ bounds: ``(xmin, ymin, xmax, ymax)``, or any object with a
116
+ ``.bounds`` attribute in that convention (shapely
117
+ geometries qualify).
118
+ x: The x/longitude column name.
119
+ y: The y/latitude column name.
120
+ pad: Optional margin added on every side (e.g. to be safe
121
+ around ``ST_DWithin``-style predicates).
122
+
123
+ Returns:
124
+ A SQL snippet ``"x" BETWEEN a AND b AND "y" BETWEEN c AND d``.
125
+ """
126
+ values = getattr(bounds, "bounds", bounds)
127
+ xmin, ymin, xmax, ymax = (float(v) for v in values)
128
+ return (
129
+ f'"{x}" BETWEEN {xmin - pad!r} AND {xmax + pad!r} '
130
+ f'AND "{y}" BETWEEN {ymin - pad!r} AND {ymax + pad!r}'
131
+ )
xarray_sql/lazyscan.py ADDED
@@ -0,0 +1,462 @@
1
+ """Re-executable engine handles behind the lazy chunked round-trip.
2
+
3
+ The lazy path of ``to_dataset(chunks=...)`` re-executes the engine's
4
+ query per accessed chunk, narrowed to that chunk's coordinate window and
5
+ columns. That requires the engine result to be *re-executable* — a
6
+ handle onto the query, not a one-shot stream of its rows. Each handle
7
+ here adapts one engine's native lazy surface to the three operations the
8
+ reconstruction needs:
9
+
10
+ * [schema][xarray_sql.lazyscan.LazyResultHandle.schema] — result column names/types, without
11
+ executing the query;
12
+ * [distinct][xarray_sql.lazyscan.LazyResultHandle.distinct] — one column's distinct values
13
+ (coordinate discovery; the caller sorts);
14
+ * [fetch][xarray_sql.lazyscan.LazyResultHandle.fetch] — the result narrowed by per-dimension
15
+ windows and projected to the requested columns, as Arrow batches.
16
+
17
+ Windows are passed as [DimSpec][xarray_sql.lazyscan.DimSpec] values instead of rendered SQL so
18
+ each engine can express them with its own *typed* expression API —
19
+ strings would re-open every literal-formatting pitfall (timestamps,
20
+ floats, quoting) per dialect.
21
+
22
+ Handles compose with the registration seam: when the wrapped query
23
+ scans a Dataset registered through xarray-sql's pushdown machinery, the
24
+ per-chunk range filter flows back through the engine into
25
+ [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset], so each
26
+ output chunk's access reads only the source chunks it maps onto.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import weakref
32
+ from collections.abc import Iterator
33
+ from concurrent.futures import ThreadPoolExecutor
34
+ from typing import Any, Literal, Protocol, cast
35
+
36
+ import numpy as np
37
+ import pandas as pd
38
+ import pyarrow as pa
39
+ import pyarrow.parquet as pq
40
+
41
+ from .df import _CAN_START_THREADS
42
+
43
+ DimSpec = tuple[Literal["range", "values"], Any, Any]
44
+ """One dimension's window: ``("range", lo, hi)`` (inclusive bounds; the
45
+ requested coordinate positions are contiguous) or ``("values", array,
46
+ None)`` (explicit value list, for stepped/fancy indexers).
47
+
48
+ The payload stays ``Any``: the values are coordinate scalars handed to
49
+ the engine's typed expression API, which does the comparing — Python
50
+ never orders them, and a concrete union over coordinate dtypes
51
+ (timestamps, cftime, numerics, strings) would stay incomplete."""
52
+
53
+
54
+ def _collect_streaming(lf: Any) -> Any:
55
+ """Collect a Polars LazyFrame on the streaming engine, if available.
56
+
57
+ ``collect(engine="streaming")`` needs polars >= 1.25 (the test
58
+ extra pins higher); an older installed polars raises TypeError on
59
+ the unknown keyword, and the plain in-memory collect is a correct,
60
+ if less memory-frugal, fallback.
61
+ """
62
+ try:
63
+ return lf.collect(engine="streaming")
64
+ except TypeError:
65
+ return lf.collect()
66
+
67
+
68
+ def _plain(value: Any) -> Any:
69
+ """A plain-Python literal (numpy scalars don't travel to engines)."""
70
+ if isinstance(value, np.datetime64):
71
+ return pd.Timestamp(value)
72
+ if isinstance(value, np.timedelta64):
73
+ return pd.Timedelta(value)
74
+ if isinstance(value, np.generic):
75
+ return value.item()
76
+ return value
77
+
78
+
79
+ class LazyResultHandle(Protocol):
80
+ """A re-executable query result (see module docstring)."""
81
+
82
+ supports_chunked: bool = True
83
+ """Whether fetch() may be driven from consumer worker threads (the
84
+ chunked reconstruction). Handles for engines that cannot safely
85
+ re-execute under foreign threads set this False; the eager path
86
+ remains available."""
87
+
88
+ def schema(self) -> pa.Schema: ...
89
+
90
+ def distinct(self, column: str) -> np.ndarray: ...
91
+
92
+ def fetch(
93
+ self, specs: dict[str, DimSpec], columns: list[str]
94
+ ) -> list[pa.RecordBatch]: ...
95
+
96
+ def spill_parquet(self, path: str) -> None: ...
97
+
98
+ # Handles may additionally offer ``stream(columns)``, yielding the
99
+ # unfiltered result's Arrow batches incrementally instead of
100
+ # materializing it first. The eager round-trip uses it to enforce
101
+ # ``max_result_bytes`` while collecting; handles without it refuse
102
+ # the budget rather than blow past it after collecting.
103
+
104
+
105
+ class DataFusionHandle:
106
+ """Handle over a ``datafusion.DataFrame``."""
107
+
108
+ supports_chunked = True
109
+
110
+ def __init__(self, df: Any) -> None:
111
+ self._df = df
112
+
113
+ def schema(self) -> pa.Schema:
114
+ return self._df.schema()
115
+
116
+ def distinct(self, column: str) -> np.ndarray:
117
+ from datafusion import col
118
+
119
+ dim_only = self._df.select(col(f'"{column}"')).distinct()
120
+ batches = [b.to_pyarrow() for b in dim_only.execute_stream()]
121
+ if not batches:
122
+ return cast(np.ndarray, np.asarray([]))
123
+ return cast(
124
+ np.ndarray,
125
+ np.concatenate(
126
+ [b.column(0).to_numpy(zero_copy_only=False) for b in batches]
127
+ ),
128
+ )
129
+
130
+ def fetch(
131
+ self, specs: dict[str, DimSpec], columns: list[str]
132
+ ) -> list[pa.RecordBatch]:
133
+ from datafusion import col, literal
134
+
135
+ predicate = None
136
+ for dim, (kind, a, b) in specs.items():
137
+ c = col(f'"{dim}"')
138
+ if kind == "range":
139
+ p = (c >= literal(a)) & (c <= literal(b))
140
+ else:
141
+ # DataFusion 52.0.0 exposes no clean ``Expr.in_list``
142
+ # from Python; OR-chained equalities constant-fold
143
+ # equivalently and stay typed.
144
+ p = c == literal(a[0])
145
+ for v in a[1:]:
146
+ p = p | (c == literal(v))
147
+ predicate = p if predicate is None else predicate & p
148
+ out = self._df if predicate is None else self._df.filter(predicate)
149
+ out = out.select(*(col(f'"{n}"') for n in columns))
150
+ return [b.to_pyarrow() for b in out.execute_stream()]
151
+
152
+ def stream(self, columns: list[str]) -> Iterator[pa.RecordBatch]:
153
+ """Execute once, yielding Arrow batches as the plan produces them."""
154
+ from datafusion import col
155
+
156
+ out = self._df.select(*(col(f'"{n}"') for n in columns))
157
+ return (b.to_pyarrow() for b in out.execute_stream())
158
+
159
+ def spill_parquet(self, path: str) -> None:
160
+ with pq.ParquetWriter(path, self.schema()) as writer:
161
+ for batch in self._df.execute_stream():
162
+ writer.write_batch(batch.to_pyarrow())
163
+
164
+
165
+ class DuckDBHandle:
166
+ """Handle over a ``duckdb.DuckDBPyRelation``.
167
+
168
+ Relations are lazy relational algebra: ``filter``/``project`` derive
169
+ new relations and every materialization runs the query again, so a
170
+ single relation can serve any number of per-chunk fetches, each
171
+ narrowed to its own window — the *re-executable* property the
172
+ module docstring requires of every handle. Predicates are built
173
+ with DuckDB's typed expression API, never rendered SQL text.
174
+
175
+ Every engine call runs on one dedicated thread owned by the handle.
176
+ A relation is bound to one connection, and a query over a table
177
+ registered through xarray-sql re-enters Python from DuckDB's
178
+ execution threads (the Arrow scan callback); driving such queries
179
+ directly from several consumer threads at once (dask computing
180
+ output chunks of a lazy round-trip) deadlocks between the
181
+ connection's serialization, the callback's need for the GIL, and
182
+ the consumer pool's own thread management. Funnelling execution
183
+ through a single pre-started thread reproduces the topology that is
184
+ known safe — one thread inside the engine, every other thread
185
+ parked on a GIL-releasing wait.
186
+ """
187
+
188
+ supports_chunked = False
189
+ """Chunked (lazy) reconstruction is disabled for DuckDB relations.
190
+
191
+ Windows of a chunked round-trip re-execute the relation from the
192
+ consumer's worker threads (dask). A DuckDB query whose source is a
193
+ Python-callback Arrow scan (any table registered through xarray-sql)
194
+ intermittently deadlocks inside duckdb-python/CPython when other
195
+ Python threads start or stop during execution — reproduced on
196
+ duckdb 1.4-1.5 / CPython 3.12 / macOS at ~50% of runs, regardless
197
+ of ``SET threads=1``, connection serialization, or pool pre-warming.
198
+ Until that upstream race is fixed, chunked DuckDB round-trips fail
199
+ fast instead of hanging; the eager path (and every other handle
200
+ operation) runs on one dedicated thread and is unaffected.
201
+ """
202
+
203
+ def __init__(self, rel: Any) -> None:
204
+ self._rel = rel
205
+ self._runner: ThreadPoolExecutor | None = None
206
+ if not _CAN_START_THREADS:
207
+ # Single-threaded interpreters (Pyodide) have no other
208
+ # threads to deadlock against; run engine calls inline.
209
+ return
210
+ self._runner = ThreadPoolExecutor(max_workers=1)
211
+ self._runner.submit(lambda: None).result() # start the thread now
212
+ # Stop the dedicated engine thread when the handle dies; it
213
+ # would otherwise linger for the life of the process, one per
214
+ # discarded handle. The callback is bound to the executor, not
215
+ # the handle, so the finalizer holds no reference back to self.
216
+ weakref.finalize(
217
+ self, self._runner.shutdown, wait=False, cancel_futures=True
218
+ )
219
+
220
+ def _run(self, fn: Any) -> Any:
221
+ if self._runner is None:
222
+ return fn()
223
+ return self._runner.submit(fn).result()
224
+
225
+ @staticmethod
226
+ def _to_arrow_table(rel: Any) -> pa.Table:
227
+ if hasattr(rel, "to_arrow_table"):
228
+ return rel.to_arrow_table()
229
+ return rel.fetch_arrow_table() # duckdb < 1.5
230
+
231
+ @staticmethod
232
+ def _to_arrow_reader(rel: Any) -> pa.RecordBatchReader:
233
+ if hasattr(rel, "to_arrow_reader"):
234
+ return rel.to_arrow_reader()
235
+ return rel.fetch_record_batch() # duckdb < 1.5
236
+
237
+ def schema(self) -> pa.Schema:
238
+ return self._run(
239
+ lambda: self._to_arrow_table(self._rel.limit(0)).schema
240
+ )
241
+
242
+ def distinct(self, column: str) -> np.ndarray:
243
+ import duckdb
244
+
245
+ table = self._run(
246
+ lambda: self._to_arrow_table(
247
+ self._rel.project(duckdb.ColumnExpression(column)).distinct()
248
+ )
249
+ )
250
+ return cast(
251
+ np.ndarray,
252
+ np.asarray(table.column(0).to_numpy(zero_copy_only=False)),
253
+ )
254
+
255
+ def fetch(
256
+ self, specs: dict[str, DimSpec], columns: list[str]
257
+ ) -> list[pa.RecordBatch]:
258
+ import duckdb
259
+
260
+ predicate = None
261
+ for dim, (kind, a, b) in specs.items():
262
+ c = duckdb.ColumnExpression(dim)
263
+ if kind == "range":
264
+ p = (c >= duckdb.ConstantExpression(_plain(a))) & (
265
+ c <= duckdb.ConstantExpression(_plain(b))
266
+ )
267
+ else:
268
+ p = c.isin(*(duckdb.ConstantExpression(_plain(v)) for v in a))
269
+ predicate = p if predicate is None else predicate & p
270
+ rel = self._rel if predicate is None else self._rel.filter(predicate)
271
+ rel = rel.project(*(duckdb.ColumnExpression(n) for n in columns))
272
+
273
+ return cast(
274
+ list[pa.RecordBatch],
275
+ self._run(lambda: list(self._to_arrow_reader(rel))),
276
+ )
277
+
278
+ def stream(self, columns: list[str]) -> Iterator[pa.RecordBatch]:
279
+ """Execute once, yielding Arrow batches as DuckDB produces them.
280
+
281
+ The reader is closed on the engine thread, even when the consumer
282
+ stops early, so the query is never torn down from another thread.
283
+ """
284
+ import duckdb
285
+
286
+ rel = self._rel.project(*(duckdb.ColumnExpression(n) for n in columns))
287
+ reader = self._run(lambda: self._to_arrow_reader(rel))
288
+ batches = iter(reader)
289
+ try:
290
+ while (batch := self._run(lambda: next(batches, None))) is not None:
291
+ yield batch
292
+ finally:
293
+ self._run(reader.close)
294
+
295
+ def spill_parquet(self, path: str) -> None:
296
+ def run() -> None:
297
+ reader = self._to_arrow_reader(self._rel)
298
+ with pq.ParquetWriter(path, reader.schema) as writer:
299
+ for batch in reader:
300
+ writer.write_batch(batch)
301
+
302
+ self._run(run)
303
+
304
+
305
+ def _nanosecond(dtype: Any) -> str | None:
306
+ """The numpy ``[ns]`` type matching a Polars time column, else ``None``."""
307
+ import polars as pl
308
+
309
+ if isinstance(dtype, pl.Datetime):
310
+ return "datetime64[ns]"
311
+ if isinstance(dtype, pl.Duration):
312
+ return "timedelta64[ns]"
313
+ return None
314
+
315
+
316
+ def _in_zone(expr: Any, dtype: Any) -> Any:
317
+ """*expr*, a naive UTC time, in *dtype*'s zone if it has one."""
318
+ zone = getattr(dtype, "time_zone", None)
319
+ if zone:
320
+ return expr.dt.replace_time_zone("UTC").dt.convert_time_zone(zone)
321
+ return expr
322
+
323
+
324
+ def _polars_times(values: np.ndarray, dtype: Any) -> Any:
325
+ """Window times or durations as a Polars Series of the column's type.
326
+
327
+ Built from the numpy values, so nanoseconds survive: Polars reads a
328
+ ``pd.Timestamp``, ``pd.Timedelta``, or ``datetime`` literal as
329
+ microseconds, which never equals a value in a nanosecond column
330
+ (``is_in`` then matches nothing, or fails as a join on mismatched
331
+ key types). A naive window time is a UTC instant; it is expressed in
332
+ the column's zone, if any.
333
+ """
334
+ import polars as pl
335
+
336
+ series = pl.Series(np.asarray(values, dtype=_nanosecond(dtype)))
337
+ return _in_zone(series, dtype).cast(dtype)
338
+
339
+
340
+ def _polars_value(value: Any, dtype: Any) -> Any:
341
+ """One window bound as a Polars literal comparable with *dtype*."""
342
+ import polars as pl
343
+
344
+ unit = _nanosecond(dtype)
345
+ if unit and isinstance(value, (np.datetime64, np.timedelta64)):
346
+ # From the integer count, which is exact; a Python object is not.
347
+ nanos = int(value.astype(unit).astype("int64"))
348
+ literal = pl.lit(nanos, dtype=pl.Int64).cast(type(dtype)("ns"))
349
+ return _in_zone(literal, dtype).cast(dtype)
350
+ return _plain(value)
351
+
352
+
353
+ class PolarsHandle:
354
+ """Handle over a ``polars.LazyFrame``.
355
+
356
+ Per-window fetches run on the streaming engine, so a window read
357
+ never materializes more than the window even when the frame scans
358
+ an out-of-core source.
359
+ """
360
+
361
+ supports_chunked = True
362
+
363
+ def __init__(self, lf: Any) -> None:
364
+ self._lf = lf
365
+
366
+ def schema(self) -> pa.Schema:
367
+ import polars as pl
368
+
369
+ return pl.DataFrame(schema=self._lf.collect_schema()).to_arrow().schema
370
+
371
+ def distinct(self, column: str) -> np.ndarray:
372
+ import polars as pl
373
+
374
+ out = _collect_streaming(self._lf.select(pl.col(column).unique()))
375
+ return cast(np.ndarray, out.to_series().to_numpy())
376
+
377
+ def fetch(
378
+ self, specs: dict[str, DimSpec], columns: list[str]
379
+ ) -> list[pa.RecordBatch]:
380
+ import polars as pl
381
+
382
+ schema = self._lf.collect_schema()
383
+ exprs = []
384
+ for dim, (kind, a, b) in specs.items():
385
+ dtype = schema.get(dim)
386
+ if kind == "range":
387
+ exprs.append(
388
+ pl.col(dim).is_between(
389
+ _polars_value(a, dtype), _polars_value(b, dtype)
390
+ )
391
+ )
392
+ elif getattr(a, "dtype", None) is not None and a.dtype.kind == "f":
393
+ # Upstream Polars translates float ``is_in`` literals
394
+ # imprecisely (silently matching nothing); degenerate
395
+ # ranges compare exactly. Reproduced on polars 1.42.
396
+ # ``any_horizontal`` keeps the disjunction flat — a
397
+ # left-deep OR chain plans quadratically in the number
398
+ # of values.
399
+ exprs.append(
400
+ pl.any_horizontal(
401
+ [pl.col(dim).is_between(*(_plain(v),) * 2) for v in a]
402
+ )
403
+ )
404
+ elif _nanosecond(dtype):
405
+ exprs.append(pl.col(dim).is_in(_polars_times(a, dtype)))
406
+ else:
407
+ exprs.append(pl.col(dim).is_in([_plain(v) for v in a]))
408
+ lf = self._lf.filter(*exprs) if exprs else self._lf
409
+ out = _collect_streaming(lf.select([pl.col(n) for n in columns]))
410
+ return cast(list[pa.RecordBatch], out.to_arrow().to_batches())
411
+
412
+ def stream(self, columns: list[str]) -> Iterator[pa.RecordBatch]:
413
+ """Execute once, yielding Arrow batches incrementally.
414
+
415
+ Unlike [fetch][xarray_sql.lazyscan.PolarsHandle.fetch], whose ``collect`` materializes the whole
416
+ result inside the engine before any batch surfaces, this yields
417
+ batches as the streaming engine produces them, so a byte budget
418
+ can fire before the result is fully in memory. Requires
419
+ ``LazyFrame.collect_batches`` (polars >= 1.33); older Polars
420
+ raises here, before anything is collected.
421
+ """
422
+ import polars as pl
423
+
424
+ lf = self._lf.select([pl.col(n) for n in columns])
425
+ if not hasattr(lf, "collect_batches"):
426
+ raise ValueError(
427
+ "streaming collection of a Polars LazyFrame requires "
428
+ "polars >= 1.33 (LazyFrame.collect_batches)."
429
+ )
430
+
431
+ def generate() -> Iterator[pa.RecordBatch]:
432
+ for frame in lf.collect_batches(engine="streaming"):
433
+ yield from frame.to_arrow().to_batches()
434
+
435
+ return generate()
436
+
437
+ def spill_parquet(self, path: str) -> None:
438
+ self._lf.sink_parquet(path)
439
+
440
+
441
+ def resolve_lazy_handle(result: Any) -> LazyResultHandle | None:
442
+ """Adapt an engine result to a handle, or ``None`` if it is one-shot.
443
+
444
+ Recognizes DuckDB relations, Polars lazy *and* eager frames (an
445
+ eager frame re-executes trivially over its in-memory data), and
446
+ DataFusion DataFrames. ``pyarrow`` tables/readers and bare
447
+ ``__arrow_c_stream__`` objects are one-shot streams: there is no
448
+ query to re-execute, so the lazy path cannot serve them.
449
+ """
450
+ root = type(result).__module__.split(".")[0]
451
+ if root in ("duckdb", "_duckdb") and hasattr(result, "filter"):
452
+ return DuckDBHandle(result)
453
+ if root == "polars":
454
+ import polars as pl
455
+
456
+ if isinstance(result, pl.LazyFrame):
457
+ return PolarsHandle(result)
458
+ if isinstance(result, pl.DataFrame):
459
+ return PolarsHandle(result.lazy())
460
+ if hasattr(result, "execute_stream") and hasattr(result, "logical_plan"):
461
+ return DataFusionHandle(result)
462
+ return None