xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
xarray_sql/geometry.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""GeoArrow point-geometry columns derived from coordinate dimensions.
|
|
2
|
+
|
|
3
|
+
A regular grid's pivot already materializes per-row x/y coordinate
|
|
4
|
+
columns; a point-geometry column is those same values under a GeoArrow
|
|
5
|
+
extension annotation. Two encodings:
|
|
6
|
+
|
|
7
|
+
* ``"wkb"`` (default) — 21-byte WKB points under the ``geoarrow.wkb``
|
|
8
|
+
extension name. DuckDB (>= 1.2, spatial loaded) ingests the column as
|
|
9
|
+
a native ``GEOMETRY`` with the CRS attached, so ``ST_Within(geometry,
|
|
10
|
+
...)`` works with no ``ST_Point(x, y)`` construction in user SQL.
|
|
11
|
+
* ``"point"`` — GeoArrow native points with *separated* coordinates
|
|
12
|
+
(``struct<x: double, y: double>`` under ``geoarrow.point``): the
|
|
13
|
+
child arrays are the coordinate columns themselves, no per-row
|
|
14
|
+
parsing for consumers that execute on native layouts (GeoPandas 1.x,
|
|
15
|
+
geoarrow-rs, lonboard, SedonaDB). DuckDB does not consume this
|
|
16
|
+
encoding.
|
|
17
|
+
|
|
18
|
+
The CRS rides in the extension metadata (GeoArrow 0.2 allows
|
|
19
|
+
authority:code strings alongside PROJJSON). ``OGC:CRS84`` is the
|
|
20
|
+
correct tag for plain longitude/latitude grids.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
import numpy as np
|
|
29
|
+
import pyarrow as pa
|
|
30
|
+
|
|
31
|
+
GEOMETRY_COLUMN = "geometry"
|
|
32
|
+
|
|
33
|
+
_ENCODINGS = ("wkb", "point")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def geometry_field(encoding: str, crs: str | None) -> pa.Field:
|
|
37
|
+
"""The schema field for the derived geometry column."""
|
|
38
|
+
if encoding not in _ENCODINGS:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"geometry_encoding must be one of {_ENCODINGS}, got {encoding!r}"
|
|
41
|
+
)
|
|
42
|
+
metadata = {
|
|
43
|
+
b"ARROW:extension:name": f"geoarrow.{encoding}".encode(),
|
|
44
|
+
}
|
|
45
|
+
if crs is not None:
|
|
46
|
+
metadata[b"ARROW:extension:metadata"] = json.dumps(
|
|
47
|
+
{"crs": crs}
|
|
48
|
+
).encode()
|
|
49
|
+
storage = (
|
|
50
|
+
pa.binary()
|
|
51
|
+
if encoding == "wkb"
|
|
52
|
+
else pa.struct([("x", pa.float64()), ("y", pa.float64())])
|
|
53
|
+
)
|
|
54
|
+
return pa.field(GEOMETRY_COLUMN, storage, metadata=metadata)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def build_geometry(encoding: str, x: pa.Array, y: pa.Array) -> pa.Array:
|
|
58
|
+
"""Point geometries for one batch's x/y coordinate columns."""
|
|
59
|
+
if encoding == "point":
|
|
60
|
+
return pa.StructArray.from_arrays(
|
|
61
|
+
[x.cast(pa.float64()), y.cast(pa.float64())], ["x", "y"]
|
|
62
|
+
)
|
|
63
|
+
return _wkb_points(
|
|
64
|
+
np.ascontiguousarray(x.to_numpy(zero_copy_only=False), "<f8"),
|
|
65
|
+
np.ascontiguousarray(y.to_numpy(zero_copy_only=False), "<f8"),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _wkb_points(x: np.ndarray, y: np.ndarray) -> pa.Array:
|
|
70
|
+
"""Vectorized 21-byte little-endian WKB point encoding."""
|
|
71
|
+
n = len(x)
|
|
72
|
+
# ``pa.binary()`` carries int32 offsets, which the final offset
|
|
73
|
+
# (n * 21) overflows past ~102M points; the buffers would build
|
|
74
|
+
# silently corrupt. Unreachable through the pivot (batch_size caps
|
|
75
|
+
# rows per batch well below this), so guard rather than widen the
|
|
76
|
+
# storage to large_binary, which DuckDB's ingestion expects not to
|
|
77
|
+
# see.
|
|
78
|
+
if n * 21 > np.iinfo(np.int32).max:
|
|
79
|
+
raise ValueError(
|
|
80
|
+
f"cannot WKB-encode {n:,} points in a single batch: "
|
|
81
|
+
"pa.binary() offsets are int32 and n * 21 bytes would "
|
|
82
|
+
"overflow them. Use a smaller batch_size."
|
|
83
|
+
)
|
|
84
|
+
buf: np.ndarray = np.empty((n, 21), dtype=np.uint8)
|
|
85
|
+
buf[:, 0] = 1 # little-endian byte order mark
|
|
86
|
+
buf[:, 1:5] = np.array([1, 0, 0, 0], dtype=np.uint8) # WKB type 1: Point
|
|
87
|
+
buf[:, 5:13] = x.view(np.uint8).reshape(n, 8)
|
|
88
|
+
buf[:, 13:21] = y.view(np.uint8).reshape(n, 8)
|
|
89
|
+
offsets = pa.py_buffer(
|
|
90
|
+
np.arange(0, (n + 1) * 21, 21, dtype=np.int32).tobytes()
|
|
91
|
+
)
|
|
92
|
+
return pa.Array.from_buffers(
|
|
93
|
+
pa.binary(), n, [None, offsets, pa.py_buffer(buf.tobytes())]
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def bbox_conjuncts(
|
|
98
|
+
bounds: Any, x: str = "x", y: str = "y", pad: float = 0.0
|
|
99
|
+
) -> str:
|
|
100
|
+
"""SQL bbox conjuncts for a geometry's envelope — the pruning half.
|
|
101
|
+
|
|
102
|
+
Engines do not push ``ST_*`` functions into the scan, so a
|
|
103
|
+
geometry-only predicate reads every chunk; pairing it with range
|
|
104
|
+
conjuncts on the coordinate columns restores pruning. This helper
|
|
105
|
+
renders those conjuncts from a geometry's envelope::
|
|
106
|
+
|
|
107
|
+
poly = shapely.from_wkt("POLYGON (...)")
|
|
108
|
+
sql = (
|
|
109
|
+
f"SELECT avg(risk) FROM eri "
|
|
110
|
+
f"WHERE {xql.bbox_conjuncts(poly, x='x', y='y')} "
|
|
111
|
+
f"AND ST_Within(geometry, ST_GeomFromText('{poly.wkt}'))"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
bounds: ``(xmin, ymin, xmax, ymax)``, or any object with a
|
|
116
|
+
``.bounds`` attribute in that convention (shapely
|
|
117
|
+
geometries qualify).
|
|
118
|
+
x: The x/longitude column name.
|
|
119
|
+
y: The y/latitude column name.
|
|
120
|
+
pad: Optional margin added on every side (e.g. to be safe
|
|
121
|
+
around ``ST_DWithin``-style predicates).
|
|
122
|
+
|
|
123
|
+
Returns:
|
|
124
|
+
A SQL snippet ``"x" BETWEEN a AND b AND "y" BETWEEN c AND d``.
|
|
125
|
+
"""
|
|
126
|
+
values = getattr(bounds, "bounds", bounds)
|
|
127
|
+
xmin, ymin, xmax, ymax = (float(v) for v in values)
|
|
128
|
+
return (
|
|
129
|
+
f'"{x}" BETWEEN {xmin - pad!r} AND {xmax + pad!r} '
|
|
130
|
+
f'AND "{y}" BETWEEN {ymin - pad!r} AND {ymax + pad!r}'
|
|
131
|
+
)
|
xarray_sql/lazyscan.py
ADDED
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
"""Re-executable engine handles behind the lazy chunked round-trip.
|
|
2
|
+
|
|
3
|
+
The lazy path of ``to_dataset(chunks=...)`` re-executes the engine's
|
|
4
|
+
query per accessed chunk, narrowed to that chunk's coordinate window and
|
|
5
|
+
columns. That requires the engine result to be *re-executable* — a
|
|
6
|
+
handle onto the query, not a one-shot stream of its rows. Each handle
|
|
7
|
+
here adapts one engine's native lazy surface to the three operations the
|
|
8
|
+
reconstruction needs:
|
|
9
|
+
|
|
10
|
+
* [schema][xarray_sql.lazyscan.LazyResultHandle.schema] — result column names/types, without
|
|
11
|
+
executing the query;
|
|
12
|
+
* [distinct][xarray_sql.lazyscan.LazyResultHandle.distinct] — one column's distinct values
|
|
13
|
+
(coordinate discovery; the caller sorts);
|
|
14
|
+
* [fetch][xarray_sql.lazyscan.LazyResultHandle.fetch] — the result narrowed by per-dimension
|
|
15
|
+
windows and projected to the requested columns, as Arrow batches.
|
|
16
|
+
|
|
17
|
+
Windows are passed as [DimSpec][xarray_sql.lazyscan.DimSpec] values instead of rendered SQL so
|
|
18
|
+
each engine can express them with its own *typed* expression API —
|
|
19
|
+
strings would re-open every literal-formatting pitfall (timestamps,
|
|
20
|
+
floats, quoting) per dialect.
|
|
21
|
+
|
|
22
|
+
Handles compose with the registration seam: when the wrapped query
|
|
23
|
+
scans a Dataset registered through xarray-sql's pushdown machinery, the
|
|
24
|
+
per-chunk range filter flows back through the engine into
|
|
25
|
+
[XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset], so each
|
|
26
|
+
output chunk's access reads only the source chunks it maps onto.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import weakref
|
|
32
|
+
from collections.abc import Iterator
|
|
33
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
34
|
+
from typing import Any, Literal, Protocol, cast
|
|
35
|
+
|
|
36
|
+
import numpy as np
|
|
37
|
+
import pandas as pd
|
|
38
|
+
import pyarrow as pa
|
|
39
|
+
import pyarrow.parquet as pq
|
|
40
|
+
|
|
41
|
+
from .df import _CAN_START_THREADS
|
|
42
|
+
|
|
43
|
+
DimSpec = tuple[Literal["range", "values"], Any, Any]
|
|
44
|
+
"""One dimension's window: ``("range", lo, hi)`` (inclusive bounds; the
|
|
45
|
+
requested coordinate positions are contiguous) or ``("values", array,
|
|
46
|
+
None)`` (explicit value list, for stepped/fancy indexers).
|
|
47
|
+
|
|
48
|
+
The payload stays ``Any``: the values are coordinate scalars handed to
|
|
49
|
+
the engine's typed expression API, which does the comparing — Python
|
|
50
|
+
never orders them, and a concrete union over coordinate dtypes
|
|
51
|
+
(timestamps, cftime, numerics, strings) would stay incomplete."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _collect_streaming(lf: Any) -> Any:
|
|
55
|
+
"""Collect a Polars LazyFrame on the streaming engine, if available.
|
|
56
|
+
|
|
57
|
+
``collect(engine="streaming")`` needs polars >= 1.25 (the test
|
|
58
|
+
extra pins higher); an older installed polars raises TypeError on
|
|
59
|
+
the unknown keyword, and the plain in-memory collect is a correct,
|
|
60
|
+
if less memory-frugal, fallback.
|
|
61
|
+
"""
|
|
62
|
+
try:
|
|
63
|
+
return lf.collect(engine="streaming")
|
|
64
|
+
except TypeError:
|
|
65
|
+
return lf.collect()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _plain(value: Any) -> Any:
|
|
69
|
+
"""A plain-Python literal (numpy scalars don't travel to engines)."""
|
|
70
|
+
if isinstance(value, np.datetime64):
|
|
71
|
+
return pd.Timestamp(value)
|
|
72
|
+
if isinstance(value, np.timedelta64):
|
|
73
|
+
return pd.Timedelta(value)
|
|
74
|
+
if isinstance(value, np.generic):
|
|
75
|
+
return value.item()
|
|
76
|
+
return value
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class LazyResultHandle(Protocol):
|
|
80
|
+
"""A re-executable query result (see module docstring)."""
|
|
81
|
+
|
|
82
|
+
supports_chunked: bool = True
|
|
83
|
+
"""Whether fetch() may be driven from consumer worker threads (the
|
|
84
|
+
chunked reconstruction). Handles for engines that cannot safely
|
|
85
|
+
re-execute under foreign threads set this False; the eager path
|
|
86
|
+
remains available."""
|
|
87
|
+
|
|
88
|
+
def schema(self) -> pa.Schema: ...
|
|
89
|
+
|
|
90
|
+
def distinct(self, column: str) -> np.ndarray: ...
|
|
91
|
+
|
|
92
|
+
def fetch(
|
|
93
|
+
self, specs: dict[str, DimSpec], columns: list[str]
|
|
94
|
+
) -> list[pa.RecordBatch]: ...
|
|
95
|
+
|
|
96
|
+
def spill_parquet(self, path: str) -> None: ...
|
|
97
|
+
|
|
98
|
+
# Handles may additionally offer ``stream(columns)``, yielding the
|
|
99
|
+
# unfiltered result's Arrow batches incrementally instead of
|
|
100
|
+
# materializing it first. The eager round-trip uses it to enforce
|
|
101
|
+
# ``max_result_bytes`` while collecting; handles without it refuse
|
|
102
|
+
# the budget rather than blow past it after collecting.
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class DataFusionHandle:
|
|
106
|
+
"""Handle over a ``datafusion.DataFrame``."""
|
|
107
|
+
|
|
108
|
+
supports_chunked = True
|
|
109
|
+
|
|
110
|
+
def __init__(self, df: Any) -> None:
|
|
111
|
+
self._df = df
|
|
112
|
+
|
|
113
|
+
def schema(self) -> pa.Schema:
|
|
114
|
+
return self._df.schema()
|
|
115
|
+
|
|
116
|
+
def distinct(self, column: str) -> np.ndarray:
|
|
117
|
+
from datafusion import col
|
|
118
|
+
|
|
119
|
+
dim_only = self._df.select(col(f'"{column}"')).distinct()
|
|
120
|
+
batches = [b.to_pyarrow() for b in dim_only.execute_stream()]
|
|
121
|
+
if not batches:
|
|
122
|
+
return cast(np.ndarray, np.asarray([]))
|
|
123
|
+
return cast(
|
|
124
|
+
np.ndarray,
|
|
125
|
+
np.concatenate(
|
|
126
|
+
[b.column(0).to_numpy(zero_copy_only=False) for b in batches]
|
|
127
|
+
),
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
def fetch(
|
|
131
|
+
self, specs: dict[str, DimSpec], columns: list[str]
|
|
132
|
+
) -> list[pa.RecordBatch]:
|
|
133
|
+
from datafusion import col, literal
|
|
134
|
+
|
|
135
|
+
predicate = None
|
|
136
|
+
for dim, (kind, a, b) in specs.items():
|
|
137
|
+
c = col(f'"{dim}"')
|
|
138
|
+
if kind == "range":
|
|
139
|
+
p = (c >= literal(a)) & (c <= literal(b))
|
|
140
|
+
else:
|
|
141
|
+
# DataFusion 52.0.0 exposes no clean ``Expr.in_list``
|
|
142
|
+
# from Python; OR-chained equalities constant-fold
|
|
143
|
+
# equivalently and stay typed.
|
|
144
|
+
p = c == literal(a[0])
|
|
145
|
+
for v in a[1:]:
|
|
146
|
+
p = p | (c == literal(v))
|
|
147
|
+
predicate = p if predicate is None else predicate & p
|
|
148
|
+
out = self._df if predicate is None else self._df.filter(predicate)
|
|
149
|
+
out = out.select(*(col(f'"{n}"') for n in columns))
|
|
150
|
+
return [b.to_pyarrow() for b in out.execute_stream()]
|
|
151
|
+
|
|
152
|
+
def stream(self, columns: list[str]) -> Iterator[pa.RecordBatch]:
|
|
153
|
+
"""Execute once, yielding Arrow batches as the plan produces them."""
|
|
154
|
+
from datafusion import col
|
|
155
|
+
|
|
156
|
+
out = self._df.select(*(col(f'"{n}"') for n in columns))
|
|
157
|
+
return (b.to_pyarrow() for b in out.execute_stream())
|
|
158
|
+
|
|
159
|
+
def spill_parquet(self, path: str) -> None:
|
|
160
|
+
with pq.ParquetWriter(path, self.schema()) as writer:
|
|
161
|
+
for batch in self._df.execute_stream():
|
|
162
|
+
writer.write_batch(batch.to_pyarrow())
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class DuckDBHandle:
|
|
166
|
+
"""Handle over a ``duckdb.DuckDBPyRelation``.
|
|
167
|
+
|
|
168
|
+
Relations are lazy relational algebra: ``filter``/``project`` derive
|
|
169
|
+
new relations and every materialization runs the query again, so a
|
|
170
|
+
single relation can serve any number of per-chunk fetches, each
|
|
171
|
+
narrowed to its own window — the *re-executable* property the
|
|
172
|
+
module docstring requires of every handle. Predicates are built
|
|
173
|
+
with DuckDB's typed expression API, never rendered SQL text.
|
|
174
|
+
|
|
175
|
+
Every engine call runs on one dedicated thread owned by the handle.
|
|
176
|
+
A relation is bound to one connection, and a query over a table
|
|
177
|
+
registered through xarray-sql re-enters Python from DuckDB's
|
|
178
|
+
execution threads (the Arrow scan callback); driving such queries
|
|
179
|
+
directly from several consumer threads at once (dask computing
|
|
180
|
+
output chunks of a lazy round-trip) deadlocks between the
|
|
181
|
+
connection's serialization, the callback's need for the GIL, and
|
|
182
|
+
the consumer pool's own thread management. Funnelling execution
|
|
183
|
+
through a single pre-started thread reproduces the topology that is
|
|
184
|
+
known safe — one thread inside the engine, every other thread
|
|
185
|
+
parked on a GIL-releasing wait.
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
supports_chunked = False
|
|
189
|
+
"""Chunked (lazy) reconstruction is disabled for DuckDB relations.
|
|
190
|
+
|
|
191
|
+
Windows of a chunked round-trip re-execute the relation from the
|
|
192
|
+
consumer's worker threads (dask). A DuckDB query whose source is a
|
|
193
|
+
Python-callback Arrow scan (any table registered through xarray-sql)
|
|
194
|
+
intermittently deadlocks inside duckdb-python/CPython when other
|
|
195
|
+
Python threads start or stop during execution — reproduced on
|
|
196
|
+
duckdb 1.4-1.5 / CPython 3.12 / macOS at ~50% of runs, regardless
|
|
197
|
+
of ``SET threads=1``, connection serialization, or pool pre-warming.
|
|
198
|
+
Until that upstream race is fixed, chunked DuckDB round-trips fail
|
|
199
|
+
fast instead of hanging; the eager path (and every other handle
|
|
200
|
+
operation) runs on one dedicated thread and is unaffected.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
def __init__(self, rel: Any) -> None:
|
|
204
|
+
self._rel = rel
|
|
205
|
+
self._runner: ThreadPoolExecutor | None = None
|
|
206
|
+
if not _CAN_START_THREADS:
|
|
207
|
+
# Single-threaded interpreters (Pyodide) have no other
|
|
208
|
+
# threads to deadlock against; run engine calls inline.
|
|
209
|
+
return
|
|
210
|
+
self._runner = ThreadPoolExecutor(max_workers=1)
|
|
211
|
+
self._runner.submit(lambda: None).result() # start the thread now
|
|
212
|
+
# Stop the dedicated engine thread when the handle dies; it
|
|
213
|
+
# would otherwise linger for the life of the process, one per
|
|
214
|
+
# discarded handle. The callback is bound to the executor, not
|
|
215
|
+
# the handle, so the finalizer holds no reference back to self.
|
|
216
|
+
weakref.finalize(
|
|
217
|
+
self, self._runner.shutdown, wait=False, cancel_futures=True
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
def _run(self, fn: Any) -> Any:
|
|
221
|
+
if self._runner is None:
|
|
222
|
+
return fn()
|
|
223
|
+
return self._runner.submit(fn).result()
|
|
224
|
+
|
|
225
|
+
@staticmethod
|
|
226
|
+
def _to_arrow_table(rel: Any) -> pa.Table:
|
|
227
|
+
if hasattr(rel, "to_arrow_table"):
|
|
228
|
+
return rel.to_arrow_table()
|
|
229
|
+
return rel.fetch_arrow_table() # duckdb < 1.5
|
|
230
|
+
|
|
231
|
+
@staticmethod
|
|
232
|
+
def _to_arrow_reader(rel: Any) -> pa.RecordBatchReader:
|
|
233
|
+
if hasattr(rel, "to_arrow_reader"):
|
|
234
|
+
return rel.to_arrow_reader()
|
|
235
|
+
return rel.fetch_record_batch() # duckdb < 1.5
|
|
236
|
+
|
|
237
|
+
def schema(self) -> pa.Schema:
|
|
238
|
+
return self._run(
|
|
239
|
+
lambda: self._to_arrow_table(self._rel.limit(0)).schema
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
def distinct(self, column: str) -> np.ndarray:
|
|
243
|
+
import duckdb
|
|
244
|
+
|
|
245
|
+
table = self._run(
|
|
246
|
+
lambda: self._to_arrow_table(
|
|
247
|
+
self._rel.project(duckdb.ColumnExpression(column)).distinct()
|
|
248
|
+
)
|
|
249
|
+
)
|
|
250
|
+
return cast(
|
|
251
|
+
np.ndarray,
|
|
252
|
+
np.asarray(table.column(0).to_numpy(zero_copy_only=False)),
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
def fetch(
|
|
256
|
+
self, specs: dict[str, DimSpec], columns: list[str]
|
|
257
|
+
) -> list[pa.RecordBatch]:
|
|
258
|
+
import duckdb
|
|
259
|
+
|
|
260
|
+
predicate = None
|
|
261
|
+
for dim, (kind, a, b) in specs.items():
|
|
262
|
+
c = duckdb.ColumnExpression(dim)
|
|
263
|
+
if kind == "range":
|
|
264
|
+
p = (c >= duckdb.ConstantExpression(_plain(a))) & (
|
|
265
|
+
c <= duckdb.ConstantExpression(_plain(b))
|
|
266
|
+
)
|
|
267
|
+
else:
|
|
268
|
+
p = c.isin(*(duckdb.ConstantExpression(_plain(v)) for v in a))
|
|
269
|
+
predicate = p if predicate is None else predicate & p
|
|
270
|
+
rel = self._rel if predicate is None else self._rel.filter(predicate)
|
|
271
|
+
rel = rel.project(*(duckdb.ColumnExpression(n) for n in columns))
|
|
272
|
+
|
|
273
|
+
return cast(
|
|
274
|
+
list[pa.RecordBatch],
|
|
275
|
+
self._run(lambda: list(self._to_arrow_reader(rel))),
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
def stream(self, columns: list[str]) -> Iterator[pa.RecordBatch]:
|
|
279
|
+
"""Execute once, yielding Arrow batches as DuckDB produces them.
|
|
280
|
+
|
|
281
|
+
The reader is closed on the engine thread, even when the consumer
|
|
282
|
+
stops early, so the query is never torn down from another thread.
|
|
283
|
+
"""
|
|
284
|
+
import duckdb
|
|
285
|
+
|
|
286
|
+
rel = self._rel.project(*(duckdb.ColumnExpression(n) for n in columns))
|
|
287
|
+
reader = self._run(lambda: self._to_arrow_reader(rel))
|
|
288
|
+
batches = iter(reader)
|
|
289
|
+
try:
|
|
290
|
+
while (batch := self._run(lambda: next(batches, None))) is not None:
|
|
291
|
+
yield batch
|
|
292
|
+
finally:
|
|
293
|
+
self._run(reader.close)
|
|
294
|
+
|
|
295
|
+
def spill_parquet(self, path: str) -> None:
|
|
296
|
+
def run() -> None:
|
|
297
|
+
reader = self._to_arrow_reader(self._rel)
|
|
298
|
+
with pq.ParquetWriter(path, reader.schema) as writer:
|
|
299
|
+
for batch in reader:
|
|
300
|
+
writer.write_batch(batch)
|
|
301
|
+
|
|
302
|
+
self._run(run)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _nanosecond(dtype: Any) -> str | None:
|
|
306
|
+
"""The numpy ``[ns]`` type matching a Polars time column, else ``None``."""
|
|
307
|
+
import polars as pl
|
|
308
|
+
|
|
309
|
+
if isinstance(dtype, pl.Datetime):
|
|
310
|
+
return "datetime64[ns]"
|
|
311
|
+
if isinstance(dtype, pl.Duration):
|
|
312
|
+
return "timedelta64[ns]"
|
|
313
|
+
return None
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _in_zone(expr: Any, dtype: Any) -> Any:
|
|
317
|
+
"""*expr*, a naive UTC time, in *dtype*'s zone if it has one."""
|
|
318
|
+
zone = getattr(dtype, "time_zone", None)
|
|
319
|
+
if zone:
|
|
320
|
+
return expr.dt.replace_time_zone("UTC").dt.convert_time_zone(zone)
|
|
321
|
+
return expr
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _polars_times(values: np.ndarray, dtype: Any) -> Any:
|
|
325
|
+
"""Window times or durations as a Polars Series of the column's type.
|
|
326
|
+
|
|
327
|
+
Built from the numpy values, so nanoseconds survive: Polars reads a
|
|
328
|
+
``pd.Timestamp``, ``pd.Timedelta``, or ``datetime`` literal as
|
|
329
|
+
microseconds, which never equals a value in a nanosecond column
|
|
330
|
+
(``is_in`` then matches nothing, or fails as a join on mismatched
|
|
331
|
+
key types). A naive window time is a UTC instant; it is expressed in
|
|
332
|
+
the column's zone, if any.
|
|
333
|
+
"""
|
|
334
|
+
import polars as pl
|
|
335
|
+
|
|
336
|
+
series = pl.Series(np.asarray(values, dtype=_nanosecond(dtype)))
|
|
337
|
+
return _in_zone(series, dtype).cast(dtype)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _polars_value(value: Any, dtype: Any) -> Any:
|
|
341
|
+
"""One window bound as a Polars literal comparable with *dtype*."""
|
|
342
|
+
import polars as pl
|
|
343
|
+
|
|
344
|
+
unit = _nanosecond(dtype)
|
|
345
|
+
if unit and isinstance(value, (np.datetime64, np.timedelta64)):
|
|
346
|
+
# From the integer count, which is exact; a Python object is not.
|
|
347
|
+
nanos = int(value.astype(unit).astype("int64"))
|
|
348
|
+
literal = pl.lit(nanos, dtype=pl.Int64).cast(type(dtype)("ns"))
|
|
349
|
+
return _in_zone(literal, dtype).cast(dtype)
|
|
350
|
+
return _plain(value)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class PolarsHandle:
|
|
354
|
+
"""Handle over a ``polars.LazyFrame``.
|
|
355
|
+
|
|
356
|
+
Per-window fetches run on the streaming engine, so a window read
|
|
357
|
+
never materializes more than the window even when the frame scans
|
|
358
|
+
an out-of-core source.
|
|
359
|
+
"""
|
|
360
|
+
|
|
361
|
+
supports_chunked = True
|
|
362
|
+
|
|
363
|
+
def __init__(self, lf: Any) -> None:
|
|
364
|
+
self._lf = lf
|
|
365
|
+
|
|
366
|
+
def schema(self) -> pa.Schema:
|
|
367
|
+
import polars as pl
|
|
368
|
+
|
|
369
|
+
return pl.DataFrame(schema=self._lf.collect_schema()).to_arrow().schema
|
|
370
|
+
|
|
371
|
+
def distinct(self, column: str) -> np.ndarray:
|
|
372
|
+
import polars as pl
|
|
373
|
+
|
|
374
|
+
out = _collect_streaming(self._lf.select(pl.col(column).unique()))
|
|
375
|
+
return cast(np.ndarray, out.to_series().to_numpy())
|
|
376
|
+
|
|
377
|
+
def fetch(
|
|
378
|
+
self, specs: dict[str, DimSpec], columns: list[str]
|
|
379
|
+
) -> list[pa.RecordBatch]:
|
|
380
|
+
import polars as pl
|
|
381
|
+
|
|
382
|
+
schema = self._lf.collect_schema()
|
|
383
|
+
exprs = []
|
|
384
|
+
for dim, (kind, a, b) in specs.items():
|
|
385
|
+
dtype = schema.get(dim)
|
|
386
|
+
if kind == "range":
|
|
387
|
+
exprs.append(
|
|
388
|
+
pl.col(dim).is_between(
|
|
389
|
+
_polars_value(a, dtype), _polars_value(b, dtype)
|
|
390
|
+
)
|
|
391
|
+
)
|
|
392
|
+
elif getattr(a, "dtype", None) is not None and a.dtype.kind == "f":
|
|
393
|
+
# Upstream Polars translates float ``is_in`` literals
|
|
394
|
+
# imprecisely (silently matching nothing); degenerate
|
|
395
|
+
# ranges compare exactly. Reproduced on polars 1.42.
|
|
396
|
+
# ``any_horizontal`` keeps the disjunction flat — a
|
|
397
|
+
# left-deep OR chain plans quadratically in the number
|
|
398
|
+
# of values.
|
|
399
|
+
exprs.append(
|
|
400
|
+
pl.any_horizontal(
|
|
401
|
+
[pl.col(dim).is_between(*(_plain(v),) * 2) for v in a]
|
|
402
|
+
)
|
|
403
|
+
)
|
|
404
|
+
elif _nanosecond(dtype):
|
|
405
|
+
exprs.append(pl.col(dim).is_in(_polars_times(a, dtype)))
|
|
406
|
+
else:
|
|
407
|
+
exprs.append(pl.col(dim).is_in([_plain(v) for v in a]))
|
|
408
|
+
lf = self._lf.filter(*exprs) if exprs else self._lf
|
|
409
|
+
out = _collect_streaming(lf.select([pl.col(n) for n in columns]))
|
|
410
|
+
return cast(list[pa.RecordBatch], out.to_arrow().to_batches())
|
|
411
|
+
|
|
412
|
+
def stream(self, columns: list[str]) -> Iterator[pa.RecordBatch]:
|
|
413
|
+
"""Execute once, yielding Arrow batches incrementally.
|
|
414
|
+
|
|
415
|
+
Unlike [fetch][xarray_sql.lazyscan.PolarsHandle.fetch], whose ``collect`` materializes the whole
|
|
416
|
+
result inside the engine before any batch surfaces, this yields
|
|
417
|
+
batches as the streaming engine produces them, so a byte budget
|
|
418
|
+
can fire before the result is fully in memory. Requires
|
|
419
|
+
``LazyFrame.collect_batches`` (polars >= 1.33); older Polars
|
|
420
|
+
raises here, before anything is collected.
|
|
421
|
+
"""
|
|
422
|
+
import polars as pl
|
|
423
|
+
|
|
424
|
+
lf = self._lf.select([pl.col(n) for n in columns])
|
|
425
|
+
if not hasattr(lf, "collect_batches"):
|
|
426
|
+
raise ValueError(
|
|
427
|
+
"streaming collection of a Polars LazyFrame requires "
|
|
428
|
+
"polars >= 1.33 (LazyFrame.collect_batches)."
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
def generate() -> Iterator[pa.RecordBatch]:
|
|
432
|
+
for frame in lf.collect_batches(engine="streaming"):
|
|
433
|
+
yield from frame.to_arrow().to_batches()
|
|
434
|
+
|
|
435
|
+
return generate()
|
|
436
|
+
|
|
437
|
+
def spill_parquet(self, path: str) -> None:
|
|
438
|
+
self._lf.sink_parquet(path)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def resolve_lazy_handle(result: Any) -> LazyResultHandle | None:
|
|
442
|
+
"""Adapt an engine result to a handle, or ``None`` if it is one-shot.
|
|
443
|
+
|
|
444
|
+
Recognizes DuckDB relations, Polars lazy *and* eager frames (an
|
|
445
|
+
eager frame re-executes trivially over its in-memory data), and
|
|
446
|
+
DataFusion DataFrames. ``pyarrow`` tables/readers and bare
|
|
447
|
+
``__arrow_c_stream__`` objects are one-shot streams: there is no
|
|
448
|
+
query to re-execute, so the lazy path cannot serve them.
|
|
449
|
+
"""
|
|
450
|
+
root = type(result).__module__.split(".")[0]
|
|
451
|
+
if root in ("duckdb", "_duckdb") and hasattr(result, "filter"):
|
|
452
|
+
return DuckDBHandle(result)
|
|
453
|
+
if root == "polars":
|
|
454
|
+
import polars as pl
|
|
455
|
+
|
|
456
|
+
if isinstance(result, pl.LazyFrame):
|
|
457
|
+
return PolarsHandle(result)
|
|
458
|
+
if isinstance(result, pl.DataFrame):
|
|
459
|
+
return PolarsHandle(result.lazy())
|
|
460
|
+
if hasattr(result, "execute_stream") and hasattr(result, "logical_plan"):
|
|
461
|
+
return DataFusionHandle(result)
|
|
462
|
+
return None
|