xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
"""ADBC engine adapter.
|
|
2
|
+
|
|
3
|
+
[ADBC](https://arrow.apache.org/adbc/) (Arrow Database Connectivity) is
|
|
4
|
+
a database-neutral API whose drivers speak Arrow natively: PostgreSQL,
|
|
5
|
+
SQLite, Snowflake, BigQuery, Flight SQL, DuckDB, and more, each behind
|
|
6
|
+
the same DBAPI-style ``Connection``. This adapter registers a Dataset on
|
|
7
|
+
any of them through ADBC's bulk-ingest call, so one code path reaches
|
|
8
|
+
every database with an ADBC driver.
|
|
9
|
+
|
|
10
|
+
Unlike the DataFusion and DuckDB adapters, registration here *copies*
|
|
11
|
+
the data: an ADBC database is typically a separate process or service
|
|
12
|
+
that cannot call back into Python to scan a lazy Dataset while a query
|
|
13
|
+
runs. The Dataset is streamed chunk by chunk (prefetched, with bounded
|
|
14
|
+
memory) into a new table, so only the ingest itself touches the source
|
|
15
|
+
data, and queries afterwards run entirely inside the database.
|
|
16
|
+
|
|
17
|
+
Results come back through ADBC's Arrow cursor API, which
|
|
18
|
+
[xarray_sql.to_dataset][] accepts directly::
|
|
19
|
+
|
|
20
|
+
cur = con.cursor()
|
|
21
|
+
cur.execute("SELECT time, lat, lon, t2m FROM era5 ORDER BY time")
|
|
22
|
+
out = xql.to_dataset(cur, template=ds)
|
|
23
|
+
|
|
24
|
+
This adapter never imports ``adbc_driver_manager`` at runtime —
|
|
25
|
+
detection is by connection type — so ADBC stays a purely optional
|
|
26
|
+
dependency (``pip install xarray-sql[adbc]`` plus a driver package).
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import contextlib
|
|
32
|
+
import warnings
|
|
33
|
+
from collections.abc import Iterator, Mapping
|
|
34
|
+
from typing import TYPE_CHECKING, Any, TypeGuard
|
|
35
|
+
|
|
36
|
+
import xarray as xr
|
|
37
|
+
|
|
38
|
+
from ..df import (
|
|
39
|
+
Chunks,
|
|
40
|
+
TableNames,
|
|
41
|
+
group_vars_by_dims,
|
|
42
|
+
resolve_table_names,
|
|
43
|
+
shared_coord_arrays,
|
|
44
|
+
)
|
|
45
|
+
from ._adbc_dialects import (
|
|
46
|
+
Dialect,
|
|
47
|
+
IngestMode,
|
|
48
|
+
dialect_for,
|
|
49
|
+
driver_error,
|
|
50
|
+
ingestable,
|
|
51
|
+
)
|
|
52
|
+
from .base import register_adapter
|
|
53
|
+
from .pyarrow import XarrayPushdownDataset
|
|
54
|
+
|
|
55
|
+
if TYPE_CHECKING:
|
|
56
|
+
from adbc_driver_manager import dbapi
|
|
57
|
+
|
|
58
|
+
__all__ = ["ADBCAdapter"]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _ingest(
|
|
62
|
+
con: dbapi.Connection,
|
|
63
|
+
table: str,
|
|
64
|
+
ds: xr.Dataset,
|
|
65
|
+
chunks: Chunks,
|
|
66
|
+
*,
|
|
67
|
+
dialect: Dialect,
|
|
68
|
+
dims: tuple[str, ...],
|
|
69
|
+
mode: IngestMode,
|
|
70
|
+
temporary: bool,
|
|
71
|
+
ingest_options: Mapping[str, str] | None,
|
|
72
|
+
db_schema_name: str | None = None,
|
|
73
|
+
**kwargs: Any,
|
|
74
|
+
) -> None:
|
|
75
|
+
"""Stream *ds* into *table* through ADBC bulk ingest.
|
|
76
|
+
|
|
77
|
+
The full scan of an
|
|
78
|
+
[XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset]
|
|
79
|
+
prefetches chunks on a thread pool while the driver writes earlier
|
|
80
|
+
batches, so the source read and the database write overlap.
|
|
81
|
+
"""
|
|
82
|
+
reader = XarrayPushdownDataset(ds, chunks, **kwargs).scanner().to_reader()
|
|
83
|
+
reader = ingestable(reader, dialect)
|
|
84
|
+
plan = None
|
|
85
|
+
if dialect.table_ddl is not None:
|
|
86
|
+
plan = dialect.table_ddl(
|
|
87
|
+
table,
|
|
88
|
+
reader.schema,
|
|
89
|
+
dims,
|
|
90
|
+
mode=mode,
|
|
91
|
+
temporary=temporary,
|
|
92
|
+
database=db_schema_name,
|
|
93
|
+
)
|
|
94
|
+
_execute(con, plan.before)
|
|
95
|
+
mode, temporary = "append", False
|
|
96
|
+
target_schema = db_schema_name
|
|
97
|
+
in_database: contextlib.AbstractContextManager[None] = (
|
|
98
|
+
contextlib.nullcontext()
|
|
99
|
+
)
|
|
100
|
+
if (
|
|
101
|
+
db_schema_name is not None
|
|
102
|
+
and dialect.target_schema == "default_database"
|
|
103
|
+
):
|
|
104
|
+
in_database = _default_database(con, db_schema_name, dialect)
|
|
105
|
+
target_schema = None
|
|
106
|
+
try:
|
|
107
|
+
with in_database, con.cursor() as cur:
|
|
108
|
+
if ingest_options:
|
|
109
|
+
cur.adbc_statement.set_options(**ingest_options)
|
|
110
|
+
cur.adbc_ingest(
|
|
111
|
+
plan.table if plan is not None else table,
|
|
112
|
+
reader,
|
|
113
|
+
mode=mode,
|
|
114
|
+
db_schema_name=target_schema,
|
|
115
|
+
temporary=temporary,
|
|
116
|
+
)
|
|
117
|
+
except BaseException:
|
|
118
|
+
if plan is not None:
|
|
119
|
+
try:
|
|
120
|
+
_execute(con, plan.on_failure)
|
|
121
|
+
except driver_error(con):
|
|
122
|
+
pass # the ingest's own error is the one to report
|
|
123
|
+
raise
|
|
124
|
+
if plan is not None:
|
|
125
|
+
_execute(con, plan.after)
|
|
126
|
+
if dialect.analyze:
|
|
127
|
+
target = dialect.quote_identifier(table)
|
|
128
|
+
if db_schema_name is not None:
|
|
129
|
+
target = f"{dialect.quote_identifier(db_schema_name)}.{target}"
|
|
130
|
+
with con.cursor() as cur:
|
|
131
|
+
cur.execute(f"ANALYZE {target}")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _execute(con: dbapi.Connection, statements: list[str]) -> None:
|
|
135
|
+
for statement in statements:
|
|
136
|
+
with con.cursor() as cur:
|
|
137
|
+
cur.execute(statement)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@contextlib.contextmanager
|
|
141
|
+
def _default_database(
|
|
142
|
+
con: dbapi.Connection, database: str, dialect: Dialect
|
|
143
|
+
) -> Iterator[None]:
|
|
144
|
+
"""Make *database* the connection's default while in the block.
|
|
145
|
+
|
|
146
|
+
For drivers that create an ingest's table in the default database
|
|
147
|
+
whatever target schema they are given. The previous default is
|
|
148
|
+
restored afterwards; MySQL cannot unset a default database, so a
|
|
149
|
+
connection that had none keeps *database*.
|
|
150
|
+
"""
|
|
151
|
+
with con.cursor() as cur:
|
|
152
|
+
cur.execute("SELECT DATABASE()")
|
|
153
|
+
(previous,) = cur.fetchone()
|
|
154
|
+
cur.execute(f"USE {dialect.quote_identifier(database)}")
|
|
155
|
+
try:
|
|
156
|
+
yield
|
|
157
|
+
finally:
|
|
158
|
+
if previous is not None:
|
|
159
|
+
with con.cursor() as cur:
|
|
160
|
+
cur.execute(f"USE {dialect.quote_identifier(previous)}")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _schema_exists(con: dbapi.Connection, name: str) -> bool:
|
|
164
|
+
"""Whether the database already has a schema named exactly *name*."""
|
|
165
|
+
try:
|
|
166
|
+
objects = con.adbc_get_objects(
|
|
167
|
+
depth="db_schemas", db_schema_filter=name
|
|
168
|
+
).read_all()
|
|
169
|
+
except driver_error(con): # metadata unsupported; assume not
|
|
170
|
+
return False
|
|
171
|
+
return any(
|
|
172
|
+
schema["db_schema_name"] == name
|
|
173
|
+
for catalog in objects.to_pylist()
|
|
174
|
+
for schema in catalog["catalog_db_schemas"] or []
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _connection_usable(con: dbapi.Connection) -> bool:
|
|
179
|
+
"""Whether *con* still runs statements after one of them failed."""
|
|
180
|
+
try:
|
|
181
|
+
with con.cursor() as cur:
|
|
182
|
+
cur.execute("SELECT 1")
|
|
183
|
+
cur.fetchall()
|
|
184
|
+
except driver_error(con):
|
|
185
|
+
return False
|
|
186
|
+
return True
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _warn_on_lost_precision(ds: xr.Dataset, dialect: Dialect) -> None:
|
|
190
|
+
"""Warn if *dialect* would truncate a time coordinate of *ds*."""
|
|
191
|
+
if dialect.timestamp_unit == "ns":
|
|
192
|
+
return
|
|
193
|
+
for name, coord in ds.coords.items():
|
|
194
|
+
if coord.dtype.kind != "M":
|
|
195
|
+
continue
|
|
196
|
+
values = coord.values.astype("datetime64[ns]")
|
|
197
|
+
if (values.astype("datetime64[us]") != values).any():
|
|
198
|
+
warnings.warn(
|
|
199
|
+
f"{dialect.name} stores times to the microsecond, so the "
|
|
200
|
+
f"sub-microsecond part of {name!r} will be truncated.",
|
|
201
|
+
RuntimeWarning,
|
|
202
|
+
stacklevel=4,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _warn_on_folded_names(
|
|
207
|
+
names: list[str], dialect: Dialect, stacklevel: int = 4
|
|
208
|
+
) -> None:
|
|
209
|
+
"""Warn about names *dialect*'s database would fold if unquoted.
|
|
210
|
+
|
|
211
|
+
*stacklevel* counts frames from here to the user's call.
|
|
212
|
+
"""
|
|
213
|
+
if dialect.folds is None:
|
|
214
|
+
return
|
|
215
|
+
fold = str.lower if dialect.folds == "lower" else str.upper
|
|
216
|
+
folded = [n for n in names if fold(n) != n]
|
|
217
|
+
if folded:
|
|
218
|
+
quoted = ", ".join(dialect.quote_identifier(n) for n in folded)
|
|
219
|
+
warnings.warn(
|
|
220
|
+
f"{dialect.name} folds unquoted names to {dialect.folds}case, "
|
|
221
|
+
f"so quote these in queries: {quoted}.",
|
|
222
|
+
RuntimeWarning,
|
|
223
|
+
stacklevel=stacklevel,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _flat(name: str, reason: str) -> bool:
|
|
228
|
+
"""Warn that *name*'s groups become flat tables; always ``False``."""
|
|
229
|
+
warnings.warn(
|
|
230
|
+
f"Registering the dimension groups of {name!r} as flat "
|
|
231
|
+
f"{name}_<group> tables: {reason}.",
|
|
232
|
+
RuntimeWarning,
|
|
233
|
+
stacklevel=4,
|
|
234
|
+
)
|
|
235
|
+
return False
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _create_schema(con: dbapi.Connection, name: str, dialect: Dialect) -> bool:
|
|
239
|
+
"""Ensure the database schema *name* exists; whether it does.
|
|
240
|
+
|
|
241
|
+
An existing schema is used as is: creating it can need privileges on
|
|
242
|
+
the whole database (PostgreSQL checks them even for ``IF NOT
|
|
243
|
+
EXISTS``) that a role granted only that schema lacks.
|
|
244
|
+
|
|
245
|
+
Creating one can fail for lack of privileges. When the connection
|
|
246
|
+
survives the failure, the caller falls back to flat table names. On
|
|
247
|
+
databases where a failed statement aborts the transaction
|
|
248
|
+
(PostgreSQL), nothing after it could run, so this raises instead.
|
|
249
|
+
"""
|
|
250
|
+
if not dialect.schemas:
|
|
251
|
+
return _flat(name, f"{dialect.name} has no schemas to hold them")
|
|
252
|
+
if _schema_exists(con, name):
|
|
253
|
+
return True
|
|
254
|
+
try:
|
|
255
|
+
with con.cursor() as cur:
|
|
256
|
+
cur.execute(dialect.create_schema_sql(name))
|
|
257
|
+
except driver_error(con) as exc:
|
|
258
|
+
if not _connection_usable(con):
|
|
259
|
+
raise RuntimeError(
|
|
260
|
+
f"Could not create the {name!r} schema to hold the dimension "
|
|
261
|
+
f"groups of {name!r} ({exc}). The failure aborted the "
|
|
262
|
+
f"connection's transaction, so call con.rollback() before "
|
|
263
|
+
f"using it again. Create the schema beforehand (or grant "
|
|
264
|
+
f"the privilege to), or pass temporary=True to register "
|
|
265
|
+
f"flat {name}_<group> tables instead."
|
|
266
|
+
) from exc
|
|
267
|
+
return _flat(name, f"could not create the {name!r} schema ({exc})")
|
|
268
|
+
return True
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@register_adapter
|
|
272
|
+
class ADBCAdapter:
|
|
273
|
+
"""Registers Datasets on ADBC DBAPI connections."""
|
|
274
|
+
|
|
275
|
+
@staticmethod
|
|
276
|
+
def matches(con: object) -> TypeGuard[dbapi.Connection]:
|
|
277
|
+
# Every driver's ``dbapi.connect`` returns this class or a
|
|
278
|
+
# subclass of it (e.g. ``adbc_driver_sqlite.dbapi``'s).
|
|
279
|
+
return any(
|
|
280
|
+
cls.__module__ == "adbc_driver_manager.dbapi"
|
|
281
|
+
and cls.__qualname__ == "Connection"
|
|
282
|
+
for cls in type(con).__mro__
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
@staticmethod
|
|
286
|
+
def register(
|
|
287
|
+
con: dbapi.Connection,
|
|
288
|
+
name: str,
|
|
289
|
+
ds: xr.Dataset,
|
|
290
|
+
*,
|
|
291
|
+
chunks: Chunks = None,
|
|
292
|
+
table_names: TableNames = None,
|
|
293
|
+
mode: IngestMode = "create",
|
|
294
|
+
temporary: bool = False,
|
|
295
|
+
ingest_options: Mapping[str, str] | None = None,
|
|
296
|
+
**kwargs: Any,
|
|
297
|
+
) -> dbapi.Connection:
|
|
298
|
+
"""Ingest ``ds`` into tables on an ADBC connection.
|
|
299
|
+
|
|
300
|
+
Datasets whose variables all share the same dimensions become a
|
|
301
|
+
single table named ``name``. Mixed-dimension datasets are split
|
|
302
|
+
into one table per dimension group, created in a database schema
|
|
303
|
+
named ``name`` so they are queried as ``name.group`` — the same
|
|
304
|
+
spelling every other engine uses::
|
|
305
|
+
|
|
306
|
+
xql.register(con, 'era5', ds, table_names={
|
|
307
|
+
('time', 'latitude', 'longitude'): 'surface',
|
|
308
|
+
('time', 'level', 'latitude', 'longitude'): 'atmosphere',
|
|
309
|
+
})
|
|
310
|
+
cur.execute('SELECT ... FROM era5.surface')
|
|
311
|
+
|
|
312
|
+
An existing schema is used as is. On databases without schemas
|
|
313
|
+
(SQLite), and for temporary tables, which most drivers cannot
|
|
314
|
+
place in a schema, the groups are created as flat
|
|
315
|
+
``name_group`` tables instead. If creating the schema fails in a
|
|
316
|
+
way that aborts the transaction (PostgreSQL without the
|
|
317
|
+
``CREATE`` privilege), this raises ``RuntimeError``; roll back,
|
|
318
|
+
then create the schema beforehand or pass ``temporary=True``.
|
|
319
|
+
|
|
320
|
+
On ClickHouse, whose driver can only append, the adapter creates
|
|
321
|
+
the tables itself: ``MergeTree`` tables sorted by their
|
|
322
|
+
dimensions (``Memory`` for temporary ones), in a ClickHouse
|
|
323
|
+
database named ``name`` for a mixed-dimension Dataset, with
|
|
324
|
+
timestamps declared ``DateTime64(p, 'UTC')`` at the coordinate's
|
|
325
|
+
precision (``p`` is 9 for ``datetime64[ns]``).
|
|
326
|
+
|
|
327
|
+
Registration runs inside the connection's current transaction:
|
|
328
|
+
the tables are visible to this connection immediately, and to
|
|
329
|
+
others once you call ``con.commit()`` (unless the connection is
|
|
330
|
+
in autocommit mode).
|
|
331
|
+
|
|
332
|
+
Args:
|
|
333
|
+
mode: What to do when a table already exists, as in ADBC's
|
|
334
|
+
``adbc_ingest``: ``"create"`` (default) raises,
|
|
335
|
+
``"replace"`` drops and recreates it, ``"append"`` and
|
|
336
|
+
``"create_append"`` add rows to it.
|
|
337
|
+
temporary: Create temporary tables, which the database drops
|
|
338
|
+
when the connection closes. Raises ``ValueError`` on
|
|
339
|
+
databases whose driver cannot (DataFusion, Trino, Spark,
|
|
340
|
+
BigQuery, Databricks, Snowflake).
|
|
341
|
+
ingest_options: Driver-specific statement options set on each
|
|
342
|
+
ingest, e.g. Spark's
|
|
343
|
+
``{"spark.ingest.staging_area_uri": "s3://bucket/path"}``.
|
|
344
|
+
**kwargs: Forwarded to
|
|
345
|
+
[XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset]
|
|
346
|
+
(``batch_size``, ``prefetch``, ``prefetch_bytes``,
|
|
347
|
+
``coalesce_rows``) to tune the ingest scan.
|
|
348
|
+
"""
|
|
349
|
+
groups = group_vars_by_dims(ds)
|
|
350
|
+
names = resolve_table_names(ds, table_names, case_insensitive=True)
|
|
351
|
+
dialect = dialect_for(con)
|
|
352
|
+
if temporary and not dialect.temporary_tables:
|
|
353
|
+
raise ValueError(
|
|
354
|
+
f"{dialect.name}'s ADBC driver does not support temporary "
|
|
355
|
+
f"tables; register without temporary=True and drop the "
|
|
356
|
+
f"tables when done."
|
|
357
|
+
)
|
|
358
|
+
_warn_on_lost_precision(ds, dialect)
|
|
359
|
+
_warn_on_folded_names(
|
|
360
|
+
[name] if len(groups) <= 1 else [name, *names.values()], dialect
|
|
361
|
+
)
|
|
362
|
+
if len(groups) <= 1:
|
|
363
|
+
_ingest(
|
|
364
|
+
con,
|
|
365
|
+
name,
|
|
366
|
+
ds,
|
|
367
|
+
chunks,
|
|
368
|
+
dialect=dialect,
|
|
369
|
+
dims=next(iter(groups), ()),
|
|
370
|
+
mode=mode,
|
|
371
|
+
temporary=temporary,
|
|
372
|
+
ingest_options=ingest_options,
|
|
373
|
+
**kwargs,
|
|
374
|
+
)
|
|
375
|
+
return con
|
|
376
|
+
|
|
377
|
+
in_schema = not temporary and _create_schema(con, name, dialect)
|
|
378
|
+
coord_arrays = shared_coord_arrays(ds)
|
|
379
|
+
for dims, var_names in groups.items():
|
|
380
|
+
group = names[dims]
|
|
381
|
+
_ingest(
|
|
382
|
+
con,
|
|
383
|
+
group if in_schema else f"{name}_{group}",
|
|
384
|
+
ds[var_names],
|
|
385
|
+
chunks,
|
|
386
|
+
dialect=dialect,
|
|
387
|
+
dims=dims,
|
|
388
|
+
mode=mode,
|
|
389
|
+
temporary=temporary,
|
|
390
|
+
ingest_options=ingest_options,
|
|
391
|
+
db_schema_name=name if in_schema else None,
|
|
392
|
+
coord_arrays=coord_arrays,
|
|
393
|
+
**kwargs,
|
|
394
|
+
)
|
|
395
|
+
return con
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Engine-adapter dispatch for [xarray_sql.register][].
|
|
2
|
+
|
|
3
|
+
An *engine adapter* implements the register seam: given an engine's native
|
|
4
|
+
connection object and a lazy ``xarray.Dataset``, register the Dataset as
|
|
5
|
+
a queryable table on that connection. The Arrow C-stream protocol is the
|
|
6
|
+
common wire between xarray and every engine; adapters differ only in how
|
|
7
|
+
a stream is attached to the connection and in what pushdown the engine
|
|
8
|
+
can do against it.
|
|
9
|
+
|
|
10
|
+
Adapters self-describe which connections they accept via ``matches``,
|
|
11
|
+
which must not require the engine's package to be importable (detection
|
|
12
|
+
is by type inspection), so optional engines stay optional.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Any, Protocol, TypeGuard, TypeVar, cast
|
|
18
|
+
|
|
19
|
+
import xarray as xr
|
|
20
|
+
|
|
21
|
+
from ..df import Chunks, TableNames
|
|
22
|
+
|
|
23
|
+
ConT = TypeVar("ConT")
|
|
24
|
+
"""An engine's native connection type (e.g. ``duckdb.DuckDBPyConnection``)."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class EngineAdapter(Protocol[ConT]):
|
|
28
|
+
"""One engine's implementation of the register seam."""
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
def matches(con: object) -> TypeGuard[ConT]:
|
|
32
|
+
"""Whether *con* is a connection this adapter can register into."""
|
|
33
|
+
...
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def register(
|
|
37
|
+
con: ConT,
|
|
38
|
+
name: str,
|
|
39
|
+
ds: xr.Dataset,
|
|
40
|
+
*,
|
|
41
|
+
chunks: Chunks = None,
|
|
42
|
+
table_names: TableNames = None,
|
|
43
|
+
**kwargs: Any,
|
|
44
|
+
) -> ConT:
|
|
45
|
+
"""Register *ds* as table *name* on *con*; returns *con*."""
|
|
46
|
+
...
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
_ADAPTERS: list[type[EngineAdapter[Any]]] = []
|
|
50
|
+
|
|
51
|
+
_A = TypeVar("_A", bound=type[EngineAdapter[Any]])
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def register_adapter(cls: _A) -> _A:
|
|
55
|
+
"""Class decorator adding an adapter to the dispatch list."""
|
|
56
|
+
_ADAPTERS.append(cls)
|
|
57
|
+
return cls
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_adapter(con: object) -> type[EngineAdapter[Any]]:
|
|
61
|
+
"""Return the first adapter whose ``matches(con)`` is true."""
|
|
62
|
+
for adapter in _ADAPTERS:
|
|
63
|
+
if adapter.matches(con):
|
|
64
|
+
return adapter
|
|
65
|
+
raise TypeError(
|
|
66
|
+
f"No xarray-sql engine adapter for connection of type "
|
|
67
|
+
f"{type(con).__module__}.{type(con).__qualname__}. "
|
|
68
|
+
f"Supported: DataFusion SessionContext, DuckDB, ADBC DBAPI "
|
|
69
|
+
"connections, and xarray_sql.FlightSQLServer."
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def register(
|
|
74
|
+
con: ConT,
|
|
75
|
+
name: str,
|
|
76
|
+
ds: xr.Dataset,
|
|
77
|
+
*,
|
|
78
|
+
chunks: Chunks = None,
|
|
79
|
+
table_names: TableNames = None,
|
|
80
|
+
**kwargs: Any,
|
|
81
|
+
) -> ConT:
|
|
82
|
+
"""Register a lazy xarray Dataset as a table on an engine connection.
|
|
83
|
+
|
|
84
|
+
The engine is inferred from the connection type. Data is not read at
|
|
85
|
+
registration time; the engine pulls Arrow record batches lazily during
|
|
86
|
+
query execution. Write your SQL in the engine's own dialect and use
|
|
87
|
+
the engine's extension ecosystem directly — xarray-sql translates the
|
|
88
|
+
data, not the queries.
|
|
89
|
+
|
|
90
|
+
Example (DuckDB)::
|
|
91
|
+
|
|
92
|
+
import duckdb
|
|
93
|
+
import xarray_sql as xql
|
|
94
|
+
|
|
95
|
+
con = duckdb.connect()
|
|
96
|
+
xql.register(con, "era5", ds)
|
|
97
|
+
rel = con.sql("SELECT time, AVG(t2m) AS t2m FROM era5 GROUP BY time")
|
|
98
|
+
result = xql.to_dataset(rel, template=ds)
|
|
99
|
+
|
|
100
|
+
A Dataset whose variables sit on different dimensions is split into
|
|
101
|
+
one table per dimension group. Name those tables with
|
|
102
|
+
``table_names``, and the same SQL runs on every engine::
|
|
103
|
+
|
|
104
|
+
xql.register(con, "era5", ds, table_names={
|
|
105
|
+
("time", "latitude", "longitude"): "surface",
|
|
106
|
+
("time", "level", "latitude", "longitude"): "atmosphere",
|
|
107
|
+
})
|
|
108
|
+
con.sql("SELECT AVG(temperature) FROM era5.atmosphere")
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
con: An engine connection: a ``datafusion.SessionContext`` (or
|
|
112
|
+
[xarray_sql.Context][]), a
|
|
113
|
+
``duckdb.DuckDBPyConnection``, or an ADBC DBAPI connection
|
|
114
|
+
(``adbc_driver_manager.dbapi.Connection``), into which the
|
|
115
|
+
Dataset is ingested as a table.
|
|
116
|
+
name: The table name to register the Dataset under. Datasets
|
|
117
|
+
whose variables have differing dimensions are split into one
|
|
118
|
+
table per dimension group, addressed as ``name.group`` on
|
|
119
|
+
every engine (DuckDB also keeps the flat ``name_group``
|
|
120
|
+
spelling, since its registration namespace is flat).
|
|
121
|
+
ds: An xarray Dataset.
|
|
122
|
+
chunks: Xarray-like chunks specification controlling partition
|
|
123
|
+
granularity. Defaults to the Dataset's existing chunks. An
|
|
124
|
+
unchunked Dataset opened lazily with ``chunks=None`` is split
|
|
125
|
+
along the storage chunks its backend recorded in
|
|
126
|
+
``encoding["preferred_chunks"]``, and is one chunk along other
|
|
127
|
+
dimensions; one built in memory, which records none, is split
|
|
128
|
+
into partitions of about a million rows. Dask is not required.
|
|
129
|
+
table_names: Maps a dimension group's exact dim tuple to the name
|
|
130
|
+
its table takes. Groups left unnamed take their dimensions
|
|
131
|
+
joined by underscores (``time_latitude_longitude``); the
|
|
132
|
+
group holding scalar variables, if any, takes ``scalar``.
|
|
133
|
+
Keys matching no group in ``ds`` are ignored, so one naming
|
|
134
|
+
map can be reused across Datasets holding different subsets
|
|
135
|
+
of the same variables.
|
|
136
|
+
**kwargs: Adapter-specific options, forwarded as-is — e.g.
|
|
137
|
+
``batch_size`` / ``prefetch`` on DuckDB.
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
The connection, to allow chaining.
|
|
141
|
+
"""
|
|
142
|
+
# The connection type is erased by the runtime dispatch; every adapter
|
|
143
|
+
# returns the connection it was given.
|
|
144
|
+
adapter: Any = get_adapter(con)
|
|
145
|
+
return cast(
|
|
146
|
+
ConT,
|
|
147
|
+
adapter.register(
|
|
148
|
+
con, name, ds, chunks=chunks, table_names=table_names, **kwargs
|
|
149
|
+
),
|
|
150
|
+
)
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""DataFusion engine adapter.
|
|
2
|
+
|
|
3
|
+
DataFusion is xarray-sql's default engine and the richest adapter: the
|
|
4
|
+
Rust ``LazyArrowStreamTable`` table provider gives partition pruning on
|
|
5
|
+
dimension predicates, projection pushdown, and exact per-partition
|
|
6
|
+
statistics for the optimizer. This module only routes the generic
|
|
7
|
+
[xarray_sql.register][] seam onto that existing machinery.
|
|
8
|
+
|
|
9
|
+
Like the other adapters, it never imports ``datafusion`` until a
|
|
10
|
+
DataFusion connection is registered, so the rest of xarray-sql works
|
|
11
|
+
where DataFusion's Python bindings are unavailable (e.g. Pyodide).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import TYPE_CHECKING, Any, TypeGuard
|
|
17
|
+
|
|
18
|
+
import xarray as xr
|
|
19
|
+
|
|
20
|
+
from ..df import (
|
|
21
|
+
Chunks,
|
|
22
|
+
TableNames,
|
|
23
|
+
group_vars_by_dims,
|
|
24
|
+
resolve_table_names,
|
|
25
|
+
shared_coord_arrays,
|
|
26
|
+
)
|
|
27
|
+
from ..reader import read_xarray_table
|
|
28
|
+
from .base import register_adapter
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from datafusion import SessionContext
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@register_adapter
|
|
35
|
+
class DataFusionAdapter:
|
|
36
|
+
"""Registers Datasets on ``datafusion.SessionContext`` connections."""
|
|
37
|
+
|
|
38
|
+
@staticmethod
|
|
39
|
+
def matches(con: object) -> TypeGuard[SessionContext]:
|
|
40
|
+
# Context and any other subclass carry this in their MRO.
|
|
41
|
+
return any(
|
|
42
|
+
cls.__module__ == "datafusion.context"
|
|
43
|
+
and cls.__qualname__ == "SessionContext"
|
|
44
|
+
for cls in type(con).__mro__
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
@staticmethod
|
|
48
|
+
def register(
|
|
49
|
+
con: SessionContext,
|
|
50
|
+
name: str,
|
|
51
|
+
ds: xr.Dataset,
|
|
52
|
+
*,
|
|
53
|
+
chunks: Chunks = None,
|
|
54
|
+
table_names: TableNames = None,
|
|
55
|
+
**kwargs: Any,
|
|
56
|
+
) -> SessionContext:
|
|
57
|
+
from datafusion.catalog import Schema
|
|
58
|
+
|
|
59
|
+
from ..sql import Context
|
|
60
|
+
|
|
61
|
+
# Context.from_dataset adds cftime UDF registration and
|
|
62
|
+
# round-trip metadata tracking on top of the split below; use it
|
|
63
|
+
# when available.
|
|
64
|
+
if isinstance(con, Context):
|
|
65
|
+
return con.from_dataset(
|
|
66
|
+
name, ds, chunks=chunks, table_names=table_names, **kwargs
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
groups = group_vars_by_dims(ds)
|
|
70
|
+
names = resolve_table_names(ds, table_names)
|
|
71
|
+
if len(groups) <= 1:
|
|
72
|
+
con.register_table(name, read_xarray_table(ds, chunks, **kwargs))
|
|
73
|
+
return con
|
|
74
|
+
|
|
75
|
+
# A mixed-dimension Dataset becomes one table per dimension group
|
|
76
|
+
# in a schema named after the Dataset, exactly as Context
|
|
77
|
+
# registers it — so ``name.group`` is the same SQL either way.
|
|
78
|
+
coord_arrays = shared_coord_arrays(ds)
|
|
79
|
+
schema = Schema.memory_schema(con)
|
|
80
|
+
con.catalog().register_schema(name, schema)
|
|
81
|
+
for dims, var_names in groups.items():
|
|
82
|
+
schema.register_table(
|
|
83
|
+
names[dims],
|
|
84
|
+
read_xarray_table(
|
|
85
|
+
ds[var_names],
|
|
86
|
+
chunks,
|
|
87
|
+
coord_arrays=coord_arrays,
|
|
88
|
+
**kwargs,
|
|
89
|
+
),
|
|
90
|
+
)
|
|
91
|
+
return con
|