xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,193 @@
1
+ """DuckDB engine adapter.
2
+
3
+ Registers a lazy ``xarray.Dataset`` on a ``duckdb.DuckDBPyConnection``
4
+ as an [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset]:
5
+ DuckDB classifies it with a real ``isinstance`` check against
6
+ ``pyarrow.dataset.Dataset`` and calls ``scanner(columns=[...],
7
+ filter=<pyarrow.compute.Expression>)`` once per query, giving
8
+ projection pushdown, coordinate-range chunk pruning, and prefetched
9
+ parallel production (see [xarray_sql.backends.pyarrow][]).
10
+
11
+ This adapter never imports the ``duckdb`` package at runtime — detection
12
+ is by connection type, and registration is a method call on the
13
+ connection — so DuckDB stays a purely optional dependency
14
+ (``pip install xarray-sql[duckdb]``).
15
+
16
+ Zarr-native scanning inside DuckDB is what the [duckdb-zarr](https://github.com/xqlsystems/duckdb-zarr) extension provides; this
17
+ adapter instead covers everything xarray can open (NetCDF, GRIB, Xee, CF
18
+ decoding, in-memory) and pairs with [xarray_sql.to_dataset][] for
19
+ the labeled round-trip.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import warnings
25
+ from typing import TYPE_CHECKING, Any, TypeGuard
26
+
27
+ import xarray as xr
28
+
29
+ from ..df import (
30
+ Chunks,
31
+ TableNames,
32
+ group_vars_by_dims,
33
+ resolve_table_names,
34
+ shared_coord_arrays,
35
+ )
36
+ from .base import register_adapter
37
+ from .pyarrow import XarrayArrowStream, XarrayPushdownDataset
38
+
39
+ if TYPE_CHECKING:
40
+ import duckdb
41
+
42
+ __all__ = ["DuckDBAdapter", "XarrayArrowStream", "XarrayPushdownDataset"]
43
+
44
+
45
+ def _quote(identifier: str) -> str:
46
+ """Render *identifier* as a quoted SQL identifier."""
47
+ escaped = identifier.replace('"', '""')
48
+ return f'"{escaped}"'
49
+
50
+
51
+ def _is_in_memory(con: duckdb.DuckDBPyConnection) -> bool:
52
+ """Whether every database attached to *con* is memory-backed.
53
+
54
+ ``con.register`` binds a Python object to the connection only for
55
+ its lifetime — never to the catalog on disk. A view created over
56
+ that binding is ordinary catalog DDL, so on a file-backed database
57
+ it persists after the flat table it selects from is gone, and a
58
+ later connection resolves it into "Table ... does not exist".
59
+ In-memory databases have nothing to outlive, so mirroring there is
60
+ safe.
61
+ """
62
+ try:
63
+ rows = con.execute("PRAGMA database_list").fetchall()
64
+ except Exception: # noqa: BLE001 — conservative: assume persistent
65
+ return False
66
+ return all(path is None for _, _, path in rows)
67
+
68
+
69
+ def _mirror_as_schema(
70
+ con: duckdb.DuckDBPyConnection, name: str, tables: dict[str, str]
71
+ ) -> None:
72
+ """Expose flat tables as views in a DuckDB schema named *name*.
73
+
74
+ This translates a view to `name_group` can be queried as `name.group`
75
+ in DuckDB.
76
+
77
+ *tables* maps each group's table name to the flat name it was
78
+ registered under.
79
+ """
80
+ if not _is_in_memory(con):
81
+ warnings.warn(
82
+ f"Registered the dimension groups of {name!r} as "
83
+ f"{', '.join(sorted(tables.values()))}, but skipped mirroring "
84
+ f"them as the {name!r} schema: this is a file-backed "
85
+ f"connection, and {name}.<group> views would persist in it "
86
+ f"after the flat tables they select from — registered only "
87
+ f"for this connection — are gone. Query the flat table names "
88
+ f"instead, or use an in-memory connection for the dotted "
89
+ f"spelling.",
90
+ RuntimeWarning,
91
+ stacklevel=3,
92
+ )
93
+ return
94
+ try:
95
+ con.execute(f"CREATE SCHEMA IF NOT EXISTS {_quote(name)}")
96
+ existing = {
97
+ row[0]
98
+ for row in con.execute(
99
+ "SELECT table_name FROM duckdb_tables() "
100
+ "WHERE schema_name = ? AND NOT internal",
101
+ [name],
102
+ ).fetchall()
103
+ }
104
+ conflicts = set(tables) & existing
105
+ if conflicts:
106
+ raise ValueError(
107
+ f"the {name!r} schema already has real tables named "
108
+ f"{sorted(conflicts)} that were not registered by "
109
+ f"xarray-sql; refusing to replace them with views"
110
+ )
111
+ for group, flat in tables.items():
112
+ con.execute(
113
+ f"CREATE OR REPLACE VIEW {_quote(name)}.{_quote(group)} "
114
+ f"AS SELECT * FROM {_quote(flat)}"
115
+ )
116
+ except Exception as exc: # noqa: BLE001 — degrade, don't fail the register
117
+ # Creating a schema needs a writable catalog, and the conflict
118
+ # check above can itself raise; either is not a reason to lose
119
+ # the registration.
120
+ warnings.warn(
121
+ f"Registered the dimension groups of {name!r} as "
122
+ f"{', '.join(sorted(tables.values()))}, but could not create "
123
+ f"the {name!r} schema mirroring them as {name}.<group> "
124
+ f"({exc}). Query the flat table names instead.",
125
+ RuntimeWarning,
126
+ stacklevel=3,
127
+ )
128
+
129
+
130
+ @register_adapter
131
+ class DuckDBAdapter:
132
+ """Registers Datasets on ``duckdb.DuckDBPyConnection`` connections."""
133
+
134
+ @staticmethod
135
+ def matches(con: object) -> TypeGuard[duckdb.DuckDBPyConnection]:
136
+ # The connection class lives in ``duckdb`` or, in newer releases,
137
+ # the ``_duckdb`` C-extension module.
138
+ root = type(con).__module__.split(".")[0]
139
+ return root in ("duckdb", "_duckdb")
140
+
141
+ @staticmethod
142
+ def register(
143
+ con: duckdb.DuckDBPyConnection,
144
+ name: str,
145
+ ds: xr.Dataset,
146
+ *,
147
+ chunks: Chunks = None,
148
+ table_names: TableNames = None,
149
+ **kwargs: Any,
150
+ ) -> duckdb.DuckDBPyConnection:
151
+ """Register ``ds`` on a DuckDB connection.
152
+
153
+ Datasets whose variables all share the same dimensions become a
154
+ single table named ``name``. Mixed-dimension datasets are split
155
+ into one table per dimension group, named after the group's
156
+ dimensions joined by underscores unless ``table_names`` gives the
157
+ group a name of its own::
158
+
159
+ xql.register(con, 'era5', ds, table_names={
160
+ ('time', 'latitude', 'longitude'): 'surface',
161
+ ('time', 'level', 'latitude', 'longitude'): 'atmosphere',
162
+ })
163
+ con.sql('SELECT ... FROM era5.surface') # or era5_surface
164
+
165
+ Each group is registered under the flat name ``<name>_<group>``
166
+ and mirrored as a view ``<name>.<group>`` in a DuckDB schema, so
167
+ the dotted spelling the DataFusion adapter uses queries the same
168
+ table here. Extra keyword arguments (``batch_size``,
169
+ ``prefetch``) are forwarded to
170
+ [XarrayPushdownDataset][xarray_sql.backends.pyarrow.XarrayPushdownDataset].
171
+ """
172
+ groups = group_vars_by_dims(ds)
173
+ names = resolve_table_names(ds, table_names, case_insensitive=True)
174
+ if len(groups) <= 1:
175
+ con.register(name, XarrayPushdownDataset(ds, chunks, **kwargs))
176
+ return con
177
+ # Materialise dim coordinates once and share across sub-tables.
178
+ coord_arrays = shared_coord_arrays(ds)
179
+ flat_names = {}
180
+ for dims, var_names in groups.items():
181
+ group = names[dims]
182
+ flat_names[group] = f"{name}_{group}"
183
+ con.register(
184
+ flat_names[group],
185
+ XarrayPushdownDataset(
186
+ ds[var_names],
187
+ chunks,
188
+ coord_arrays=coord_arrays,
189
+ **kwargs,
190
+ ),
191
+ )
192
+ _mirror_as_schema(con, name, flat_names)
193
+ return con
@@ -0,0 +1,307 @@
1
+ """Serve lazy xarray Datasets over Arrow Flight SQL.
2
+
3
+ The other adapters bring an engine to the data: the Dataset becomes a
4
+ table on a connection in this process. A
5
+ [FlightSQLServer][xarray_sql.backends.flight.FlightSQLServer] turns that
6
+ around and brings the data to remote clients. It hosts a DataFusion
7
+ session over the same lazy tables
8
+ [Context][xarray_sql.Context] uses and speaks
9
+ [Flight SQL](https://arrow.apache.org/docs/format/FlightSql.html), so any
10
+ Flight SQL client — ADBC's Flight SQL driver in Python, R, Go, or Java,
11
+ the Flight SQL JDBC and ODBC drivers, and the SQL tools built on them —
12
+ can query the Dataset without a copy::
13
+
14
+ server = xql.serve({"era5": ds}, port=8815)
15
+
16
+ # elsewhere
17
+ import adbc_driver_flightsql.dbapi as flight_sql
18
+ con = flight_sql.connect("grpc://server-host:8815")
19
+ cur = con.cursor()
20
+ cur.execute("SELECT AVG(t2m) FROM era5 WHERE time >= '2020-01-01'")
21
+
22
+ Queries keep partition pruning on dimension predicates and projection
23
+ pushdown: only the chunks and variables a query touches are read from
24
+ the source, only while it runs, and results stream back as Arrow record
25
+ batches.
26
+
27
+ The server has no authentication or TLS, and binds to ``127.0.0.1``
28
+ unless told otherwise. Its SQL is read-only: DDL, DML, and other
29
+ statements are rejected, so clients cannot read or write the server's
30
+ filesystem through it.
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import time
36
+ from typing import Any, TypeGuard
37
+
38
+ import xarray as xr
39
+
40
+ from ..df import (
41
+ Chunks,
42
+ TableNames,
43
+ group_vars_by_dims,
44
+ resolve_table_names,
45
+ shared_coord_arrays,
46
+ )
47
+ from ..reader import read_xarray_table
48
+ from ._adbc_dialects import DIALECTS
49
+ from .adbc import _warn_on_folded_names
50
+ from .base import register_adapter
51
+
52
+ try:
53
+ from .._native import FlightSqlServer as _NativeFlightSqlServer
54
+ except ImportError: # WebAssembly builds have no sockets or threads.
55
+ _NativeFlightSqlServer = None
56
+
57
+ __all__ = ["FlightSQLServer", "serve"]
58
+
59
+ _WILDCARD_HOSTS = ("", "0.0.0.0", "::")
60
+
61
+
62
+ class FlightSQLServer:
63
+ """A Flight SQL endpoint over lazily registered xarray Datasets.
64
+
65
+ Register Datasets with
66
+ [register][xarray_sql.backends.flight.FlightSQLServer.register] (or
67
+ [xarray_sql.register][], which dispatches here), then start it with
68
+ [serve][xarray_sql.backends.flight.FlightSQLServer.serve]. Datasets
69
+ registered while the server runs are visible to the next query.
70
+
71
+ The server runs on a background thread; the Python process must stay
72
+ alive while it serves. In a script, block with
73
+ [wait][xarray_sql.backends.flight.FlightSQLServer.wait]. It is also a
74
+ context manager that shuts down on exit.
75
+ """
76
+
77
+ def __init__(self, memory_limit: int | None = None) -> None:
78
+ """Create a server with no tables.
79
+
80
+ Args:
81
+ memory_limit: Bytes that queries' sorts, joins, and
82
+ aggregations may hold at once. A query that needs more
83
+ spills to disk where it can and fails otherwise; the
84
+ server keeps serving. ``None`` (default) sets no limit.
85
+ Reading chunks isn't counted, so cap the process
86
+ externally too (e.g. a container memory limit) before
87
+ serving untrusted clients.
88
+ """
89
+ if _NativeFlightSqlServer is None:
90
+ raise NotImplementedError(
91
+ "This build of xarray-sql has no Flight SQL server "
92
+ "(it is not available under WebAssembly)."
93
+ )
94
+ self._native = _NativeFlightSqlServer(memory_limit)
95
+ self._host: str | None = None
96
+ self._port: int | None = None
97
+
98
+ def register(
99
+ self,
100
+ name: str,
101
+ ds: xr.Dataset,
102
+ *,
103
+ chunks: Chunks = None,
104
+ table_names: TableNames = None,
105
+ ) -> FlightSQLServer:
106
+ """Register ``ds`` as a table named ``name``.
107
+
108
+ A Dataset whose variables sit on different dimensions is split
109
+ into one table per dimension group, in a SQL schema named
110
+ ``name`` (``name.group``), exactly as
111
+ [Context.from_dataset][xarray_sql.Context.from_dataset]
112
+ registers it.
113
+
114
+ Names are registered exactly as given. The server's SQL folds
115
+ unquoted names to lowercase, so a mixed-case name warns and must
116
+ be quoted in queries.
117
+
118
+ Args:
119
+ name: The table name, or the schema name for a
120
+ mixed-dimension Dataset.
121
+ ds: An xarray Dataset.
122
+ chunks: Xarray-like chunks specification controlling partition
123
+ granularity. Defaults to the Dataset's existing chunks, or
124
+ for an unchunked Dataset its storage chunks.
125
+ table_names: Maps a dimension group's exact dim tuple to the
126
+ name its table takes.
127
+
128
+ Returns:
129
+ The server, to allow chaining.
130
+ """
131
+ return self._register(name, ds, chunks, table_names, stacklevel=4)
132
+
133
+ def _register(
134
+ self,
135
+ name: str,
136
+ ds: xr.Dataset,
137
+ chunks: Chunks,
138
+ table_names: TableNames,
139
+ stacklevel: int,
140
+ ) -> FlightSQLServer:
141
+ """``register``, warning *stacklevel* frames up (the user's call)."""
142
+ groups = group_vars_by_dims(ds)
143
+ names = resolve_table_names(ds, table_names)
144
+ _warn_on_folded_names(
145
+ [name] if len(groups) <= 1 else [name, *names.values()],
146
+ DIALECTS["datafusion"],
147
+ stacklevel=stacklevel,
148
+ )
149
+ if len(groups) <= 1:
150
+ self._native.register_table(name, read_xarray_table(ds, chunks))
151
+ return self
152
+
153
+ # Every group's table is built before any is registered, so a
154
+ # failure leaves the server as it was.
155
+ coord_arrays = shared_coord_arrays(ds)
156
+ tables = [
157
+ (
158
+ names[dims],
159
+ read_xarray_table(
160
+ ds[var_names], chunks, coord_arrays=coord_arrays
161
+ ),
162
+ )
163
+ for dims, var_names in groups.items()
164
+ ]
165
+ self._native.register_tables(name, tables)
166
+ return self
167
+
168
+ def serve(self, host: str = "127.0.0.1", port: int = 0) -> FlightSQLServer:
169
+ """Start accepting connections on a background thread.
170
+
171
+ Args:
172
+ host: The interface to bind. The default accepts only local
173
+ connections; ``"0.0.0.0"`` accepts them from anywhere the
174
+ network allows (the server has no authentication).
175
+ port: The TCP port. ``0`` (default) picks a free one; read it
176
+ back from [port][xarray_sql.backends.flight.FlightSQLServer.port].
177
+
178
+ Returns:
179
+ The server, to allow chaining.
180
+ """
181
+ self._port = self._native.serve(host, port)
182
+ self._host = host
183
+ return self
184
+
185
+ @property
186
+ def port(self) -> int:
187
+ """The TCP port the server is bound to."""
188
+ if self._port is None:
189
+ raise RuntimeError("the server has not been started")
190
+ return self._port
191
+
192
+ @property
193
+ def uri(self) -> str:
194
+ """A ``grpc://`` URI a local client can connect to."""
195
+ host = self._host if self._host not in _WILDCARD_HOSTS else "localhost"
196
+ if host is not None and ":" in host:
197
+ host = f"[{host}]"
198
+ return f"grpc://{host}:{self.port}"
199
+
200
+ @property
201
+ def is_running(self) -> bool:
202
+ """Whether the server is accepting connections."""
203
+ return bool(self._native.is_running())
204
+
205
+ def wait(self, poll_interval: float = 0.5) -> None:
206
+ """Block until the server stops; Ctrl+C shuts it down.
207
+
208
+ Raises:
209
+ RuntimeError: The server stopped on an error.
210
+ """
211
+ try:
212
+ while self._native.is_running():
213
+ time.sleep(poll_interval)
214
+ except KeyboardInterrupt:
215
+ pass
216
+ # Joins the server thread, raising what stopped it, if anything.
217
+ self.shutdown()
218
+
219
+ def shutdown(self, timeout: float = 5.0) -> None:
220
+ """Stop accepting connections.
221
+
222
+ Args:
223
+ timeout: Seconds in-flight queries may keep running, after
224
+ which their connections are closed. A client that stops
225
+ reading a result partway otherwise holds its stream open
226
+ indefinitely.
227
+
228
+ Raises:
229
+ RuntimeError: The server had stopped on an error.
230
+ """
231
+ self._native.shutdown(timeout)
232
+
233
+ def __enter__(self) -> FlightSQLServer:
234
+ return self
235
+
236
+ def __exit__(self, *exc: object) -> None:
237
+ self.shutdown()
238
+
239
+ def __repr__(self) -> str:
240
+ state = f"serving on {self.uri}" if self.is_running else "stopped"
241
+ return f"FlightSQLServer({state})"
242
+
243
+
244
+ def serve(
245
+ datasets: dict[str, xr.Dataset],
246
+ host: str = "127.0.0.1",
247
+ port: int = 0,
248
+ *,
249
+ chunks: Chunks = None,
250
+ table_names: TableNames = None,
251
+ memory_limit: int | None = None,
252
+ ) -> FlightSQLServer:
253
+ """Serve Datasets over Arrow Flight SQL, without copying them.
254
+
255
+ Starts a [FlightSQLServer][xarray_sql.backends.flight.FlightSQLServer]
256
+ on a background thread with each Dataset registered under its key::
257
+
258
+ server = xql.serve({"era5": ds}, port=8815)
259
+ print(server.uri) # grpc://127.0.0.1:8815
260
+ server.wait() # in a script: block until Ctrl+C
261
+
262
+ Any Flight SQL client can then query the Datasets; with ADBC,
263
+ ``xql.to_dataset(cursor, template=ds)`` turns a result back into a
264
+ labeled Dataset on the client.
265
+
266
+ Args:
267
+ datasets: Table name to Dataset.
268
+ host: The interface to bind; ``127.0.0.1`` by default.
269
+ port: The TCP port; ``0`` (default) picks a free one.
270
+ chunks: Chunks specification applied to every Dataset.
271
+ table_names: Dimension-group naming applied to every Dataset.
272
+ memory_limit: Bytes queries may hold at once; see
273
+ [FlightSQLServer][xarray_sql.backends.flight.FlightSQLServer].
274
+
275
+ Returns:
276
+ The running server.
277
+ """
278
+ server = FlightSQLServer(memory_limit)
279
+ for name, ds in datasets.items():
280
+ server.register(name, ds, chunks=chunks, table_names=table_names)
281
+ return server.serve(host, port)
282
+
283
+
284
+ @register_adapter
285
+ class FlightSQLAdapter:
286
+ """Registers Datasets on a [FlightSQLServer][xarray_sql.backends.flight.FlightSQLServer]."""
287
+
288
+ @staticmethod
289
+ def matches(con: object) -> TypeGuard[FlightSQLServer]:
290
+ return isinstance(con, FlightSQLServer)
291
+
292
+ @staticmethod
293
+ def register(
294
+ con: FlightSQLServer,
295
+ name: str,
296
+ ds: xr.Dataset,
297
+ *,
298
+ chunks: Chunks = None,
299
+ table_names: TableNames = None,
300
+ **kwargs: Any,
301
+ ) -> FlightSQLServer:
302
+ if kwargs:
303
+ raise TypeError(
304
+ f"unexpected options for a FlightSQLServer: {sorted(kwargs)}"
305
+ )
306
+ # Called by xarray_sql.register, one frame further from the user.
307
+ return con._register(name, ds, chunks, table_names, stacklevel=5)