xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/__init__.py ADDED
@@ -0,0 +1,46 @@
1
+ from . import cftime
2
+ from .backends import (
3
+ FlightSQLServer,
4
+ arrow_dataset,
5
+ arrow_datasets,
6
+ register,
7
+ serve,
8
+ )
9
+ from .geometry import bbox_conjuncts
10
+ from .df import from_map
11
+ from .reader import read_xarray, read_xarray_table
12
+ from .roundtrip import to_dataset
13
+
14
+ __all__ = [
15
+ "cftime",
16
+ "Context",
17
+ "XarrayContext",
18
+ "FlightSQLServer",
19
+ "read_xarray_table",
20
+ "read_xarray",
21
+ "arrow_dataset",
22
+ "arrow_datasets",
23
+ "bbox_conjuncts",
24
+ "register",
25
+ "serve",
26
+ "to_dataset",
27
+ "from_map", # deprecated
28
+ ]
29
+
30
+ try:
31
+ from .sql import Context, XarrayContext
32
+ except ModuleNotFoundError as e:
33
+ # DataFusion's Python bindings have no wheels for some platforms
34
+ # (e.g. Pyodide); every other engine adapter still works there.
35
+ if e.name != "datafusion":
36
+ raise
37
+ __all__.remove("Context")
38
+ __all__.remove("XarrayContext")
39
+
40
+ def __getattr__(name: str):
41
+ if name in ("Context", "XarrayContext"):
42
+ raise AttributeError(
43
+ f"{name} requires the datafusion package, which is "
44
+ "not installed."
45
+ )
46
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
Binary file
@@ -0,0 +1,40 @@
1
+ """Engine adapters — the *register* seam of xarray-sql.
2
+
3
+ xarray-sql translates data, not queries, across two seams — the two
4
+ boundaries between xarray and a query engine that neither side builds
5
+ for itself: *register* (a lazy ``xarray.Dataset`` becomes a table on
6
+ the engine's own connection; this package) and *round-trip* (an Arrow
7
+ result becomes a labeled Dataset again; [xarray_sql.to_dataset][]).
8
+ SQL dialects, geometry, H3, and optimizers belong to each engine and
9
+ its extension ecosystem.
10
+
11
+ Adapters register themselves on import via
12
+ [register_adapter][xarray_sql.backends.base.register_adapter];
13
+ [register][xarray_sql.backends.base.register] dispatches on the connection
14
+ type.
15
+ """
16
+
17
+ from .base import EngineAdapter, get_adapter, register, register_adapter
18
+ from . import adbc as _adbc # noqa: F401 (self-registers)
19
+ from . import datafusion as _datafusion # noqa: F401 (self-registers)
20
+ from . import duckdb as _duckdb # noqa: F401 (self-registers)
21
+ from .flight import FlightSQLServer, serve # also self-registers
22
+ from .pyarrow import (
23
+ XarrayArrowStream,
24
+ XarrayPushdownDataset,
25
+ arrow_dataset,
26
+ arrow_datasets,
27
+ )
28
+
29
+ __all__ = [
30
+ "EngineAdapter",
31
+ "FlightSQLServer",
32
+ "XarrayArrowStream",
33
+ "XarrayPushdownDataset",
34
+ "arrow_dataset",
35
+ "arrow_datasets",
36
+ "get_adapter",
37
+ "register",
38
+ "register_adapter",
39
+ "serve",
40
+ ]
@@ -0,0 +1,439 @@
1
+ """How the databases behind ADBC drivers differ, in one table.
2
+
3
+ ADBC gives every database the same bulk-ingest call, but the SQL around
4
+ it and the drivers' support for it vary: how identifiers are quoted,
5
+ whether the database has schemas, whether its driver can create tables
6
+ or honor ``temporary=True``, and which Arrow types it can store. Each
7
+ [Dialect][xarray_sql.backends._adbc_dialects.Dialect] records those
8
+ facts for one database, so the adapter has one code path and adding a
9
+ database means adding a row.
10
+
11
+ A database missing from the table gets the defaults, which are what
12
+ ANSI SQL and the ADBC specification prescribe.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import dataclasses
18
+ import sys
19
+ from collections.abc import Callable
20
+ from typing import TYPE_CHECKING, Literal
21
+
22
+ import numpy as np
23
+ import pyarrow as pa
24
+
25
+ if TYPE_CHECKING:
26
+ from adbc_driver_manager import dbapi
27
+
28
+ IngestMode = Literal["create", "append", "replace", "create_append"]
29
+
30
+
31
+ @dataclasses.dataclass(frozen=True)
32
+ class IngestPlan:
33
+ """The DDL around an ingest, for drivers that can only append."""
34
+
35
+ table: str
36
+ """The table the rows are appended to."""
37
+
38
+ before: list[str]
39
+ """Run before the ingest."""
40
+
41
+ after: list[str] = dataclasses.field(default_factory=list)
42
+ """Run once the ingest succeeds."""
43
+
44
+ on_failure: list[str] = dataclasses.field(default_factory=list)
45
+ """Run if the ingest fails, before the error propagates."""
46
+
47
+
48
+ TableDDL = Callable[..., IngestPlan]
49
+ """Plans the DDL around an append-only ingest into a table."""
50
+
51
+
52
+ @dataclasses.dataclass(frozen=True)
53
+ class Dialect:
54
+ """What the ADBC adapter needs to know about one database."""
55
+
56
+ name: str
57
+ """The database's name, as used in error messages."""
58
+
59
+ quote: str = '"'
60
+ """The identifier quote: ``"`` in ANSI SQL, a backtick in MySQL-family
61
+ and Hive-family SQL, where ``"era5"`` is a string."""
62
+
63
+ schemas: bool = True
64
+ """Whether a mixed-dimension Dataset can go in a schema of its own
65
+ (``era5.surface``); otherwise its tables are flat (``era5_surface``)."""
66
+
67
+ schema_kind: str = "SCHEMA"
68
+ """The object ``CREATE ... IF NOT EXISTS`` makes to hold them."""
69
+
70
+ target_schema: Literal["option", "default_database"] = "option"
71
+ """How an ingest reaches a table in that schema: ADBC's target-schema
72
+ option, or, where the driver ignores that option, by making the
73
+ schema the connection's default database for the ingest."""
74
+
75
+ temporary_tables: bool = True
76
+ """Whether the driver honors ``temporary=True``."""
77
+
78
+ durations: bool = True
79
+ """Whether the database stores Arrow durations; if not, timedeltas are
80
+ ingested as integer counts of their unit."""
81
+
82
+ unsigned: bool = True
83
+ """Whether the database stores unsigned integers. If not, they are
84
+ widened to the next signed width, and a ``uint64`` above the int64
85
+ range is refused rather than wrapped."""
86
+
87
+ timestamps_as_text: bool = False
88
+ """For databases without a time type: write times as text like
89
+ ``2021-01-01 04:00:00.000000000``, which compares correctly with a
90
+ literal such as ``'2021-01-01 04:00:00'`` (SQLite's own format)."""
91
+
92
+ timestamp_unit: Literal["ns", "us"] = "ns"
93
+ """The finest time resolution the database stores."""
94
+
95
+ folds: Literal["lower", "upper"] | None = None
96
+ """How the database folds unquoted identifiers; names it would fold
97
+ must be quoted in queries."""
98
+
99
+ analyze: bool = False
100
+ """Whether to ``ANALYZE`` a table after ingest. PostgreSQL gathers
101
+ statistics only in the background, after a commit; until then its
102
+ planner guesses, and a join over a freshly registered table can pick
103
+ a nested loop that runs for hours instead of seconds."""
104
+
105
+ table_ddl: TableDDL | None = None
106
+ """For drivers that can only append: creates each table beforehand."""
107
+
108
+ schema_ddl: Callable[[str], str] | None = None
109
+ """For SQL without ``CREATE ... IF NOT EXISTS``: the statement that
110
+ creates the schema of a given name unless it exists."""
111
+
112
+ def quote_identifier(self, identifier: str) -> str:
113
+ """*identifier* as a quoted identifier in this database's SQL."""
114
+ escaped = identifier.replace(self.quote, self.quote * 2)
115
+ return f"{self.quote}{escaped}{self.quote}"
116
+
117
+ def create_schema_sql(self, name: str) -> str:
118
+ """The statement that creates schema *name* unless it exists."""
119
+ if self.schema_ddl is not None:
120
+ return self.schema_ddl(name)
121
+ target = self.quote_identifier(name)
122
+ return f"CREATE {self.schema_kind} IF NOT EXISTS {target}"
123
+
124
+
125
+ def _tsql_literal(text: str) -> str:
126
+ """*text* as a T-SQL Unicode string literal."""
127
+ return "N'" + text.replace("'", "''") + "'"
128
+
129
+
130
+ def _sql_server_schema_ddl(name: str) -> str:
131
+ """Creates schema *name* unless it exists, in T-SQL.
132
+
133
+ T-SQL has no ``CREATE SCHEMA IF NOT EXISTS``, and ``CREATE SCHEMA``
134
+ must be alone in its batch, hence the dynamic ``EXEC``.
135
+ """
136
+ create = f"CREATE SCHEMA {SQL_SERVER.quote_identifier(name)}"
137
+ return (
138
+ f"IF SCHEMA_ID({_tsql_literal(name)}) IS NULL "
139
+ f"EXEC({_tsql_literal(create)})"
140
+ )
141
+
142
+
143
+ _CLICKHOUSE_TYPES = {
144
+ pa.bool_(): "Bool",
145
+ pa.int8(): "Int8",
146
+ pa.int16(): "Int16",
147
+ pa.int32(): "Int32",
148
+ pa.int64(): "Int64",
149
+ pa.uint8(): "UInt8",
150
+ pa.uint16(): "UInt16",
151
+ pa.uint32(): "UInt32",
152
+ pa.uint64(): "UInt64",
153
+ pa.float16(): "Float32",
154
+ pa.float32(): "Float32",
155
+ pa.float64(): "Float64",
156
+ pa.string(): "String",
157
+ pa.large_string(): "String",
158
+ pa.binary(): "String",
159
+ pa.large_binary(): "String",
160
+ pa.date32(): "Date32",
161
+ }
162
+
163
+ _TIMESTAMP_PRECISION = {"s": 0, "ms": 3, "us": 6, "ns": 9}
164
+
165
+
166
+ def _clickhouse_type(field: pa.Field, key: bool) -> str:
167
+ """The ClickHouse column type for an Arrow field.
168
+
169
+ Timestamps without a zone are declared UTC, which is what their
170
+ values mean; ClickHouse also parses string literals compared with a
171
+ column in that column's zone, so ``time >= '2020-01-01'`` means UTC
172
+ rather than the server's local time. Sort-key columns are not
173
+ ``Nullable``. Floats are: the scan writes NaN as null so aggregates
174
+ skip missing values, and a plain ``Float64`` column would store that
175
+ null as 0.
176
+ """
177
+ arrow_type = field.type
178
+ if pa.types.is_timestamp(arrow_type):
179
+ precision = _TIMESTAMP_PRECISION[arrow_type.unit]
180
+ zone = arrow_type.tz or "UTC"
181
+ name = f"DateTime64({precision}, '{zone}')"
182
+ elif arrow_type in _CLICKHOUSE_TYPES:
183
+ name = _CLICKHOUSE_TYPES[arrow_type]
184
+ else:
185
+ raise TypeError(
186
+ f"no ClickHouse column type for {field.name!r} of Arrow type "
187
+ f"{arrow_type}; create the table yourself and register with "
188
+ f'mode="append"'
189
+ )
190
+ if key or not field.nullable:
191
+ return name
192
+ return f"Nullable({name})"
193
+
194
+
195
+ def _clickhouse_ddl(
196
+ table: str,
197
+ schema: pa.Schema,
198
+ dims: tuple[str, ...],
199
+ *,
200
+ mode: IngestMode,
201
+ temporary: bool,
202
+ database: str | None,
203
+ ) -> IngestPlan:
204
+ """The DDL around an append-mode ingest into *table*.
205
+
206
+ ClickHouse's ADBC driver only appends, so the table is created here
207
+ for every other mode. Tables are sorted by their dimensions, so
208
+ ClickHouse's primary index skips data on dimension predicates the
209
+ way chunk pruning does in the other engines.
210
+
211
+ ClickHouse has no transactions, so ``replace`` ingests into a staging
212
+ table and swaps it in with ``EXCHANGE TABLES`` (atomic) only once the
213
+ ingest has succeeded: a failed ingest leaves the old table as it was.
214
+ Temporary tables cannot be exchanged and are dropped and recreated.
215
+ """
216
+ quote = CLICKHOUSE.quote_identifier
217
+
218
+ def qualified(name: str) -> str:
219
+ if database is None:
220
+ return quote(name)
221
+ return f"{quote(database)}.{quote(name)}"
222
+
223
+ if mode == "append":
224
+ return IngestPlan(table, [])
225
+ columns = ", ".join(
226
+ f"{quote(field.name)} {_clickhouse_type(field, field.name in dims)}"
227
+ for field in schema
228
+ )
229
+ kind = "TEMPORARY TABLE" if temporary else "TABLE"
230
+ if temporary:
231
+ engine = "ENGINE = Memory"
232
+ else:
233
+ order = ", ".join(quote(dim) for dim in dims) or "tuple()"
234
+ engine = f"ENGINE = MergeTree ORDER BY ({order})"
235
+ target = qualified(table)
236
+ if mode == "replace" and not temporary:
237
+ staging_name = f"{table}__xarray_sql_replace"
238
+ staging = qualified(staging_name)
239
+ return IngestPlan(
240
+ staging_name,
241
+ before=[
242
+ f"DROP TABLE IF EXISTS {staging}",
243
+ f"CREATE TABLE {staging} ({columns}) {engine}",
244
+ ],
245
+ after=[
246
+ f"CREATE TABLE IF NOT EXISTS {target} AS {staging}",
247
+ f"EXCHANGE TABLES {target} AND {staging}",
248
+ f"DROP TABLE {staging}",
249
+ ],
250
+ on_failure=[f"DROP TABLE IF EXISTS {staging}"],
251
+ )
252
+ statements = []
253
+ if mode == "replace":
254
+ statements.append(f"DROP {kind} IF EXISTS {target}")
255
+ exists = " IF NOT EXISTS" if mode == "create_append" else ""
256
+ statements.append(f"CREATE {kind}{exists} {target} ({columns}) {engine}")
257
+ return IngestPlan(table, statements)
258
+
259
+
260
+ CLICKHOUSE = Dialect(
261
+ "ClickHouse",
262
+ schema_kind="DATABASE",
263
+ durations=False,
264
+ table_ddl=_clickhouse_ddl,
265
+ )
266
+
267
+ SQL_SERVER = Dialect(
268
+ "SQL Server",
269
+ durations=False,
270
+ unsigned=False,
271
+ timestamp_unit="us",
272
+ schema_ddl=_sql_server_schema_ddl,
273
+ )
274
+
275
+ DIALECTS: dict[str, Dialect] = {
276
+ # Exercised by the test suite, against a live database or driver.
277
+ "sqlite": Dialect(
278
+ "SQLite",
279
+ schemas=False,
280
+ durations=False,
281
+ unsigned=False,
282
+ timestamps_as_text=True,
283
+ ),
284
+ "duckdb": Dialect("DuckDB"),
285
+ # PostgreSQL wraps a uint64 above the int64 range without an error.
286
+ "postgresql": Dialect(
287
+ "PostgreSQL",
288
+ unsigned=False,
289
+ timestamp_unit="us",
290
+ folds="lower",
291
+ analyze=True,
292
+ ),
293
+ # MariaDB's server reports itself as MySQL.
294
+ "mysql": Dialect(
295
+ "MySQL",
296
+ quote="`",
297
+ schema_kind="DATABASE",
298
+ target_schema="default_database",
299
+ unsigned=False,
300
+ timestamp_unit="us",
301
+ ),
302
+ "clickhouse": CLICKHOUSE,
303
+ "datafusion": Dialect("DataFusion", temporary_tables=False, folds="lower"),
304
+ # Trino's driver ignores temporary=True and creates a permanent table.
305
+ "trino": Dialect("Trino", temporary_tables=False, unsigned=False),
306
+ # Temporary tables are queried as #name.
307
+ "sql server": SQL_SERVER,
308
+ # From the drivers' published feature tables and the databases' SQL
309
+ # references; not exercised by the test suite.
310
+ "spark": Dialect("Spark", quote="`", schemas=False, temporary_tables=False),
311
+ "bigquery": Dialect(
312
+ "BigQuery",
313
+ quote="`",
314
+ temporary_tables=False,
315
+ durations=False,
316
+ unsigned=False,
317
+ ),
318
+ "databricks": Dialect("Databricks", quote="`", temporary_tables=False),
319
+ "snowflake": Dialect("Snowflake", temporary_tables=False, folds="upper"),
320
+ }
321
+ """Known databases, keyed by a name their drivers' vendor names contain."""
322
+
323
+
324
+ def driver_error(con: dbapi.Connection) -> type[Exception]:
325
+ """The DB-API ``Error`` every ADBC driver's failures derive from.
326
+
327
+ Looked up rather than imported, so ADBC stays an optional dependency:
328
+ whenever an ADBC connection exists, its driver manager is loaded.
329
+ Catching only this lets bugs and other surprises propagate instead of
330
+ being mistaken for an unsupported feature.
331
+ """
332
+ error: type[Exception] = sys.modules["adbc_driver_manager.dbapi"].Error
333
+ return error
334
+
335
+
336
+ def dialect_for(con: dbapi.Connection) -> Dialect:
337
+ """The [Dialect][xarray_sql.backends._adbc_dialects.Dialect] of *con*.
338
+
339
+ Drivers name their database in ``adbc_get_info``. The ClickHouse
340
+ driver does not implement it, so only a driver without it is probed
341
+ with a query against ClickHouse's ``system.one`` table.
342
+ """
343
+ try:
344
+ info = con.adbc_get_info()
345
+ except driver_error(con): # unimplemented; probe instead
346
+ pass
347
+ else:
348
+ vendor = str(info.get("vendor_name") or "").lower()
349
+ for key, dialect in DIALECTS.items():
350
+ if key in vendor:
351
+ return dialect
352
+ return Dialect(str(info.get("vendor_name") or "the database"))
353
+ try:
354
+ with con.cursor() as cur:
355
+ cur.execute("SELECT 1 FROM system.one")
356
+ cur.fetchall()
357
+ except driver_error(con):
358
+ return Dialect("the database")
359
+ return CLICKHOUSE
360
+
361
+
362
+ _SIGNED_WIDTH = {
363
+ pa.uint8(): pa.int16(),
364
+ pa.uint16(): pa.int32(),
365
+ pa.uint32(): pa.int64(),
366
+ pa.uint64(): pa.int64(),
367
+ }
368
+
369
+
370
+ def _timestamps_as_text(array: pa.Array) -> pa.Array:
371
+ """Times as space-separated ISO text.
372
+
373
+ Whole seconds are written exactly as SQLite's ``datetime()`` writes
374
+ them (``2021-01-01 04:00:00``), so equality and inclusive bounds
375
+ against such a literal hold; a fraction of a second is kept where
376
+ there is one, which still orders correctly.
377
+ """
378
+ values = array.to_numpy(zero_copy_only=False)
379
+ whole = np.datetime_as_string(values, unit="s")
380
+ exact = np.datetime_as_string(values)
381
+ fractional = values != values.astype("datetime64[s]")
382
+ text = np.where(fractional, exact, whole)
383
+ text = np.char.replace(text, "T", " ", count=1)
384
+ return pa.array(text, pa.string(), mask=np.asarray(array.is_null()))
385
+
386
+
387
+ def _conversion(field: pa.Field, dialect: Dialect) -> pa.DataType | None:
388
+ """The type *field* must be ingested as, or ``None`` to keep it."""
389
+ arrow_type = field.type
390
+ if pa.types.is_duration(arrow_type) and not dialect.durations:
391
+ return pa.int64()
392
+ if pa.types.is_unsigned_integer(arrow_type) and not dialect.unsigned:
393
+ return _SIGNED_WIDTH[arrow_type]
394
+ if pa.types.is_timestamp(arrow_type) and dialect.timestamps_as_text:
395
+ return pa.string()
396
+ return None
397
+
398
+
399
+ def _convert(
400
+ array: pa.Array, target: pa.DataType, dialect: Dialect
401
+ ) -> pa.Array:
402
+ if pa.types.is_string(target):
403
+ return _timestamps_as_text(array)
404
+ try:
405
+ return array.cast(target)
406
+ except pa.ArrowInvalid as exc:
407
+ raise ValueError(
408
+ f"{dialect.name} cannot store {array.type} values above the "
409
+ f"{target} range, which it would otherwise wrap around; convert "
410
+ f"the variable (e.g. to float64) before registering it."
411
+ ) from exc
412
+
413
+
414
+ def ingestable(
415
+ reader: pa.RecordBatchReader, dialect: Dialect
416
+ ) -> pa.RecordBatchReader:
417
+ """*reader* with every column in a type *dialect*'s database stores.
418
+
419
+ Durations become integer counts of their unit (the template's
420
+ ``timedelta64`` unit matches it, so [xarray_sql.to_dataset][] reads
421
+ them back exactly), unsigned integers widen to the next signed width,
422
+ and times become text where there is no time type.
423
+ """
424
+ targets = {i: _conversion(f, dialect) for i, f in enumerate(reader.schema)}
425
+ targets = {i: t for i, t in targets.items() if t is not None}
426
+ if not targets:
427
+ return reader
428
+ schema = reader.schema
429
+ for i, target in targets.items():
430
+ schema = schema.set(i, schema.field(i).with_type(target))
431
+
432
+ def batches():
433
+ for batch in reader:
434
+ columns = list(batch.columns)
435
+ for i, target in targets.items():
436
+ columns[i] = _convert(columns[i], target, dialect)
437
+ yield pa.RecordBatch.from_arrays(columns, schema=schema)
438
+
439
+ return pa.RecordBatchReader.from_batches(schema, batches())