xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
xarray_sql/__init__.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from . import cftime
|
|
2
|
+
from .backends import (
|
|
3
|
+
FlightSQLServer,
|
|
4
|
+
arrow_dataset,
|
|
5
|
+
arrow_datasets,
|
|
6
|
+
register,
|
|
7
|
+
serve,
|
|
8
|
+
)
|
|
9
|
+
from .geometry import bbox_conjuncts
|
|
10
|
+
from .df import from_map
|
|
11
|
+
from .reader import read_xarray, read_xarray_table
|
|
12
|
+
from .roundtrip import to_dataset
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"cftime",
|
|
16
|
+
"Context",
|
|
17
|
+
"XarrayContext",
|
|
18
|
+
"FlightSQLServer",
|
|
19
|
+
"read_xarray_table",
|
|
20
|
+
"read_xarray",
|
|
21
|
+
"arrow_dataset",
|
|
22
|
+
"arrow_datasets",
|
|
23
|
+
"bbox_conjuncts",
|
|
24
|
+
"register",
|
|
25
|
+
"serve",
|
|
26
|
+
"to_dataset",
|
|
27
|
+
"from_map", # deprecated
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
from .sql import Context, XarrayContext
|
|
32
|
+
except ModuleNotFoundError as e:
|
|
33
|
+
# DataFusion's Python bindings have no wheels for some platforms
|
|
34
|
+
# (e.g. Pyodide); every other engine adapter still works there.
|
|
35
|
+
if e.name != "datafusion":
|
|
36
|
+
raise
|
|
37
|
+
__all__.remove("Context")
|
|
38
|
+
__all__.remove("XarrayContext")
|
|
39
|
+
|
|
40
|
+
def __getattr__(name: str):
|
|
41
|
+
if name in ("Context", "XarrayContext"):
|
|
42
|
+
raise AttributeError(
|
|
43
|
+
f"{name} requires the datafusion package, which is "
|
|
44
|
+
"not installed."
|
|
45
|
+
)
|
|
46
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
Binary file
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Engine adapters — the *register* seam of xarray-sql.
|
|
2
|
+
|
|
3
|
+
xarray-sql translates data, not queries, across two seams — the two
|
|
4
|
+
boundaries between xarray and a query engine that neither side builds
|
|
5
|
+
for itself: *register* (a lazy ``xarray.Dataset`` becomes a table on
|
|
6
|
+
the engine's own connection; this package) and *round-trip* (an Arrow
|
|
7
|
+
result becomes a labeled Dataset again; [xarray_sql.to_dataset][]).
|
|
8
|
+
SQL dialects, geometry, H3, and optimizers belong to each engine and
|
|
9
|
+
its extension ecosystem.
|
|
10
|
+
|
|
11
|
+
Adapters register themselves on import via
|
|
12
|
+
[register_adapter][xarray_sql.backends.base.register_adapter];
|
|
13
|
+
[register][xarray_sql.backends.base.register] dispatches on the connection
|
|
14
|
+
type.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from .base import EngineAdapter, get_adapter, register, register_adapter
|
|
18
|
+
from . import adbc as _adbc # noqa: F401 (self-registers)
|
|
19
|
+
from . import datafusion as _datafusion # noqa: F401 (self-registers)
|
|
20
|
+
from . import duckdb as _duckdb # noqa: F401 (self-registers)
|
|
21
|
+
from .flight import FlightSQLServer, serve # also self-registers
|
|
22
|
+
from .pyarrow import (
|
|
23
|
+
XarrayArrowStream,
|
|
24
|
+
XarrayPushdownDataset,
|
|
25
|
+
arrow_dataset,
|
|
26
|
+
arrow_datasets,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"EngineAdapter",
|
|
31
|
+
"FlightSQLServer",
|
|
32
|
+
"XarrayArrowStream",
|
|
33
|
+
"XarrayPushdownDataset",
|
|
34
|
+
"arrow_dataset",
|
|
35
|
+
"arrow_datasets",
|
|
36
|
+
"get_adapter",
|
|
37
|
+
"register",
|
|
38
|
+
"register_adapter",
|
|
39
|
+
"serve",
|
|
40
|
+
]
|
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
"""How the databases behind ADBC drivers differ, in one table.
|
|
2
|
+
|
|
3
|
+
ADBC gives every database the same bulk-ingest call, but the SQL around
|
|
4
|
+
it and the drivers' support for it vary: how identifiers are quoted,
|
|
5
|
+
whether the database has schemas, whether its driver can create tables
|
|
6
|
+
or honor ``temporary=True``, and which Arrow types it can store. Each
|
|
7
|
+
[Dialect][xarray_sql.backends._adbc_dialects.Dialect] records those
|
|
8
|
+
facts for one database, so the adapter has one code path and adding a
|
|
9
|
+
database means adding a row.
|
|
10
|
+
|
|
11
|
+
A database missing from the table gets the defaults, which are what
|
|
12
|
+
ANSI SQL and the ADBC specification prescribe.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import dataclasses
|
|
18
|
+
import sys
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from typing import TYPE_CHECKING, Literal
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
import pyarrow as pa
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from adbc_driver_manager import dbapi
|
|
27
|
+
|
|
28
|
+
IngestMode = Literal["create", "append", "replace", "create_append"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclasses.dataclass(frozen=True)
|
|
32
|
+
class IngestPlan:
|
|
33
|
+
"""The DDL around an ingest, for drivers that can only append."""
|
|
34
|
+
|
|
35
|
+
table: str
|
|
36
|
+
"""The table the rows are appended to."""
|
|
37
|
+
|
|
38
|
+
before: list[str]
|
|
39
|
+
"""Run before the ingest."""
|
|
40
|
+
|
|
41
|
+
after: list[str] = dataclasses.field(default_factory=list)
|
|
42
|
+
"""Run once the ingest succeeds."""
|
|
43
|
+
|
|
44
|
+
on_failure: list[str] = dataclasses.field(default_factory=list)
|
|
45
|
+
"""Run if the ingest fails, before the error propagates."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
TableDDL = Callable[..., IngestPlan]
|
|
49
|
+
"""Plans the DDL around an append-only ingest into a table."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclasses.dataclass(frozen=True)
|
|
53
|
+
class Dialect:
|
|
54
|
+
"""What the ADBC adapter needs to know about one database."""
|
|
55
|
+
|
|
56
|
+
name: str
|
|
57
|
+
"""The database's name, as used in error messages."""
|
|
58
|
+
|
|
59
|
+
quote: str = '"'
|
|
60
|
+
"""The identifier quote: ``"`` in ANSI SQL, a backtick in MySQL-family
|
|
61
|
+
and Hive-family SQL, where ``"era5"`` is a string."""
|
|
62
|
+
|
|
63
|
+
schemas: bool = True
|
|
64
|
+
"""Whether a mixed-dimension Dataset can go in a schema of its own
|
|
65
|
+
(``era5.surface``); otherwise its tables are flat (``era5_surface``)."""
|
|
66
|
+
|
|
67
|
+
schema_kind: str = "SCHEMA"
|
|
68
|
+
"""The object ``CREATE ... IF NOT EXISTS`` makes to hold them."""
|
|
69
|
+
|
|
70
|
+
target_schema: Literal["option", "default_database"] = "option"
|
|
71
|
+
"""How an ingest reaches a table in that schema: ADBC's target-schema
|
|
72
|
+
option, or, where the driver ignores that option, by making the
|
|
73
|
+
schema the connection's default database for the ingest."""
|
|
74
|
+
|
|
75
|
+
temporary_tables: bool = True
|
|
76
|
+
"""Whether the driver honors ``temporary=True``."""
|
|
77
|
+
|
|
78
|
+
durations: bool = True
|
|
79
|
+
"""Whether the database stores Arrow durations; if not, timedeltas are
|
|
80
|
+
ingested as integer counts of their unit."""
|
|
81
|
+
|
|
82
|
+
unsigned: bool = True
|
|
83
|
+
"""Whether the database stores unsigned integers. If not, they are
|
|
84
|
+
widened to the next signed width, and a ``uint64`` above the int64
|
|
85
|
+
range is refused rather than wrapped."""
|
|
86
|
+
|
|
87
|
+
timestamps_as_text: bool = False
|
|
88
|
+
"""For databases without a time type: write times as text like
|
|
89
|
+
``2021-01-01 04:00:00.000000000``, which compares correctly with a
|
|
90
|
+
literal such as ``'2021-01-01 04:00:00'`` (SQLite's own format)."""
|
|
91
|
+
|
|
92
|
+
timestamp_unit: Literal["ns", "us"] = "ns"
|
|
93
|
+
"""The finest time resolution the database stores."""
|
|
94
|
+
|
|
95
|
+
folds: Literal["lower", "upper"] | None = None
|
|
96
|
+
"""How the database folds unquoted identifiers; names it would fold
|
|
97
|
+
must be quoted in queries."""
|
|
98
|
+
|
|
99
|
+
analyze: bool = False
|
|
100
|
+
"""Whether to ``ANALYZE`` a table after ingest. PostgreSQL gathers
|
|
101
|
+
statistics only in the background, after a commit; until then its
|
|
102
|
+
planner guesses, and a join over a freshly registered table can pick
|
|
103
|
+
a nested loop that runs for hours instead of seconds."""
|
|
104
|
+
|
|
105
|
+
table_ddl: TableDDL | None = None
|
|
106
|
+
"""For drivers that can only append: creates each table beforehand."""
|
|
107
|
+
|
|
108
|
+
schema_ddl: Callable[[str], str] | None = None
|
|
109
|
+
"""For SQL without ``CREATE ... IF NOT EXISTS``: the statement that
|
|
110
|
+
creates the schema of a given name unless it exists."""
|
|
111
|
+
|
|
112
|
+
def quote_identifier(self, identifier: str) -> str:
|
|
113
|
+
"""*identifier* as a quoted identifier in this database's SQL."""
|
|
114
|
+
escaped = identifier.replace(self.quote, self.quote * 2)
|
|
115
|
+
return f"{self.quote}{escaped}{self.quote}"
|
|
116
|
+
|
|
117
|
+
def create_schema_sql(self, name: str) -> str:
|
|
118
|
+
"""The statement that creates schema *name* unless it exists."""
|
|
119
|
+
if self.schema_ddl is not None:
|
|
120
|
+
return self.schema_ddl(name)
|
|
121
|
+
target = self.quote_identifier(name)
|
|
122
|
+
return f"CREATE {self.schema_kind} IF NOT EXISTS {target}"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _tsql_literal(text: str) -> str:
|
|
126
|
+
"""*text* as a T-SQL Unicode string literal."""
|
|
127
|
+
return "N'" + text.replace("'", "''") + "'"
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _sql_server_schema_ddl(name: str) -> str:
|
|
131
|
+
"""Creates schema *name* unless it exists, in T-SQL.
|
|
132
|
+
|
|
133
|
+
T-SQL has no ``CREATE SCHEMA IF NOT EXISTS``, and ``CREATE SCHEMA``
|
|
134
|
+
must be alone in its batch, hence the dynamic ``EXEC``.
|
|
135
|
+
"""
|
|
136
|
+
create = f"CREATE SCHEMA {SQL_SERVER.quote_identifier(name)}"
|
|
137
|
+
return (
|
|
138
|
+
f"IF SCHEMA_ID({_tsql_literal(name)}) IS NULL "
|
|
139
|
+
f"EXEC({_tsql_literal(create)})"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
_CLICKHOUSE_TYPES = {
|
|
144
|
+
pa.bool_(): "Bool",
|
|
145
|
+
pa.int8(): "Int8",
|
|
146
|
+
pa.int16(): "Int16",
|
|
147
|
+
pa.int32(): "Int32",
|
|
148
|
+
pa.int64(): "Int64",
|
|
149
|
+
pa.uint8(): "UInt8",
|
|
150
|
+
pa.uint16(): "UInt16",
|
|
151
|
+
pa.uint32(): "UInt32",
|
|
152
|
+
pa.uint64(): "UInt64",
|
|
153
|
+
pa.float16(): "Float32",
|
|
154
|
+
pa.float32(): "Float32",
|
|
155
|
+
pa.float64(): "Float64",
|
|
156
|
+
pa.string(): "String",
|
|
157
|
+
pa.large_string(): "String",
|
|
158
|
+
pa.binary(): "String",
|
|
159
|
+
pa.large_binary(): "String",
|
|
160
|
+
pa.date32(): "Date32",
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
_TIMESTAMP_PRECISION = {"s": 0, "ms": 3, "us": 6, "ns": 9}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _clickhouse_type(field: pa.Field, key: bool) -> str:
|
|
167
|
+
"""The ClickHouse column type for an Arrow field.
|
|
168
|
+
|
|
169
|
+
Timestamps without a zone are declared UTC, which is what their
|
|
170
|
+
values mean; ClickHouse also parses string literals compared with a
|
|
171
|
+
column in that column's zone, so ``time >= '2020-01-01'`` means UTC
|
|
172
|
+
rather than the server's local time. Sort-key columns are not
|
|
173
|
+
``Nullable``. Floats are: the scan writes NaN as null so aggregates
|
|
174
|
+
skip missing values, and a plain ``Float64`` column would store that
|
|
175
|
+
null as 0.
|
|
176
|
+
"""
|
|
177
|
+
arrow_type = field.type
|
|
178
|
+
if pa.types.is_timestamp(arrow_type):
|
|
179
|
+
precision = _TIMESTAMP_PRECISION[arrow_type.unit]
|
|
180
|
+
zone = arrow_type.tz or "UTC"
|
|
181
|
+
name = f"DateTime64({precision}, '{zone}')"
|
|
182
|
+
elif arrow_type in _CLICKHOUSE_TYPES:
|
|
183
|
+
name = _CLICKHOUSE_TYPES[arrow_type]
|
|
184
|
+
else:
|
|
185
|
+
raise TypeError(
|
|
186
|
+
f"no ClickHouse column type for {field.name!r} of Arrow type "
|
|
187
|
+
f"{arrow_type}; create the table yourself and register with "
|
|
188
|
+
f'mode="append"'
|
|
189
|
+
)
|
|
190
|
+
if key or not field.nullable:
|
|
191
|
+
return name
|
|
192
|
+
return f"Nullable({name})"
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _clickhouse_ddl(
|
|
196
|
+
table: str,
|
|
197
|
+
schema: pa.Schema,
|
|
198
|
+
dims: tuple[str, ...],
|
|
199
|
+
*,
|
|
200
|
+
mode: IngestMode,
|
|
201
|
+
temporary: bool,
|
|
202
|
+
database: str | None,
|
|
203
|
+
) -> IngestPlan:
|
|
204
|
+
"""The DDL around an append-mode ingest into *table*.
|
|
205
|
+
|
|
206
|
+
ClickHouse's ADBC driver only appends, so the table is created here
|
|
207
|
+
for every other mode. Tables are sorted by their dimensions, so
|
|
208
|
+
ClickHouse's primary index skips data on dimension predicates the
|
|
209
|
+
way chunk pruning does in the other engines.
|
|
210
|
+
|
|
211
|
+
ClickHouse has no transactions, so ``replace`` ingests into a staging
|
|
212
|
+
table and swaps it in with ``EXCHANGE TABLES`` (atomic) only once the
|
|
213
|
+
ingest has succeeded: a failed ingest leaves the old table as it was.
|
|
214
|
+
Temporary tables cannot be exchanged and are dropped and recreated.
|
|
215
|
+
"""
|
|
216
|
+
quote = CLICKHOUSE.quote_identifier
|
|
217
|
+
|
|
218
|
+
def qualified(name: str) -> str:
|
|
219
|
+
if database is None:
|
|
220
|
+
return quote(name)
|
|
221
|
+
return f"{quote(database)}.{quote(name)}"
|
|
222
|
+
|
|
223
|
+
if mode == "append":
|
|
224
|
+
return IngestPlan(table, [])
|
|
225
|
+
columns = ", ".join(
|
|
226
|
+
f"{quote(field.name)} {_clickhouse_type(field, field.name in dims)}"
|
|
227
|
+
for field in schema
|
|
228
|
+
)
|
|
229
|
+
kind = "TEMPORARY TABLE" if temporary else "TABLE"
|
|
230
|
+
if temporary:
|
|
231
|
+
engine = "ENGINE = Memory"
|
|
232
|
+
else:
|
|
233
|
+
order = ", ".join(quote(dim) for dim in dims) or "tuple()"
|
|
234
|
+
engine = f"ENGINE = MergeTree ORDER BY ({order})"
|
|
235
|
+
target = qualified(table)
|
|
236
|
+
if mode == "replace" and not temporary:
|
|
237
|
+
staging_name = f"{table}__xarray_sql_replace"
|
|
238
|
+
staging = qualified(staging_name)
|
|
239
|
+
return IngestPlan(
|
|
240
|
+
staging_name,
|
|
241
|
+
before=[
|
|
242
|
+
f"DROP TABLE IF EXISTS {staging}",
|
|
243
|
+
f"CREATE TABLE {staging} ({columns}) {engine}",
|
|
244
|
+
],
|
|
245
|
+
after=[
|
|
246
|
+
f"CREATE TABLE IF NOT EXISTS {target} AS {staging}",
|
|
247
|
+
f"EXCHANGE TABLES {target} AND {staging}",
|
|
248
|
+
f"DROP TABLE {staging}",
|
|
249
|
+
],
|
|
250
|
+
on_failure=[f"DROP TABLE IF EXISTS {staging}"],
|
|
251
|
+
)
|
|
252
|
+
statements = []
|
|
253
|
+
if mode == "replace":
|
|
254
|
+
statements.append(f"DROP {kind} IF EXISTS {target}")
|
|
255
|
+
exists = " IF NOT EXISTS" if mode == "create_append" else ""
|
|
256
|
+
statements.append(f"CREATE {kind}{exists} {target} ({columns}) {engine}")
|
|
257
|
+
return IngestPlan(table, statements)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
CLICKHOUSE = Dialect(
|
|
261
|
+
"ClickHouse",
|
|
262
|
+
schema_kind="DATABASE",
|
|
263
|
+
durations=False,
|
|
264
|
+
table_ddl=_clickhouse_ddl,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
SQL_SERVER = Dialect(
|
|
268
|
+
"SQL Server",
|
|
269
|
+
durations=False,
|
|
270
|
+
unsigned=False,
|
|
271
|
+
timestamp_unit="us",
|
|
272
|
+
schema_ddl=_sql_server_schema_ddl,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
DIALECTS: dict[str, Dialect] = {
|
|
276
|
+
# Exercised by the test suite, against a live database or driver.
|
|
277
|
+
"sqlite": Dialect(
|
|
278
|
+
"SQLite",
|
|
279
|
+
schemas=False,
|
|
280
|
+
durations=False,
|
|
281
|
+
unsigned=False,
|
|
282
|
+
timestamps_as_text=True,
|
|
283
|
+
),
|
|
284
|
+
"duckdb": Dialect("DuckDB"),
|
|
285
|
+
# PostgreSQL wraps a uint64 above the int64 range without an error.
|
|
286
|
+
"postgresql": Dialect(
|
|
287
|
+
"PostgreSQL",
|
|
288
|
+
unsigned=False,
|
|
289
|
+
timestamp_unit="us",
|
|
290
|
+
folds="lower",
|
|
291
|
+
analyze=True,
|
|
292
|
+
),
|
|
293
|
+
# MariaDB's server reports itself as MySQL.
|
|
294
|
+
"mysql": Dialect(
|
|
295
|
+
"MySQL",
|
|
296
|
+
quote="`",
|
|
297
|
+
schema_kind="DATABASE",
|
|
298
|
+
target_schema="default_database",
|
|
299
|
+
unsigned=False,
|
|
300
|
+
timestamp_unit="us",
|
|
301
|
+
),
|
|
302
|
+
"clickhouse": CLICKHOUSE,
|
|
303
|
+
"datafusion": Dialect("DataFusion", temporary_tables=False, folds="lower"),
|
|
304
|
+
# Trino's driver ignores temporary=True and creates a permanent table.
|
|
305
|
+
"trino": Dialect("Trino", temporary_tables=False, unsigned=False),
|
|
306
|
+
# Temporary tables are queried as #name.
|
|
307
|
+
"sql server": SQL_SERVER,
|
|
308
|
+
# From the drivers' published feature tables and the databases' SQL
|
|
309
|
+
# references; not exercised by the test suite.
|
|
310
|
+
"spark": Dialect("Spark", quote="`", schemas=False, temporary_tables=False),
|
|
311
|
+
"bigquery": Dialect(
|
|
312
|
+
"BigQuery",
|
|
313
|
+
quote="`",
|
|
314
|
+
temporary_tables=False,
|
|
315
|
+
durations=False,
|
|
316
|
+
unsigned=False,
|
|
317
|
+
),
|
|
318
|
+
"databricks": Dialect("Databricks", quote="`", temporary_tables=False),
|
|
319
|
+
"snowflake": Dialect("Snowflake", temporary_tables=False, folds="upper"),
|
|
320
|
+
}
|
|
321
|
+
"""Known databases, keyed by a name their drivers' vendor names contain."""
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def driver_error(con: dbapi.Connection) -> type[Exception]:
|
|
325
|
+
"""The DB-API ``Error`` every ADBC driver's failures derive from.
|
|
326
|
+
|
|
327
|
+
Looked up rather than imported, so ADBC stays an optional dependency:
|
|
328
|
+
whenever an ADBC connection exists, its driver manager is loaded.
|
|
329
|
+
Catching only this lets bugs and other surprises propagate instead of
|
|
330
|
+
being mistaken for an unsupported feature.
|
|
331
|
+
"""
|
|
332
|
+
error: type[Exception] = sys.modules["adbc_driver_manager.dbapi"].Error
|
|
333
|
+
return error
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def dialect_for(con: dbapi.Connection) -> Dialect:
|
|
337
|
+
"""The [Dialect][xarray_sql.backends._adbc_dialects.Dialect] of *con*.
|
|
338
|
+
|
|
339
|
+
Drivers name their database in ``adbc_get_info``. The ClickHouse
|
|
340
|
+
driver does not implement it, so only a driver without it is probed
|
|
341
|
+
with a query against ClickHouse's ``system.one`` table.
|
|
342
|
+
"""
|
|
343
|
+
try:
|
|
344
|
+
info = con.adbc_get_info()
|
|
345
|
+
except driver_error(con): # unimplemented; probe instead
|
|
346
|
+
pass
|
|
347
|
+
else:
|
|
348
|
+
vendor = str(info.get("vendor_name") or "").lower()
|
|
349
|
+
for key, dialect in DIALECTS.items():
|
|
350
|
+
if key in vendor:
|
|
351
|
+
return dialect
|
|
352
|
+
return Dialect(str(info.get("vendor_name") or "the database"))
|
|
353
|
+
try:
|
|
354
|
+
with con.cursor() as cur:
|
|
355
|
+
cur.execute("SELECT 1 FROM system.one")
|
|
356
|
+
cur.fetchall()
|
|
357
|
+
except driver_error(con):
|
|
358
|
+
return Dialect("the database")
|
|
359
|
+
return CLICKHOUSE
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
_SIGNED_WIDTH = {
|
|
363
|
+
pa.uint8(): pa.int16(),
|
|
364
|
+
pa.uint16(): pa.int32(),
|
|
365
|
+
pa.uint32(): pa.int64(),
|
|
366
|
+
pa.uint64(): pa.int64(),
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _timestamps_as_text(array: pa.Array) -> pa.Array:
|
|
371
|
+
"""Times as space-separated ISO text.
|
|
372
|
+
|
|
373
|
+
Whole seconds are written exactly as SQLite's ``datetime()`` writes
|
|
374
|
+
them (``2021-01-01 04:00:00``), so equality and inclusive bounds
|
|
375
|
+
against such a literal hold; a fraction of a second is kept where
|
|
376
|
+
there is one, which still orders correctly.
|
|
377
|
+
"""
|
|
378
|
+
values = array.to_numpy(zero_copy_only=False)
|
|
379
|
+
whole = np.datetime_as_string(values, unit="s")
|
|
380
|
+
exact = np.datetime_as_string(values)
|
|
381
|
+
fractional = values != values.astype("datetime64[s]")
|
|
382
|
+
text = np.where(fractional, exact, whole)
|
|
383
|
+
text = np.char.replace(text, "T", " ", count=1)
|
|
384
|
+
return pa.array(text, pa.string(), mask=np.asarray(array.is_null()))
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _conversion(field: pa.Field, dialect: Dialect) -> pa.DataType | None:
|
|
388
|
+
"""The type *field* must be ingested as, or ``None`` to keep it."""
|
|
389
|
+
arrow_type = field.type
|
|
390
|
+
if pa.types.is_duration(arrow_type) and not dialect.durations:
|
|
391
|
+
return pa.int64()
|
|
392
|
+
if pa.types.is_unsigned_integer(arrow_type) and not dialect.unsigned:
|
|
393
|
+
return _SIGNED_WIDTH[arrow_type]
|
|
394
|
+
if pa.types.is_timestamp(arrow_type) and dialect.timestamps_as_text:
|
|
395
|
+
return pa.string()
|
|
396
|
+
return None
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _convert(
|
|
400
|
+
array: pa.Array, target: pa.DataType, dialect: Dialect
|
|
401
|
+
) -> pa.Array:
|
|
402
|
+
if pa.types.is_string(target):
|
|
403
|
+
return _timestamps_as_text(array)
|
|
404
|
+
try:
|
|
405
|
+
return array.cast(target)
|
|
406
|
+
except pa.ArrowInvalid as exc:
|
|
407
|
+
raise ValueError(
|
|
408
|
+
f"{dialect.name} cannot store {array.type} values above the "
|
|
409
|
+
f"{target} range, which it would otherwise wrap around; convert "
|
|
410
|
+
f"the variable (e.g. to float64) before registering it."
|
|
411
|
+
) from exc
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def ingestable(
|
|
415
|
+
reader: pa.RecordBatchReader, dialect: Dialect
|
|
416
|
+
) -> pa.RecordBatchReader:
|
|
417
|
+
"""*reader* with every column in a type *dialect*'s database stores.
|
|
418
|
+
|
|
419
|
+
Durations become integer counts of their unit (the template's
|
|
420
|
+
``timedelta64`` unit matches it, so [xarray_sql.to_dataset][] reads
|
|
421
|
+
them back exactly), unsigned integers widen to the next signed width,
|
|
422
|
+
and times become text where there is no time type.
|
|
423
|
+
"""
|
|
424
|
+
targets = {i: _conversion(f, dialect) for i, f in enumerate(reader.schema)}
|
|
425
|
+
targets = {i: t for i, t in targets.items() if t is not None}
|
|
426
|
+
if not targets:
|
|
427
|
+
return reader
|
|
428
|
+
schema = reader.schema
|
|
429
|
+
for i, target in targets.items():
|
|
430
|
+
schema = schema.set(i, schema.field(i).with_type(target))
|
|
431
|
+
|
|
432
|
+
def batches():
|
|
433
|
+
for batch in reader:
|
|
434
|
+
columns = list(batch.columns)
|
|
435
|
+
for i, target in targets.items():
|
|
436
|
+
columns[i] = _convert(columns[i], target, dialect)
|
|
437
|
+
yield pa.RecordBatch.from_arrays(columns, schema=schema)
|
|
438
|
+
|
|
439
|
+
return pa.RecordBatchReader.from_batches(schema, batches())
|