xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +46 -0
- xarray_sql/_native.abi3.so +0 -0
- xarray_sql/backends/__init__.py +40 -0
- xarray_sql/backends/_adbc_dialects.py +439 -0
- xarray_sql/backends/adbc.py +395 -0
- xarray_sql/backends/base.py +150 -0
- xarray_sql/backends/datafusion.py +91 -0
- xarray_sql/backends/duckdb.py +193 -0
- xarray_sql/backends/flight.py +307 -0
- xarray_sql/backends/pyarrow.py +1239 -0
- xarray_sql/cftime.py +261 -0
- xarray_sql/core.py +49 -0
- xarray_sql/df.py +832 -0
- xarray_sql/ds.py +1286 -0
- xarray_sql/geometry.py +131 -0
- xarray_sql/lazyscan.py +462 -0
- xarray_sql/proj.py +235 -0
- xarray_sql/reader.py +343 -0
- xarray_sql/roundtrip.py +614 -0
- xarray_sql/sql.py +200 -0
- xarray_sql-0.6.0.dist-info/METADATA +411 -0
- xarray_sql-0.6.0.dist-info/RECORD +25 -0
- xarray_sql-0.6.0.dist-info/WHEEL +5 -0
- xarray_sql-0.6.0.dist-info/licenses/LICENSE +202 -0
- xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json +9574 -0
xarray_sql/sql.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import xarray as xr
|
|
2
|
+
from datafusion import SessionContext
|
|
3
|
+
from datafusion.catalog import Schema
|
|
4
|
+
from types import ModuleType
|
|
5
|
+
|
|
6
|
+
from . import cftime as cft
|
|
7
|
+
from .df import (
|
|
8
|
+
Chunks,
|
|
9
|
+
TableNames,
|
|
10
|
+
group_vars_by_dims,
|
|
11
|
+
resolve_table_names,
|
|
12
|
+
shared_coord_arrays,
|
|
13
|
+
)
|
|
14
|
+
from .ds import XarrayDataFrame
|
|
15
|
+
from .reader import read_xarray_table
|
|
16
|
+
|
|
17
|
+
_proj: ModuleType | None
|
|
18
|
+
try: # pyproj is an optional dependency (`pip install xarray-sql[geo]`).
|
|
19
|
+
from . import proj
|
|
20
|
+
|
|
21
|
+
_proj = proj
|
|
22
|
+
except ImportError: # pragma: no cover - depends on the environment
|
|
23
|
+
_proj = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Context(SessionContext):
|
|
27
|
+
"""A datafusion `SessionContext` that also supports `xarray.Dataset`s.
|
|
28
|
+
|
|
29
|
+
Also available under its original name, ``XarrayContext``.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, *args, **kwargs):
|
|
33
|
+
super().__init__(*args, **kwargs)
|
|
34
|
+
# Track registered xarray Datasets so XarrayDataFrame can recover
|
|
35
|
+
# defaults (dimension_columns) and metadata (var/dataset attrs,
|
|
36
|
+
# non-dim coords, dim-coord dtype) that the forward pivot drops.
|
|
37
|
+
# Keys are the fully-qualified table names users will reference
|
|
38
|
+
# in SQL (e.g. ``"air"`` for a uniform-dim Dataset, or
|
|
39
|
+
# ``"era5.surface"`` for one entry from a multi-dim-group split).
|
|
40
|
+
self._registered_datasets: dict[str, xr.Dataset] = {}
|
|
41
|
+
# With pyproj installed, every context speaks CRS out of the box:
|
|
42
|
+
# reproject(x, y, src_crs, dst_crs), à la PostGIS ST_Transform.
|
|
43
|
+
if _proj is not None:
|
|
44
|
+
_proj.register(self)
|
|
45
|
+
|
|
46
|
+
def from_dataset(
|
|
47
|
+
self,
|
|
48
|
+
name: str,
|
|
49
|
+
input_table: xr.Dataset,
|
|
50
|
+
*,
|
|
51
|
+
table_names: TableNames = None,
|
|
52
|
+
chunks: Chunks = None,
|
|
53
|
+
):
|
|
54
|
+
"""Register an xarray Dataset as one or more queryable SQL tables.
|
|
55
|
+
|
|
56
|
+
When all data variables share the same dimensions, the dataset is
|
|
57
|
+
registered as a single table named ``name``. When variables have
|
|
58
|
+
differing dimensions (e.g. some on a 3D grid and others on a 4D
|
|
59
|
+
grid), the dataset is split into one table per dimension group.
|
|
60
|
+
The tables are registered under a SQL schema (namespace) named
|
|
61
|
+
``name`` and named ``<dim1>_<dim2>_...`` by default::
|
|
62
|
+
|
|
63
|
+
ctx.from_dataset('era5', ds, chunks={'time': 24})
|
|
64
|
+
# registers tables: 'era5.time_lat_lon' and
|
|
65
|
+
# 'era5.time_lat_lon_level'
|
|
66
|
+
ctx.sql('SELECT AVG(temperature_2m) FROM era5.time_lat_lon')
|
|
67
|
+
|
|
68
|
+
Use ``table_names`` to override the name for specific dimension
|
|
69
|
+
tuples::
|
|
70
|
+
|
|
71
|
+
ctx.from_dataset(
|
|
72
|
+
'era5', ds,
|
|
73
|
+
table_names={('time', 'lat', 'lon'): 'surface'},
|
|
74
|
+
)
|
|
75
|
+
ctx.sql('SELECT * FROM era5.surface')
|
|
76
|
+
|
|
77
|
+
For datasets with non-Gregorian cftime coordinates (e.g. 360_day,
|
|
78
|
+
julian), a ``cftime()`` scalar UDF is automatically registered so
|
|
79
|
+
you can write ergonomic SQL filters::
|
|
80
|
+
|
|
81
|
+
ctx.from_dataset("ds360", ds, chunks={"time": 6})
|
|
82
|
+
ctx.sql("SELECT * FROM ds360 WHERE time >= cftime('2000-07-01')")
|
|
83
|
+
|
|
84
|
+
.. note::
|
|
85
|
+
|
|
86
|
+
Only one ``cftime()`` UDF is registered per context, using the
|
|
87
|
+
units and calendar of the *first* non-Gregorian coordinate
|
|
88
|
+
encountered. If you register multiple datasets with *different*
|
|
89
|
+
non-Gregorian calendars (e.g. one 360_day and one julian), the
|
|
90
|
+
UDF from the first registration will be used for all subsequent
|
|
91
|
+
``cftime()`` calls and may produce incorrect offsets for the
|
|
92
|
+
other dataset. In that case, create a separate ``Context``
|
|
93
|
+
for each calendar.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
name: The SQL identifier under which the dataset is registered.
|
|
97
|
+
For datasets with uniform dimensions, this is the table
|
|
98
|
+
name. For datasets with mixed dimensions, this is the name
|
|
99
|
+
of a SQL schema (namespace) containing one table per
|
|
100
|
+
dimension group.
|
|
101
|
+
input_table: An xarray Dataset.
|
|
102
|
+
table_names: Optional mapping from dimension tuples to custom
|
|
103
|
+
table names within the schema, used when the dataset has
|
|
104
|
+
variables with differing dimensions.
|
|
105
|
+
chunks: Xarray-like chunks specification. If not provided, uses
|
|
106
|
+
the Dataset's existing chunks, or for an unchunked Dataset
|
|
107
|
+
its storage chunks (see [register][xarray_sql.register]).
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
self, to allow chaining.
|
|
111
|
+
"""
|
|
112
|
+
groups = group_vars_by_dims(input_table)
|
|
113
|
+
names = resolve_table_names(input_table, table_names)
|
|
114
|
+
|
|
115
|
+
# Materialise dim coordinates once and share across every sub-table.
|
|
116
|
+
# For Zarr-backed parents (e.g. ARCO-ERA5 on GCS) this saves one
|
|
117
|
+
# network round-trip per dim per dim-group.
|
|
118
|
+
coord_arrays = shared_coord_arrays(input_table)
|
|
119
|
+
|
|
120
|
+
if len(groups) <= 1:
|
|
121
|
+
self._registered_datasets[name] = input_table
|
|
122
|
+
return self._from_dataset(
|
|
123
|
+
name, input_table, chunks, coord_arrays=coord_arrays
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
schema = Schema.memory_schema(self)
|
|
127
|
+
self.catalog().register_schema(name, schema)
|
|
128
|
+
|
|
129
|
+
for dims, var_names in groups.items():
|
|
130
|
+
sub_name = names[dims]
|
|
131
|
+
sub_ds = input_table[var_names]
|
|
132
|
+
self._from_dataset(
|
|
133
|
+
sub_name,
|
|
134
|
+
sub_ds,
|
|
135
|
+
chunks,
|
|
136
|
+
schema=schema,
|
|
137
|
+
coord_arrays=coord_arrays,
|
|
138
|
+
)
|
|
139
|
+
# Track the fully-qualified name so XarrayDataFrame metadata
|
|
140
|
+
# recovery can find this Dataset on round-trip.
|
|
141
|
+
self._registered_datasets[f"{name}.{sub_name}"] = sub_ds
|
|
142
|
+
|
|
143
|
+
return self
|
|
144
|
+
|
|
145
|
+
def _from_dataset(
|
|
146
|
+
self,
|
|
147
|
+
table_name: str,
|
|
148
|
+
input_table: xr.Dataset,
|
|
149
|
+
chunks: Chunks = None,
|
|
150
|
+
schema: Schema | None = None,
|
|
151
|
+
coord_arrays: dict | None = None,
|
|
152
|
+
):
|
|
153
|
+
"""Register a Dataset as a single SQL table.
|
|
154
|
+
|
|
155
|
+
Registers a top-level table by default, or a table inside ``schema``
|
|
156
|
+
(a SQL namespace) when one is given.
|
|
157
|
+
"""
|
|
158
|
+
register = (
|
|
159
|
+
self.register_table if schema is None else schema.register_table
|
|
160
|
+
)
|
|
161
|
+
register(
|
|
162
|
+
table_name,
|
|
163
|
+
read_xarray_table(input_table, chunks, coord_arrays=coord_arrays),
|
|
164
|
+
)
|
|
165
|
+
self._maybe_register_cftime_udf(input_table)
|
|
166
|
+
return self
|
|
167
|
+
|
|
168
|
+
def _maybe_register_cftime_udf(self, ds: xr.Dataset) -> None:
|
|
169
|
+
"""Auto-register a cftime() UDF for non-Gregorian cftime coordinates."""
|
|
170
|
+
for coord_name in ds.dims:
|
|
171
|
+
if cft.is_cftime_index(ds, coord_name):
|
|
172
|
+
units, cal = cft.encoding(ds, coord_name)
|
|
173
|
+
if not cft.is_gregorian_like(cal):
|
|
174
|
+
self.register_udf(cft.make_cftime_udf(units, cal))
|
|
175
|
+
break # One UDF per context is enough.
|
|
176
|
+
|
|
177
|
+
def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame:
|
|
178
|
+
"""Run a SQL query, returning an [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapper.
|
|
179
|
+
|
|
180
|
+
Identical to ``datafusion.SessionContext.sql`` except the returned
|
|
181
|
+
object wraps the DataFusion DataFrame. The wrapper exposes
|
|
182
|
+
``.to_pandas()`` (unchanged), forwards every other DataFusion
|
|
183
|
+
method via ``__getattr__``, and adds
|
|
184
|
+
``.to_dataset(dimension_columns=[...])`` for round-tripping the
|
|
185
|
+
result back to an ``xr.Dataset``.
|
|
186
|
+
|
|
187
|
+
Args:
|
|
188
|
+
query: A SQL query string.
|
|
189
|
+
*args: Forwarded to ``SessionContext.sql``.
|
|
190
|
+
**kwargs: Forwarded to ``SessionContext.sql``.
|
|
191
|
+
|
|
192
|
+
Returns:
|
|
193
|
+
An [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapping the DataFusion DataFrame.
|
|
194
|
+
"""
|
|
195
|
+
inner = super().sql(query, *args, **kwargs)
|
|
196
|
+
return XarrayDataFrame(inner, templates=self._registered_datasets)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
XarrayContext = Context
|
|
200
|
+
"""The original name of [Context][xarray_sql.Context], kept for compatibility."""
|
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xarray_sql
|
|
3
|
+
Version: 0.6.0
|
|
4
|
+
Classifier: Development Status :: 4 - Beta
|
|
5
|
+
Classifier: Intended Audience :: Science/Research
|
|
6
|
+
Classifier: Intended Audience :: Developers
|
|
7
|
+
Classifier: Intended Audience :: Information Technology
|
|
8
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
9
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
10
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
11
|
+
Classifier: Operating System :: POSIX
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Rust
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
18
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
19
|
+
Requires-Dist: datafusion==54.0.0 ; sys_platform != 'emscripten'
|
|
20
|
+
Requires-Dist: pyarrow>=16.0.0
|
|
21
|
+
Requires-Dist: xarray>=2024.7.0
|
|
22
|
+
Requires-Dist: adbc-driver-manager>=1.12 ; extra == 'adbc'
|
|
23
|
+
Requires-Dist: dask>=2024.8.0 ; extra == 'dask'
|
|
24
|
+
Requires-Dist: xarray-sql[docs] ; extra == 'dev'
|
|
25
|
+
Requires-Dist: pre-commit ; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest ; extra == 'dev'
|
|
27
|
+
Requires-Dist: watchfiles ; extra == 'dev'
|
|
28
|
+
Requires-Dist: zensical ; extra == 'docs'
|
|
29
|
+
Requires-Dist: mkdocstrings[python] ; extra == 'docs'
|
|
30
|
+
Requires-Dist: duckdb>=1.4.0 ; extra == 'duckdb'
|
|
31
|
+
Requires-Dist: pyproj ; extra == 'geo'
|
|
32
|
+
Requires-Dist: polars>=1.33 ; extra == 'polars'
|
|
33
|
+
Requires-Dist: adbc-driver-flightsql>=1.12 ; extra == 'test'
|
|
34
|
+
Requires-Dist: adbc-driver-sqlite>=1.12 ; extra == 'test'
|
|
35
|
+
Requires-Dist: cftime ; extra == 'test'
|
|
36
|
+
Requires-Dist: xarray-sql[adbc,dask,duckdb,polars,geo] ; extra == 'test'
|
|
37
|
+
Requires-Dist: pytest ; extra == 'test'
|
|
38
|
+
Requires-Dist: xarray[io] ; extra == 'test'
|
|
39
|
+
Requires-Dist: gcsfs ; extra == 'test'
|
|
40
|
+
Provides-Extra: adbc
|
|
41
|
+
Provides-Extra: dask
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Provides-Extra: docs
|
|
44
|
+
Provides-Extra: duckdb
|
|
45
|
+
Provides-Extra: geo
|
|
46
|
+
Provides-Extra: polars
|
|
47
|
+
Provides-Extra: test
|
|
48
|
+
License-File: LICENSE
|
|
49
|
+
Summary: Query Xarray with SQL.
|
|
50
|
+
Author-email: Alexander Merose <al@merose.com>
|
|
51
|
+
License: Apache-2.0
|
|
52
|
+
Requires-Python: >=3.10
|
|
53
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
54
|
+
Project-URL: Homepage, https://github.com/alxmrs/xarray-sql
|
|
55
|
+
Project-URL: Issues, https://github.com/alxmrs/xarray-sql/issues
|
|
56
|
+
|
|
57
|
+
# xarray-sql
|
|
58
|
+
|
|
59
|
+
_Query [Xarray](https://xarray.dev/) with SQL_
|
|
60
|
+
|
|
61
|
+

|
|
62
|
+
[](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci.yml)
|
|
63
|
+
[](https://github.com/xqlsystems/xarray-sql/actions/workflows/lint.yml)
|
|
64
|
+
[](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-build.yml)
|
|
65
|
+
[](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-rust.yml)
|
|
66
|
+
[](https://pepy.tech/projects/xarray-sql)
|
|
67
|
+
[](https://pepy.tech/projects/xarray-sql)
|
|
68
|
+
|
|
69
|
+
```shell
|
|
70
|
+
pip install xarray-sql
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## What is this?
|
|
74
|
+
|
|
75
|
+
This is an experiment to provide a SQL interface for array datasets.
|
|
76
|
+
Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
|
|
77
|
+
SQL queries against them — on the query engine of your choice. xarray-sql
|
|
78
|
+
translates data, not queries: it registers a lazy Dataset as a table on
|
|
79
|
+
DataFusion (built in), DuckDB, or Polars, and turns any engine's Arrow result
|
|
80
|
+
back into a labeled Dataset. Dialects, geometry functions, and optimizers stay
|
|
81
|
+
with the engine.
|
|
82
|
+
|
|
83
|
+
## Quickstart
|
|
84
|
+
|
|
85
|
+
Open a Dataset, register it as a table with `from_dataset`, compute a
|
|
86
|
+
climatology in SQL, then write the result back to Xarray and plot it:
|
|
87
|
+
|
|
88
|
+
> **Note:** this example also needs `pooch` and a netCDF backend (for the
|
|
89
|
+
> tutorial download) and `matplotlib` (for the plot):
|
|
90
|
+
> `pip install pooch netCDF4 matplotlib`.
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
import xarray as xr
|
|
94
|
+
import xarray_sql as xql
|
|
95
|
+
|
|
96
|
+
# 4x-daily surface air temperature on a lat/lon grid, 2013-2014.
|
|
97
|
+
ds = xr.tutorial.open_dataset('air_temperature')
|
|
98
|
+
|
|
99
|
+
ctx = xql.Context()
|
|
100
|
+
ctx.from_dataset('air', ds, chunks=dict(time=100))
|
|
101
|
+
|
|
102
|
+
# A climatology — the mean annual cycle — computed in SQL: average air
|
|
103
|
+
# temperature for each month of the year, over all grid cells and years.
|
|
104
|
+
clim = ctx.sql('''
|
|
105
|
+
SELECT
|
|
106
|
+
CAST(date_part('month', "time") AS INTEGER) AS month,
|
|
107
|
+
AVG("air") AS air
|
|
108
|
+
FROM "air"
|
|
109
|
+
GROUP BY CAST(date_part('month', "time") AS INTEGER)
|
|
110
|
+
ORDER BY month
|
|
111
|
+
''')
|
|
112
|
+
|
|
113
|
+
# Round-trip the result back to Xarray. `month` is a derived column, so name
|
|
114
|
+
# it as the dimension.
|
|
115
|
+
clim_ds = clim.to_dataset(dims=["month"])
|
|
116
|
+
|
|
117
|
+
# Plot the annual cycle as a time series.
|
|
118
|
+
clim_ds["air"].plot() # in a script, call matplotlib.pyplot.show() to display
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
That's the round trip — Xarray in, SQL in the middle, Xarray (and a plot) back
|
|
122
|
+
out.
|
|
123
|
+
|
|
124
|
+
The same Dataset registers on other engines with one call — DuckDB gets a
|
|
125
|
+
native lazy table with predicate pushdown, Polars scans the same object:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
import duckdb
|
|
129
|
+
|
|
130
|
+
con = duckdb.connect()
|
|
131
|
+
xql.register(con, 'air', ds, chunks=dict(time=100))
|
|
132
|
+
rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY time')
|
|
133
|
+
xql.to_dataset(rel, template=ds) # any engine's Arrow result round-trips
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Any database with an [ADBC](https://arrow.apache.org/adbc/) driver
|
|
137
|
+
(PostgreSQL, SQLite, Snowflake, BigQuery, ...) works too: `xql.register`
|
|
138
|
+
ingests the Dataset into a table there, and `xql.to_dataset(cursor, ...)`
|
|
139
|
+
brings results back. Or go the other way without copying anything:
|
|
140
|
+
`xql.serve({'air': ds})` starts an Arrow Flight SQL server that any Flight SQL
|
|
141
|
+
client (ADBC, JDBC, ODBC) can query, with the same chunk pruning as in process.
|
|
142
|
+
|
|
143
|
+
`table_names` (below) works the same way on every engine, so a query written
|
|
144
|
+
against `era5.surface` is not tied to the engine it was written for.
|
|
145
|
+
|
|
146
|
+
See [Engines](https://xqlsystems.github.io/xarray-sql/latest/engines/) for the support matrix, DuckDB/Polars/ADBC details,
|
|
147
|
+
and the lazy chunked round-trip.
|
|
148
|
+
|
|
149
|
+
## A bigger example: ARCO-ERA5
|
|
150
|
+
|
|
151
|
+
The same interface scales to cloud-native datasets with hundreds of variables,
|
|
152
|
+
like [ARCO-ERA5](https://github.com/google-research/arco-era5).
|
|
153
|
+
|
|
154
|
+
> **Note:** reading from `gs://` requires `gcsfs` (`pip install gcsfs`).
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
import xarray as xr
|
|
158
|
+
import xarray_sql as xql
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# Open ARCO-ERA5 — a weather dataset with 273 variables since 1940.
|
|
162
|
+
# Turning off dask means we don't have to wait to construct a task graph.
|
|
163
|
+
ds = xr.open_zarr(
|
|
164
|
+
'gs://gcp-public-data-arco-era5/ar/full_37-1h-0p25deg-chunk-1.zarr-v3',
|
|
165
|
+
chunks=None, # Turn dask off
|
|
166
|
+
storage_options={'token': 'anon'} # Anonymous read from the public GCS bucket — no auth required.
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
ctx = xql.Context()
|
|
170
|
+
# `chunks` sets the partition size; without it, partitions follow the
|
|
171
|
+
# store's own chunks (one hour each here).
|
|
172
|
+
ctx.from_dataset('era5', ds, chunks=dict(time=6), table_names={
|
|
173
|
+
('time', 'latitude', 'longitude'): 'surface',
|
|
174
|
+
('time', 'level', 'latitude', 'longitude'): 'atmosphere',
|
|
175
|
+
})
|
|
176
|
+
# Registration takes ~10s on my machine.
|
|
177
|
+
|
|
178
|
+
# Heads up: ARCO-ERA5 has 262 surface + 11 atmospheric variables. The library
|
|
179
|
+
# pushes column projection down to Zarr, so SELECT only fetches what you ask
|
|
180
|
+
# for — but `SELECT * FROM era5.surface` would try to pull every variable
|
|
181
|
+
# across the year (terabytes from GCS).
|
|
182
|
+
# ---> Always SELECT specific columns. <---
|
|
183
|
+
|
|
184
|
+
# Average 2m-temperature over NYC on the morning of 2020-01-01. The library
|
|
185
|
+
# pushes WHERE clauses on dimension columns down to partition pruning.
|
|
186
|
+
ctx.sql('''
|
|
187
|
+
SELECT AVG("2m_temperature") - 273.15 AS avg_c
|
|
188
|
+
FROM era5.surface
|
|
189
|
+
WHERE time BETWEEN TIMESTAMP '2020-01-01'
|
|
190
|
+
AND TIMESTAMP '2020-01-01 05:00:00'
|
|
191
|
+
AND latitude BETWEEN 39 AND 40
|
|
192
|
+
AND longitude BETWEEN 286 AND 287 -- ERA5 uses 0-360 longitudes
|
|
193
|
+
''').to_pandas()
|
|
194
|
+
# avg_c
|
|
195
|
+
# 0 8.640069
|
|
196
|
+
|
|
197
|
+
# Average temperature per pressure level, globally.
|
|
198
|
+
result = ctx.sql('''
|
|
199
|
+
SELECT level, AVG(temperature) - 273.15 AS avg_c
|
|
200
|
+
FROM era5.atmosphere
|
|
201
|
+
WHERE time BETWEEN TIMESTAMP '2020-01-01'
|
|
202
|
+
AND TIMESTAMP '2020-01-01 05:00:00'
|
|
203
|
+
GROUP BY level
|
|
204
|
+
ORDER BY level DESC
|
|
205
|
+
''')
|
|
206
|
+
# DataFrame()
|
|
207
|
+
# +-------+----------------------+
|
|
208
|
+
# | level | avg_c |
|
|
209
|
+
# +-------+----------------------+
|
|
210
|
+
# | 1000 | 6.6210120796502565 |
|
|
211
|
+
# | 975 | 5.185637919348153 |
|
|
212
|
+
# | 950 | 4.028428657263021 |
|
|
213
|
+
# | 925 | 3.0828117974912743 |
|
|
214
|
+
# | 900 | 2.2109172992531967 |
|
|
215
|
+
# | 875 | 1.395017610194202 |
|
|
216
|
+
# | 850 | 0.6342670572626616 |
|
|
217
|
+
# | 825 | -0.21037158786759846 |
|
|
218
|
+
# | 800 | -1.1810754318269687 |
|
|
219
|
+
# | 775 | -2.3064649711534457 |
|
|
220
|
+
# +-------+----------------------+
|
|
221
|
+
|
|
222
|
+
# `latitude`/`longitude` are inferred from the registered table's surviving
|
|
223
|
+
# dims; `template` is kept only to recover metadata (attrs, encoding).
|
|
224
|
+
ctx.sql('''
|
|
225
|
+
SELECT latitude, longitude, AVG("2m_temperature") - 273.15 AS avg_c
|
|
226
|
+
FROM era5.surface
|
|
227
|
+
WHERE time BETWEEN TIMESTAMP '2020-01-01'
|
|
228
|
+
AND TIMESTAMP '2020-01-01 05:00:00'
|
|
229
|
+
GROUP BY latitude, longitude
|
|
230
|
+
ORDER BY latitude DESC, longitude
|
|
231
|
+
''').to_dataset(template=ds)
|
|
232
|
+
# <xarray.Dataset> Size: 8MB
|
|
233
|
+
# Dimensions: (latitude: 721, longitude: 1440)
|
|
234
|
+
# Coordinates:
|
|
235
|
+
# * latitude (latitude) float32 3kB 90.0 89.75 89.5 ... -89.5 -89.75 -90.0
|
|
236
|
+
# * longitude (longitude) float32 6kB 0.0 0.25 0.5 0.75 ... 359.2 359.5 359.8
|
|
237
|
+
# Data variables:
|
|
238
|
+
# avg_c (latitude, longitude) float64 8MB -26.84 -26.84 ... -27.38 -27.38
|
|
239
|
+
# Attributes:
|
|
240
|
+
# last_updated: 2026-06-20 02:33:34.265980+00:00
|
|
241
|
+
# valid_time_start: 1940-01-01
|
|
242
|
+
# valid_time_stop: 2025-12-31
|
|
243
|
+
# valid_time_stop_era5t: 2026-06-14
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
_(A runnable version of this example lives at
|
|
247
|
+
[`perf_tests/era5_temp_profile.py`](https://github.com/xqlsystems/xarray-sql/blob/main/perf_tests/era5_temp_profile.py).)_
|
|
248
|
+
|
|
249
|
+
## Why build this?
|
|
250
|
+
|
|
251
|
+
A few reasons:
|
|
252
|
+
|
|
253
|
+
* Even though SQL is the lingua franca of data, scientific datasets are often
|
|
254
|
+
inaccessible to non-scientists (SQL users).
|
|
255
|
+
* Joining tabular data with raster data is common yet difficult. It could be
|
|
256
|
+
easy.
|
|
257
|
+
* There are many cloud-native, Xarray-openable datasets,
|
|
258
|
+
from [Google Earth Engine](https://github.com/google/Xee)
|
|
259
|
+
to the [Source Cooperative](https://source.coop/products?tags=zarr). Wouldn’t it be great if these
|
|
260
|
+
were also SQL-accessible? How can the bridge be built with minimal effort?
|
|
261
|
+
|
|
262
|
+
This is a light-weight way to prove the value of the interface.
|
|
263
|
+
|
|
264
|
+
The larger goal is to explore the hypothesis that the [Pangeo
|
|
265
|
+
ecosystem is a scientific database](https://www.hytradboi.com/2025/c18b8cdc-fd17-4099-9c03-eb107217f627-pangeo-is-a-database). Here, xarray-sql can be thought of as a missing
|
|
266
|
+
DB front end.
|
|
267
|
+
|
|
268
|
+
## How does it work?
|
|
269
|
+
|
|
270
|
+
All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
|
|
271
|
+
`from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
|
|
272
|
+
That's it!
|
|
273
|
+
|
|
274
|
+
_2025 update_: This library now implements a Dask-like `from_map` interface in
|
|
275
|
+
pure DataFusion and PyArrow, but works with the same principle!
|
|
276
|
+
|
|
277
|
+
_2026 update_: Instead of `from_map()`, we create a way to translate Xarray chunks
|
|
278
|
+
into Arrow RecordBatches. We pass a Python callback into a DataFusion `TableProvider`
|
|
279
|
+
that lets the DB engine translate the underlying Dataset arrays into DataFusion partitions.
|
|
280
|
+
The same chunks-to-batches translation is also exposed as a
|
|
281
|
+
`pyarrow.dataset.Dataset` with predicate and projection pushdown, which is how
|
|
282
|
+
DuckDB and Polars consume registered Datasets with no engine-specific code.
|
|
283
|
+
Ultimately, the initial insight of the `pivot()` function -- that any ndarray can be
|
|
284
|
+
translated into a 2D table -- underlies this performant query mechanism.
|
|
285
|
+
|
|
286
|
+
## Does it work?
|
|
287
|
+
|
|
288
|
+
Yes. The recurring worry is that the SQL interface is a toy — fine for `SELECT`s,
|
|
289
|
+
but not for the operations geoscience actually runs. So we wrote a suite that
|
|
290
|
+
takes the staples of geospatial and climate analysis — the ones we assume *need*
|
|
291
|
+
an array library — and expresses each one in SQL, then **checks the SQL answer
|
|
292
|
+
against an xarray/array reference** to floating-point tolerance:
|
|
293
|
+
|
|
294
|
+
* **Spectral indices** (NDVI) — column arithmetic over a real Sentinel-2 scene.
|
|
295
|
+
* **Climatology, anomalies, zonal means** — `GROUP BY` and self-`JOIN` against
|
|
296
|
+
the 0.25° **ARCO-ERA5** archive registered as a lazy table. Each query is
|
|
297
|
+
bounded to a small window (a few days over a region) and reads only that
|
|
298
|
+
slice — the point is that you can aim a query at a multi-decade archive and
|
|
299
|
+
pay only for the data it asks for, not that the query scans the whole record.
|
|
300
|
+
* **Forecast skill** — scoring the **Pangu-Weather** and **GraphCast** ML models
|
|
301
|
+
against ERA5 (WeatherBench 2) as a `JOIN` on `valid_time = init + lead`; it
|
|
302
|
+
reproduces the published result that GraphCast beats Pangu at every lead.
|
|
303
|
+
* **Raster × vector zonal stats** — a range `JOIN` of the ERA5 grid against a
|
|
304
|
+
table of regions.
|
|
305
|
+
* **Reprojection and regridding** — a `reproject(x, y, src_crs, dst_crs)`
|
|
306
|
+
scalar PROJ UDF, shipped as the optional geo extension
|
|
307
|
+
(`pip install xarray-sql[geo]`, validated against Earth Engine's own
|
|
308
|
+
geodesy via [Xee](https://github.com/google/Xee)) and a
|
|
309
|
+
sparse-weight-table `JOIN` (regridding real SRTM terrain).
|
|
310
|
+
|
|
311
|
+
Every case matches its array reference. The headline finding: these operations
|
|
312
|
+
are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
|
|
313
|
+
functions, and `CASE` in disguise, and a query engine runs them at scale. See
|
|
314
|
+
[`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
|
|
315
|
+
[Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/latest/geospatial/).
|
|
316
|
+
|
|
317
|
+
## Why does this work?
|
|
318
|
+
|
|
319
|
+
Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
|
|
320
|
+
chunks and represented contiguously in memory. It is only a matter of metadata
|
|
321
|
+
that breaks them up into ndarrays. `pivot()`, which uses `to_dataframe()`,
|
|
322
|
+
just changes this metadata (via a `ravel()`/`reshape()`), back into a column
|
|
323
|
+
amenable to a DataFrame. We take advantage of this lightweight metadata change to
|
|
324
|
+
make chunked information scannable by a DB engine (DataFusion, DuckDB, Polars —
|
|
325
|
+
anything that speaks Arrow).
|
|
326
|
+
|
|
327
|
+
## What are the current limitations?
|
|
328
|
+
|
|
329
|
+
The sharp edges we know about — per engine and fundamental — are cataloged in
|
|
330
|
+
[Known issues & limitations](https://xqlsystems.github.io/xarray-sql/latest/limitations/). Currently, we're looking for
|
|
331
|
+
early users – "tire kickers", if you will. We'd love your input to shape the direction of this
|
|
332
|
+
project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
|
|
333
|
+
you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/latest/contributing/), too 😉.
|
|
334
|
+
|
|
335
|
+
## What would a deeper integration look like?
|
|
336
|
+
|
|
337
|
+
I have a few ideas so far. One approach involves applying operations directly on
|
|
338
|
+
Xarray Datasets. This approach is being pursued
|
|
339
|
+
[here](https://github.com/google/weather-tools/tree/main/xql), as `xql`.
|
|
340
|
+
|
|
341
|
+
Deeper still: I was thinking we could make
|
|
342
|
+
a [virtual](https://fsspec.github.io/kerchunk/)
|
|
343
|
+
filesystem for parquet that would internally map to Zarr. Raster-backed virtual
|
|
344
|
+
parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
|
|
345
|
+
and BigQuery. More thoughts on this
|
|
346
|
+
in [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
|
|
347
|
+
|
|
348
|
+
_2025 update_: Something like this is being built across a few projects! The ones I know about are:
|
|
349
|
+
|
|
350
|
+
- [CartoDB's Raquet](https://github.com/CartoDB/raquet)
|
|
351
|
+
- The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
|
|
352
|
+
|
|
353
|
+
_2026 update_: A colleague and I are experimenting with native Zarr RDBMS engines. Check out:
|
|
354
|
+
|
|
355
|
+
- [Zarr-Datafusion](https://lib.rs/crates/zarr-datafusion)
|
|
356
|
+
- [DuckDB-Zarr](https://github.com/xqlsystems/duckdb-zarr)
|
|
357
|
+
|
|
358
|
+
## Roadmap
|
|
359
|
+
|
|
360
|
+
- [x] ~Lazy evaluation via the pyarrow Dataset interface [#93](https://github.com/xqlsystems/xarray-sql/issues/93).~ _Implemented in [#100](https://github.com/xqlsystems/xarray-sql/pull/100)_
|
|
361
|
+
- [x] Support proper parallelism via proper partition handling on the rust/datafusion side. [#106](https://github.com/xqlsystems/xarray-sql/issues/106)
|
|
362
|
+
- [x] Support core datafusion optimizations to scan less data, like [#104](https://github.com/xqlsystems/xarray-sql/issues/104), ...
|
|
363
|
+
- [x] Translate a single Zarr to a collection of tables [#85](https://github.com/xqlsystems/xarray-sql/issues/85).
|
|
364
|
+
- [ ] Distributed beyond a single node through the DataFusion integration with Ray Datasets [#68](https://github.com/xqlsystems/xarray-sql/issues/68) or Apache Ballista [#98](https://github.com/xqlsystems/xarray-sql/issues/98).
|
|
365
|
+
- [ ] Demo: calculate Sea Surface Temperature from 1940 - Present in SQL [#36](https://github.com/xqlsystems/xarray-sql/issues/36).
|
|
366
|
+
- [ ] Provide an option to integrate DataFusion directly to Zarr via Rust [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
|
|
367
|
+
- [ ] (To be formally announced eventually): The 100 Trillion Row Challenge [#34](https://github.com/xqlsystems/xarray-sql/issues/34).
|
|
368
|
+
|
|
369
|
+
## Sponsors & Contributors
|
|
370
|
+
|
|
371
|
+
I want to give a special thanks to the following folks and institutions:
|
|
372
|
+
|
|
373
|
+
- Pramod Gupta and the Anthromet Team at Google Research for the problem
|
|
374
|
+
formation and design inspiration.
|
|
375
|
+
- Jake Wall and AI2/Ecoscope for compute resources and key use cases.
|
|
376
|
+
- Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
|
|
377
|
+
for the early review and discussion of this project.
|
|
378
|
+
- Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
|
|
379
|
+
Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
|
|
380
|
+
and DataFusion-specific collaboration.
|
|
381
|
+
- The gracious volunteer data science students at [UCSD's DS3](https://www.ds3atucsd.com/) org,
|
|
382
|
+
who are working to make this library better.
|
|
383
|
+
- Andrew Huang for the sense of taste he brings to the project and consummate code
|
|
384
|
+
changes.
|
|
385
|
+
- Aman Kumar for spending a considerable amount of his GSoC internship
|
|
386
|
+
contributing to this project.
|
|
387
|
+
- Miguel Moncada Isla for reimagining this project to be a cross SQL engine
|
|
388
|
+
interface to and from Xarray. He brought new life to this experiment, in my
|
|
389
|
+
opinion.
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
## License
|
|
393
|
+
|
|
394
|
+
```
|
|
395
|
+
Copyright 2024 Alexander Merose
|
|
396
|
+
|
|
397
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
398
|
+
you may not use this file except in compliance with the License.
|
|
399
|
+
You may obtain a copy of the License at
|
|
400
|
+
|
|
401
|
+
https://www.apache.org/licenses/LICENSE-2.0
|
|
402
|
+
|
|
403
|
+
Unless required by applicable law or agreed to in writing, software
|
|
404
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
405
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
406
|
+
See the License for the specific language governing permissions and
|
|
407
|
+
limitations under the License.
|
|
408
|
+
```
|
|
409
|
+
|
|
410
|
+
All vendored code has proper license attribution.
|
|
411
|
+
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
xarray_sql/__init__.py,sha256=uWROeltFlV-y11FggA4juVA3Wa__3AhMdDqzO6n0BA8,1195
|
|
2
|
+
xarray_sql/_native.abi3.so,sha256=TDLVCLMa-Qq-NxlcmvCmHd38F1TWBQnvk4mN8XYmpIg,15084579
|
|
3
|
+
xarray_sql/cftime.py,sha256=a8jIslG29iDfv4_yP3INAbMRT6fNMKg9iMr5wCznEkg,9082
|
|
4
|
+
xarray_sql/core.py,sha256=eQU89mn71POcppgPXr5Cf_N19G9Sn6gMD8YPbYfgous,1304
|
|
5
|
+
xarray_sql/df.py,sha256=wi3Fnif8LxTOR4pTmb6TJusuHSA1z6O4UL12aXJyMwc,33072
|
|
6
|
+
xarray_sql/ds.py,sha256=jJ9JdfCF8S-Koh_I5AoLLLmx6RppEv3uAytfpKMGdnk,53831
|
|
7
|
+
xarray_sql/geometry.py,sha256=sX2VH0ids8Qfp3m1n40IOVUY4G4APt3G0MCa2kLKUzQ,5040
|
|
8
|
+
xarray_sql/lazyscan.py,sha256=WTplgJFJ1YoqWL-8WLSUdH40ktafaohsiohJiN6w1xs,18245
|
|
9
|
+
xarray_sql/proj.py,sha256=sKAJw1kO28_U54FpSlGABU628Ux8xxdTKfEPzI1w2lQ,9525
|
|
10
|
+
xarray_sql/reader.py,sha256=aHsuzt4_FTQCTW2E2czbpkSJH1PDy9rDLSC1TF959Vs,13660
|
|
11
|
+
xarray_sql/roundtrip.py,sha256=-KQ8lwkHAQdMxIwpcx7596L6bESupqR52X9gj13IjEQ,25516
|
|
12
|
+
xarray_sql/sql.py,sha256=Ipzfti5Httm53DQ-J3Fb9IX1wVkoxP3r_AM1yTTBDG0,8006
|
|
13
|
+
xarray_sql/backends/__init__.py,sha256=SowGCRe2AmM6tfnqZYsjS2So22Z4ymBjw3ed9uHEx6M,1370
|
|
14
|
+
xarray_sql/backends/_adbc_dialects.py,sha256=0hwxohgSRXNoTYNnV5ymjloxuujRqdaMhVyiEDk89CI,15581
|
|
15
|
+
xarray_sql/backends/adbc.py,sha256=hpz6RfObUgUIfnbsifkOAuJ6BakKHrn7djd87eh33mc,14567
|
|
16
|
+
xarray_sql/backends/base.py,sha256=kfbN5NN6rd8ZKkNcLBlfwMZTf9N--WtvV5U1oaJ5Zl4,5627
|
|
17
|
+
xarray_sql/backends/datafusion.py,sha256=dfPKu994DttMuNkUPLZpSFnRBjDkieQ1GilfKRasYew,2942
|
|
18
|
+
xarray_sql/backends/duckdb.py,sha256=kC7lwQfQLmITLFwvgACvniDPR5bqyAq1w9kooorfRcA,7514
|
|
19
|
+
xarray_sql/backends/flight.py,sha256=7AZRen-4g29Z-Hwc2cECeBjVRAmHc92iQUYhLvuDmpk,10944
|
|
20
|
+
xarray_sql/backends/pyarrow.py,sha256=uDsqEOrIXmptBFGm2-2nbZADU0LidOlJJu2zU9u7GW8,52468
|
|
21
|
+
xarray_sql-0.6.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
22
|
+
xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json,sha256=d84Mn5sE0zZdfofsYJtK5NuS2GdGH7S_spLY8eKh0j8,327254
|
|
23
|
+
xarray_sql-0.6.0.dist-info/METADATA,sha256=hy_uXLIY538ECi8WOHbqvcX3FYEOftD8OblgPJSiyuY,19266
|
|
24
|
+
xarray_sql-0.6.0.dist-info/WHEEL,sha256=JvzZS3vTSCSj6m0z_Oqq9HFALOUfSVFBoMGY4a5Sw88,114
|
|
25
|
+
xarray_sql-0.6.0.dist-info/RECORD,,
|