xarray-sql 0.6.0__cp310-abi3-pyemscripten_2026_0_wasm32.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/sql.py ADDED
@@ -0,0 +1,200 @@
1
+ import xarray as xr
2
+ from datafusion import SessionContext
3
+ from datafusion.catalog import Schema
4
+ from types import ModuleType
5
+
6
+ from . import cftime as cft
7
+ from .df import (
8
+ Chunks,
9
+ TableNames,
10
+ group_vars_by_dims,
11
+ resolve_table_names,
12
+ shared_coord_arrays,
13
+ )
14
+ from .ds import XarrayDataFrame
15
+ from .reader import read_xarray_table
16
+
17
+ _proj: ModuleType | None
18
+ try: # pyproj is an optional dependency (`pip install xarray-sql[geo]`).
19
+ from . import proj
20
+
21
+ _proj = proj
22
+ except ImportError: # pragma: no cover - depends on the environment
23
+ _proj = None
24
+
25
+
26
+ class Context(SessionContext):
27
+ """A datafusion `SessionContext` that also supports `xarray.Dataset`s.
28
+
29
+ Also available under its original name, ``XarrayContext``.
30
+ """
31
+
32
+ def __init__(self, *args, **kwargs):
33
+ super().__init__(*args, **kwargs)
34
+ # Track registered xarray Datasets so XarrayDataFrame can recover
35
+ # defaults (dimension_columns) and metadata (var/dataset attrs,
36
+ # non-dim coords, dim-coord dtype) that the forward pivot drops.
37
+ # Keys are the fully-qualified table names users will reference
38
+ # in SQL (e.g. ``"air"`` for a uniform-dim Dataset, or
39
+ # ``"era5.surface"`` for one entry from a multi-dim-group split).
40
+ self._registered_datasets: dict[str, xr.Dataset] = {}
41
+ # With pyproj installed, every context speaks CRS out of the box:
42
+ # reproject(x, y, src_crs, dst_crs), à la PostGIS ST_Transform.
43
+ if _proj is not None:
44
+ _proj.register(self)
45
+
46
+ def from_dataset(
47
+ self,
48
+ name: str,
49
+ input_table: xr.Dataset,
50
+ *,
51
+ table_names: TableNames = None,
52
+ chunks: Chunks = None,
53
+ ):
54
+ """Register an xarray Dataset as one or more queryable SQL tables.
55
+
56
+ When all data variables share the same dimensions, the dataset is
57
+ registered as a single table named ``name``. When variables have
58
+ differing dimensions (e.g. some on a 3D grid and others on a 4D
59
+ grid), the dataset is split into one table per dimension group.
60
+ The tables are registered under a SQL schema (namespace) named
61
+ ``name`` and named ``<dim1>_<dim2>_...`` by default::
62
+
63
+ ctx.from_dataset('era5', ds, chunks={'time': 24})
64
+ # registers tables: 'era5.time_lat_lon' and
65
+ # 'era5.time_lat_lon_level'
66
+ ctx.sql('SELECT AVG(temperature_2m) FROM era5.time_lat_lon')
67
+
68
+ Use ``table_names`` to override the name for specific dimension
69
+ tuples::
70
+
71
+ ctx.from_dataset(
72
+ 'era5', ds,
73
+ table_names={('time', 'lat', 'lon'): 'surface'},
74
+ )
75
+ ctx.sql('SELECT * FROM era5.surface')
76
+
77
+ For datasets with non-Gregorian cftime coordinates (e.g. 360_day,
78
+ julian), a ``cftime()`` scalar UDF is automatically registered so
79
+ you can write ergonomic SQL filters::
80
+
81
+ ctx.from_dataset("ds360", ds, chunks={"time": 6})
82
+ ctx.sql("SELECT * FROM ds360 WHERE time >= cftime('2000-07-01')")
83
+
84
+ .. note::
85
+
86
+ Only one ``cftime()`` UDF is registered per context, using the
87
+ units and calendar of the *first* non-Gregorian coordinate
88
+ encountered. If you register multiple datasets with *different*
89
+ non-Gregorian calendars (e.g. one 360_day and one julian), the
90
+ UDF from the first registration will be used for all subsequent
91
+ ``cftime()`` calls and may produce incorrect offsets for the
92
+ other dataset. In that case, create a separate ``Context``
93
+ for each calendar.
94
+
95
+ Args:
96
+ name: The SQL identifier under which the dataset is registered.
97
+ For datasets with uniform dimensions, this is the table
98
+ name. For datasets with mixed dimensions, this is the name
99
+ of a SQL schema (namespace) containing one table per
100
+ dimension group.
101
+ input_table: An xarray Dataset.
102
+ table_names: Optional mapping from dimension tuples to custom
103
+ table names within the schema, used when the dataset has
104
+ variables with differing dimensions.
105
+ chunks: Xarray-like chunks specification. If not provided, uses
106
+ the Dataset's existing chunks, or for an unchunked Dataset
107
+ its storage chunks (see [register][xarray_sql.register]).
108
+
109
+ Returns:
110
+ self, to allow chaining.
111
+ """
112
+ groups = group_vars_by_dims(input_table)
113
+ names = resolve_table_names(input_table, table_names)
114
+
115
+ # Materialise dim coordinates once and share across every sub-table.
116
+ # For Zarr-backed parents (e.g. ARCO-ERA5 on GCS) this saves one
117
+ # network round-trip per dim per dim-group.
118
+ coord_arrays = shared_coord_arrays(input_table)
119
+
120
+ if len(groups) <= 1:
121
+ self._registered_datasets[name] = input_table
122
+ return self._from_dataset(
123
+ name, input_table, chunks, coord_arrays=coord_arrays
124
+ )
125
+
126
+ schema = Schema.memory_schema(self)
127
+ self.catalog().register_schema(name, schema)
128
+
129
+ for dims, var_names in groups.items():
130
+ sub_name = names[dims]
131
+ sub_ds = input_table[var_names]
132
+ self._from_dataset(
133
+ sub_name,
134
+ sub_ds,
135
+ chunks,
136
+ schema=schema,
137
+ coord_arrays=coord_arrays,
138
+ )
139
+ # Track the fully-qualified name so XarrayDataFrame metadata
140
+ # recovery can find this Dataset on round-trip.
141
+ self._registered_datasets[f"{name}.{sub_name}"] = sub_ds
142
+
143
+ return self
144
+
145
+ def _from_dataset(
146
+ self,
147
+ table_name: str,
148
+ input_table: xr.Dataset,
149
+ chunks: Chunks = None,
150
+ schema: Schema | None = None,
151
+ coord_arrays: dict | None = None,
152
+ ):
153
+ """Register a Dataset as a single SQL table.
154
+
155
+ Registers a top-level table by default, or a table inside ``schema``
156
+ (a SQL namespace) when one is given.
157
+ """
158
+ register = (
159
+ self.register_table if schema is None else schema.register_table
160
+ )
161
+ register(
162
+ table_name,
163
+ read_xarray_table(input_table, chunks, coord_arrays=coord_arrays),
164
+ )
165
+ self._maybe_register_cftime_udf(input_table)
166
+ return self
167
+
168
+ def _maybe_register_cftime_udf(self, ds: xr.Dataset) -> None:
169
+ """Auto-register a cftime() UDF for non-Gregorian cftime coordinates."""
170
+ for coord_name in ds.dims:
171
+ if cft.is_cftime_index(ds, coord_name):
172
+ units, cal = cft.encoding(ds, coord_name)
173
+ if not cft.is_gregorian_like(cal):
174
+ self.register_udf(cft.make_cftime_udf(units, cal))
175
+ break # One UDF per context is enough.
176
+
177
+ def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame:
178
+ """Run a SQL query, returning an [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapper.
179
+
180
+ Identical to ``datafusion.SessionContext.sql`` except the returned
181
+ object wraps the DataFusion DataFrame. The wrapper exposes
182
+ ``.to_pandas()`` (unchanged), forwards every other DataFusion
183
+ method via ``__getattr__``, and adds
184
+ ``.to_dataset(dimension_columns=[...])`` for round-tripping the
185
+ result back to an ``xr.Dataset``.
186
+
187
+ Args:
188
+ query: A SQL query string.
189
+ *args: Forwarded to ``SessionContext.sql``.
190
+ **kwargs: Forwarded to ``SessionContext.sql``.
191
+
192
+ Returns:
193
+ An [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapping the DataFusion DataFrame.
194
+ """
195
+ inner = super().sql(query, *args, **kwargs)
196
+ return XarrayDataFrame(inner, templates=self._registered_datasets)
197
+
198
+
199
+ XarrayContext = Context
200
+ """The original name of [Context][xarray_sql.Context], kept for compatibility."""
@@ -0,0 +1,411 @@
1
+ Metadata-Version: 2.4
2
+ Name: xarray_sql
3
+ Version: 0.6.0
4
+ Classifier: Development Status :: 4 - Beta
5
+ Classifier: Intended Audience :: Science/Research
6
+ Classifier: Intended Audience :: Developers
7
+ Classifier: Intended Audience :: Information Technology
8
+ Classifier: License :: OSI Approved :: Apache Software License
9
+ Classifier: Operating System :: MacOS :: MacOS X
10
+ Classifier: Operating System :: Microsoft :: Windows
11
+ Classifier: Operating System :: POSIX
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Rust
17
+ Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
18
+ Classifier: Topic :: Database :: Front-Ends
19
+ Requires-Dist: datafusion==54.0.0 ; sys_platform != 'emscripten'
20
+ Requires-Dist: pyarrow>=16.0.0
21
+ Requires-Dist: xarray>=2024.7.0
22
+ Requires-Dist: adbc-driver-manager>=1.12 ; extra == 'adbc'
23
+ Requires-Dist: dask>=2024.8.0 ; extra == 'dask'
24
+ Requires-Dist: xarray-sql[docs] ; extra == 'dev'
25
+ Requires-Dist: pre-commit ; extra == 'dev'
26
+ Requires-Dist: pytest ; extra == 'dev'
27
+ Requires-Dist: watchfiles ; extra == 'dev'
28
+ Requires-Dist: zensical ; extra == 'docs'
29
+ Requires-Dist: mkdocstrings[python] ; extra == 'docs'
30
+ Requires-Dist: duckdb>=1.4.0 ; extra == 'duckdb'
31
+ Requires-Dist: pyproj ; extra == 'geo'
32
+ Requires-Dist: polars>=1.33 ; extra == 'polars'
33
+ Requires-Dist: adbc-driver-flightsql>=1.12 ; extra == 'test'
34
+ Requires-Dist: adbc-driver-sqlite>=1.12 ; extra == 'test'
35
+ Requires-Dist: cftime ; extra == 'test'
36
+ Requires-Dist: xarray-sql[adbc,dask,duckdb,polars,geo] ; extra == 'test'
37
+ Requires-Dist: pytest ; extra == 'test'
38
+ Requires-Dist: xarray[io] ; extra == 'test'
39
+ Requires-Dist: gcsfs ; extra == 'test'
40
+ Provides-Extra: adbc
41
+ Provides-Extra: dask
42
+ Provides-Extra: dev
43
+ Provides-Extra: docs
44
+ Provides-Extra: duckdb
45
+ Provides-Extra: geo
46
+ Provides-Extra: polars
47
+ Provides-Extra: test
48
+ License-File: LICENSE
49
+ Summary: Query Xarray with SQL.
50
+ Author-email: Alexander Merose <al@merose.com>
51
+ License: Apache-2.0
52
+ Requires-Python: >=3.10
53
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
54
+ Project-URL: Homepage, https://github.com/alxmrs/xarray-sql
55
+ Project-URL: Issues, https://github.com/alxmrs/xarray-sql/issues
56
+
57
+ # xarray-sql
58
+
59
+ _Query [Xarray](https://xarray.dev/) with SQL_
60
+
61
+ ![PyPI Version](https://img.shields.io/pypi/v/xarray-sql?color=green)
62
+ [![ci](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci.yml)
63
+ [![lint](https://github.com/xqlsystems/xarray-sql/actions/workflows/lint.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/lint.yml)
64
+ [![ci-build](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-build.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-build.yml)
65
+ [![ci-rust](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-rust.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-rust.yml)
66
+ [![PyPI Downloads](https://static.pepy.tech/personalized-badge/xarray-sql?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/xarray-sql)
67
+ [![PyPI Downloads](https://static.pepy.tech/personalized-badge/xarray-sql?period=monthly&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads%2Fmonth)](https://pepy.tech/projects/xarray-sql)
68
+
69
+ ```shell
70
+ pip install xarray-sql
71
+ ```
72
+
73
+ ## What is this?
74
+
75
+ This is an experiment to provide a SQL interface for array datasets.
76
+ Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
77
+ SQL queries against them — on the query engine of your choice. xarray-sql
78
+ translates data, not queries: it registers a lazy Dataset as a table on
79
+ DataFusion (built in), DuckDB, or Polars, and turns any engine's Arrow result
80
+ back into a labeled Dataset. Dialects, geometry functions, and optimizers stay
81
+ with the engine.
82
+
83
+ ## Quickstart
84
+
85
+ Open a Dataset, register it as a table with `from_dataset`, compute a
86
+ climatology in SQL, then write the result back to Xarray and plot it:
87
+
88
+ > **Note:** this example also needs `pooch` and a netCDF backend (for the
89
+ > tutorial download) and `matplotlib` (for the plot):
90
+ > `pip install pooch netCDF4 matplotlib`.
91
+
92
+ ```python
93
+ import xarray as xr
94
+ import xarray_sql as xql
95
+
96
+ # 4x-daily surface air temperature on a lat/lon grid, 2013-2014.
97
+ ds = xr.tutorial.open_dataset('air_temperature')
98
+
99
+ ctx = xql.Context()
100
+ ctx.from_dataset('air', ds, chunks=dict(time=100))
101
+
102
+ # A climatology — the mean annual cycle — computed in SQL: average air
103
+ # temperature for each month of the year, over all grid cells and years.
104
+ clim = ctx.sql('''
105
+ SELECT
106
+ CAST(date_part('month', "time") AS INTEGER) AS month,
107
+ AVG("air") AS air
108
+ FROM "air"
109
+ GROUP BY CAST(date_part('month', "time") AS INTEGER)
110
+ ORDER BY month
111
+ ''')
112
+
113
+ # Round-trip the result back to Xarray. `month` is a derived column, so name
114
+ # it as the dimension.
115
+ clim_ds = clim.to_dataset(dims=["month"])
116
+
117
+ # Plot the annual cycle as a time series.
118
+ clim_ds["air"].plot() # in a script, call matplotlib.pyplot.show() to display
119
+ ```
120
+
121
+ That's the round trip — Xarray in, SQL in the middle, Xarray (and a plot) back
122
+ out.
123
+
124
+ The same Dataset registers on other engines with one call — DuckDB gets a
125
+ native lazy table with predicate pushdown, Polars scans the same object:
126
+
127
+ ```python
128
+ import duckdb
129
+
130
+ con = duckdb.connect()
131
+ xql.register(con, 'air', ds, chunks=dict(time=100))
132
+ rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY time')
133
+ xql.to_dataset(rel, template=ds) # any engine's Arrow result round-trips
134
+ ```
135
+
136
+ Any database with an [ADBC](https://arrow.apache.org/adbc/) driver
137
+ (PostgreSQL, SQLite, Snowflake, BigQuery, ...) works too: `xql.register`
138
+ ingests the Dataset into a table there, and `xql.to_dataset(cursor, ...)`
139
+ brings results back. Or go the other way without copying anything:
140
+ `xql.serve({'air': ds})` starts an Arrow Flight SQL server that any Flight SQL
141
+ client (ADBC, JDBC, ODBC) can query, with the same chunk pruning as in process.
142
+
143
+ `table_names` (below) works the same way on every engine, so a query written
144
+ against `era5.surface` is not tied to the engine it was written for.
145
+
146
+ See [Engines](https://xqlsystems.github.io/xarray-sql/latest/engines/) for the support matrix, DuckDB/Polars/ADBC details,
147
+ and the lazy chunked round-trip.
148
+
149
+ ## A bigger example: ARCO-ERA5
150
+
151
+ The same interface scales to cloud-native datasets with hundreds of variables,
152
+ like [ARCO-ERA5](https://github.com/google-research/arco-era5).
153
+
154
+ > **Note:** reading from `gs://` requires `gcsfs` (`pip install gcsfs`).
155
+
156
+ ```python
157
+ import xarray as xr
158
+ import xarray_sql as xql
159
+
160
+
161
+ # Open ARCO-ERA5 — a weather dataset with 273 variables since 1940.
162
+ # Turning off dask means we don't have to wait to construct a task graph.
163
+ ds = xr.open_zarr(
164
+ 'gs://gcp-public-data-arco-era5/ar/full_37-1h-0p25deg-chunk-1.zarr-v3',
165
+ chunks=None, # Turn dask off
166
+ storage_options={'token': 'anon'} # Anonymous read from the public GCS bucket — no auth required.
167
+ )
168
+
169
+ ctx = xql.Context()
170
+ # `chunks` sets the partition size; without it, partitions follow the
171
+ # store's own chunks (one hour each here).
172
+ ctx.from_dataset('era5', ds, chunks=dict(time=6), table_names={
173
+ ('time', 'latitude', 'longitude'): 'surface',
174
+ ('time', 'level', 'latitude', 'longitude'): 'atmosphere',
175
+ })
176
+ # Registration takes ~10s on my machine.
177
+
178
+ # Heads up: ARCO-ERA5 has 262 surface + 11 atmospheric variables. The library
179
+ # pushes column projection down to Zarr, so SELECT only fetches what you ask
180
+ # for — but `SELECT * FROM era5.surface` would try to pull every variable
181
+ # across the year (terabytes from GCS).
182
+ # ---> Always SELECT specific columns. <---
183
+
184
+ # Average 2m-temperature over NYC on the morning of 2020-01-01. The library
185
+ # pushes WHERE clauses on dimension columns down to partition pruning.
186
+ ctx.sql('''
187
+ SELECT AVG("2m_temperature") - 273.15 AS avg_c
188
+ FROM era5.surface
189
+ WHERE time BETWEEN TIMESTAMP '2020-01-01'
190
+ AND TIMESTAMP '2020-01-01 05:00:00'
191
+ AND latitude BETWEEN 39 AND 40
192
+ AND longitude BETWEEN 286 AND 287 -- ERA5 uses 0-360 longitudes
193
+ ''').to_pandas()
194
+ # avg_c
195
+ # 0 8.640069
196
+
197
+ # Average temperature per pressure level, globally.
198
+ result = ctx.sql('''
199
+ SELECT level, AVG(temperature) - 273.15 AS avg_c
200
+ FROM era5.atmosphere
201
+ WHERE time BETWEEN TIMESTAMP '2020-01-01'
202
+ AND TIMESTAMP '2020-01-01 05:00:00'
203
+ GROUP BY level
204
+ ORDER BY level DESC
205
+ ''')
206
+ # DataFrame()
207
+ # +-------+----------------------+
208
+ # | level | avg_c |
209
+ # +-------+----------------------+
210
+ # | 1000 | 6.6210120796502565 |
211
+ # | 975 | 5.185637919348153 |
212
+ # | 950 | 4.028428657263021 |
213
+ # | 925 | 3.0828117974912743 |
214
+ # | 900 | 2.2109172992531967 |
215
+ # | 875 | 1.395017610194202 |
216
+ # | 850 | 0.6342670572626616 |
217
+ # | 825 | -0.21037158786759846 |
218
+ # | 800 | -1.1810754318269687 |
219
+ # | 775 | -2.3064649711534457 |
220
+ # +-------+----------------------+
221
+
222
+ # `latitude`/`longitude` are inferred from the registered table's surviving
223
+ # dims; `template` is kept only to recover metadata (attrs, encoding).
224
+ ctx.sql('''
225
+ SELECT latitude, longitude, AVG("2m_temperature") - 273.15 AS avg_c
226
+ FROM era5.surface
227
+ WHERE time BETWEEN TIMESTAMP '2020-01-01'
228
+ AND TIMESTAMP '2020-01-01 05:00:00'
229
+ GROUP BY latitude, longitude
230
+ ORDER BY latitude DESC, longitude
231
+ ''').to_dataset(template=ds)
232
+ # <xarray.Dataset> Size: 8MB
233
+ # Dimensions: (latitude: 721, longitude: 1440)
234
+ # Coordinates:
235
+ # * latitude (latitude) float32 3kB 90.0 89.75 89.5 ... -89.5 -89.75 -90.0
236
+ # * longitude (longitude) float32 6kB 0.0 0.25 0.5 0.75 ... 359.2 359.5 359.8
237
+ # Data variables:
238
+ # avg_c (latitude, longitude) float64 8MB -26.84 -26.84 ... -27.38 -27.38
239
+ # Attributes:
240
+ # last_updated: 2026-06-20 02:33:34.265980+00:00
241
+ # valid_time_start: 1940-01-01
242
+ # valid_time_stop: 2025-12-31
243
+ # valid_time_stop_era5t: 2026-06-14
244
+ ```
245
+
246
+ _(A runnable version of this example lives at
247
+ [`perf_tests/era5_temp_profile.py`](https://github.com/xqlsystems/xarray-sql/blob/main/perf_tests/era5_temp_profile.py).)_
248
+
249
+ ## Why build this?
250
+
251
+ A few reasons:
252
+
253
+ * Even though SQL is the lingua franca of data, scientific datasets are often
254
+ inaccessible to non-scientists (SQL users).
255
+ * Joining tabular data with raster data is common yet difficult. It could be
256
+ easy.
257
+ * There are many cloud-native, Xarray-openable datasets,
258
+ from [Google Earth Engine](https://github.com/google/Xee)
259
+ to the [Source Cooperative](https://source.coop/products?tags=zarr). Wouldn’t it be great if these
260
+ were also SQL-accessible? How can the bridge be built with minimal effort?
261
+
262
+ This is a light-weight way to prove the value of the interface.
263
+
264
+ The larger goal is to explore the hypothesis that the [Pangeo
265
+ ecosystem is a scientific database](https://www.hytradboi.com/2025/c18b8cdc-fd17-4099-9c03-eb107217f627-pangeo-is-a-database). Here, xarray-sql can be thought of as a missing
266
+ DB front end.
267
+
268
+ ## How does it work?
269
+
270
+ All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
271
+ `from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
272
+ That's it!
273
+
274
+ _2025 update_: This library now implements a Dask-like `from_map` interface in
275
+ pure DataFusion and PyArrow, but works with the same principle!
276
+
277
+ _2026 update_: Instead of `from_map()`, we create a way to translate Xarray chunks
278
+ into Arrow RecordBatches. We pass a Python callback into a DataFusion `TableProvider`
279
+ that lets the DB engine translate the underlying Dataset arrays into DataFusion partitions.
280
+ The same chunks-to-batches translation is also exposed as a
281
+ `pyarrow.dataset.Dataset` with predicate and projection pushdown, which is how
282
+ DuckDB and Polars consume registered Datasets with no engine-specific code.
283
+ Ultimately, the initial insight of the `pivot()` function -- that any ndarray can be
284
+ translated into a 2D table -- underlies this performant query mechanism.
285
+
286
+ ## Does it work?
287
+
288
+ Yes. The recurring worry is that the SQL interface is a toy — fine for `SELECT`s,
289
+ but not for the operations geoscience actually runs. So we wrote a suite that
290
+ takes the staples of geospatial and climate analysis — the ones we assume *need*
291
+ an array library — and expresses each one in SQL, then **checks the SQL answer
292
+ against an xarray/array reference** to floating-point tolerance:
293
+
294
+ * **Spectral indices** (NDVI) — column arithmetic over a real Sentinel-2 scene.
295
+ * **Climatology, anomalies, zonal means** — `GROUP BY` and self-`JOIN` against
296
+ the 0.25° **ARCO-ERA5** archive registered as a lazy table. Each query is
297
+ bounded to a small window (a few days over a region) and reads only that
298
+ slice — the point is that you can aim a query at a multi-decade archive and
299
+ pay only for the data it asks for, not that the query scans the whole record.
300
+ * **Forecast skill** — scoring the **Pangu-Weather** and **GraphCast** ML models
301
+ against ERA5 (WeatherBench 2) as a `JOIN` on `valid_time = init + lead`; it
302
+ reproduces the published result that GraphCast beats Pangu at every lead.
303
+ * **Raster × vector zonal stats** — a range `JOIN` of the ERA5 grid against a
304
+ table of regions.
305
+ * **Reprojection and regridding** — a `reproject(x, y, src_crs, dst_crs)`
306
+ scalar PROJ UDF, shipped as the optional geo extension
307
+ (`pip install xarray-sql[geo]`, validated against Earth Engine's own
308
+ geodesy via [Xee](https://github.com/google/Xee)) and a
309
+ sparse-weight-table `JOIN` (regridding real SRTM terrain).
310
+
311
+ Every case matches its array reference. The headline finding: these operations
312
+ are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
313
+ functions, and `CASE` in disguise, and a query engine runs them at scale. See
314
+ [`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
315
+ [Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/latest/geospatial/).
316
+
317
+ ## Why does this work?
318
+
319
+ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
320
+ chunks and represented contiguously in memory. It is only a matter of metadata
321
+ that breaks them up into ndarrays. `pivot()`, which uses `to_dataframe()`,
322
+ just changes this metadata (via a `ravel()`/`reshape()`), back into a column
323
+ amenable to a DataFrame. We take advantage of this lightweight metadata change to
324
+ make chunked information scannable by a DB engine (DataFusion, DuckDB, Polars —
325
+ anything that speaks Arrow).
326
+
327
+ ## What are the current limitations?
328
+
329
+ The sharp edges we know about — per engine and fundamental — are cataloged in
330
+ [Known issues & limitations](https://xqlsystems.github.io/xarray-sql/latest/limitations/). Currently, we're looking for
331
+ early users – "tire kickers", if you will. We'd love your input to shape the direction of this
332
+ project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
333
+ you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/latest/contributing/), too 😉.
334
+
335
+ ## What would a deeper integration look like?
336
+
337
+ I have a few ideas so far. One approach involves applying operations directly on
338
+ Xarray Datasets. This approach is being pursued
339
+ [here](https://github.com/google/weather-tools/tree/main/xql), as `xql`.
340
+
341
+ Deeper still: I was thinking we could make
342
+ a [virtual](https://fsspec.github.io/kerchunk/)
343
+ filesystem for parquet that would internally map to Zarr. Raster-backed virtual
344
+ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
345
+ and BigQuery. More thoughts on this
346
+ in [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
347
+
348
+ _2025 update_: Something like this is being built across a few projects! The ones I know about are:
349
+
350
+ - [CartoDB's Raquet](https://github.com/CartoDB/raquet)
351
+ - The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
352
+
353
+ _2026 update_: A colleague and I are experimenting with native Zarr RDBMS engines. Check out:
354
+
355
+ - [Zarr-Datafusion](https://lib.rs/crates/zarr-datafusion)
356
+ - [DuckDB-Zarr](https://github.com/xqlsystems/duckdb-zarr)
357
+
358
+ ## Roadmap
359
+
360
+ - [x] ~Lazy evaluation via the pyarrow Dataset interface [#93](https://github.com/xqlsystems/xarray-sql/issues/93).~ _Implemented in [#100](https://github.com/xqlsystems/xarray-sql/pull/100)_
361
+ - [x] Support proper parallelism via proper partition handling on the rust/datafusion side. [#106](https://github.com/xqlsystems/xarray-sql/issues/106)
362
+ - [x] Support core datafusion optimizations to scan less data, like [#104](https://github.com/xqlsystems/xarray-sql/issues/104), ...
363
+ - [x] Translate a single Zarr to a collection of tables [#85](https://github.com/xqlsystems/xarray-sql/issues/85).
364
+ - [ ] Distributed beyond a single node through the DataFusion integration with Ray Datasets [#68](https://github.com/xqlsystems/xarray-sql/issues/68) or Apache Ballista [#98](https://github.com/xqlsystems/xarray-sql/issues/98).
365
+ - [ ] Demo: calculate Sea Surface Temperature from 1940 - Present in SQL [#36](https://github.com/xqlsystems/xarray-sql/issues/36).
366
+ - [ ] Provide an option to integrate DataFusion directly to Zarr via Rust [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
367
+ - [ ] (To be formally announced eventually): The 100 Trillion Row Challenge [#34](https://github.com/xqlsystems/xarray-sql/issues/34).
368
+
369
+ ## Sponsors & Contributors
370
+
371
+ I want to give a special thanks to the following folks and institutions:
372
+
373
+ - Pramod Gupta and the Anthromet Team at Google Research for the problem
374
+ formation and design inspiration.
375
+ - Jake Wall and AI2/Ecoscope for compute resources and key use cases.
376
+ - Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
377
+ for the early review and discussion of this project.
378
+ - Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
379
+ Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
380
+ and DataFusion-specific collaboration.
381
+ - The gracious volunteer data science students at [UCSD's DS3](https://www.ds3atucsd.com/) org,
382
+ who are working to make this library better.
383
+ - Andrew Huang for the sense of taste he brings to the project and consummate code
384
+ changes.
385
+ - Aman Kumar for spending a considerable amount of his GSoC internship
386
+ contributing to this project.
387
+ - Miguel Moncada Isla for reimagining this project to be a cross SQL engine
388
+ interface to and from Xarray. He brought new life to this experiment, in my
389
+ opinion.
390
+
391
+
392
+ ## License
393
+
394
+ ```
395
+ Copyright 2024 Alexander Merose
396
+
397
+ Licensed under the Apache License, Version 2.0 (the "License");
398
+ you may not use this file except in compliance with the License.
399
+ You may obtain a copy of the License at
400
+
401
+ https://www.apache.org/licenses/LICENSE-2.0
402
+
403
+ Unless required by applicable law or agreed to in writing, software
404
+ distributed under the License is distributed on an "AS IS" BASIS,
405
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
406
+ See the License for the specific language governing permissions and
407
+ limitations under the License.
408
+ ```
409
+
410
+ All vendored code has proper license attribution.
411
+
@@ -0,0 +1,25 @@
1
+ xarray_sql/__init__.py,sha256=uWROeltFlV-y11FggA4juVA3Wa__3AhMdDqzO6n0BA8,1195
2
+ xarray_sql/_native.abi3.so,sha256=TDLVCLMa-Qq-NxlcmvCmHd38F1TWBQnvk4mN8XYmpIg,15084579
3
+ xarray_sql/cftime.py,sha256=a8jIslG29iDfv4_yP3INAbMRT6fNMKg9iMr5wCznEkg,9082
4
+ xarray_sql/core.py,sha256=eQU89mn71POcppgPXr5Cf_N19G9Sn6gMD8YPbYfgous,1304
5
+ xarray_sql/df.py,sha256=wi3Fnif8LxTOR4pTmb6TJusuHSA1z6O4UL12aXJyMwc,33072
6
+ xarray_sql/ds.py,sha256=jJ9JdfCF8S-Koh_I5AoLLLmx6RppEv3uAytfpKMGdnk,53831
7
+ xarray_sql/geometry.py,sha256=sX2VH0ids8Qfp3m1n40IOVUY4G4APt3G0MCa2kLKUzQ,5040
8
+ xarray_sql/lazyscan.py,sha256=WTplgJFJ1YoqWL-8WLSUdH40ktafaohsiohJiN6w1xs,18245
9
+ xarray_sql/proj.py,sha256=sKAJw1kO28_U54FpSlGABU628Ux8xxdTKfEPzI1w2lQ,9525
10
+ xarray_sql/reader.py,sha256=aHsuzt4_FTQCTW2E2czbpkSJH1PDy9rDLSC1TF959Vs,13660
11
+ xarray_sql/roundtrip.py,sha256=-KQ8lwkHAQdMxIwpcx7596L6bESupqR52X9gj13IjEQ,25516
12
+ xarray_sql/sql.py,sha256=Ipzfti5Httm53DQ-J3Fb9IX1wVkoxP3r_AM1yTTBDG0,8006
13
+ xarray_sql/backends/__init__.py,sha256=SowGCRe2AmM6tfnqZYsjS2So22Z4ymBjw3ed9uHEx6M,1370
14
+ xarray_sql/backends/_adbc_dialects.py,sha256=0hwxohgSRXNoTYNnV5ymjloxuujRqdaMhVyiEDk89CI,15581
15
+ xarray_sql/backends/adbc.py,sha256=hpz6RfObUgUIfnbsifkOAuJ6BakKHrn7djd87eh33mc,14567
16
+ xarray_sql/backends/base.py,sha256=kfbN5NN6rd8ZKkNcLBlfwMZTf9N--WtvV5U1oaJ5Zl4,5627
17
+ xarray_sql/backends/datafusion.py,sha256=dfPKu994DttMuNkUPLZpSFnRBjDkieQ1GilfKRasYew,2942
18
+ xarray_sql/backends/duckdb.py,sha256=kC7lwQfQLmITLFwvgACvniDPR5bqyAq1w9kooorfRcA,7514
19
+ xarray_sql/backends/flight.py,sha256=7AZRen-4g29Z-Hwc2cECeBjVRAmHc92iQUYhLvuDmpk,10944
20
+ xarray_sql/backends/pyarrow.py,sha256=uDsqEOrIXmptBFGm2-2nbZADU0LidOlJJu2zU9u7GW8,52468
21
+ xarray_sql-0.6.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
22
+ xarray_sql-0.6.0.dist-info/sboms/xarray_sql.cyclonedx.json,sha256=d84Mn5sE0zZdfofsYJtK5NuS2GdGH7S_spLY8eKh0j8,327254
23
+ xarray_sql-0.6.0.dist-info/METADATA,sha256=hy_uXLIY538ECi8WOHbqvcX3FYEOftD8OblgPJSiyuY,19266
24
+ xarray_sql-0.6.0.dist-info/WHEEL,sha256=JvzZS3vTSCSj6m0z_Oqq9HFALOUfSVFBoMGY4a5Sw88,114
25
+ xarray_sql-0.6.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: maturin (1.15.0)
3
+ Root-Is-Purelib: false
4
+ Tag: cp310-abi3-pyemscripten_2026_0_wasm32
5
+