xarray_sql 0.3.3__tar.gz → 0.4.0rc1__tar.gz

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/Cargo.lock

@@ -3367,7 +3367,7 @@ checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
 
 [[package]]
 name = "xarray_sql"
-version = "0.3.3"
+version = "0.4.0-rc.1"
 dependencies = [
  "arrow",
  "async-stream",

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xarray_sql"
-version = "0.3.3"
+version = "0.4.0-rc.1"
 authors = ["Alex Merose"]
 edition = "2021"
 exclude = [

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xarray_sql
-Version: 0.3.3
+Version: 0.4.0rc1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Developers
@@ -25,18 +25,22 @@ Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: watchfiles ; extra == 'dev'
 Requires-Dist: zensical ; extra == 'docs'
 Requires-Dist: mkdocstrings[python] ; extra == 'docs'
+Requires-Dist: duckdb>=1.4.0 ; extra == 'duckdb'
 Requires-Dist: pyproj ; extra == 'geo'
+Requires-Dist: polars>=1.33 ; extra == 'polars'
 Requires-Dist: cftime ; extra == 'test'
-Requires-Dist: xarray-sql[geo] ; extra == 'test'
+Requires-Dist: xarray-sql[duckdb,polars,geo] ; extra == 'test'
 Requires-Dist: pytest ; extra == 'test'
 Requires-Dist: xarray[io] ; extra == 'test'
 Requires-Dist: gcsfs ; extra == 'test'
 Provides-Extra: dev
 Provides-Extra: docs
+Provides-Extra: duckdb
 Provides-Extra: geo
+Provides-Extra: polars
 Provides-Extra: test
 License-File: LICENSE
-Summary: Querry Xarray with SQL.
+Summary: Query Xarray with SQL.
 Author-email: Alexander Merose <al@merose.com>
 License: Apache-2.0
 Requires-Python: >=3.10
@@ -49,10 +53,10 @@ Project-URL: Issues, https://github.com/alxmrs/xarray-sql/issues
 _Query [Xarray](https://xarray.dev/) with SQL_
 
 ![PyPI Version](https://img.shields.io/pypi/v/xarray-sql?color=green)
-[![ci](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml)
-[![lint](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml)
-[![ci-build](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-build.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-build.yml)
-[![ci-rust](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-rust.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-rust.yml)
+[![ci](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci.yml)
+[![lint](https://github.com/xqlsystems/xarray-sql/actions/workflows/lint.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/lint.yml)
+[![ci-build](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-build.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-build.yml)
+[![ci-rust](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-rust.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-rust.yml)
 [![PyPI Downloads](https://static.pepy.tech/personalized-badge/xarray-sql?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/xarray-sql)
 [![PyPI Downloads](https://static.pepy.tech/personalized-badge/xarray-sql?period=monthly&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads%2Fmonth)](https://pepy.tech/projects/xarray-sql)
 
@@ -64,7 +68,11 @@ pip install xarray-sql
 
 This is an experiment to provide a SQL interface for array datasets.
 Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
-SQL queries against them.
+SQL queries against them — on the query engine of your choice. xarray-sql
+translates data, not queries: it registers a lazy Dataset as a table on
+DataFusion (built in), DuckDB, or Polars, and turns any engine's Arrow result
+back into a labeled Dataset. Dialects, geometry functions, and optimizers stay
+with the engine.
 
 ## Quickstart
 
@@ -107,6 +115,21 @@ clim_ds["air"].plot()  # in a script, call matplotlib.pyplot.show() to display
 That's the round trip — Xarray in, SQL in the middle, Xarray (and a plot) back
 out.
 
+The same Dataset registers on other engines with one call — DuckDB gets a
+native lazy table with predicate pushdown, Polars scans the same object:
+
+```python
+import duckdb
+
+con = duckdb.connect()
+xql.register(con, 'air', ds, chunks=dict(time=100))
+rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY time')
+xql.to_dataset(rel, template=ds)   # any engine's Arrow result round-trips
+```
+
+See [Engines](https://xqlsystems.github.io/xarray-sql/engines/) for the support matrix, DuckDB/Polars details,
+and the lazy chunked round-trip.
+
 ## A bigger example: ARCO-ERA5
 
 The same interface scales to cloud-native datasets with hundreds of variables,
@@ -179,6 +202,8 @@ result = ctx.sql('''
 # | 775   | -2.3064649711534457  |
 # +-------+----------------------+
 
+# `latitude`/`longitude` are inferred from the registered table's surviving
+# dims; `template` is kept only to recover metadata (attrs, encoding).
 ctx.sql('''
   SELECT latitude, longitude, AVG("2m_temperature") - 273.15 AS avg_c
   FROM era5.surface
@@ -186,8 +211,6 @@ ctx.sql('''
                  AND TIMESTAMP '2020-01-01 05:00:00'
   GROUP BY latitude, longitude
   ORDER BY latitude DESC, longitude
-# `latitude`/`longitude` are inferred from the registered table's surviving
-# dims; `template` is kept only to recover metadata (attrs, encoding).
 ''').to_dataset(template=ds)
 # <xarray.Dataset> Size: 8MB
 # Dimensions:    (latitude: 721, longitude: 1440)
@@ -204,7 +227,7 @@ ctx.sql('''
 ```
 
 _(A runnable version of this example lives at
-[`perf_tests/era5_temp_profile.py`](perf_tests/era5_temp_profile.py).)_
+[`perf_tests/era5_temp_profile.py`](https://github.com/xqlsystems/xarray-sql/blob/main/perf_tests/era5_temp_profile.py).)_
 
 ## Why build this?
 
@@ -237,6 +260,9 @@ pure DataFusion and PyArrow, but works with the same principle!
 _2026 update_: Instead of `from_map()`, we create a way to translate Xarray chunks
 into Arrow RecordBatches. We pass a Python callback into a DataFusion `TableProvider`
 that lets the DB engine translate the underlying Dataset arrays into DataFusion partitions.
+The same chunks-to-batches translation is also exposed as a
+`pyarrow.dataset.Dataset` with predicate and projection pushdown, which is how
+DuckDB and Polars consume registered Datasets with no engine-specific code.
 Ultimately, the initial insight of the `pivot()` function -- that any ndarray can be
 translated into a 2D table -- underlies this performant query mechanism. 
 
@@ -268,8 +294,8 @@ against an xarray/array reference** to floating-point tolerance:
 Every case matches its array reference. The headline finding: these operations
 are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
 functions, and `CASE` in disguise, and a query engine runs them at scale. See
-[`benchmarks/geospatial/`](benchmarks/geospatial/) and the write-up,
-[Geospatial operations are relational operations](docs/geospatial.md).
+[`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
+[Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/geospatial/).
 
 ## Why does this work?
 
@@ -277,15 +303,17 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
 chunks and represented contiguously in memory. It is only a matter of metadata
 that breaks them up into ndarrays. `pivot()`, which uses `to_dataframe()`,
 just changes this metadata (via a `ravel()`/`reshape()`), back into a column
-amenable to a DataFrame. We take advantage of this light weight metadata change to
-make chunked information scannable by a DB engine (DataFusion).
+amenable to a DataFrame. We take advantage of this lightweight metadata change to
+make chunked information scannable by a DB engine (DataFusion, DuckDB, Polars —
+anything that speaks Arrow).
 
 ## What are the current limitations?
 
-TBD, DataFusion provides a whole new world! Currently, we're looking for
+The sharp edges we know about — per engine and fundamental — are cataloged in
+[Known issues & limitations](https://xqlsystems.github.io/xarray-sql/limitations/). Currently, we're looking for
 early users – "tire kickers", if you will. We'd love your input to shape the direction of this
-project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
-you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
+project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
+you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/contributing/), too 😉.
 
 ## What would a deeper integration look like?
 
@@ -298,7 +326,7 @@ a [virtual](https://fsspec.github.io/kerchunk/)
 filesystem for parquet that would internally map to Zarr. Raster-backed virtual
 parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
 and BigQuery. More thoughts on this
-in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
+in [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
 
 _2025 update_: Something like this is being built across a few projects! The ones I know about are:
 
@@ -308,18 +336,18 @@ _2025 update_: Something like this is being built across a few projects! The one
 _2026 update_: A colleague and I are experimenting with native Zarr RDBMS engines. Check out:
 
 - [Zarr-Datafusion](https://lib.rs/crates/zarr-datafusion)
-- [DuckDB-Zarr](https://github.com/alxmrs/duckdb-zarr)
+- [DuckDB-Zarr](https://github.com/xqlsystems/duckdb-zarr)
 
 ## Roadmap
 
-- [x] ~Lazy evaluation via the pyarrow Dataset interface [#93](https://github.com/alxmrs/xarray-sql/issues/93).~ _Implemented in [#100](https://github.com/alxmrs/xarray-sql/pull/100)_
-- [x] Support proper parallelism via proper partition handling on the rust/datafusion side. [#106](https://github.com/alxmrs/xarray-sql/issues/106)
-- [x] Support core datafusion optimizations to scan less data, like [104](https://github.com/alxmrs/xarray-sql/issues/104), ...
-- [x] Translate a single Zarr to a collection of tables [#85](https://github.com/alxmrs/xarray-sql/issues/85).
-- [ ] Distributed beyond a single node through the DataFusion integration with Ray Datasets [#68](https://github.com/alxmrs/xarray-sql/issues/68) or Apache Ballista [#98](https://github.com/alxmrs/xarray-sql/issues/98).
-- [ ] Demo: calculate Sea Surface Temperature from 1940 - Present in SQL [#36](https://github.com/alxmrs/xarray-sql/issues/36).
-- [ ] Provide an option to integrate DataFusion directly to Zarr via Rust [#4](https://github.com/alxmrs/xarray-sql/issues/4).
-- [ ] (To be formally announced eventually): The 100 Trillion Row Challenge [#34](https://github.com/alxmrs/xarray-sql/issues/34).
+- [x] ~Lazy evaluation via the pyarrow Dataset interface [#93](https://github.com/xqlsystems/xarray-sql/issues/93).~ _Implemented in [#100](https://github.com/xqlsystems/xarray-sql/pull/100)_
+- [x] Support proper parallelism via proper partition handling on the rust/datafusion side. [#106](https://github.com/xqlsystems/xarray-sql/issues/106)
+- [x] Support core datafusion optimizations to scan less data, like [#104](https://github.com/xqlsystems/xarray-sql/issues/104), ...
+- [x] Translate a single Zarr to a collection of tables [#85](https://github.com/xqlsystems/xarray-sql/issues/85).
+- [ ] Distributed beyond a single node through the DataFusion integration with Ray Datasets [#68](https://github.com/xqlsystems/xarray-sql/issues/68) or Apache Ballista [#98](https://github.com/xqlsystems/xarray-sql/issues/98).
+- [ ] Demo: calculate Sea Surface Temperature from 1940 - Present in SQL [#36](https://github.com/xqlsystems/xarray-sql/issues/36).
+- [ ] Provide an option to integrate DataFusion directly to Zarr via Rust [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
+- [ ] (To be formally announced eventually): The 100 Trillion Row Challenge [#34](https://github.com/xqlsystems/xarray-sql/issues/34).
 
 ## Sponsors & Contributors
 

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/README.md

@@ -3,10 +3,10 @@
 _Query [Xarray](https://xarray.dev/) with SQL_
 
 ![PyPI Version](https://img.shields.io/pypi/v/xarray-sql?color=green)
-[![ci](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml)
-[![lint](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml)
-[![ci-build](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-build.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-build.yml)
-[![ci-rust](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-rust.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci-rust.yml)
+[![ci](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci.yml)
+[![lint](https://github.com/xqlsystems/xarray-sql/actions/workflows/lint.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/lint.yml)
+[![ci-build](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-build.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-build.yml)
+[![ci-rust](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-rust.yml/badge.svg)](https://github.com/xqlsystems/xarray-sql/actions/workflows/ci-rust.yml)
 [![PyPI Downloads](https://static.pepy.tech/personalized-badge/xarray-sql?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/xarray-sql)
 [![PyPI Downloads](https://static.pepy.tech/personalized-badge/xarray-sql?period=monthly&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads%2Fmonth)](https://pepy.tech/projects/xarray-sql)
 
@@ -18,7 +18,11 @@ pip install xarray-sql
 
 This is an experiment to provide a SQL interface for array datasets.
 Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
-SQL queries against them.
+SQL queries against them — on the query engine of your choice. xarray-sql
+translates data, not queries: it registers a lazy Dataset as a table on
+DataFusion (built in), DuckDB, or Polars, and turns any engine's Arrow result
+back into a labeled Dataset. Dialects, geometry functions, and optimizers stay
+with the engine.
 
 ## Quickstart
 
@@ -61,6 +65,21 @@ clim_ds["air"].plot()  # in a script, call matplotlib.pyplot.show() to display
 That's the round trip — Xarray in, SQL in the middle, Xarray (and a plot) back
 out.
 
+The same Dataset registers on other engines with one call — DuckDB gets a
+native lazy table with predicate pushdown, Polars scans the same object:
+
+```python
+import duckdb
+
+con = duckdb.connect()
+xql.register(con, 'air', ds, chunks=dict(time=100))
+rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY time')
+xql.to_dataset(rel, template=ds)   # any engine's Arrow result round-trips
+```
+
+See [Engines](https://xqlsystems.github.io/xarray-sql/engines/) for the support matrix, DuckDB/Polars details,
+and the lazy chunked round-trip.
+
 ## A bigger example: ARCO-ERA5
 
 The same interface scales to cloud-native datasets with hundreds of variables,
@@ -133,6 +152,8 @@ result = ctx.sql('''
 # | 775   | -2.3064649711534457  |
 # +-------+----------------------+
 
+# `latitude`/`longitude` are inferred from the registered table's surviving
+# dims; `template` is kept only to recover metadata (attrs, encoding).
 ctx.sql('''
   SELECT latitude, longitude, AVG("2m_temperature") - 273.15 AS avg_c
   FROM era5.surface
@@ -140,8 +161,6 @@ ctx.sql('''
                  AND TIMESTAMP '2020-01-01 05:00:00'
   GROUP BY latitude, longitude
   ORDER BY latitude DESC, longitude
-# `latitude`/`longitude` are inferred from the registered table's surviving
-# dims; `template` is kept only to recover metadata (attrs, encoding).
 ''').to_dataset(template=ds)
 # <xarray.Dataset> Size: 8MB
 # Dimensions:    (latitude: 721, longitude: 1440)
@@ -158,7 +177,7 @@ ctx.sql('''
 ```
 
 _(A runnable version of this example lives at
-[`perf_tests/era5_temp_profile.py`](perf_tests/era5_temp_profile.py).)_
+[`perf_tests/era5_temp_profile.py`](https://github.com/xqlsystems/xarray-sql/blob/main/perf_tests/era5_temp_profile.py).)_
 
 ## Why build this?
 
@@ -191,6 +210,9 @@ pure DataFusion and PyArrow, but works with the same principle!
 _2026 update_: Instead of `from_map()`, we create a way to translate Xarray chunks
 into Arrow RecordBatches. We pass a Python callback into a DataFusion `TableProvider`
 that lets the DB engine translate the underlying Dataset arrays into DataFusion partitions.
+The same chunks-to-batches translation is also exposed as a
+`pyarrow.dataset.Dataset` with predicate and projection pushdown, which is how
+DuckDB and Polars consume registered Datasets with no engine-specific code.
 Ultimately, the initial insight of the `pivot()` function -- that any ndarray can be
 translated into a 2D table -- underlies this performant query mechanism. 
 
@@ -222,8 +244,8 @@ against an xarray/array reference** to floating-point tolerance:
 Every case matches its array reference. The headline finding: these operations
 are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
 functions, and `CASE` in disguise, and a query engine runs them at scale. See
-[`benchmarks/geospatial/`](benchmarks/geospatial/) and the write-up,
-[Geospatial operations are relational operations](docs/geospatial.md).
+[`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
+[Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/geospatial/).
 
 ## Why does this work?
 
@@ -231,15 +253,17 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
 chunks and represented contiguously in memory. It is only a matter of metadata
 that breaks them up into ndarrays. `pivot()`, which uses `to_dataframe()`,
 just changes this metadata (via a `ravel()`/`reshape()`), back into a column
-amenable to a DataFrame. We take advantage of this light weight metadata change to
-make chunked information scannable by a DB engine (DataFusion).
+amenable to a DataFrame. We take advantage of this lightweight metadata change to
+make chunked information scannable by a DB engine (DataFusion, DuckDB, Polars —
+anything that speaks Arrow).
 
 ## What are the current limitations?
 
-TBD, DataFusion provides a whole new world! Currently, we're looking for
+The sharp edges we know about — per engine and fundamental — are cataloged in
+[Known issues & limitations](https://xqlsystems.github.io/xarray-sql/limitations/). Currently, we're looking for
 early users – "tire kickers", if you will. We'd love your input to shape the direction of this
-project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
-you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
+project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
+you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/contributing/), too 😉.
 
 ## What would a deeper integration look like?
 
@@ -252,7 +276,7 @@ a [virtual](https://fsspec.github.io/kerchunk/)
 filesystem for parquet that would internally map to Zarr. Raster-backed virtual
 parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
 and BigQuery. More thoughts on this
-in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
+in [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
 
 _2025 update_: Something like this is being built across a few projects! The ones I know about are:
 
@@ -262,18 +286,18 @@ _2025 update_: Something like this is being built across a few projects! The one
 _2026 update_: A colleague and I are experimenting with native Zarr RDBMS engines. Check out:
 
 - [Zarr-Datafusion](https://lib.rs/crates/zarr-datafusion)
-- [DuckDB-Zarr](https://github.com/alxmrs/duckdb-zarr)
+- [DuckDB-Zarr](https://github.com/xqlsystems/duckdb-zarr)
 
 ## Roadmap
 
-- [x] ~Lazy evaluation via the pyarrow Dataset interface [#93](https://github.com/alxmrs/xarray-sql/issues/93).~ _Implemented in [#100](https://github.com/alxmrs/xarray-sql/pull/100)_
-- [x] Support proper parallelism via proper partition handling on the rust/datafusion side. [#106](https://github.com/alxmrs/xarray-sql/issues/106)
-- [x] Support core datafusion optimizations to scan less data, like [104](https://github.com/alxmrs/xarray-sql/issues/104), ...
-- [x] Translate a single Zarr to a collection of tables [#85](https://github.com/alxmrs/xarray-sql/issues/85).
-- [ ] Distributed beyond a single node through the DataFusion integration with Ray Datasets [#68](https://github.com/alxmrs/xarray-sql/issues/68) or Apache Ballista [#98](https://github.com/alxmrs/xarray-sql/issues/98).
-- [ ] Demo: calculate Sea Surface Temperature from 1940 - Present in SQL [#36](https://github.com/alxmrs/xarray-sql/issues/36).
-- [ ] Provide an option to integrate DataFusion directly to Zarr via Rust [#4](https://github.com/alxmrs/xarray-sql/issues/4).
-- [ ] (To be formally announced eventually): The 100 Trillion Row Challenge [#34](https://github.com/alxmrs/xarray-sql/issues/34).
+- [x] ~Lazy evaluation via the pyarrow Dataset interface [#93](https://github.com/xqlsystems/xarray-sql/issues/93).~ _Implemented in [#100](https://github.com/xqlsystems/xarray-sql/pull/100)_
+- [x] Support proper parallelism via proper partition handling on the rust/datafusion side. [#106](https://github.com/xqlsystems/xarray-sql/issues/106)
+- [x] Support core datafusion optimizations to scan less data, like [#104](https://github.com/xqlsystems/xarray-sql/issues/104), ...
+- [x] Translate a single Zarr to a collection of tables [#85](https://github.com/xqlsystems/xarray-sql/issues/85).
+- [ ] Distributed beyond a single node through the DataFusion integration with Ray Datasets [#68](https://github.com/xqlsystems/xarray-sql/issues/68) or Apache Ballista [#98](https://github.com/xqlsystems/xarray-sql/issues/98).
+- [ ] Demo: calculate Sea Surface Temperature from 1940 - Present in SQL [#36](https://github.com/xqlsystems/xarray-sql/issues/36).
+- [ ] Provide an option to integrate DataFusion directly to Zarr via Rust [#4](https://github.com/xqlsystems/xarray-sql/issues/4).
+- [ ] (To be formally announced eventually): The 100 Trillion Row Challenge [#34](https://github.com/xqlsystems/xarray-sql/issues/34).
 
 ## Sponsors & Contributors
 

xarray_sql-0.4.0rc1/benchmarks/duckdb_pushdown.py

@@ -0,0 +1,103 @@
+"""Benchmark: DuckDB re-scannable stream vs pushdown dataset vs ceiling.
+
+Times the three ways DuckDB can consume the same 10M-row synthetic
+dataset — the re-scannable stream (no pushdown), the default
+``register()`` pushdown dataset, and an in-memory ``pyarrow.dataset``
+as the ceiling — and asserts at the end that all three returned the
+same answers. Cross-engine comparisons live in
+``benchmarks/geospatial/``; this measures the adapter paths within one
+engine.
+
+Usage: python benchmarks/duckdb_pushdown.py  (needs duckdb installed)
+"""
+
+import math
+import statistics
+import time
+
+import duckdb
+import numpy as np
+import pandas as pd
+import pyarrow.dataset as pads
+import xarray as xr
+
+import xarray_sql as xql
+from xarray_sql.backends.duckdb import XarrayArrowStream
+
+np.random.seed(0)
+N_TIME, N_LAT, N_LON = 1000, 100, 100  # 10M rows
+ds = xr.Dataset(
+    {
+        "temperature": (
+            ["time", "lat", "lon"],
+            np.random.rand(N_TIME, N_LAT, N_LON),
+        ),
+        "humidity": (
+            ["time", "lat", "lon"],
+            np.random.rand(N_TIME, N_LAT, N_LON),
+        ),
+    },
+    coords={
+        "time": pd.date_range("2020-01-01", periods=N_TIME, freq="h"),
+        "lat": np.linspace(-90, 90, N_LAT),
+        "lon": np.linspace(-180, 180, N_LON),
+    },
+).chunk({"time": 50})  # 20 partitions
+
+con = duckdb.connect()
+
+QUERIES = {
+    "full AVG scan": "SELECT AVG(temperature) FROM {t}",
+    "1pct time filter": (
+        "SELECT AVG(temperature) FROM {t} WHERE time < '2020-01-01 10:00:00'"
+    ),
+    "bbox filter": (
+        "SELECT AVG(temperature) FROM {t} "
+        "WHERE lat BETWEEN 0 AND 10 AND lon BETWEEN 0 AND 20"
+    ),
+    "projection (1 of 2 vars)": "SELECT AVG(humidity) FROM {t}",
+    "count only": "SELECT COUNT(*) FROM {t}",
+}
+
+
+def bench(table, label, n=5):
+    """Times each query; returns {query: answer} for equivalence checks."""
+    print(f"\n== {label} ==")
+    answers = {}
+    for qname, q in QUERIES.items():
+        sql = q.format(t=table)
+        times = []
+        for _ in range(n):
+            t0 = time.perf_counter()
+            r = con.sql(sql).fetchall()
+            times.append(time.perf_counter() - t0)
+        answers[qname] = r[0][0]
+        med = statistics.median(times)
+        print(
+            f"  {qname:28s} {med:8.3f}s "
+            f"(min {min(times):.3f} / max {max(times):.3f})   -> {r[0][0]:.6g}"
+        )
+    return answers
+
+
+# re-scannable stream, registered via the stream wrapper explicitly:
+# DuckDB scans every row, no filter/projection pushdown
+con.register("t_stream", XarrayArrowStream(ds))
+stream = bench("t_stream", "stream (no pushdown)")
+
+# default register(): the pushdown pyarrow-dataset path
+xql.register(con, "t_pushdown", ds)
+pushdown = bench("t_pushdown", "register() [pushdown]")
+
+# ceiling: materialized pa.Table via pyarrow.dataset
+table = xql.read_xarray(ds).read_all()
+con.register("t_ceiling", pads.dataset(table))
+ceiling = bench("t_ceiling", "ceiling: in-memory pyarrow.dataset")
+
+# The timings are only meaningful if every path computed the same thing.
+for qname in QUERIES:
+    a, b, c = stream[qname], pushdown[qname], ceiling[qname]
+    assert math.isclose(a, b, rel_tol=1e-9) and math.isclose(
+        a, c, rel_tol=1e-9
+    ), f"{qname}: paths disagree — stream={a} pushdown={b} ceiling={c}"
+print("\nall paths agree")

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/01_ndvi.py

@@ -45,8 +45,7 @@ from __future__ import annotations
 
 import xarray as xr
 
-import xarray_sql as xql
-
+from _engines import EngineContext
 from _harness import (
     CaseSkipped,
     assert_grid_close,
@@ -111,7 +110,8 @@ def main() -> None:
         f"  scene window: {dict(scene.sizes)}  ({n:,} pixels, B04=red/B08=NIR)"
     )
 
-    ctx = xql.XarrayContext()
+    ctx = EngineContext()
+    print(f"  engine: {ctx.flavor}")
     ctx.from_dataset("scene", scene, chunks={"y": 256, "x": 256})
 
     sql = """
@@ -122,7 +122,7 @@ def main() -> None:
     show_sql(sql)
 
     for _ in measured("SQL NDVI"):
-        got = ctx.sql(sql).to_dataset(dims=["y", "x"]).ndvi
+        got = ctx.sql_to_dataset(sql, dims=["y", "x"]).ndvi
 
     # Array reference: the same formula in pure xarray. ``.compute()`` reads the
     # window and evaluates it here (the scene is lazy), so this measures the same

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/02_climatology.py

@@ -42,8 +42,7 @@ import datetime
 
 import xarray as xr
 
-import xarray_sql as xql
-
+from _engines import EngineContext
 from _harness import (
     CaseSkipped,
     assert_grid_close,
@@ -81,7 +80,8 @@ def main() -> None:
     except Exception as exc:  # noqa: BLE001 — any failure → skip, not crash
         raise CaseSkipped(f"ARCO-ERA5 unavailable ({exc})") from exc
 
-    ctx = xql.XarrayContext()
+    ctx = EngineContext()
+    print(f"  engine: {ctx.flavor}")
     with timed("register full ERA5 (lazy)"):
         ctx.from_dataset(
             "era5",
@@ -110,8 +110,10 @@ def main() -> None:
     # A climatology is a gridded product: round-trip the result back to an
     # xarray Dataset keyed by (latitude, longitude, hour) — how it is used.
     for _ in measured("SQL diurnal climatology (lazy read)"):
-        got = ctx.sql(sql, param_values=_PARAMS).to_dataset(
-            dims=["latitude", "longitude", "hour"]
+        got = ctx.sql_to_dataset(
+            sql,
+            dims=["latitude", "longitude", "hour"],
+            param_values=_PARAMS,
         )
 
     # Array reference: the textbook groupby-over-the-cycle reduction, in °C —

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/03_zonal_mean.py

@@ -36,8 +36,7 @@ import datetime
 
 import xarray as xr
 
-import xarray_sql as xql
-
+from _engines import EngineContext
 from _harness import (
     CaseSkipped,
     assert_grid_close,
@@ -75,7 +74,8 @@ def main() -> None:
 
     # ERA5 mixes surface (time, lat, lon) and atmospheric (… level …) variables,
     # so register it as two tables under an ``era5`` schema.
-    ctx = xql.XarrayContext()
+    ctx = EngineContext()
+    print(f"  engine: {ctx.flavor}")
     with timed("register full ERA5"):
         ctx.from_dataset(
             "era5",
@@ -101,9 +101,11 @@ def main() -> None:
 
     # Round-trip the profile back to an xarray Dataset keyed by latitude.
     for _ in measured("SQL zonal mean (reads one day)"):
-        got = ctx.sql(
-            sql, param_values={"start": _START, "end": _END}
-        ).to_dataset(dims=["latitude"])
+        got = ctx.sql_to_dataset(
+            sql,
+            dims=["latitude"],
+            param_values={"start": _START, "end": _END},
+        )
 
     # Array reference: reduce the same day over the two un-grouped axes.
     for _ in measured("xarray reference"):

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/04_anomaly.py

@@ -40,8 +40,7 @@ import datetime
 
 import xarray as xr
 
-import xarray_sql as xql
-
+from _engines import EngineContext
 from _harness import (
     CaseSkipped,
     assert_grid_close,
@@ -74,7 +73,8 @@ def main() -> None:
     except Exception as exc:  # noqa: BLE001 — any failure → skip, not crash
         raise CaseSkipped(f"ARCO-ERA5 unavailable ({exc})") from exc
 
-    ctx = xql.XarrayContext()
+    ctx = EngineContext()
+    print(f"  engine: {ctx.flavor}")
     with timed("register full ERA5 (lazy)"):
         ctx.from_dataset(
             "era5",
@@ -113,8 +113,10 @@ def main() -> None:
 
     # The anomaly is a gridded field; round-trip it to (time, lat, lon).
     for _ in measured("SQL anomaly (climatology CTE self-join, lazy read)"):
-        got = ctx.sql(sql, param_values=_PARAMS).to_dataset(
-            dims=["time", "latitude", "longitude"]
+        got = ctx.sql_to_dataset(
+            sql,
+            dims=["time", "latitude", "longitude"],
+            param_values=_PARAMS,
         )
 
     # Array reference: grouped broadcast-subtract, in pure xarray (lazy window).

{xarray_sql-0.3.3 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/05_forecast_skill.py

@@ -49,8 +49,7 @@ import numpy as np
 import pandas as pd
 import xarray as xr
 
-import xarray_sql as xql
-
+from _engines import EngineContext
 from _harness import (
     CaseSkipped,
     assert_grid_close,
@@ -144,7 +143,8 @@ def main() -> None:
         f"leads × 2 models)"
     )
 
-    ctx = xql.XarrayContext()
+    ctx = EngineContext()
+    print(f"  engine: {ctx.flavor}")
     # chunks here is the Arrow batch (partition) size each table streams in, not a
     # filter — no data is dropped. Both windows are small, so one partition each is
     # fastest (fewer partitions = fewer Python→Arrow round-trips for the same
@@ -172,7 +172,7 @@ def main() -> None:
     show_sql(sql)
 
     for _ in measured("SQL RMSE by (model, lead) — lazy JOIN"):
-        got = ctx.sql(sql).to_dataset(dims=["model", "lead"]).rmse
+        got = ctx.sql_to_dataset(sql, dims=["model", "lead"]).rmse
 
     for _ in measured("xarray reference"):
         ref = _reference_rmse(forecasts, truth)