xarray-sql 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/.gitignore +2 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/Cargo.lock +1 -1
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/Cargo.toml +23 -6
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/PKG-INFO +10 -7
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/README.md +4 -3
- xarray_sql-0.6.0/docs/browser.md +153 -0
- xarray_sql-0.6.0/docs/demo/app.py +217 -0
- xarray_sql-0.6.0/docs/demo/chunk_cache.py +238 -0
- xarray_sql-0.6.0/docs/demo/fetch_store.py +148 -0
- xarray_sql-0.6.0/docs/demo/index.html +444 -0
- xarray_sql-0.6.0/docs/demo/smoke.mjs +52 -0
- xarray_sql-0.6.0/docs/demo/worker.js +67 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/engines.md +41 -6
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/examples.md +9 -9
- xarray_sql-0.6.0/docs/explainer/era5.json +1 -0
- xarray_sql-0.6.0/docs/explainer/index.html +537 -0
- xarray_sql-0.6.0/docs/explainer/make_snapshot.py +46 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/geospatial.md +1 -1
- xarray_sql-0.6.0/docs/how-it-works.md +41 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/limitations.md +7 -6
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/performance.md +1 -1
- xarray_sql-0.6.0/flake.nix +109 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/pyproject.toml +9 -3
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/src/lib.rs +2 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_arrow_dataset.py +24 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_df.py +2 -4
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_ds.py +10 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_duckdb_backend.py +8 -3
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_lazy_roundtrip.py +74 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_sql.py +5 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_to_dataset_perf.py +3 -3
- xarray_sql-0.6.0/xarray_sql/__init__.py +46 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/base.py +7 -2
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/datafusion.py +21 -8
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/flight.py +14 -4
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/pyarrow.py +8 -8
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/cftime.py +1 -1
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/df.py +48 -9
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/ds.py +98 -48
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/lazyscan.py +32 -1
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/proj.py +2 -2
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/reader.py +6 -3
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/roundtrip.py +75 -22
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/sql.py +12 -4
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/zensical.toml +2 -0
- xarray_sql-0.5.0/flake.nix +0 -44
- xarray_sql-0.5.0/xarray_sql/__init__.py +0 -28
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/.envrc +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/AGENTS.md +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/LICENSE +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/duckdb_pushdown.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/01_ndvi.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/02_climatology.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/03_zonal_mean.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/04_anomaly.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/05_forecast_skill.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/06_zonal_vector.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/07_reproject_udf.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/08_regrid_weights.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/09_warp.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/README.md +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/_engines.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/_harness.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/engine_suite.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/perf_summary.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/run_all.sh +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/run_perf.sh +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/assets/logo.svg +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/contributing.md +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/index.md +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/reference/xarray_sql.md +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/flake.lock +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/src/flight.rs +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/__init__.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/_adbc.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/conftest.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_adbc_backend.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_adbc_era5_integration.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_arrow_dataset_integration.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_cft.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_coord_lookup.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_flight_sql.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_geometry.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_proj.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_reader.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_sql_recipes.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_stats.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_table_names.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/__init__.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/_adbc_dialects.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/adbc.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/duckdb.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/core.py +0 -0
- {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/geometry.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "xarray_sql"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.6.0"
|
|
4
4
|
authors = ["Alex Merose"]
|
|
5
5
|
edition = "2021"
|
|
6
6
|
exclude = [
|
|
@@ -20,22 +20,39 @@ readme = "README.md"
|
|
|
20
20
|
|
|
21
21
|
[dependencies]
|
|
22
22
|
arrow = { version = "58", features = ["pyarrow"] }
|
|
23
|
-
arrow-flight = { version = "58", features = ["flight-sql"] }
|
|
23
|
+
arrow-flight = { version = "58", features = ["flight-sql"], optional = true }
|
|
24
24
|
async-stream = "0.3"
|
|
25
25
|
async-trait = "0.1"
|
|
26
|
-
datafusion = { version = "54.0.0" }
|
|
26
|
+
datafusion = { version = "54.0.0", default-features = false }
|
|
27
27
|
datafusion-ffi = { version = "54.0.0" }
|
|
28
28
|
futures = { version = "0.3" }
|
|
29
29
|
half = "2.7"
|
|
30
|
-
prost = "0.14"
|
|
30
|
+
prost = { version = "0.14", optional = true }
|
|
31
31
|
# `abi3-py310` builds against CPython's stable ABI, so a single wheel per
|
|
32
32
|
# platform works on all CPython >= 3.10 (matching `requires-python`). Maturin
|
|
33
33
|
# enables `pyo3/extension-module` through pyproject.toml for wheel builds; it
|
|
34
34
|
# must stay disabled for ordinary Cargo test binaries so they link libpython.
|
|
35
35
|
pyo3 = { version = "0.28.0", features = ["abi3-py310"] }
|
|
36
|
-
tokio = { version = "1.46.1", features = ["macros", "
|
|
37
|
-
tonic = { version = "0.14", features = ["transport"] }
|
|
36
|
+
tokio = { version = "1.46.1", features = ["macros", "rt", "sync", "time"] }
|
|
37
|
+
tonic = { version = "0.14", features = ["transport"], optional = true }
|
|
38
38
|
|
|
39
|
+
[features]
|
|
40
|
+
default = ["flight"]
|
|
41
|
+
# The Arrow Flight SQL server needs TCP sockets and OS threads, neither of
|
|
42
|
+
# which exist under WebAssembly. Pyodide builds disable default features.
|
|
43
|
+
# The server plans SQL, so it also needs DataFusion's default features
|
|
44
|
+
# (SQL, function libraries, Parquet, compression).
|
|
45
|
+
flight = ["dep:arrow-flight", "dep:prost", "dep:tonic", "datafusion/default", "tokio/net", "tokio/rt-multi-thread"]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# Pyodide downloads the wheel on every page load, so wasm builds trade
|
|
49
|
+
# speed for size: `MATURIN_PEP517_ARGS="--no-default-features --profile wasm"`.
|
|
50
|
+
[profile.wasm]
|
|
51
|
+
inherits = "release"
|
|
52
|
+
opt-level = "z"
|
|
53
|
+
lto = true
|
|
54
|
+
codegen-units = 1
|
|
55
|
+
strip = true
|
|
39
56
|
|
|
40
57
|
[build-dependencies]
|
|
41
58
|
pyo3-build-config = "0.28"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xarray_sql
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Science/Research
|
|
6
6
|
Classifier: Intended Audience :: Developers
|
|
@@ -16,10 +16,11 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
16
16
|
Classifier: Programming Language :: Rust
|
|
17
17
|
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
18
18
|
Classifier: Topic :: Database :: Front-Ends
|
|
19
|
-
Requires-Dist:
|
|
20
|
-
Requires-Dist:
|
|
19
|
+
Requires-Dist: datafusion==54.0.0 ; sys_platform != 'emscripten'
|
|
20
|
+
Requires-Dist: pyarrow>=16.0.0
|
|
21
21
|
Requires-Dist: xarray>=2024.7.0
|
|
22
22
|
Requires-Dist: adbc-driver-manager>=1.12 ; extra == 'adbc'
|
|
23
|
+
Requires-Dist: dask>=2024.8.0 ; extra == 'dask'
|
|
23
24
|
Requires-Dist: xarray-sql[docs] ; extra == 'dev'
|
|
24
25
|
Requires-Dist: pre-commit ; extra == 'dev'
|
|
25
26
|
Requires-Dist: pytest ; extra == 'dev'
|
|
@@ -32,11 +33,12 @@ Requires-Dist: polars>=1.33 ; extra == 'polars'
|
|
|
32
33
|
Requires-Dist: adbc-driver-flightsql>=1.12 ; extra == 'test'
|
|
33
34
|
Requires-Dist: adbc-driver-sqlite>=1.12 ; extra == 'test'
|
|
34
35
|
Requires-Dist: cftime ; extra == 'test'
|
|
35
|
-
Requires-Dist: xarray-sql[adbc,duckdb,polars,geo] ; extra == 'test'
|
|
36
|
+
Requires-Dist: xarray-sql[adbc,dask,duckdb,polars,geo] ; extra == 'test'
|
|
36
37
|
Requires-Dist: pytest ; extra == 'test'
|
|
37
38
|
Requires-Dist: xarray[io] ; extra == 'test'
|
|
38
39
|
Requires-Dist: gcsfs ; extra == 'test'
|
|
39
40
|
Provides-Extra: adbc
|
|
41
|
+
Provides-Extra: dask
|
|
40
42
|
Provides-Extra: dev
|
|
41
43
|
Provides-Extra: docs
|
|
42
44
|
Provides-Extra: duckdb
|
|
@@ -94,7 +96,7 @@ import xarray_sql as xql
|
|
|
94
96
|
# 4x-daily surface air temperature on a lat/lon grid, 2013-2014.
|
|
95
97
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
96
98
|
|
|
97
|
-
ctx = xql.
|
|
99
|
+
ctx = xql.Context()
|
|
98
100
|
ctx.from_dataset('air', ds, chunks=dict(time=100))
|
|
99
101
|
|
|
100
102
|
# A climatology — the mean annual cycle — computed in SQL: average air
|
|
@@ -164,8 +166,9 @@ ds = xr.open_zarr(
|
|
|
164
166
|
storage_options={'token': 'anon'} # Anonymous read from the public GCS bucket — no auth required.
|
|
165
167
|
)
|
|
166
168
|
|
|
167
|
-
ctx = xql.
|
|
168
|
-
#
|
|
169
|
+
ctx = xql.Context()
|
|
170
|
+
# `chunks` sets the partition size; without it, partitions follow the
|
|
171
|
+
# store's own chunks (one hour each here).
|
|
169
172
|
ctx.from_dataset('era5', ds, chunks=dict(time=6), table_names={
|
|
170
173
|
('time', 'latitude', 'longitude'): 'surface',
|
|
171
174
|
('time', 'level', 'latitude', 'longitude'): 'atmosphere',
|
|
@@ -40,7 +40,7 @@ import xarray_sql as xql
|
|
|
40
40
|
# 4x-daily surface air temperature on a lat/lon grid, 2013-2014.
|
|
41
41
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
42
42
|
|
|
43
|
-
ctx = xql.
|
|
43
|
+
ctx = xql.Context()
|
|
44
44
|
ctx.from_dataset('air', ds, chunks=dict(time=100))
|
|
45
45
|
|
|
46
46
|
# A climatology — the mean annual cycle — computed in SQL: average air
|
|
@@ -110,8 +110,9 @@ ds = xr.open_zarr(
|
|
|
110
110
|
storage_options={'token': 'anon'} # Anonymous read from the public GCS bucket — no auth required.
|
|
111
111
|
)
|
|
112
112
|
|
|
113
|
-
ctx = xql.
|
|
114
|
-
#
|
|
113
|
+
ctx = xql.Context()
|
|
114
|
+
# `chunks` sets the partition size; without it, partitions follow the
|
|
115
|
+
# store's own chunks (one hour each here).
|
|
115
116
|
ctx.from_dataset('era5', ds, chunks=dict(time=6), table_names={
|
|
116
117
|
('time', 'latitude', 'longitude'): 'surface',
|
|
117
118
|
('time', 'level', 'latitude', 'longitude'): 'atmosphere',
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# Run in the browser
|
|
2
|
+
|
|
3
|
+
xarray-sql runs in [Pyodide](https://pyodide.org), Python compiled to
|
|
4
|
+
WebAssembly. The demo below runs SQL over NOAA GFS forecasts read straight
|
|
5
|
+
from [dynamical.org](https://dynamical.org)'s public Zarr store. xarray-sql
|
|
6
|
+
and DuckDB run in a Web Worker in your browser, so there is no server. Each
|
|
7
|
+
query downloads only the chunks that its filters on `init_time`, `latitude`,
|
|
8
|
+
and `longitude` reach.
|
|
9
|
+
|
|
10
|
+
The demo needs Chrome or Edge 137 or newer. It takes about 15 seconds to
|
|
11
|
+
start, most of it downloading and loading Pyodide, pyarrow, DuckDB, and
|
|
12
|
+
pandas.
|
|
13
|
+
|
|
14
|
+
<iframe id="demo" src="../demo/" title="xarray-sql browser demo" loading="lazy"
|
|
15
|
+
style="width: 100%; height: 1150px; border: 1px solid var(--md-default-fg-color--lightest); border-radius: 8px;"></iframe>
|
|
16
|
+
|
|
17
|
+
[Open the demo in its own page](demo/){ #demo-link target="_blank" }
|
|
18
|
+
|
|
19
|
+
<script>
|
|
20
|
+
// Pass ?wheel=<url> through to the demo, to try a locally built wheel.
|
|
21
|
+
(() => {
|
|
22
|
+
const wheel = new URLSearchParams(location.search).get("wheel");
|
|
23
|
+
if (!wheel) return;
|
|
24
|
+
for (const [id, attr] of [["demo", "src"], ["demo-link", "href"]]) {
|
|
25
|
+
const el = document.getElementById(id);
|
|
26
|
+
const url = new URL(el.getAttribute(attr), location.href);
|
|
27
|
+
url.searchParams.set("wheel", new URL(wheel, location.href).href);
|
|
28
|
+
el.setAttribute(attr, url.href);
|
|
29
|
+
}
|
|
30
|
+
})();
|
|
31
|
+
</script>
|
|
32
|
+
|
|
33
|
+
## Install in Pyodide
|
|
34
|
+
|
|
35
|
+
Releases from 0.6.0 on publish a Pyodide wheel to PyPI. The DuckDB engine
|
|
36
|
+
adapter works in Pyodide, including the round-trip back to a Dataset with
|
|
37
|
+
[to_dataset][xarray_sql.to_dataset]. DataFusion does not yet, because its
|
|
38
|
+
Python bindings have no Pyodide build, so [Context][xarray_sql.Context] is
|
|
39
|
+
unavailable there.
|
|
40
|
+
|
|
41
|
+
Copy [`fetch_store.py`](https://github.com/xqlsystems/xarray-sql/tree/main/docs/demo/fetch_store.py)
|
|
42
|
+
from the demo to open Zarr stores over HTTP in Pyodide, since fsspec's HTTP
|
|
43
|
+
filesystem needs sockets. Then open a window of the dataset, read it into
|
|
44
|
+
memory, and query it:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import micropip
|
|
48
|
+
await micropip.install(["xarray-sql", "duckdb", "zarr"])
|
|
49
|
+
|
|
50
|
+
import duckdb
|
|
51
|
+
import xarray as xr
|
|
52
|
+
import xarray_sql as xql
|
|
53
|
+
from fetch_store import FetchStore
|
|
54
|
+
|
|
55
|
+
store = FetchStore("https://data.dynamical.org/noaa/gfs/forecast/latest.zarr")
|
|
56
|
+
ds = (
|
|
57
|
+
xr.open_zarr(store, chunks=None)[["temperature_2m"]]
|
|
58
|
+
.isel(init_time=-1, lead_time=slice(0, 24))
|
|
59
|
+
.sel(latitude=slice(50, 25), longitude=slice(-100, -75))
|
|
60
|
+
)
|
|
61
|
+
ds = await ds.load_async()
|
|
62
|
+
|
|
63
|
+
con = duckdb.connect()
|
|
64
|
+
xql.register(con, "gfs", ds)
|
|
65
|
+
hottest = con.sql("""
|
|
66
|
+
SELECT latitude, longitude, max(temperature_2m) AS t2m_max
|
|
67
|
+
FROM gfs
|
|
68
|
+
GROUP BY latitude, longitude
|
|
69
|
+
""")
|
|
70
|
+
out = xql.to_dataset(hottest, template=ds) # dims latitude, longitude
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The window is read into memory first because a DuckDB scan cannot wait on
|
|
74
|
+
the network in Pyodide (see the next section). Pyodide also cannot start
|
|
75
|
+
threads, so xarray-sql loads chunks one at a time there. To query a larger
|
|
76
|
+
store, read it through a cache like the demo's.
|
|
77
|
+
|
|
78
|
+
## How lazy reads work in the browser
|
|
79
|
+
|
|
80
|
+
Browsers can only read over the network asynchronously. zarr's synchronous
|
|
81
|
+
API waits on those reads with Pyodide's JavaScript Promise Integration
|
|
82
|
+
(JSPI), but JSPI cannot suspend while DuckDB or pyarrow is calling back into
|
|
83
|
+
Python: their C++ frames sit between the suspension point and Python. So a
|
|
84
|
+
query cannot read chunks in the middle of a scan.
|
|
85
|
+
|
|
86
|
+
The demo's `ChunkCache` runs each query against a cache of decoded chunks
|
|
87
|
+
instead:
|
|
88
|
+
|
|
89
|
+
1. Run the query. A chunk that is not cached is recorded as missing and
|
|
90
|
+
scanned as fill values, so one run records every chunk that survives
|
|
91
|
+
xarray-sql's chunk pruning.
|
|
92
|
+
2. Fetch the missing chunks concurrently with `Dataset.load_async`, then run
|
|
93
|
+
the query again.
|
|
94
|
+
3. A run with no missing chunks used only real data, so its result is
|
|
95
|
+
exact.
|
|
96
|
+
|
|
97
|
+
Most queries need two runs. The stats line under the editor shows the runs,
|
|
98
|
+
chunks, requests, and bytes for each query.
|
|
99
|
+
|
|
100
|
+
The demo's files are in
|
|
101
|
+
[`docs/demo/`](https://github.com/xqlsystems/xarray-sql/tree/main/docs/demo):
|
|
102
|
+
|
|
103
|
+
| File | Role |
|
|
104
|
+
|------|------|
|
|
105
|
+
| `index.html` | The page: example queries, SQL editor, results table, and an interactive map ([deck.gl](https://deck.gl) over a [MapLibre](https://maplibre.org) basemap) for `latitude, longitude, value` results. |
|
|
106
|
+
| `worker.js` | Loads Pyodide, installs xarray-sql, DuckDB, and zarr, and runs queries off the main thread. |
|
|
107
|
+
| `app.py` | Opens the dataset, registers it on DuckDB as `gfs`, and runs queries. |
|
|
108
|
+
| `chunk_cache.py` | Lazy reads that work in the browser, as described above. |
|
|
109
|
+
| `fetch_store.py` | A read-only Zarr store that uses `fetch` with `Range` requests. |
|
|
110
|
+
| `smoke.mjs` | Runs every example query headlessly in Node. |
|
|
111
|
+
|
|
112
|
+
## Run it locally
|
|
113
|
+
|
|
114
|
+
Serve `docs/demo` over HTTP and open it in Chrome or Edge:
|
|
115
|
+
|
|
116
|
+
```shell
|
|
117
|
+
python -m http.server 8000 --directory docs/demo
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
By default the worker installs xarray-sql from PyPI. To test a local build,
|
|
121
|
+
copy the wheel from `pyodide build` (see "Building for WebAssembly" in the
|
|
122
|
+
[contributing guide](contributing.md)) into `docs/demo` and pass it in the
|
|
123
|
+
URL:
|
|
124
|
+
|
|
125
|
+
```
|
|
126
|
+
http://localhost:8000/?wheel=./xarray_sql-0.6.0-cp310-abi3-pyemscripten_2026_0_wasm32.whl
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
On this docs page, `?wheel=` is passed through to the embedded demo, so a
|
|
130
|
+
locally built docs site can use a local wheel too (for example,
|
|
131
|
+
`/browser/?wheel=../demo/xarray_sql-...whl` after copying the wheel into the
|
|
132
|
+
built `demo/` directory).
|
|
133
|
+
|
|
134
|
+
To run the examples without a browser:
|
|
135
|
+
|
|
136
|
+
```shell
|
|
137
|
+
cd docs/demo
|
|
138
|
+
npm install --no-save pyodide@314.0.7
|
|
139
|
+
node smoke.mjs path/to/xarray_sql-*-pyemscripten_*.whl
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Known limits
|
|
143
|
+
|
|
144
|
+
- **`lead_time` filters do not prune chunks.** DuckDB does not push
|
|
145
|
+
`INTERVAL` predicates down to Arrow scans. The demo registers only the
|
|
146
|
+
first lead-time chunk (forecast hours 0–104), so every query reads at most
|
|
147
|
+
that one.
|
|
148
|
+
- **Queries that touch the whole grid are slow.** For example,
|
|
149
|
+
`SELECT max(init_time) FROM gfs` builds every coordinate row (about 440
|
|
150
|
+
million) even though it downloads nothing. Use the literal timestamps that
|
|
151
|
+
the example queries use.
|
|
152
|
+
- **The cache holds 768 MiB of decoded chunks.** A query that needs more
|
|
153
|
+
fails with a message to narrow its filters.
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""The Python half of the browser demo: open a remote Zarr, query it with SQL.
|
|
2
|
+
|
|
3
|
+
``worker.js`` loads this module into Pyodide and calls :func:`setup` once
|
|
4
|
+
and :func:`run` per query. Both return JSON strings so the worker can hand
|
|
5
|
+
results to the page without converting Python objects.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import math
|
|
12
|
+
from decimal import Decimal
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import duckdb
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import pyarrow as pa
|
|
19
|
+
import xarray as xr
|
|
20
|
+
from chunk_cache import ChunkCache
|
|
21
|
+
from fetch_store import FetchStore
|
|
22
|
+
|
|
23
|
+
import xarray_sql as xql
|
|
24
|
+
|
|
25
|
+
GFS_URL = "https://data.dynamical.org/noaa/gfs/forecast/latest.zarr"
|
|
26
|
+
GFS_VARIABLES = [
|
|
27
|
+
"temperature_2m",
|
|
28
|
+
"precipitation_surface",
|
|
29
|
+
"wind_u_10m",
|
|
30
|
+
"wind_v_10m",
|
|
31
|
+
]
|
|
32
|
+
# The store's inner chunks; one cached chunk is then one range request.
|
|
33
|
+
GFS_CHUNKS = {
|
|
34
|
+
"init_time": 1,
|
|
35
|
+
"lead_time": 105,
|
|
36
|
+
"latitude": 121,
|
|
37
|
+
"longitude": 121,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
_state: dict[str, Any] = {}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _ts(value: Any) -> str:
|
|
44
|
+
return str(pd.Timestamp(value))[:19]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _examples(ds: xr.Dataset) -> list[dict[str, str]]:
|
|
48
|
+
latest = _ts(ds.init_time.values[-1])
|
|
49
|
+
earliest = _ts(ds.init_time.values[0])
|
|
50
|
+
return [
|
|
51
|
+
{
|
|
52
|
+
"title": "San Francisco: 2 m temperature for the next 3 days",
|
|
53
|
+
"sql": f"""SELECT init_time + lead_time AS valid_time,
|
|
54
|
+
round(avg(temperature_2m), 1) AS temperature_c
|
|
55
|
+
FROM gfs
|
|
56
|
+
WHERE init_time = TIMESTAMP '{latest}'
|
|
57
|
+
AND lead_time <= INTERVAL 72 HOUR
|
|
58
|
+
AND latitude BETWEEN 37.5 AND 38 AND longitude BETWEEN -122.75 AND -122.25
|
|
59
|
+
GROUP BY ALL ORDER BY valid_time""",
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"title": "Map: Europe 2 m temperature, 24 h ahead",
|
|
63
|
+
"sql": f"""SELECT latitude, longitude, temperature_2m
|
|
64
|
+
FROM gfs
|
|
65
|
+
WHERE init_time = TIMESTAMP '{latest}'
|
|
66
|
+
AND lead_time = INTERVAL 24 HOUR
|
|
67
|
+
AND latitude BETWEEN 35 AND 70 AND longitude BETWEEN -10 AND 40""",
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"title": "Map: contiguous US wind speed, 12 h ahead",
|
|
71
|
+
"sql": f"""SELECT latitude, longitude,
|
|
72
|
+
round(sqrt(wind_u_10m ^ 2 + wind_v_10m ^ 2), 1) AS wind_speed
|
|
73
|
+
FROM gfs
|
|
74
|
+
WHERE init_time = TIMESTAMP '{latest}'
|
|
75
|
+
AND lead_time = INTERVAL 12 HOUR
|
|
76
|
+
AND latitude BETWEEN 24 AND 50 AND longitude BETWEEN -125 AND -66""",
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"title": "Wettest places in the contiguous US, next 24 h",
|
|
80
|
+
"sql": f"""SELECT latitude, longitude,
|
|
81
|
+
round(sum(precipitation_surface) * 3600, 1) AS approx_precip_mm
|
|
82
|
+
FROM gfs
|
|
83
|
+
WHERE init_time = TIMESTAMP '{latest}'
|
|
84
|
+
AND lead_time BETWEEN INTERVAL 1 HOUR AND INTERVAL 24 HOUR
|
|
85
|
+
AND latitude BETWEEN 24 AND 50 AND longitude BETWEEN -125 AND -66
|
|
86
|
+
GROUP BY ALL ORDER BY approx_precip_mm DESC LIMIT 10""",
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"title": "How the forecast for San Francisco changed across runs",
|
|
90
|
+
"sql": f"""SELECT init_time,
|
|
91
|
+
round(max(temperature_2m), 1) AS max_temperature_c
|
|
92
|
+
FROM gfs
|
|
93
|
+
WHERE init_time BETWEEN TIMESTAMP '{earliest}' AND TIMESTAMP '{latest}'
|
|
94
|
+
AND init_time + lead_time BETWEEN TIMESTAMP '{latest}' + INTERVAL 24 HOUR
|
|
95
|
+
AND TIMESTAMP '{latest}' + INTERVAL 48 HOUR
|
|
96
|
+
AND lead_time <= INTERVAL 72 HOUR
|
|
97
|
+
AND latitude BETWEEN 37.5 AND 38 AND longitude BETWEEN -122.75 AND -122.25
|
|
98
|
+
GROUP BY ALL ORDER BY init_time""",
|
|
99
|
+
},
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
async def setup(
|
|
104
|
+
url: str = GFS_URL,
|
|
105
|
+
variables: list[str] | None = None,
|
|
106
|
+
runs: int = 4,
|
|
107
|
+
max_cache_mib: int = 768,
|
|
108
|
+
) -> str:
|
|
109
|
+
"""Open the dataset and register it on a DuckDB connection as ``gfs``."""
|
|
110
|
+
store = FetchStore(url)
|
|
111
|
+
ds = xr.open_zarr(store, chunks=None)
|
|
112
|
+
# DuckDB does not push INTERVAL predicates down to Arrow scans, so a
|
|
113
|
+
# lead_time filter cannot prune chunks. Keeping only the first lead-time
|
|
114
|
+
# chunk (forecast hours 0-104) halves what every query downloads.
|
|
115
|
+
ds = ds[variables or GFS_VARIABLES].isel(
|
|
116
|
+
init_time=slice(-runs, None),
|
|
117
|
+
lead_time=slice(0, GFS_CHUNKS["lead_time"]),
|
|
118
|
+
)
|
|
119
|
+
cache = ChunkCache(ds, GFS_CHUNKS, max_bytes=max_cache_mib * 2**20)
|
|
120
|
+
con = duckdb.connect()
|
|
121
|
+
con.sql("SET enable_progress_bar = false")
|
|
122
|
+
xql.register(con, "gfs", cache.dataset(), chunks=GFS_CHUNKS)
|
|
123
|
+
_state.update(store=store, cache=cache, con=con)
|
|
124
|
+
return json.dumps(
|
|
125
|
+
{
|
|
126
|
+
"url": url,
|
|
127
|
+
"variables": {
|
|
128
|
+
name: {
|
|
129
|
+
"dims": list(var.dims),
|
|
130
|
+
"units": var.attrs.get("units", ""),
|
|
131
|
+
"long_name": var.attrs.get("long_name", name),
|
|
132
|
+
}
|
|
133
|
+
for name, var in ds.data_vars.items()
|
|
134
|
+
},
|
|
135
|
+
"sizes": dict(ds.sizes),
|
|
136
|
+
"init_times": [_ts(t) for t in ds.init_time.values],
|
|
137
|
+
"examples": _examples(ds),
|
|
138
|
+
}
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _jsonable(value: Any) -> Any:
|
|
143
|
+
"""A Python value from Arrow, made safe for JSON (which has no NaN)."""
|
|
144
|
+
if isinstance(value, float):
|
|
145
|
+
return value if math.isfinite(value) else None
|
|
146
|
+
if isinstance(value, Decimal):
|
|
147
|
+
return float(value)
|
|
148
|
+
# Exactly tuple: Arrow intervals are named tuples, shown as text below.
|
|
149
|
+
if isinstance(value, list) or type(value) is tuple:
|
|
150
|
+
return [_jsonable(v) for v in value]
|
|
151
|
+
if isinstance(value, dict):
|
|
152
|
+
return {str(k): _jsonable(v) for k, v in value.items()}
|
|
153
|
+
if value is None or isinstance(value, (str, int)):
|
|
154
|
+
return value
|
|
155
|
+
return str(value) # Timestamps, intervals, bytes, ...
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _grid(table: pa.Table) -> dict[str, Any] | None:
|
|
159
|
+
"""Columns for the page's map: latitude, longitude, and one number."""
|
|
160
|
+
names = table.column_names
|
|
161
|
+
if "latitude" not in names or "longitude" not in names:
|
|
162
|
+
return None
|
|
163
|
+
values = [
|
|
164
|
+
n
|
|
165
|
+
for n in names
|
|
166
|
+
if n not in ("latitude", "longitude")
|
|
167
|
+
and (
|
|
168
|
+
pa.types.is_floating(table[n].type)
|
|
169
|
+
or pa.types.is_integer(table[n].type)
|
|
170
|
+
or pa.types.is_decimal(table[n].type)
|
|
171
|
+
)
|
|
172
|
+
]
|
|
173
|
+
if len(values) != 1 or table.num_rows < 4:
|
|
174
|
+
return None
|
|
175
|
+
lat = table["latitude"].to_numpy().astype("float64")
|
|
176
|
+
lon = table["longitude"].to_numpy().astype("float64")
|
|
177
|
+
# One pixel per point: with several rows per point (e.g. every lead
|
|
178
|
+
# time), the table is the honest view.
|
|
179
|
+
if len(np.unique(np.stack([lat, lon]), axis=1)[0]) != table.num_rows:
|
|
180
|
+
return None
|
|
181
|
+
value = table[values[0]].to_numpy().astype("float64")
|
|
182
|
+
finite = np.isfinite(value)
|
|
183
|
+
if not finite.any():
|
|
184
|
+
return None
|
|
185
|
+
return {
|
|
186
|
+
"name": values[0],
|
|
187
|
+
"latitude": lat[finite].tolist(),
|
|
188
|
+
"longitude": lon[finite].tolist(),
|
|
189
|
+
"value": value[finite].tolist(),
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
async def run(sql: str, max_rows: int = 500) -> str:
|
|
194
|
+
"""Run ``sql``, fetching whatever chunks it needs, and return the result."""
|
|
195
|
+
store, cache, con = _state["store"], _state["cache"], _state["con"]
|
|
196
|
+
requests, nbytes = store.requests, store.bytes_read
|
|
197
|
+
table, stats = await cache.query(lambda: con.sql(sql).to_arrow_table())
|
|
198
|
+
shown = table.slice(0, max_rows)
|
|
199
|
+
return json.dumps(
|
|
200
|
+
{
|
|
201
|
+
"columns": shown.column_names,
|
|
202
|
+
"rows": _jsonable(
|
|
203
|
+
list(zip(*(shown[c].to_pylist() for c in shown.column_names)))
|
|
204
|
+
),
|
|
205
|
+
"num_rows": table.num_rows,
|
|
206
|
+
"grid": _grid(table),
|
|
207
|
+
"stats": {
|
|
208
|
+
"runs": stats.rounds,
|
|
209
|
+
"chunks_fetched": stats.chunks_fetched,
|
|
210
|
+
"requests": store.requests - requests,
|
|
211
|
+
"mb_downloaded": round((store.bytes_read - nbytes) / 1e6, 1),
|
|
212
|
+
"seconds_fetching": round(stats.seconds_fetching, 2),
|
|
213
|
+
"seconds_querying": round(stats.seconds_querying, 2),
|
|
214
|
+
"cached_chunks": len(cache),
|
|
215
|
+
},
|
|
216
|
+
}
|
|
217
|
+
)
|