xarray-sql 0.5.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/.gitignore +2 -0
  2. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/Cargo.lock +1 -1
  3. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/Cargo.toml +23 -6
  4. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/PKG-INFO +10 -7
  5. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/README.md +4 -3
  6. xarray_sql-0.6.0/docs/browser.md +153 -0
  7. xarray_sql-0.6.0/docs/demo/app.py +217 -0
  8. xarray_sql-0.6.0/docs/demo/chunk_cache.py +238 -0
  9. xarray_sql-0.6.0/docs/demo/fetch_store.py +148 -0
  10. xarray_sql-0.6.0/docs/demo/index.html +444 -0
  11. xarray_sql-0.6.0/docs/demo/smoke.mjs +52 -0
  12. xarray_sql-0.6.0/docs/demo/worker.js +67 -0
  13. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/engines.md +41 -6
  14. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/examples.md +9 -9
  15. xarray_sql-0.6.0/docs/explainer/era5.json +1 -0
  16. xarray_sql-0.6.0/docs/explainer/index.html +537 -0
  17. xarray_sql-0.6.0/docs/explainer/make_snapshot.py +46 -0
  18. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/geospatial.md +1 -1
  19. xarray_sql-0.6.0/docs/how-it-works.md +41 -0
  20. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/limitations.md +7 -6
  21. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/performance.md +1 -1
  22. xarray_sql-0.6.0/flake.nix +109 -0
  23. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/pyproject.toml +9 -3
  24. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/src/lib.rs +2 -0
  25. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_arrow_dataset.py +24 -0
  26. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_df.py +2 -4
  27. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_ds.py +10 -0
  28. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_duckdb_backend.py +8 -3
  29. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_lazy_roundtrip.py +74 -0
  30. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_sql.py +5 -0
  31. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_to_dataset_perf.py +3 -3
  32. xarray_sql-0.6.0/xarray_sql/__init__.py +46 -0
  33. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/base.py +7 -2
  34. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/datafusion.py +21 -8
  35. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/flight.py +14 -4
  36. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/pyarrow.py +8 -8
  37. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/cftime.py +1 -1
  38. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/df.py +48 -9
  39. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/ds.py +98 -48
  40. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/lazyscan.py +32 -1
  41. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/proj.py +2 -2
  42. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/reader.py +6 -3
  43. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/roundtrip.py +75 -22
  44. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/sql.py +12 -4
  45. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/zensical.toml +2 -0
  46. xarray_sql-0.5.0/flake.nix +0 -44
  47. xarray_sql-0.5.0/xarray_sql/__init__.py +0 -28
  48. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/.envrc +0 -0
  49. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/AGENTS.md +0 -0
  50. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/LICENSE +0 -0
  51. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/duckdb_pushdown.py +0 -0
  52. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/01_ndvi.py +0 -0
  53. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/02_climatology.py +0 -0
  54. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/03_zonal_mean.py +0 -0
  55. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/04_anomaly.py +0 -0
  56. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/05_forecast_skill.py +0 -0
  57. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/06_zonal_vector.py +0 -0
  58. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/07_reproject_udf.py +0 -0
  59. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/08_regrid_weights.py +0 -0
  60. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/09_warp.py +0 -0
  61. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/README.md +0 -0
  62. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/_engines.py +0 -0
  63. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/_harness.py +0 -0
  64. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/engine_suite.py +0 -0
  65. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/perf_summary.py +0 -0
  66. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/run_all.sh +0 -0
  67. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/benchmarks/geospatial/run_perf.sh +0 -0
  68. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/assets/logo.svg +0 -0
  69. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/contributing.md +0 -0
  70. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/index.md +0 -0
  71. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/docs/reference/xarray_sql.md +0 -0
  72. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/flake.lock +0 -0
  73. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/src/flight.rs +0 -0
  74. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/__init__.py +0 -0
  75. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/_adbc.py +0 -0
  76. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/conftest.py +0 -0
  77. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_adbc_backend.py +0 -0
  78. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_adbc_era5_integration.py +0 -0
  79. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_arrow_dataset_integration.py +0 -0
  80. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_cft.py +0 -0
  81. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_coord_lookup.py +0 -0
  82. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_flight_sql.py +0 -0
  83. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_geometry.py +0 -0
  84. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_proj.py +0 -0
  85. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_reader.py +0 -0
  86. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_sql_recipes.py +0 -0
  87. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_stats.py +0 -0
  88. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/tests/test_table_names.py +0 -0
  89. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/__init__.py +0 -0
  90. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/_adbc_dialects.py +0 -0
  91. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/adbc.py +0 -0
  92. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/backends/duckdb.py +0 -0
  93. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/core.py +0 -0
  94. {xarray_sql-0.5.0 → xarray_sql-0.6.0}/xarray_sql/geometry.py +0 -0
@@ -15,3 +15,5 @@ CHANGELOG.md
15
15
  *.ipynb
16
16
  /site
17
17
  .direnv
18
+ # Pyodide wheels copied in to test the browser demo locally
19
+ docs/demo/*.whl
@@ -3703,7 +3703,7 @@ checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
3703
3703
 
3704
3704
  [[package]]
3705
3705
  name = "xarray_sql"
3706
- version = "0.5.0"
3706
+ version = "0.6.0"
3707
3707
  dependencies = [
3708
3708
  "arrow",
3709
3709
  "arrow-flight",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "xarray_sql"
3
- version = "0.5.0"
3
+ version = "0.6.0"
4
4
  authors = ["Alex Merose"]
5
5
  edition = "2021"
6
6
  exclude = [
@@ -20,22 +20,39 @@ readme = "README.md"
20
20
 
21
21
  [dependencies]
22
22
  arrow = { version = "58", features = ["pyarrow"] }
23
- arrow-flight = { version = "58", features = ["flight-sql"] }
23
+ arrow-flight = { version = "58", features = ["flight-sql"], optional = true }
24
24
  async-stream = "0.3"
25
25
  async-trait = "0.1"
26
- datafusion = { version = "54.0.0" }
26
+ datafusion = { version = "54.0.0", default-features = false }
27
27
  datafusion-ffi = { version = "54.0.0" }
28
28
  futures = { version = "0.3" }
29
29
  half = "2.7"
30
- prost = "0.14"
30
+ prost = { version = "0.14", optional = true }
31
31
  # `abi3-py310` builds against CPython's stable ABI, so a single wheel per
32
32
  # platform works on all CPython >= 3.10 (matching `requires-python`). Maturin
33
33
  # enables `pyo3/extension-module` through pyproject.toml for wheel builds; it
34
34
  # must stay disabled for ordinary Cargo test binaries so they link libpython.
35
35
  pyo3 = { version = "0.28.0", features = ["abi3-py310"] }
36
- tokio = { version = "1.46.1", features = ["macros", "net", "rt", "rt-multi-thread", "sync", "time"] }
37
- tonic = { version = "0.14", features = ["transport"] }
36
+ tokio = { version = "1.46.1", features = ["macros", "rt", "sync", "time"] }
37
+ tonic = { version = "0.14", features = ["transport"], optional = true }
38
38
 
39
+ [features]
40
+ default = ["flight"]
41
+ # The Arrow Flight SQL server needs TCP sockets and OS threads, neither of
42
+ # which exist under WebAssembly. Pyodide builds disable default features.
43
+ # The server plans SQL, so it also needs DataFusion's default features
44
+ # (SQL, function libraries, Parquet, compression).
45
+ flight = ["dep:arrow-flight", "dep:prost", "dep:tonic", "datafusion/default", "tokio/net", "tokio/rt-multi-thread"]
46
+
47
+
48
+ # Pyodide downloads the wheel on every page load, so wasm builds trade
49
+ # speed for size: `MATURIN_PEP517_ARGS="--no-default-features --profile wasm"`.
50
+ [profile.wasm]
51
+ inherits = "release"
52
+ opt-level = "z"
53
+ lto = true
54
+ codegen-units = 1
55
+ strip = true
39
56
 
40
57
  [build-dependencies]
41
58
  pyo3-build-config = "0.28"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.5.0
3
+ Version: 0.6.0
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Science/Research
6
6
  Classifier: Intended Audience :: Developers
@@ -16,10 +16,11 @@ Classifier: Programming Language :: Python :: 3.13
16
16
  Classifier: Programming Language :: Rust
17
17
  Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
18
18
  Classifier: Topic :: Database :: Front-Ends
19
- Requires-Dist: dask>=2024.8.0
20
- Requires-Dist: datafusion==54.0.0
19
+ Requires-Dist: datafusion==54.0.0 ; sys_platform != 'emscripten'
20
+ Requires-Dist: pyarrow>=16.0.0
21
21
  Requires-Dist: xarray>=2024.7.0
22
22
  Requires-Dist: adbc-driver-manager>=1.12 ; extra == 'adbc'
23
+ Requires-Dist: dask>=2024.8.0 ; extra == 'dask'
23
24
  Requires-Dist: xarray-sql[docs] ; extra == 'dev'
24
25
  Requires-Dist: pre-commit ; extra == 'dev'
25
26
  Requires-Dist: pytest ; extra == 'dev'
@@ -32,11 +33,12 @@ Requires-Dist: polars>=1.33 ; extra == 'polars'
32
33
  Requires-Dist: adbc-driver-flightsql>=1.12 ; extra == 'test'
33
34
  Requires-Dist: adbc-driver-sqlite>=1.12 ; extra == 'test'
34
35
  Requires-Dist: cftime ; extra == 'test'
35
- Requires-Dist: xarray-sql[adbc,duckdb,polars,geo] ; extra == 'test'
36
+ Requires-Dist: xarray-sql[adbc,dask,duckdb,polars,geo] ; extra == 'test'
36
37
  Requires-Dist: pytest ; extra == 'test'
37
38
  Requires-Dist: xarray[io] ; extra == 'test'
38
39
  Requires-Dist: gcsfs ; extra == 'test'
39
40
  Provides-Extra: adbc
41
+ Provides-Extra: dask
40
42
  Provides-Extra: dev
41
43
  Provides-Extra: docs
42
44
  Provides-Extra: duckdb
@@ -94,7 +96,7 @@ import xarray_sql as xql
94
96
  # 4x-daily surface air temperature on a lat/lon grid, 2013-2014.
95
97
  ds = xr.tutorial.open_dataset('air_temperature')
96
98
 
97
- ctx = xql.XarrayContext()
99
+ ctx = xql.Context()
98
100
  ctx.from_dataset('air', ds, chunks=dict(time=100))
99
101
 
100
102
  # A climatology — the mean annual cycle — computed in SQL: average air
@@ -164,8 +166,9 @@ ds = xr.open_zarr(
164
166
  storage_options={'token': 'anon'} # Anonymous read from the public GCS bucket — no auth required.
165
167
  )
166
168
 
167
- ctx = xql.XarrayContext()
168
- # Make sure to pass `chunks`!
169
+ ctx = xql.Context()
170
+ # `chunks` sets the partition size; without it, partitions follow the
171
+ # store's own chunks (one hour each here).
169
172
  ctx.from_dataset('era5', ds, chunks=dict(time=6), table_names={
170
173
  ('time', 'latitude', 'longitude'): 'surface',
171
174
  ('time', 'level', 'latitude', 'longitude'): 'atmosphere',
@@ -40,7 +40,7 @@ import xarray_sql as xql
40
40
  # 4x-daily surface air temperature on a lat/lon grid, 2013-2014.
41
41
  ds = xr.tutorial.open_dataset('air_temperature')
42
42
 
43
- ctx = xql.XarrayContext()
43
+ ctx = xql.Context()
44
44
  ctx.from_dataset('air', ds, chunks=dict(time=100))
45
45
 
46
46
  # A climatology — the mean annual cycle — computed in SQL: average air
@@ -110,8 +110,9 @@ ds = xr.open_zarr(
110
110
  storage_options={'token': 'anon'} # Anonymous read from the public GCS bucket — no auth required.
111
111
  )
112
112
 
113
- ctx = xql.XarrayContext()
114
- # Make sure to pass `chunks`!
113
+ ctx = xql.Context()
114
+ # `chunks` sets the partition size; without it, partitions follow the
115
+ # store's own chunks (one hour each here).
115
116
  ctx.from_dataset('era5', ds, chunks=dict(time=6), table_names={
116
117
  ('time', 'latitude', 'longitude'): 'surface',
117
118
  ('time', 'level', 'latitude', 'longitude'): 'atmosphere',
@@ -0,0 +1,153 @@
1
+ # Run in the browser
2
+
3
+ xarray-sql runs in [Pyodide](https://pyodide.org), Python compiled to
4
+ WebAssembly. The demo below runs SQL over NOAA GFS forecasts read straight
5
+ from [dynamical.org](https://dynamical.org)'s public Zarr store. xarray-sql
6
+ and DuckDB run in a Web Worker in your browser, so there is no server. Each
7
+ query downloads only the chunks that its filters on `init_time`, `latitude`,
8
+ and `longitude` reach.
9
+
10
+ The demo needs Chrome or Edge 137 or newer. It takes about 15 seconds to
11
+ start, most of it downloading and loading Pyodide, pyarrow, DuckDB, and
12
+ pandas.
13
+
14
+ <iframe id="demo" src="../demo/" title="xarray-sql browser demo" loading="lazy"
15
+ style="width: 100%; height: 1150px; border: 1px solid var(--md-default-fg-color--lightest); border-radius: 8px;"></iframe>
16
+
17
+ [Open the demo in its own page](demo/){ #demo-link target="_blank" }
18
+
19
+ <script>
20
+ // Pass ?wheel=<url> through to the demo, to try a locally built wheel.
21
+ (() => {
22
+ const wheel = new URLSearchParams(location.search).get("wheel");
23
+ if (!wheel) return;
24
+ for (const [id, attr] of [["demo", "src"], ["demo-link", "href"]]) {
25
+ const el = document.getElementById(id);
26
+ const url = new URL(el.getAttribute(attr), location.href);
27
+ url.searchParams.set("wheel", new URL(wheel, location.href).href);
28
+ el.setAttribute(attr, url.href);
29
+ }
30
+ })();
31
+ </script>
32
+
33
+ ## Install in Pyodide
34
+
35
+ Releases from 0.6.0 on publish a Pyodide wheel to PyPI. The DuckDB engine
36
+ adapter works in Pyodide, including the round-trip back to a Dataset with
37
+ [to_dataset][xarray_sql.to_dataset]. DataFusion does not yet, because its
38
+ Python bindings have no Pyodide build, so [Context][xarray_sql.Context] is
39
+ unavailable there.
40
+
41
+ Copy [`fetch_store.py`](https://github.com/xqlsystems/xarray-sql/tree/main/docs/demo/fetch_store.py)
42
+ from the demo to open Zarr stores over HTTP in Pyodide, since fsspec's HTTP
43
+ filesystem needs sockets. Then open a window of the dataset, read it into
44
+ memory, and query it:
45
+
46
+ ```python
47
+ import micropip
48
+ await micropip.install(["xarray-sql", "duckdb", "zarr"])
49
+
50
+ import duckdb
51
+ import xarray as xr
52
+ import xarray_sql as xql
53
+ from fetch_store import FetchStore
54
+
55
+ store = FetchStore("https://data.dynamical.org/noaa/gfs/forecast/latest.zarr")
56
+ ds = (
57
+ xr.open_zarr(store, chunks=None)[["temperature_2m"]]
58
+ .isel(init_time=-1, lead_time=slice(0, 24))
59
+ .sel(latitude=slice(50, 25), longitude=slice(-100, -75))
60
+ )
61
+ ds = await ds.load_async()
62
+
63
+ con = duckdb.connect()
64
+ xql.register(con, "gfs", ds)
65
+ hottest = con.sql("""
66
+ SELECT latitude, longitude, max(temperature_2m) AS t2m_max
67
+ FROM gfs
68
+ GROUP BY latitude, longitude
69
+ """)
70
+ out = xql.to_dataset(hottest, template=ds) # dims latitude, longitude
71
+ ```
72
+
73
+ The window is read into memory first because a DuckDB scan cannot wait on
74
+ the network in Pyodide (see the next section). Pyodide also cannot start
75
+ threads, so xarray-sql loads chunks one at a time there. To query a larger
76
+ store, read it through a cache like the demo's.
77
+
78
+ ## How lazy reads work in the browser
79
+
80
+ Browsers can only read over the network asynchronously. zarr's synchronous
81
+ API waits on those reads with Pyodide's JavaScript Promise Integration
82
+ (JSPI), but JSPI cannot suspend while DuckDB or pyarrow is calling back into
83
+ Python: their C++ frames sit between the suspension point and Python. So a
84
+ query cannot read chunks in the middle of a scan.
85
+
86
+ The demo's `ChunkCache` runs each query against a cache of decoded chunks
87
+ instead:
88
+
89
+ 1. Run the query. A chunk that is not cached is recorded as missing and
90
+ scanned as fill values, so one run records every chunk that survives
91
+ xarray-sql's chunk pruning.
92
+ 2. Fetch the missing chunks concurrently with `Dataset.load_async`, then run
93
+ the query again.
94
+ 3. A run with no missing chunks used only real data, so its result is
95
+ exact.
96
+
97
+ Most queries need two runs. The stats line under the editor shows the runs,
98
+ chunks, requests, and bytes for each query.
99
+
100
+ The demo's files are in
101
+ [`docs/demo/`](https://github.com/xqlsystems/xarray-sql/tree/main/docs/demo):
102
+
103
+ | File | Role |
104
+ |------|------|
105
+ | `index.html` | The page: example queries, SQL editor, results table, and an interactive map ([deck.gl](https://deck.gl) over a [MapLibre](https://maplibre.org) basemap) for `latitude, longitude, value` results. |
106
+ | `worker.js` | Loads Pyodide, installs xarray-sql, DuckDB, and zarr, and runs queries off the main thread. |
107
+ | `app.py` | Opens the dataset, registers it on DuckDB as `gfs`, and runs queries. |
108
+ | `chunk_cache.py` | Lazy reads that work in the browser, as described above. |
109
+ | `fetch_store.py` | A read-only Zarr store that uses `fetch` with `Range` requests. |
110
+ | `smoke.mjs` | Runs every example query headlessly in Node. |
111
+
112
+ ## Run it locally
113
+
114
+ Serve `docs/demo` over HTTP and open it in Chrome or Edge:
115
+
116
+ ```shell
117
+ python -m http.server 8000 --directory docs/demo
118
+ ```
119
+
120
+ By default the worker installs xarray-sql from PyPI. To test a local build,
121
+ copy the wheel from `pyodide build` (see "Building for WebAssembly" in the
122
+ [contributing guide](contributing.md)) into `docs/demo` and pass it in the
123
+ URL:
124
+
125
+ ```
126
+ http://localhost:8000/?wheel=./xarray_sql-0.6.0-cp310-abi3-pyemscripten_2026_0_wasm32.whl
127
+ ```
128
+
129
+ On this docs page, `?wheel=` is passed through to the embedded demo, so a
130
+ locally built docs site can use a local wheel too (for example,
131
+ `/browser/?wheel=../demo/xarray_sql-...whl` after copying the wheel into the
132
+ built `demo/` directory).
133
+
134
+ To run the examples without a browser:
135
+
136
+ ```shell
137
+ cd docs/demo
138
+ npm install --no-save pyodide@314.0.7
139
+ node smoke.mjs path/to/xarray_sql-*-pyemscripten_*.whl
140
+ ```
141
+
142
+ ## Known limits
143
+
144
+ - **`lead_time` filters do not prune chunks.** DuckDB does not push
145
+ `INTERVAL` predicates down to Arrow scans. The demo registers only the
146
+ first lead-time chunk (forecast hours 0–104), so every query reads at most
147
+ that one.
148
+ - **Queries that touch the whole grid are slow.** For example,
149
+ `SELECT max(init_time) FROM gfs` builds every coordinate row (about 440
150
+ million) even though it downloads nothing. Use the literal timestamps that
151
+ the example queries use.
152
+ - **The cache holds 768 MiB of decoded chunks.** A query that needs more
153
+ fails with a message to narrow its filters.
@@ -0,0 +1,217 @@
1
+ """The Python half of the browser demo: open a remote Zarr, query it with SQL.
2
+
3
+ ``worker.js`` loads this module into Pyodide and calls :func:`setup` once
4
+ and :func:`run` per query. Both return JSON strings so the worker can hand
5
+ results to the page without converting Python objects.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import math
12
+ from decimal import Decimal
13
+ from typing import Any
14
+
15
+ import duckdb
16
+ import numpy as np
17
+ import pandas as pd
18
+ import pyarrow as pa
19
+ import xarray as xr
20
+ from chunk_cache import ChunkCache
21
+ from fetch_store import FetchStore
22
+
23
+ import xarray_sql as xql
24
+
25
+ GFS_URL = "https://data.dynamical.org/noaa/gfs/forecast/latest.zarr"
26
+ GFS_VARIABLES = [
27
+ "temperature_2m",
28
+ "precipitation_surface",
29
+ "wind_u_10m",
30
+ "wind_v_10m",
31
+ ]
32
+ # The store's inner chunks; one cached chunk is then one range request.
33
+ GFS_CHUNKS = {
34
+ "init_time": 1,
35
+ "lead_time": 105,
36
+ "latitude": 121,
37
+ "longitude": 121,
38
+ }
39
+
40
+ _state: dict[str, Any] = {}
41
+
42
+
43
+ def _ts(value: Any) -> str:
44
+ return str(pd.Timestamp(value))[:19]
45
+
46
+
47
+ def _examples(ds: xr.Dataset) -> list[dict[str, str]]:
48
+ latest = _ts(ds.init_time.values[-1])
49
+ earliest = _ts(ds.init_time.values[0])
50
+ return [
51
+ {
52
+ "title": "San Francisco: 2 m temperature for the next 3 days",
53
+ "sql": f"""SELECT init_time + lead_time AS valid_time,
54
+ round(avg(temperature_2m), 1) AS temperature_c
55
+ FROM gfs
56
+ WHERE init_time = TIMESTAMP '{latest}'
57
+ AND lead_time <= INTERVAL 72 HOUR
58
+ AND latitude BETWEEN 37.5 AND 38 AND longitude BETWEEN -122.75 AND -122.25
59
+ GROUP BY ALL ORDER BY valid_time""",
60
+ },
61
+ {
62
+ "title": "Map: Europe 2 m temperature, 24 h ahead",
63
+ "sql": f"""SELECT latitude, longitude, temperature_2m
64
+ FROM gfs
65
+ WHERE init_time = TIMESTAMP '{latest}'
66
+ AND lead_time = INTERVAL 24 HOUR
67
+ AND latitude BETWEEN 35 AND 70 AND longitude BETWEEN -10 AND 40""",
68
+ },
69
+ {
70
+ "title": "Map: contiguous US wind speed, 12 h ahead",
71
+ "sql": f"""SELECT latitude, longitude,
72
+ round(sqrt(wind_u_10m ^ 2 + wind_v_10m ^ 2), 1) AS wind_speed
73
+ FROM gfs
74
+ WHERE init_time = TIMESTAMP '{latest}'
75
+ AND lead_time = INTERVAL 12 HOUR
76
+ AND latitude BETWEEN 24 AND 50 AND longitude BETWEEN -125 AND -66""",
77
+ },
78
+ {
79
+ "title": "Wettest places in the contiguous US, next 24 h",
80
+ "sql": f"""SELECT latitude, longitude,
81
+ round(sum(precipitation_surface) * 3600, 1) AS approx_precip_mm
82
+ FROM gfs
83
+ WHERE init_time = TIMESTAMP '{latest}'
84
+ AND lead_time BETWEEN INTERVAL 1 HOUR AND INTERVAL 24 HOUR
85
+ AND latitude BETWEEN 24 AND 50 AND longitude BETWEEN -125 AND -66
86
+ GROUP BY ALL ORDER BY approx_precip_mm DESC LIMIT 10""",
87
+ },
88
+ {
89
+ "title": "How the forecast for San Francisco changed across runs",
90
+ "sql": f"""SELECT init_time,
91
+ round(max(temperature_2m), 1) AS max_temperature_c
92
+ FROM gfs
93
+ WHERE init_time BETWEEN TIMESTAMP '{earliest}' AND TIMESTAMP '{latest}'
94
+ AND init_time + lead_time BETWEEN TIMESTAMP '{latest}' + INTERVAL 24 HOUR
95
+ AND TIMESTAMP '{latest}' + INTERVAL 48 HOUR
96
+ AND lead_time <= INTERVAL 72 HOUR
97
+ AND latitude BETWEEN 37.5 AND 38 AND longitude BETWEEN -122.75 AND -122.25
98
+ GROUP BY ALL ORDER BY init_time""",
99
+ },
100
+ ]
101
+
102
+
103
+ async def setup(
104
+ url: str = GFS_URL,
105
+ variables: list[str] | None = None,
106
+ runs: int = 4,
107
+ max_cache_mib: int = 768,
108
+ ) -> str:
109
+ """Open the dataset and register it on a DuckDB connection as ``gfs``."""
110
+ store = FetchStore(url)
111
+ ds = xr.open_zarr(store, chunks=None)
112
+ # DuckDB does not push INTERVAL predicates down to Arrow scans, so a
113
+ # lead_time filter cannot prune chunks. Keeping only the first lead-time
114
+ # chunk (forecast hours 0-104) halves what every query downloads.
115
+ ds = ds[variables or GFS_VARIABLES].isel(
116
+ init_time=slice(-runs, None),
117
+ lead_time=slice(0, GFS_CHUNKS["lead_time"]),
118
+ )
119
+ cache = ChunkCache(ds, GFS_CHUNKS, max_bytes=max_cache_mib * 2**20)
120
+ con = duckdb.connect()
121
+ con.sql("SET enable_progress_bar = false")
122
+ xql.register(con, "gfs", cache.dataset(), chunks=GFS_CHUNKS)
123
+ _state.update(store=store, cache=cache, con=con)
124
+ return json.dumps(
125
+ {
126
+ "url": url,
127
+ "variables": {
128
+ name: {
129
+ "dims": list(var.dims),
130
+ "units": var.attrs.get("units", ""),
131
+ "long_name": var.attrs.get("long_name", name),
132
+ }
133
+ for name, var in ds.data_vars.items()
134
+ },
135
+ "sizes": dict(ds.sizes),
136
+ "init_times": [_ts(t) for t in ds.init_time.values],
137
+ "examples": _examples(ds),
138
+ }
139
+ )
140
+
141
+
142
+ def _jsonable(value: Any) -> Any:
143
+ """A Python value from Arrow, made safe for JSON (which has no NaN)."""
144
+ if isinstance(value, float):
145
+ return value if math.isfinite(value) else None
146
+ if isinstance(value, Decimal):
147
+ return float(value)
148
+ # Exactly tuple: Arrow intervals are named tuples, shown as text below.
149
+ if isinstance(value, list) or type(value) is tuple:
150
+ return [_jsonable(v) for v in value]
151
+ if isinstance(value, dict):
152
+ return {str(k): _jsonable(v) for k, v in value.items()}
153
+ if value is None or isinstance(value, (str, int)):
154
+ return value
155
+ return str(value) # Timestamps, intervals, bytes, ...
156
+
157
+
158
+ def _grid(table: pa.Table) -> dict[str, Any] | None:
159
+ """Columns for the page's map: latitude, longitude, and one number."""
160
+ names = table.column_names
161
+ if "latitude" not in names or "longitude" not in names:
162
+ return None
163
+ values = [
164
+ n
165
+ for n in names
166
+ if n not in ("latitude", "longitude")
167
+ and (
168
+ pa.types.is_floating(table[n].type)
169
+ or pa.types.is_integer(table[n].type)
170
+ or pa.types.is_decimal(table[n].type)
171
+ )
172
+ ]
173
+ if len(values) != 1 or table.num_rows < 4:
174
+ return None
175
+ lat = table["latitude"].to_numpy().astype("float64")
176
+ lon = table["longitude"].to_numpy().astype("float64")
177
+ # One pixel per point: with several rows per point (e.g. every lead
178
+ # time), the table is the honest view.
179
+ if len(np.unique(np.stack([lat, lon]), axis=1)[0]) != table.num_rows:
180
+ return None
181
+ value = table[values[0]].to_numpy().astype("float64")
182
+ finite = np.isfinite(value)
183
+ if not finite.any():
184
+ return None
185
+ return {
186
+ "name": values[0],
187
+ "latitude": lat[finite].tolist(),
188
+ "longitude": lon[finite].tolist(),
189
+ "value": value[finite].tolist(),
190
+ }
191
+
192
+
193
+ async def run(sql: str, max_rows: int = 500) -> str:
194
+ """Run ``sql``, fetching whatever chunks it needs, and return the result."""
195
+ store, cache, con = _state["store"], _state["cache"], _state["con"]
196
+ requests, nbytes = store.requests, store.bytes_read
197
+ table, stats = await cache.query(lambda: con.sql(sql).to_arrow_table())
198
+ shown = table.slice(0, max_rows)
199
+ return json.dumps(
200
+ {
201
+ "columns": shown.column_names,
202
+ "rows": _jsonable(
203
+ list(zip(*(shown[c].to_pylist() for c in shown.column_names)))
204
+ ),
205
+ "num_rows": table.num_rows,
206
+ "grid": _grid(table),
207
+ "stats": {
208
+ "runs": stats.rounds,
209
+ "chunks_fetched": stats.chunks_fetched,
210
+ "requests": store.requests - requests,
211
+ "mb_downloaded": round((store.bytes_read - nbytes) / 1e6, 1),
212
+ "seconds_fetching": round(stats.seconds_fetching, 2),
213
+ "seconds_querying": round(stats.seconds_querying, 2),
214
+ "cached_chunks": len(cache),
215
+ },
216
+ }
217
+ )