xarray-sql 0.4.0__tar.gz → 0.4.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/.gitignore +0 -1
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/Cargo.lock +1 -2
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/Cargo.toml +5 -6
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/PKG-INFO +5 -11
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/README.md +4 -10
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/_engines.py +30 -15
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/engines.md +1 -39
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/examples.md +3 -46
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/limitations.md +6 -29
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/pyproject.toml +0 -2
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/src/lib.rs +94 -466
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_df.py +13 -96
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_duckdb_backend.py +1 -1
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_reader.py +0 -268
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_sql.py +0 -31
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_stats.py +0 -48
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/__init__.py +1 -2
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/backends/__init__.py +0 -2
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/backends/base.py +6 -30
- xarray_sql-0.4.0rc1/xarray_sql/backends/datafusion.py +46 -0
- xarray_sql-0.4.0rc1/xarray_sql/backends/duckdb.py +89 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/backends/pyarrow.py +0 -96
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/cftime.py +2 -4
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/df.py +38 -175
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/ds.py +36 -93
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/geometry.py +1 -1
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/lazyscan.py +5 -11
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/sql.py +9 -11
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/zensical.toml +1 -5
- xarray_sql-0.4.0/.envrc +0 -1
- xarray_sql-0.4.0/flake.lock +0 -61
- xarray_sql-0.4.0/flake.nix +0 -44
- xarray_sql-0.4.0/tests/test_coord_lookup.py +0 -167
- xarray_sql-0.4.0/tests/test_table_names.py +0 -368
- xarray_sql-0.4.0/xarray_sql/backends/datafusion.py +0 -78
- xarray_sql-0.4.0/xarray_sql/backends/duckdb.py +0 -193
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/AGENTS.md +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/LICENSE +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/duckdb_pushdown.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/01_ndvi.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/02_climatology.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/03_zonal_mean.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/04_anomaly.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/05_forecast_skill.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/06_zonal_vector.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/07_reproject_udf.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/08_regrid_weights.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/09_warp.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/README.md +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/_harness.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/engine_suite.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/perf_summary.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/run_all.sh +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/run_perf.sh +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/assets/logo.svg +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/contributing.md +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/geospatial.md +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/index.md +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/performance.md +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/reference/xarray_sql.md +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/__init__.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/conftest.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_arrow_dataset.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_arrow_dataset_integration.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_cft.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_ds.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_geometry.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_lazy_roundtrip.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_proj.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_sql_recipes.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_to_dataset_perf.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/core.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/proj.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/reader.py +0 -0
- {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/roundtrip.py +0 -0
|
@@ -3367,7 +3367,7 @@ checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
|
|
|
3367
3367
|
|
|
3368
3368
|
[[package]]
|
|
3369
3369
|
name = "xarray_sql"
|
|
3370
|
-
version = "0.4.0"
|
|
3370
|
+
version = "0.4.0-rc.1"
|
|
3371
3371
|
dependencies = [
|
|
3372
3372
|
"arrow",
|
|
3373
3373
|
"async-stream",
|
|
@@ -3375,7 +3375,6 @@ dependencies = [
|
|
|
3375
3375
|
"datafusion",
|
|
3376
3376
|
"datafusion-ffi",
|
|
3377
3377
|
"futures",
|
|
3378
|
-
"half",
|
|
3379
3378
|
"pyo3",
|
|
3380
3379
|
"pyo3-build-config",
|
|
3381
3380
|
"tokio",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "xarray_sql"
|
|
3
|
-
version = "0.4.0"
|
|
3
|
+
version = "0.4.0-rc.1"
|
|
4
4
|
authors = ["Alex Merose"]
|
|
5
5
|
edition = "2021"
|
|
6
6
|
exclude = [
|
|
@@ -25,12 +25,11 @@ async-trait = "0.1"
|
|
|
25
25
|
datafusion = { version = "54.0.0" }
|
|
26
26
|
datafusion-ffi = { version = "54.0.0" }
|
|
27
27
|
futures = { version = "0.3" }
|
|
28
|
-
half = "2.7"
|
|
29
28
|
# `abi3-py310` builds against CPython's stable ABI, so a single wheel per
|
|
30
|
-
# platform works on all CPython >= 3.10 (matching `requires-python`).
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
pyo3 = { version = "0.28.0", features = ["abi3-py310"] }
|
|
29
|
+
# platform works on all CPython >= 3.10 (matching `requires-python`). This
|
|
30
|
+
# lets the release workflow ship pre-built wheels for every interpreter
|
|
31
|
+
# without compiling per-version, avoiding local rebuilds on install.
|
|
32
|
+
pyo3 = { version = "0.28.0", features = ["extension-module", "abi3-py310"] }
|
|
34
33
|
tokio = { version = "1.46.1", features = ["rt"] }
|
|
35
34
|
|
|
36
35
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xarray_sql
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.0rc1
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Science/Research
|
|
6
6
|
Classifier: Intended Audience :: Developers
|
|
@@ -127,10 +127,7 @@ rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY ti
|
|
|
127
127
|
xql.to_dataset(rel, template=ds) # any engine's Arrow result round-trips
|
|
128
128
|
```
|
|
129
129
|
|
|
130
|
-
|
|
131
|
-
against `era5.surface` is not tied to the engine it was written for.
|
|
132
|
-
|
|
133
|
-
See [Engines](https://xqlsystems.github.io/xarray-sql/latest/engines/) for the support matrix, DuckDB/Polars details,
|
|
130
|
+
See [Engines](https://xqlsystems.github.io/xarray-sql/engines/) for the support matrix, DuckDB/Polars details,
|
|
134
131
|
and the lazy chunked round-trip.
|
|
135
132
|
|
|
136
133
|
## A bigger example: ARCO-ERA5
|
|
@@ -298,7 +295,7 @@ Every case matches its array reference. The headline finding: these operations
|
|
|
298
295
|
are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
|
|
299
296
|
functions, and `CASE` in disguise, and a query engine runs them at scale. See
|
|
300
297
|
[`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
|
|
301
|
-
[Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/
|
|
298
|
+
[Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/geospatial/).
|
|
302
299
|
|
|
303
300
|
## Why does this work?
|
|
304
301
|
|
|
@@ -313,10 +310,10 @@ anything that speaks Arrow).
|
|
|
313
310
|
## What are the current limitations?
|
|
314
311
|
|
|
315
312
|
The sharp edges we know about — per engine and fundamental — are cataloged in
|
|
316
|
-
[Known issues & limitations](https://xqlsystems.github.io/xarray-sql/
|
|
313
|
+
[Known issues & limitations](https://xqlsystems.github.io/xarray-sql/limitations/). Currently, we're looking for
|
|
317
314
|
early users – "tire kickers", if you will. We'd love your input to shape the direction of this
|
|
318
315
|
project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
|
|
319
|
-
you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/
|
|
316
|
+
you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/contributing/), too 😉.
|
|
320
317
|
|
|
321
318
|
## What would a deeper integration look like?
|
|
322
319
|
|
|
@@ -370,9 +367,6 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
370
367
|
changes.
|
|
371
368
|
- Aman Kumar for spending a considerable amount of his GSoC internship
|
|
372
369
|
contributing to this project.
|
|
373
|
-
- Miguel Moncada Isla for reimagining this project to be a cross SQL engine
|
|
374
|
-
interface to and from Xarray. He brought new life to this experiment, in my
|
|
375
|
-
opinion.
|
|
376
370
|
|
|
377
371
|
|
|
378
372
|
## License
|
|
@@ -77,10 +77,7 @@ rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY ti
|
|
|
77
77
|
xql.to_dataset(rel, template=ds) # any engine's Arrow result round-trips
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
-
|
|
81
|
-
against `era5.surface` is not tied to the engine it was written for.
|
|
82
|
-
|
|
83
|
-
See [Engines](https://xqlsystems.github.io/xarray-sql/latest/engines/) for the support matrix, DuckDB/Polars details,
|
|
80
|
+
See [Engines](https://xqlsystems.github.io/xarray-sql/engines/) for the support matrix, DuckDB/Polars details,
|
|
84
81
|
and the lazy chunked round-trip.
|
|
85
82
|
|
|
86
83
|
## A bigger example: ARCO-ERA5
|
|
@@ -248,7 +245,7 @@ Every case matches its array reference. The headline finding: these operations
|
|
|
248
245
|
are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
|
|
249
246
|
functions, and `CASE` in disguise, and a query engine runs them at scale. See
|
|
250
247
|
[`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
|
|
251
|
-
[Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/
|
|
248
|
+
[Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/geospatial/).
|
|
252
249
|
|
|
253
250
|
## Why does this work?
|
|
254
251
|
|
|
@@ -263,10 +260,10 @@ anything that speaks Arrow).
|
|
|
263
260
|
## What are the current limitations?
|
|
264
261
|
|
|
265
262
|
The sharp edges we know about — per engine and fundamental — are cataloged in
|
|
266
|
-
[Known issues & limitations](https://xqlsystems.github.io/xarray-sql/
|
|
263
|
+
[Known issues & limitations](https://xqlsystems.github.io/xarray-sql/limitations/). Currently, we're looking for
|
|
267
264
|
early users – "tire kickers", if you will. We'd love your input to shape the direction of this
|
|
268
265
|
project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
|
|
269
|
-
you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/
|
|
266
|
+
you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/contributing/), too 😉.
|
|
270
267
|
|
|
271
268
|
## What would a deeper integration look like?
|
|
272
269
|
|
|
@@ -320,9 +317,6 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
320
317
|
changes.
|
|
321
318
|
- Aman Kumar for spending a considerable amount of his GSoC internship
|
|
322
319
|
contributing to this project.
|
|
323
|
-
- Miguel Moncada Isla for reimagining this project to be a cross SQL engine
|
|
324
|
-
interface to and from Xarray. He brought new life to this experiment, in my
|
|
325
|
-
opinion.
|
|
326
320
|
|
|
327
321
|
|
|
328
322
|
## License
|
|
@@ -18,11 +18,9 @@ as it always registered them on ``XarrayContext``, and calls
|
|
|
18
18
|
:meth:`EngineContext.sql_to_dataset`. On the ``datafusion`` path this
|
|
19
19
|
is byte-for-byte the original behavior (``from_dataset`` + ``sql`` +
|
|
20
20
|
``XarrayDataFrame.to_dataset``); the other engines register one pyarrow
|
|
21
|
-
dataset per dimension group under
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
time rather than through a schema) and the result rows are
|
|
25
|
-
round-tripped to an ``xr.Dataset`` through pandas.
|
|
21
|
+
dataset per dimension group under flattened table names
|
|
22
|
+
(``era5.surface`` → ``era5_surface`` — rewritten in the SQL text) and the
|
|
23
|
+
result rows are round-tripped to an ``xr.Dataset`` through pandas.
|
|
26
24
|
|
|
27
25
|
The DataFusion-only UDF cases (07 and the UDF half of 09) build
|
|
28
26
|
``xql.XarrayContext`` directly rather than through this layer; the suite
|
|
@@ -49,6 +47,24 @@ def engine_name() -> str:
|
|
|
49
47
|
return engine
|
|
50
48
|
|
|
51
49
|
|
|
50
|
+
def _group_tables(name, ds, table_names):
|
|
51
|
+
"""Split ``ds`` into per-dimension-group tables like XarrayContext does.
|
|
52
|
+
|
|
53
|
+
Returns ``[(flat_name, dotted_name, sub_dataset)]``; a uniform dataset
|
|
54
|
+
keeps its plain name (flat == dotted == name).
|
|
55
|
+
"""
|
|
56
|
+
groups: dict[tuple, list] = {}
|
|
57
|
+
for var, v in ds.data_vars.items():
|
|
58
|
+
groups.setdefault(tuple(v.dims), []).append(var)
|
|
59
|
+
if len(groups) == 1:
|
|
60
|
+
return [(name, name, ds)]
|
|
61
|
+
out = []
|
|
62
|
+
for dims, variables in groups.items():
|
|
63
|
+
sub = (table_names or {}).get(dims) or "_".join(dims)
|
|
64
|
+
out.append((f"{name}_{sub}", f"{name}.{sub}", ds[variables]))
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
52
68
|
def _literal(value: Any) -> str:
|
|
53
69
|
"""Render a parameter value as a SQL literal (for engines without binds)."""
|
|
54
70
|
if isinstance(value, (datetime.datetime, pd.Timestamp, np.datetime64)):
|
|
@@ -118,16 +134,15 @@ class EngineContext:
|
|
|
118
134
|
"""Register ``ds`` as SQL table(s), mirroring XarrayContext naming."""
|
|
119
135
|
import xarray_sql as xql
|
|
120
136
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
self._register(flat, dataset)
|
|
137
|
+
for flat, dotted, sub in _group_tables(name, ds, table_names):
|
|
138
|
+
if dotted != flat:
|
|
139
|
+
self._renames[dotted] = flat
|
|
140
|
+
sub_chunks = (
|
|
141
|
+
{d: c for d, c in chunks.items() if d in sub.dims}
|
|
142
|
+
if isinstance(chunks, dict)
|
|
143
|
+
else chunks
|
|
144
|
+
) or None
|
|
145
|
+
self._register(flat, xql.arrow_dataset(sub, sub_chunks))
|
|
131
146
|
|
|
132
147
|
# -- querying ----------------------------------------------------------
|
|
133
148
|
|
|
@@ -90,30 +90,6 @@ exact expression via pyarrow — pruning is only an optimization on top.
|
|
|
90
90
|
`XarrayArrowStream`, the dependency-light re-scannable C-stream wrapper
|
|
91
91
|
without pushdown, remains available as a fallback.
|
|
92
92
|
|
|
93
|
-
As is standard in for all Xarray-SQL engines, users may provide a mapping of
|
|
94
|
-
groups of dimensions to their preferred table names, like so:
|
|
95
|
-
|
|
96
|
-
```python
|
|
97
|
-
xql.register(con, "era5", ds, table_names={
|
|
98
|
-
("time", "latitude", "longitude"): "surface",
|
|
99
|
-
("time", "level", "latitude", "longitude"): "atmosphere",
|
|
100
|
-
})
|
|
101
|
-
|
|
102
|
-
con.sql("SELECT AVG(temperature) FROM era5.atmosphere WHERE level = 500")
|
|
103
|
-
```
|
|
104
|
-
|
|
105
|
-
Mixed-dimension Datasets split as they do everywhere else, with one
|
|
106
|
-
DuckDB-specific wrinkle: `con.register` binds each flat table
|
|
107
|
-
(`era5_surface`) only for the connection's lifetime, never to the
|
|
108
|
-
catalog on disk, so it is mirrored as a view `era5.surface` in a schema
|
|
109
|
-
of its own only on **in-memory** connections. Both spellings hit the
|
|
110
|
-
same scan — pushdown and projection travel through the view — and the
|
|
111
|
-
dotted one is what keeps the SQL portable. On a file-backed connection
|
|
112
|
-
that view would either fail to create (read-only) or persist after its
|
|
113
|
-
flat table is gone (writable, once the connection is reopened), so
|
|
114
|
-
registration there warns and leaves the flat tables, which always work
|
|
115
|
-
either way.
|
|
116
|
-
|
|
117
93
|
Details that matter in production:
|
|
118
94
|
|
|
119
95
|
- **Finely partitioned axes** (e.g. hourly-chunked reanalysis time with
|
|
@@ -170,19 +146,6 @@ out = (
|
|
|
170
146
|
xql.to_dataset(out, template=ds) # polars frames speak Arrow PyCapsule
|
|
171
147
|
```
|
|
172
148
|
|
|
173
|
-
`arrow_dataset` wants a Dataset whose variables share one set of
|
|
174
|
-
dimensions; `xql.arrow_datasets(ds, "era5", table_names=...)` splits a
|
|
175
|
-
mixed-dimension one and hands back the tables named, reading the shared
|
|
176
|
-
dimension coordinates once for all of them.
|
|
177
|
-
|
|
178
|
-
```python
|
|
179
|
-
tables = xql.arrow_datasets(ds, "era5", table_names={...})
|
|
180
|
-
|
|
181
|
-
ctx = pl.SQLContext()
|
|
182
|
-
for table, dataset in tables.items(): # 'era5_surface', ...
|
|
183
|
-
ctx.register(table, pl.scan_pyarrow_dataset(dataset))
|
|
184
|
-
```
|
|
185
|
-
|
|
186
149
|
Polars pushes its predicate and column selection into the dataset scan
|
|
187
150
|
(verified: a filtered group-by read 1 of 20 chunks and 3 of 5 columns),
|
|
188
151
|
and its results round-trip through `xql.to_dataset` unchanged. The
|
|
@@ -203,8 +166,7 @@ What each integration provides. Known issues and constraints live on
|
|
|
203
166
|
| Eager round-trip (`xql.to_dataset`) | yes | yes | yes |
|
|
204
167
|
| Chunked round-trip (`chunks=`) | re-execution | `spill=True` [^spill-only] | re-execution (streaming engine) |
|
|
205
168
|
| `geometry` column ([geospatial](geospatial.md#geoarrow-point-geometry-columns)) | annotated WKB passes through | native `GEOMETRY` (`"wkb"` encoding) | plain binary/struct |
|
|
206
|
-
| Mixed-dimension datasets | one schema, `name.group` tables |
|
|
207
|
-
| Naming those tables (`table_names=`) | yes | yes | yes |
|
|
169
|
+
| Mixed-dimension datasets | one schema, `name.group` tables | `<name>_<dims>` tables | filter `data_vars` before `arrow_dataset` |
|
|
208
170
|
| Version floor | bundled (core dependency) | `duckdb >= 1.4` (tested on 1.5) | tested on `polars 1.42` |
|
|
209
171
|
|
|
210
172
|
[^spill-only]: Why DuckDB relations do not re-execute — and two other
|
|
@@ -107,11 +107,7 @@ ctx.sql('''
|
|
|
107
107
|
|
|
108
108
|
If you omit `table_names`, each table is named by joining its dimension names
|
|
109
109
|
with underscores, e.g. `era5.time_latitude_longitude` and
|
|
110
|
-
`era5.time_level_latitude_longitude`.
|
|
111
|
-
does not have are ignored, so one naming map can be reused across Datasets
|
|
112
|
-
holding different subsets of the same variables. The keyword is not
|
|
113
|
-
DataFusion-only — `xql.register` takes it on every engine (see
|
|
114
|
-
[the same tables on DuckDB and Polars](#the-same-tables-on-duckdb-and-polars)).
|
|
110
|
+
`era5.time_level_latitude_longitude`.
|
|
115
111
|
|
|
116
112
|
## GOES satellite imagery (scalar variables)
|
|
117
113
|
|
|
@@ -158,44 +154,5 @@ not DataFusion-specific: `xql.register(con, name, ds)` attaches the same lazy,
|
|
|
158
154
|
pushdown-scanned table to a DuckDB connection, and
|
|
159
155
|
`pl.scan_pyarrow_dataset(xql.arrow_dataset(ds))` serves Polars — same
|
|
160
156
|
splitting rules for mixed-dimension Datasets, same round-trip through
|
|
161
|
-
`xql.to_dataset(result, template=ds)`.
|
|
162
|
-
|
|
163
|
-
`table_names` travels with them, so the ERA5 example above changes engine
|
|
164
|
-
without changing a word of its SQL:
|
|
165
|
-
|
|
166
|
-
```python
|
|
167
|
-
import duckdb
|
|
168
|
-
|
|
169
|
-
con = duckdb.connect()
|
|
170
|
-
xql.register(con, 'era5', ds, table_names={
|
|
171
|
-
('time', 'latitude', 'longitude'): 'surface',
|
|
172
|
-
('time', 'level', 'latitude', 'longitude'): 'atmosphere',
|
|
173
|
-
})
|
|
174
|
-
|
|
175
|
-
rel = con.sql('''
|
|
176
|
-
SELECT level, AVG(temperature) - 273.15 AS avg_c
|
|
177
|
-
FROM era5.atmosphere
|
|
178
|
-
WHERE time BETWEEN TIMESTAMP '2020-01-01'
|
|
179
|
-
AND TIMESTAMP '2020-01-01 05:00:00'
|
|
180
|
-
GROUP BY level
|
|
181
|
-
ORDER BY level DESC
|
|
182
|
-
''')
|
|
183
|
-
xql.to_dataset(rel, template=ds, dims=['level'])
|
|
184
|
-
```
|
|
185
|
-
|
|
186
|
-
Polars registers table by table, so it takes the split datasets by name:
|
|
187
|
-
|
|
188
|
-
```python
|
|
189
|
-
import polars as pl
|
|
190
|
-
|
|
191
|
-
tables = xql.arrow_datasets(ds, 'era5', table_names={
|
|
192
|
-
('time', 'latitude', 'longitude'): 'surface',
|
|
193
|
-
('time', 'level', 'latitude', 'longitude'): 'atmosphere',
|
|
194
|
-
})
|
|
195
|
-
|
|
196
|
-
ctx = pl.SQLContext()
|
|
197
|
-
for table, dataset in tables.items(): # 'era5_surface', 'era5_atmosphere'
|
|
198
|
-
ctx.register(table, pl.scan_pyarrow_dataset(dataset))
|
|
199
|
-
```
|
|
200
|
-
|
|
201
|
-
See [Engines](engines.md) for the support matrix and per-engine details.
|
|
157
|
+
`xql.to_dataset(result, template=ds)`. See [Engines](engines.md) for the
|
|
158
|
+
support matrix and per-engine details.
|
|
@@ -144,35 +144,12 @@ A materialized table or bare C-stream has no query behind it, so the
|
|
|
144
144
|
re-execution form of `chunks=` cannot serve it — `spill=True` (one
|
|
145
145
|
pass to a temporary Parquet file) is the chunked path for these.
|
|
146
146
|
|
|
147
|
-
###
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
on disk. Creating the view still needs a writable catalog (a read-only
|
|
154
|
-
connection can't), and even on a writable one the view would persist
|
|
155
|
-
after those flat tables are gone — a later connection would resolve it
|
|
156
|
-
into "Table ... does not exist". Registering on any file-backed
|
|
157
|
-
connection therefore warns and leaves the flat `name_group` tables,
|
|
158
|
-
which are always registered and always work; the dotted spelling is
|
|
159
|
-
only mirrored on in-memory connections.
|
|
160
|
-
|
|
161
|
-
This is about *DuckDB's own* database file (`duckdb.connect("x.db")`
|
|
162
|
-
vs. `duckdb.connect()`), checked via `PRAGMA database_list` — not
|
|
163
|
-
about where the xarray Dataset's data lives. A Dataset backed by local
|
|
164
|
-
NetCDF, remote Zarr, or plain in-memory arrays is all the same
|
|
165
|
-
`con.register`-ed Python object either way; only the *connection's*
|
|
166
|
-
catalog file, if any, decides whether the view would dangle.
|
|
167
|
-
|
|
168
|
-
### `geometry=` on a mixed-dimension Dataset skips groups without those dims
|
|
169
|
-
|
|
170
|
-
`arrow_datasets(ds, table_names=..., geometry=(x_dim, y_dim))` forwards
|
|
171
|
-
`geometry` to every returned table, but a mixed-dimension Dataset can
|
|
172
|
-
have groups the geometry dims aren't columns of (a scalar metadata
|
|
173
|
-
group, or a group on an entirely different grid). Those groups come
|
|
174
|
-
back without a `geometry` column rather than raising; only the groups
|
|
175
|
-
that have both dims get one.
|
|
147
|
+
### Mixed-dimension datasets split into one DuckDB table per dim group
|
|
148
|
+
|
|
149
|
+
DuckDB registration has no schema namespace, so variables with
|
|
150
|
+
different dims land in suffixed tables (`<name>_<dims>`), sharing one
|
|
151
|
+
set of coordinate reads. DataFusion registers the same layout as
|
|
152
|
+
`name.group` tables inside one schema.
|
|
176
153
|
|
|
177
154
|
### Pointwise indexers on lazy round-trip arrays are slower
|
|
178
155
|
|