xarray-sql 0.4.0__tar.gz → 0.4.0rc1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/.gitignore +0 -1
  2. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/Cargo.lock +1 -2
  3. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/Cargo.toml +5 -6
  4. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/PKG-INFO +5 -11
  5. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/README.md +4 -10
  6. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/_engines.py +30 -15
  7. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/engines.md +1 -39
  8. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/examples.md +3 -46
  9. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/limitations.md +6 -29
  10. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/pyproject.toml +0 -2
  11. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/src/lib.rs +94 -466
  12. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_df.py +13 -96
  13. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_duckdb_backend.py +1 -1
  14. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_reader.py +0 -268
  15. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_sql.py +0 -31
  16. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_stats.py +0 -48
  17. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/__init__.py +1 -2
  18. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/backends/__init__.py +0 -2
  19. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/backends/base.py +6 -30
  20. xarray_sql-0.4.0rc1/xarray_sql/backends/datafusion.py +46 -0
  21. xarray_sql-0.4.0rc1/xarray_sql/backends/duckdb.py +89 -0
  22. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/backends/pyarrow.py +0 -96
  23. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/cftime.py +2 -4
  24. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/df.py +38 -175
  25. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/ds.py +36 -93
  26. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/geometry.py +1 -1
  27. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/lazyscan.py +5 -11
  28. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/sql.py +9 -11
  29. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/zensical.toml +1 -5
  30. xarray_sql-0.4.0/.envrc +0 -1
  31. xarray_sql-0.4.0/flake.lock +0 -61
  32. xarray_sql-0.4.0/flake.nix +0 -44
  33. xarray_sql-0.4.0/tests/test_coord_lookup.py +0 -167
  34. xarray_sql-0.4.0/tests/test_table_names.py +0 -368
  35. xarray_sql-0.4.0/xarray_sql/backends/datafusion.py +0 -78
  36. xarray_sql-0.4.0/xarray_sql/backends/duckdb.py +0 -193
  37. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/AGENTS.md +0 -0
  38. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/LICENSE +0 -0
  39. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/duckdb_pushdown.py +0 -0
  40. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/01_ndvi.py +0 -0
  41. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/02_climatology.py +0 -0
  42. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/03_zonal_mean.py +0 -0
  43. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/04_anomaly.py +0 -0
  44. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/05_forecast_skill.py +0 -0
  45. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/06_zonal_vector.py +0 -0
  46. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/07_reproject_udf.py +0 -0
  47. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/08_regrid_weights.py +0 -0
  48. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/09_warp.py +0 -0
  49. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/README.md +0 -0
  50. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/_harness.py +0 -0
  51. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/engine_suite.py +0 -0
  52. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/perf_summary.py +0 -0
  53. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/run_all.sh +0 -0
  54. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/benchmarks/geospatial/run_perf.sh +0 -0
  55. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/assets/logo.svg +0 -0
  56. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/contributing.md +0 -0
  57. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/geospatial.md +0 -0
  58. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/index.md +0 -0
  59. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/performance.md +0 -0
  60. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/docs/reference/xarray_sql.md +0 -0
  61. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/__init__.py +0 -0
  62. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/conftest.py +0 -0
  63. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_arrow_dataset.py +0 -0
  64. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_arrow_dataset_integration.py +0 -0
  65. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_cft.py +0 -0
  66. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_ds.py +0 -0
  67. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_geometry.py +0 -0
  68. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_lazy_roundtrip.py +0 -0
  69. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_proj.py +0 -0
  70. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_sql_recipes.py +0 -0
  71. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/tests/test_to_dataset_perf.py +0 -0
  72. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/core.py +0 -0
  73. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/proj.py +0 -0
  74. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/reader.py +0 -0
  75. {xarray_sql-0.4.0 → xarray_sql-0.4.0rc1}/xarray_sql/roundtrip.py +0 -0
@@ -14,4 +14,3 @@ test_data
14
14
  CHANGELOG.md
15
15
  *.ipynb
16
16
  /site
17
- .direnv
@@ -3367,7 +3367,7 @@ checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
3367
3367
 
3368
3368
  [[package]]
3369
3369
  name = "xarray_sql"
3370
- version = "0.4.0"
3370
+ version = "0.4.0-rc.1"
3371
3371
  dependencies = [
3372
3372
  "arrow",
3373
3373
  "async-stream",
@@ -3375,7 +3375,6 @@ dependencies = [
3375
3375
  "datafusion",
3376
3376
  "datafusion-ffi",
3377
3377
  "futures",
3378
- "half",
3379
3378
  "pyo3",
3380
3379
  "pyo3-build-config",
3381
3380
  "tokio",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "xarray_sql"
3
- version = "0.4.0"
3
+ version = "0.4.0-rc.1"
4
4
  authors = ["Alex Merose"]
5
5
  edition = "2021"
6
6
  exclude = [
@@ -25,12 +25,11 @@ async-trait = "0.1"
25
25
  datafusion = { version = "54.0.0" }
26
26
  datafusion-ffi = { version = "54.0.0" }
27
27
  futures = { version = "0.3" }
28
- half = "2.7"
29
28
  # `abi3-py310` builds against CPython's stable ABI, so a single wheel per
30
- # platform works on all CPython >= 3.10 (matching `requires-python`). Maturin
31
- # enables `pyo3/extension-module` through pyproject.toml for wheel builds; it
32
- # must stay disabled for ordinary Cargo test binaries so they link libpython.
33
- pyo3 = { version = "0.28.0", features = ["abi3-py310"] }
29
+ # platform works on all CPython >= 3.10 (matching `requires-python`). This
30
+ # lets the release workflow ship pre-built wheels for every interpreter
31
+ # without compiling per-version, avoiding local rebuilds on install.
32
+ pyo3 = { version = "0.28.0", features = ["extension-module", "abi3-py310"] }
34
33
  tokio = { version = "1.46.1", features = ["rt"] }
35
34
 
36
35
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.4.0
3
+ Version: 0.4.0rc1
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Science/Research
6
6
  Classifier: Intended Audience :: Developers
@@ -127,10 +127,7 @@ rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY ti
127
127
  xql.to_dataset(rel, template=ds) # any engine's Arrow result round-trips
128
128
  ```
129
129
 
130
- `table_names` (below) works the same way on every engine, so a query written
131
- against `era5.surface` is not tied to the engine it was written for.
132
-
133
- See [Engines](https://xqlsystems.github.io/xarray-sql/latest/engines/) for the support matrix, DuckDB/Polars details,
130
+ See [Engines](https://xqlsystems.github.io/xarray-sql/engines/) for the support matrix, DuckDB/Polars details,
134
131
  and the lazy chunked round-trip.
135
132
 
136
133
  ## A bigger example: ARCO-ERA5
@@ -298,7 +295,7 @@ Every case matches its array reference. The headline finding: these operations
298
295
  are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
299
296
  functions, and `CASE` in disguise, and a query engine runs them at scale. See
300
297
  [`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
301
- [Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/latest/geospatial/).
298
+ [Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/geospatial/).
302
299
 
303
300
  ## Why does this work?
304
301
 
@@ -313,10 +310,10 @@ anything that speaks Arrow).
313
310
  ## What are the current limitations?
314
311
 
315
312
  The sharp edges we know about — per engine and fundamental — are cataloged in
316
- [Known issues & limitations](https://xqlsystems.github.io/xarray-sql/latest/limitations/). Currently, we're looking for
313
+ [Known issues & limitations](https://xqlsystems.github.io/xarray-sql/limitations/). Currently, we're looking for
317
314
  early users – "tire kickers", if you will. We'd love your input to shape the direction of this
318
315
  project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
319
- you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/latest/contributing/), too 😉.
316
+ you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/contributing/), too 😉.
320
317
 
321
318
  ## What would a deeper integration look like?
322
319
 
@@ -370,9 +367,6 @@ I want to give a special thanks to the following folks and institutions:
370
367
  changes.
371
368
  - Aman Kumar for spending a considerable amount of his GSoC internship
372
369
  contributing to this project.
373
- - Miguel Moncada Isla for reimagining this project to be a cross SQL engine
374
- interface to and from Xarray. He brought new life to this experiment, in my
375
- opinion.
376
370
 
377
371
 
378
372
  ## License
@@ -77,10 +77,7 @@ rel = con.sql('SELECT time, AVG("air") AS air FROM air GROUP BY time ORDER BY ti
77
77
  xql.to_dataset(rel, template=ds) # any engine's Arrow result round-trips
78
78
  ```
79
79
 
80
- `table_names` (below) works the same way on every engine, so a query written
81
- against `era5.surface` is not tied to the engine it was written for.
82
-
83
- See [Engines](https://xqlsystems.github.io/xarray-sql/latest/engines/) for the support matrix, DuckDB/Polars details,
80
+ See [Engines](https://xqlsystems.github.io/xarray-sql/engines/) for the support matrix, DuckDB/Polars details,
84
81
  and the lazy chunked round-trip.
85
82
 
86
83
  ## A bigger example: ARCO-ERA5
@@ -248,7 +245,7 @@ Every case matches its array reference. The headline finding: these operations
248
245
  are not really "array" operations at all — they are `GROUP BY`, `JOIN`, window
249
246
  functions, and `CASE` in disguise, and a query engine runs them at scale. See
250
247
  [`benchmarks/geospatial/`](https://github.com/xqlsystems/xarray-sql/tree/main/benchmarks/geospatial/) and the write-up,
251
- [Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/latest/geospatial/).
248
+ [Geospatial operations are relational operations](https://xqlsystems.github.io/xarray-sql/geospatial/).
252
249
 
253
250
  ## Why does this work?
254
251
 
@@ -263,10 +260,10 @@ anything that speaks Arrow).
263
260
  ## What are the current limitations?
264
261
 
265
262
  The sharp edges we know about — per engine and fundamental — are cataloged in
266
- [Known issues & limitations](https://xqlsystems.github.io/xarray-sql/latest/limitations/). Currently, we're looking for
263
+ [Known issues & limitations](https://xqlsystems.github.io/xarray-sql/limitations/). Currently, we're looking for
267
264
  early users – "tire kickers", if you will. We'd love your input to shape the direction of this
268
265
  project! Please, give this a try and [file issues](https://github.com/xqlsystems/xarray-sql/issues) as
269
- you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/latest/contributing/), too 😉.
266
+ you see fit. Check out our [contributing guide](https://xqlsystems.github.io/xarray-sql/contributing/), too 😉.
270
267
 
271
268
  ## What would a deeper integration look like?
272
269
 
@@ -320,9 +317,6 @@ I want to give a special thanks to the following folks and institutions:
320
317
  changes.
321
318
  - Aman Kumar for spending a considerable amount of his GSoC internship
322
319
  contributing to this project.
323
- - Miguel Moncada Isla for reimagining this project to be a cross SQL engine
324
- interface to and from Xarray. He brought new life to this experiment, in my
325
- opinion.
326
320
 
327
321
 
328
322
  ## License
@@ -18,11 +18,9 @@ as it always registered them on ``XarrayContext``, and calls
18
18
  :meth:`EngineContext.sql_to_dataset`. On the ``datafusion`` path this
19
19
  is byte-for-byte the original behavior (``from_dataset`` + ``sql`` +
20
20
  ``XarrayDataFrame.to_dataset``); the other engines register one pyarrow
21
- dataset per dimension group under the flat table names
22
- ``xql.arrow_datasets`` returns (``era5.surface`` → ``era5_surface`` —
23
- rewritten in the SQL text, since these paths register frames one at a
24
- time rather than through a schema) and the result rows are
25
- round-tripped to an ``xr.Dataset`` through pandas.
21
+ dataset per dimension group under flattened table names
22
+ (``era5.surface`` → ``era5_surface`` — rewritten in the SQL text) and the
23
+ result rows are round-tripped to an ``xr.Dataset`` through pandas.
26
24
 
27
25
  The DataFusion-only UDF cases (07 and the UDF half of 09) build
28
26
  ``xql.XarrayContext`` directly rather than through this layer; the suite
@@ -49,6 +47,24 @@ def engine_name() -> str:
49
47
  return engine
50
48
 
51
49
 
50
+ def _group_tables(name, ds, table_names):
51
+ """Split ``ds`` into per-dimension-group tables like XarrayContext does.
52
+
53
+ Returns ``[(flat_name, dotted_name, sub_dataset)]``; a uniform dataset
54
+ keeps its plain name (flat == dotted == name).
55
+ """
56
+ groups: dict[tuple, list] = {}
57
+ for var, v in ds.data_vars.items():
58
+ groups.setdefault(tuple(v.dims), []).append(var)
59
+ if len(groups) == 1:
60
+ return [(name, name, ds)]
61
+ out = []
62
+ for dims, variables in groups.items():
63
+ sub = (table_names or {}).get(dims) or "_".join(dims)
64
+ out.append((f"{name}_{sub}", f"{name}.{sub}", ds[variables]))
65
+ return out
66
+
67
+
52
68
  def _literal(value: Any) -> str:
53
69
  """Render a parameter value as a SQL literal (for engines without binds)."""
54
70
  if isinstance(value, (datetime.datetime, pd.Timestamp, np.datetime64)):
@@ -118,16 +134,15 @@ class EngineContext:
118
134
  """Register ``ds`` as SQL table(s), mirroring XarrayContext naming."""
119
135
  import xarray_sql as xql
120
136
 
121
- tables = xql.arrow_datasets(
122
- ds, name, chunks=chunks, table_names=table_names
123
- )
124
- for flat, dataset in tables.items():
125
- if flat != name:
126
- # `era5_surface` here is `era5.surface` in the case's SQL,
127
- # which was written against XarrayContext's schema split.
128
- group = flat[len(name) + 1 :]
129
- self._renames[f"{name}.{group}"] = flat
130
- self._register(flat, dataset)
137
+ for flat, dotted, sub in _group_tables(name, ds, table_names):
138
+ if dotted != flat:
139
+ self._renames[dotted] = flat
140
+ sub_chunks = (
141
+ {d: c for d, c in chunks.items() if d in sub.dims}
142
+ if isinstance(chunks, dict)
143
+ else chunks
144
+ ) or None
145
+ self._register(flat, xql.arrow_dataset(sub, sub_chunks))
131
146
 
132
147
  # -- querying ----------------------------------------------------------
133
148
 
@@ -90,30 +90,6 @@ exact expression via pyarrow — pruning is only an optimization on top.
90
90
  `XarrayArrowStream`, the dependency-light re-scannable C-stream wrapper
91
91
  without pushdown, remains available as a fallback.
92
92
 
93
- As is standard in for all Xarray-SQL engines, users may provide a mapping of
94
- groups of dimensions to their preferred table names, like so:
95
-
96
- ```python
97
- xql.register(con, "era5", ds, table_names={
98
- ("time", "latitude", "longitude"): "surface",
99
- ("time", "level", "latitude", "longitude"): "atmosphere",
100
- })
101
-
102
- con.sql("SELECT AVG(temperature) FROM era5.atmosphere WHERE level = 500")
103
- ```
104
-
105
- Mixed-dimension Datasets split as they do everywhere else, with one
106
- DuckDB-specific wrinkle: `con.register` binds each flat table
107
- (`era5_surface`) only for the connection's lifetime, never to the
108
- catalog on disk, so it is mirrored as a view `era5.surface` in a schema
109
- of its own only on **in-memory** connections. Both spellings hit the
110
- same scan — pushdown and projection travel through the view — and the
111
- dotted one is what keeps the SQL portable. On a file-backed connection
112
- that view would either fail to create (read-only) or persist after its
113
- flat table is gone (writable, once the connection is reopened), so
114
- registration there warns and leaves the flat tables, which always work
115
- either way.
116
-
117
93
  Details that matter in production:
118
94
 
119
95
  - **Finely partitioned axes** (e.g. hourly-chunked reanalysis time with
@@ -170,19 +146,6 @@ out = (
170
146
  xql.to_dataset(out, template=ds) # polars frames speak Arrow PyCapsule
171
147
  ```
172
148
 
173
- `arrow_dataset` wants a Dataset whose variables share one set of
174
- dimensions; `xql.arrow_datasets(ds, "era5", table_names=...)` splits a
175
- mixed-dimension one and hands back the tables named, reading the shared
176
- dimension coordinates once for all of them.
177
-
178
- ```python
179
- tables = xql.arrow_datasets(ds, "era5", table_names={...})
180
-
181
- ctx = pl.SQLContext()
182
- for table, dataset in tables.items(): # 'era5_surface', ...
183
- ctx.register(table, pl.scan_pyarrow_dataset(dataset))
184
- ```
185
-
186
149
  Polars pushes its predicate and column selection into the dataset scan
187
150
  (verified: a filtered group-by read 1 of 20 chunks and 3 of 5 columns),
188
151
  and its results round-trip through `xql.to_dataset` unchanged. The
@@ -203,8 +166,7 @@ What each integration provides. Known issues and constraints live on
203
166
  | Eager round-trip (`xql.to_dataset`) | yes | yes | yes |
204
167
  | Chunked round-trip (`chunks=`) | re-execution | `spill=True` [^spill-only] | re-execution (streaming engine) |
205
168
  | `geometry` column ([geospatial](geospatial.md#geoarrow-point-geometry-columns)) | annotated WKB passes through | native `GEOMETRY` (`"wkb"` encoding) | plain binary/struct |
206
- | Mixed-dimension datasets | one schema, `name.group` tables | `name.group` views over `name_group` tables | `xql.arrow_datasets(ds, name)`, one per group |
207
- | Naming those tables (`table_names=`) | yes | yes | yes |
169
+ | Mixed-dimension datasets | one schema, `name.group` tables | `<name>_<dims>` tables | filter `data_vars` before `arrow_dataset` |
208
170
  | Version floor | bundled (core dependency) | `duckdb >= 1.4` (tested on 1.5) | tested on `polars 1.42` |
209
171
 
210
172
  [^spill-only]: Why DuckDB relations do not re-execute — and two other
@@ -107,11 +107,7 @@ ctx.sql('''
107
107
 
108
108
  If you omit `table_names`, each table is named by joining its dimension names
109
109
  with underscores, e.g. `era5.time_latitude_longitude` and
110
- `era5.time_level_latitude_longitude`. Keys naming a dimension group the Dataset
111
- does not have are ignored, so one naming map can be reused across Datasets
112
- holding different subsets of the same variables. The keyword is not
113
- DataFusion-only — `xql.register` takes it on every engine (see
114
- [the same tables on DuckDB and Polars](#the-same-tables-on-duckdb-and-polars)).
110
+ `era5.time_level_latitude_longitude`.
115
111
 
116
112
  ## GOES satellite imagery (scalar variables)
117
113
 
@@ -158,44 +154,5 @@ not DataFusion-specific: `xql.register(con, name, ds)` attaches the same lazy,
158
154
  pushdown-scanned table to a DuckDB connection, and
159
155
  `pl.scan_pyarrow_dataset(xql.arrow_dataset(ds))` serves Polars — same
160
156
  splitting rules for mixed-dimension Datasets, same round-trip through
161
- `xql.to_dataset(result, template=ds)`.
162
-
163
- `table_names` travels with them, so the ERA5 example above changes engine
164
- without changing a word of its SQL:
165
-
166
- ```python
167
- import duckdb
168
-
169
- con = duckdb.connect()
170
- xql.register(con, 'era5', ds, table_names={
171
- ('time', 'latitude', 'longitude'): 'surface',
172
- ('time', 'level', 'latitude', 'longitude'): 'atmosphere',
173
- })
174
-
175
- rel = con.sql('''
176
- SELECT level, AVG(temperature) - 273.15 AS avg_c
177
- FROM era5.atmosphere
178
- WHERE time BETWEEN TIMESTAMP '2020-01-01'
179
- AND TIMESTAMP '2020-01-01 05:00:00'
180
- GROUP BY level
181
- ORDER BY level DESC
182
- ''')
183
- xql.to_dataset(rel, template=ds, dims=['level'])
184
- ```
185
-
186
- Polars registers table by table, so it takes the split datasets by name:
187
-
188
- ```python
189
- import polars as pl
190
-
191
- tables = xql.arrow_datasets(ds, 'era5', table_names={
192
- ('time', 'latitude', 'longitude'): 'surface',
193
- ('time', 'level', 'latitude', 'longitude'): 'atmosphere',
194
- })
195
-
196
- ctx = pl.SQLContext()
197
- for table, dataset in tables.items(): # 'era5_surface', 'era5_atmosphere'
198
- ctx.register(table, pl.scan_pyarrow_dataset(dataset))
199
- ```
200
-
201
- See [Engines](engines.md) for the support matrix and per-engine details.
157
+ `xql.to_dataset(result, template=ds)`. See [Engines](engines.md) for the
158
+ support matrix and per-engine details.
@@ -144,35 +144,12 @@ A materialized table or bare C-stream has no query behind it, so the
144
144
  re-execution form of `chunks=` cannot serve it — `spill=True` (one
145
145
  pass to a temporary Parquet file) is the chunked path for these.
146
146
 
147
- ### A file-backed DuckDB connection loses the dotted table names
148
-
149
- Mixed-dimension Datasets split into one table per dimension group on
150
- every engine, addressed as `name.group`. On DuckDB that dotted spelling
151
- is a view in a schema, because `con.register` binds the flat tables it
152
- selects from only for the connection's lifetime, never to the catalog
153
- on disk. Creating the view still needs a writable catalog (a read-only
154
- connection can't), and even on a writable one the view would persist
155
- after those flat tables are gone — a later connection would resolve it
156
- into "Table ... does not exist". Registering on any file-backed
157
- connection therefore warns and leaves the flat `name_group` tables,
158
- which are always registered and always work; the dotted spelling is
159
- only mirrored on in-memory connections.
160
-
161
- This is about *DuckDB's own* database file (`duckdb.connect("x.db")`
162
- vs. `duckdb.connect()`), checked via `PRAGMA database_list` — not
163
- about where the xarray Dataset's data lives. A Dataset backed by local
164
- NetCDF, remote Zarr, or plain in-memory arrays is all the same
165
- `con.register`-ed Python object either way; only the *connection's*
166
- catalog file, if any, decides whether the view would dangle.
167
-
168
- ### `geometry=` on a mixed-dimension Dataset skips groups without those dims
169
-
170
- `arrow_datasets(ds, table_names=..., geometry=(x_dim, y_dim))` forwards
171
- `geometry` to every returned table, but a mixed-dimension Dataset can
172
- have groups the geometry dims aren't columns of (a scalar metadata
173
- group, or a group on an entirely different grid). Those groups come
174
- back without a `geometry` column rather than raising; only the groups
175
- that have both dims get one.
147
+ ### Mixed-dimension datasets split into one DuckDB table per dim group
148
+
149
+ DuckDB registration has no schema namespace, so variables with
150
+ different dims land in suffixed tables (`<name>_<dims>`), sharing one
151
+ set of coordinate reads. DataFusion registers the same layout as
152
+ `name.group` tables inside one schema.
176
153
 
177
154
  ### Pointwise indexers on lazy round-trip arrays are slower
178
155
 
@@ -104,8 +104,6 @@ module = [
104
104
  "datafusion.*",
105
105
  "xarray.*",
106
106
  "pandas.*",
107
- "cftime.*",
108
- "xarray_sql._native",
109
107
  ]
110
108
  ignore_missing_imports = true
111
109