xarray-sql 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/CONTRIBUTING.md +1 -2
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/PKG-INFO +70 -23
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/README.md +69 -22
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql/df.py +24 -10
- xarray_sql-0.1.2/xarray_sql/sql_test.py +813 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql.egg-info/PKG-INFO +70 -23
- xarray_sql-0.1.1/xarray_sql/sql_test.py +0 -64
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/.github/workflows/ci.yml +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/.github/workflows/lint.yml +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/.github/workflows/publish.yml +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/.gitignore +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/.python-version +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/LICENSE +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/README.md +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/compute_air.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/compute_air.py-2024-02-26T22:57:59+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/compute_air.py-2024-03-03T07:42:36+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/compute_air.py-2024-03-03T07:50:18+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-15T13:01:22+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-15T15:03:53+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-17T18:42:14+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-26T23:04:34+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-03-03T07:43:22+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-03-03T07:50:06+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py-2024-02-26T23:06:38+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py-2024-03-03T07:44:17+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py-2024-03-03T07:49:53+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/open_era5.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/open_era5.py-2024-02-18T17:21:44+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/open_era5.py-2024-02-18T17:33:02+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/open_era5.py-2024-02-19T17:52:51+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/profile.sh +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T09:08:28+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T09:15:07+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T12:51:37+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T15:04:48+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-17T18:42:08+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-26T23:00:00+07:00.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-03-03T07:45:47+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-03-03T07:49:44+05:30.svg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/pyproject.toml +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/setup.cfg +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/uv.lock +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql/__init__.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql/core.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql/df_integrationtest.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql/df_test.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql/sql.py +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql.egg-info/SOURCES.txt +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql.egg-info/dependency_links.txt +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql.egg-info/requires.txt +0 -0
- {xarray_sql-0.1.1 → xarray_sql-0.1.2}/xarray_sql.egg-info/top_level.txt +0 -0
|
@@ -12,8 +12,7 @@ reading [Xarray's contributing guide](https://docs.xarray.dev/en/stable/contribu
|
|
|
12
12
|
|
|
13
13
|
## Developer setup
|
|
14
14
|
|
|
15
|
-
0.
|
|
16
|
-
environment with `uv`: https://docs.astral.sh/uv/getting-started/installation/
|
|
15
|
+
0. We use `uv` to manage the project: https://docs.astral.sh/uv/getting-started/installation/
|
|
17
16
|
1. Clone the repository (bonus: [via SSH](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account))
|
|
18
17
|
and `cd xarray_sql` (the project root).
|
|
19
18
|
2. Install dev dependencies via: `uv sync --dev`
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xarray_sql
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Querry Xarray with SQL.
|
|
5
5
|
Author-email: Alexander Merose <al@merose.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -46,19 +46,35 @@ pip install xarray-sql
|
|
|
46
46
|
|
|
47
47
|
## What is this?
|
|
48
48
|
|
|
49
|
-
This is an experiment to provide a SQL interface for
|
|
49
|
+
This is an experiment to provide a SQL interface for array datasets.
|
|
50
50
|
|
|
51
51
|
```python
|
|
52
52
|
import xarray as xr
|
|
53
|
-
import xarray_sql as
|
|
53
|
+
import xarray_sql as xql
|
|
54
54
|
|
|
55
55
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
56
56
|
|
|
57
57
|
# The same as a dask-sql Context; i.e. an Apache DataFusion Context.
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
ctx = xql.XarrayContext()
|
|
59
|
+
ctx.from_dataset('air', ds, chunks=dict(time=24)) # the dataset needs to be chunked!
|
|
60
|
+
# DataFrame()
|
|
61
|
+
# +------+---------------------+-------+--------------------+
|
|
62
|
+
# | lat | time | lon | air |
|
|
63
|
+
# +------+---------------------+-------+--------------------+
|
|
64
|
+
# | 75.0 | 2013-01-01T00:00:00 | 200.0 | 241.20000000000002 |
|
|
65
|
+
# | 75.0 | 2013-01-01T00:00:00 | 202.5 | 242.5 |
|
|
66
|
+
# | 75.0 | 2013-01-01T00:00:00 | 205.0 | 243.5 |
|
|
67
|
+
# | 75.0 | 2013-01-01T00:00:00 | 207.5 | 244.0 |
|
|
68
|
+
# | 75.0 | 2013-01-01T00:00:00 | 210.0 | 244.1 |
|
|
69
|
+
# | 75.0 | 2013-01-01T00:00:00 | 212.5 | 243.89000000000001 |
|
|
70
|
+
# | 75.0 | 2013-01-01T00:00:00 | 215.0 | 243.6 |
|
|
71
|
+
# | 75.0 | 2013-01-01T00:00:00 | 217.5 | 243.1 |
|
|
72
|
+
# | 75.0 | 2013-01-01T00:00:00 | 220.0 | 242.5 |
|
|
73
|
+
# | 75.0 | 2013-01-01T00:00:00 | 222.5 | 241.89000000000001 |
|
|
74
|
+
# +------+---------------------+-------+--------------------+
|
|
75
|
+
# Data truncated.
|
|
76
|
+
|
|
77
|
+
result = ctx.sql('''
|
|
62
78
|
SELECT
|
|
63
79
|
"lat", "lon", AVG("air") as air_total
|
|
64
80
|
FROM
|
|
@@ -66,16 +82,36 @@ df = c.sql('''
|
|
|
66
82
|
GROUP BY
|
|
67
83
|
"lat", "lon"
|
|
68
84
|
''')
|
|
85
|
+
# DataFrame()
|
|
86
|
+
# +------+-------+--------------------+
|
|
87
|
+
# | lat | lon | air_total |
|
|
88
|
+
# +------+-------+--------------------+
|
|
89
|
+
# | 75.0 | 205.0 | 259.88662671232834 |
|
|
90
|
+
# | 75.0 | 207.5 | 259.48268150684896 |
|
|
91
|
+
# | 75.0 | 230.0 | 258.9192123287667 |
|
|
92
|
+
# | 75.0 | 275.0 | 257.07574315068456 |
|
|
93
|
+
# | 75.0 | 322.5 | 250.11792123287654 |
|
|
94
|
+
# | 75.0 | 325.0 | 250.81590068493125 |
|
|
95
|
+
# | 72.5 | 205.0 | 262.74933904109537 |
|
|
96
|
+
# | 72.5 | 207.5 | 262.5384315068488 |
|
|
97
|
+
# | 72.5 | 230.0 | 260.8287945205475 |
|
|
98
|
+
# | 72.5 | 275.0 | 257.30633219178037 |
|
|
99
|
+
# +------+-------+--------------------+
|
|
100
|
+
# Data truncated.
|
|
101
|
+
#
|
|
69
102
|
|
|
70
103
|
# A table of the average temperature for each location across time.
|
|
71
|
-
df.to_pandas()
|
|
72
|
-
|
|
73
|
-
# Alternatively, you can just create the DataFrame from the Dataset:
|
|
74
|
-
df = qr.read_xarray(ds).to_pandas()
|
|
104
|
+
df = result.to_pandas()
|
|
75
105
|
df.head()
|
|
106
|
+
# lat lon air_total
|
|
107
|
+
# 0 75.0 210.0 259.016562
|
|
108
|
+
# 1 75.0 222.5 258.362212
|
|
109
|
+
# 2 75.0 237.5 258.318240
|
|
110
|
+
# 3 75.0 267.5 256.928497
|
|
111
|
+
# 4 75.0 285.0 261.614103
|
|
76
112
|
```
|
|
77
113
|
|
|
78
|
-
Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
|
|
114
|
+
Succinctly, we "pivot" Xarray Datasets (with consistent dimensions) to treat them like tables so we can run
|
|
79
115
|
SQL queries against them.
|
|
80
116
|
|
|
81
117
|
## Why build this?
|
|
@@ -93,18 +129,18 @@ A few reasons:
|
|
|
93
129
|
|
|
94
130
|
This is a light-weight way to prove the value of the interface.
|
|
95
131
|
|
|
96
|
-
The larger goal is to explore the hypothesis that [Pangeo](https://pangeo.io/)
|
|
97
|
-
is a scientific database. Here, xarray-sql can be thought of as a missing
|
|
98
|
-
front end.
|
|
132
|
+
The larger goal is to explore the hypothesis that the [Pangeo](https://pangeo.io/)
|
|
133
|
+
ecosystem is a scientific database. Here, xarray-sql can be thought of as a missing
|
|
134
|
+
DB front end.
|
|
99
135
|
|
|
100
136
|
## How does it work?
|
|
101
137
|
|
|
102
|
-
All chunks in
|
|
138
|
+
All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
|
|
103
139
|
`from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
|
|
104
140
|
That's it!
|
|
105
141
|
|
|
106
|
-
_2025 update_: This library now implements a
|
|
107
|
-
pure
|
|
142
|
+
_2025 update_: This library now implements a Dask-like `from_map` interface in
|
|
143
|
+
pure DataFusion and PyArrow, but works with the same principle!
|
|
108
144
|
|
|
109
145
|
## Why does this work?
|
|
110
146
|
|
|
@@ -112,14 +148,15 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
|
|
|
112
148
|
chunks and represented contiguously in memory. It is only a matter of metadata
|
|
113
149
|
that breaks them up into ndarrays. `to_dataframe()`
|
|
114
150
|
just changes this metadata (via a `ravel()`/`reshape()`), back into a column
|
|
115
|
-
amenable to a DataFrame.
|
|
116
|
-
|
|
117
|
-
There is added overhead from duplicating dimensions as columns, which we see as
|
|
118
|
-
worth the convenience of DataFrames.
|
|
151
|
+
amenable to a DataFrame. We take advantage of this light weight metadata change to
|
|
152
|
+
make chunked information scannable by a DB engine (DataFusion).
|
|
119
153
|
|
|
120
154
|
## What are the current limitations?
|
|
121
155
|
|
|
122
|
-
_2025 update_: TBD,
|
|
156
|
+
_2025 update_: TBD, DataFusion provides a whole new world! Currently, we're looking for
|
|
157
|
+
early users – "tire kickers", if you will. We'd love your input to shape the direction of this
|
|
158
|
+
project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
|
|
159
|
+
you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
|
|
123
160
|
|
|
124
161
|
## What would a deeper integration look like?
|
|
125
162
|
|
|
@@ -134,6 +171,13 @@ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
|
|
|
134
171
|
and BigQuery. More thoughts on this
|
|
135
172
|
in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
|
|
136
173
|
|
|
174
|
+
_2025 update_: Something like this is being built across a few projects! The ones I know about are:
|
|
175
|
+
- [CartoDB's Raquet](https://github.com/CartoDB/raquet)
|
|
176
|
+
- The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
|
|
177
|
+
|
|
178
|
+
As of writing, this project is [amid integrating](https://github.com/alxmrs/xarray-sql/pull/69) a
|
|
179
|
+
rust-based DataFusion backend provided by arrow-zarr.
|
|
180
|
+
|
|
137
181
|
## Sponsors & Contributors
|
|
138
182
|
|
|
139
183
|
I want to give a special thanks to the following folks and institutions:
|
|
@@ -143,6 +187,9 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
143
187
|
- Jake Wall and AI2/Ecoscope for compute resources and key use cases.
|
|
144
188
|
- Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
|
|
145
189
|
for the early review and discussion of this project.
|
|
190
|
+
- Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
|
|
191
|
+
Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
|
|
192
|
+
and DataFusion-specific collaboration.
|
|
146
193
|
|
|
147
194
|
## License
|
|
148
195
|
|
|
@@ -11,19 +11,35 @@ pip install xarray-sql
|
|
|
11
11
|
|
|
12
12
|
## What is this?
|
|
13
13
|
|
|
14
|
-
This is an experiment to provide a SQL interface for
|
|
14
|
+
This is an experiment to provide a SQL interface for array datasets.
|
|
15
15
|
|
|
16
16
|
```python
|
|
17
17
|
import xarray as xr
|
|
18
|
-
import xarray_sql as
|
|
18
|
+
import xarray_sql as xql
|
|
19
19
|
|
|
20
20
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
21
21
|
|
|
22
22
|
# The same as a dask-sql Context; i.e. an Apache DataFusion Context.
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
ctx = xql.XarrayContext()
|
|
24
|
+
ctx.from_dataset('air', ds, chunks=dict(time=24)) # the dataset needs to be chunked!
|
|
25
|
+
# DataFrame()
|
|
26
|
+
# +------+---------------------+-------+--------------------+
|
|
27
|
+
# | lat | time | lon | air |
|
|
28
|
+
# +------+---------------------+-------+--------------------+
|
|
29
|
+
# | 75.0 | 2013-01-01T00:00:00 | 200.0 | 241.20000000000002 |
|
|
30
|
+
# | 75.0 | 2013-01-01T00:00:00 | 202.5 | 242.5 |
|
|
31
|
+
# | 75.0 | 2013-01-01T00:00:00 | 205.0 | 243.5 |
|
|
32
|
+
# | 75.0 | 2013-01-01T00:00:00 | 207.5 | 244.0 |
|
|
33
|
+
# | 75.0 | 2013-01-01T00:00:00 | 210.0 | 244.1 |
|
|
34
|
+
# | 75.0 | 2013-01-01T00:00:00 | 212.5 | 243.89000000000001 |
|
|
35
|
+
# | 75.0 | 2013-01-01T00:00:00 | 215.0 | 243.6 |
|
|
36
|
+
# | 75.0 | 2013-01-01T00:00:00 | 217.5 | 243.1 |
|
|
37
|
+
# | 75.0 | 2013-01-01T00:00:00 | 220.0 | 242.5 |
|
|
38
|
+
# | 75.0 | 2013-01-01T00:00:00 | 222.5 | 241.89000000000001 |
|
|
39
|
+
# +------+---------------------+-------+--------------------+
|
|
40
|
+
# Data truncated.
|
|
41
|
+
|
|
42
|
+
result = ctx.sql('''
|
|
27
43
|
SELECT
|
|
28
44
|
"lat", "lon", AVG("air") as air_total
|
|
29
45
|
FROM
|
|
@@ -31,16 +47,36 @@ df = c.sql('''
|
|
|
31
47
|
GROUP BY
|
|
32
48
|
"lat", "lon"
|
|
33
49
|
''')
|
|
50
|
+
# DataFrame()
|
|
51
|
+
# +------+-------+--------------------+
|
|
52
|
+
# | lat | lon | air_total |
|
|
53
|
+
# +------+-------+--------------------+
|
|
54
|
+
# | 75.0 | 205.0 | 259.88662671232834 |
|
|
55
|
+
# | 75.0 | 207.5 | 259.48268150684896 |
|
|
56
|
+
# | 75.0 | 230.0 | 258.9192123287667 |
|
|
57
|
+
# | 75.0 | 275.0 | 257.07574315068456 |
|
|
58
|
+
# | 75.0 | 322.5 | 250.11792123287654 |
|
|
59
|
+
# | 75.0 | 325.0 | 250.81590068493125 |
|
|
60
|
+
# | 72.5 | 205.0 | 262.74933904109537 |
|
|
61
|
+
# | 72.5 | 207.5 | 262.5384315068488 |
|
|
62
|
+
# | 72.5 | 230.0 | 260.8287945205475 |
|
|
63
|
+
# | 72.5 | 275.0 | 257.30633219178037 |
|
|
64
|
+
# +------+-------+--------------------+
|
|
65
|
+
# Data truncated.
|
|
66
|
+
#
|
|
34
67
|
|
|
35
68
|
# A table of the average temperature for each location across time.
|
|
36
|
-
df.to_pandas()
|
|
37
|
-
|
|
38
|
-
# Alternatively, you can just create the DataFrame from the Dataset:
|
|
39
|
-
df = qr.read_xarray(ds).to_pandas()
|
|
69
|
+
df = result.to_pandas()
|
|
40
70
|
df.head()
|
|
71
|
+
# lat lon air_total
|
|
72
|
+
# 0 75.0 210.0 259.016562
|
|
73
|
+
# 1 75.0 222.5 258.362212
|
|
74
|
+
# 2 75.0 237.5 258.318240
|
|
75
|
+
# 3 75.0 267.5 256.928497
|
|
76
|
+
# 4 75.0 285.0 261.614103
|
|
41
77
|
```
|
|
42
78
|
|
|
43
|
-
Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
|
|
79
|
+
Succinctly, we "pivot" Xarray Datasets (with consistent dimensions) to treat them like tables so we can run
|
|
44
80
|
SQL queries against them.
|
|
45
81
|
|
|
46
82
|
## Why build this?
|
|
@@ -58,18 +94,18 @@ A few reasons:
|
|
|
58
94
|
|
|
59
95
|
This is a light-weight way to prove the value of the interface.
|
|
60
96
|
|
|
61
|
-
The larger goal is to explore the hypothesis that [Pangeo](https://pangeo.io/)
|
|
62
|
-
is a scientific database. Here, xarray-sql can be thought of as a missing
|
|
63
|
-
front end.
|
|
97
|
+
The larger goal is to explore the hypothesis that the [Pangeo](https://pangeo.io/)
|
|
98
|
+
ecosystem is a scientific database. Here, xarray-sql can be thought of as a missing
|
|
99
|
+
DB front end.
|
|
64
100
|
|
|
65
101
|
## How does it work?
|
|
66
102
|
|
|
67
|
-
All chunks in
|
|
103
|
+
All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
|
|
68
104
|
`from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
|
|
69
105
|
That's it!
|
|
70
106
|
|
|
71
|
-
_2025 update_: This library now implements a
|
|
72
|
-
pure
|
|
107
|
+
_2025 update_: This library now implements a Dask-like `from_map` interface in
|
|
108
|
+
pure DataFusion and PyArrow, but works with the same principle!
|
|
73
109
|
|
|
74
110
|
## Why does this work?
|
|
75
111
|
|
|
@@ -77,14 +113,15 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
|
|
|
77
113
|
chunks and represented contiguously in memory. It is only a matter of metadata
|
|
78
114
|
that breaks them up into ndarrays. `to_dataframe()`
|
|
79
115
|
just changes this metadata (via a `ravel()`/`reshape()`), back into a column
|
|
80
|
-
amenable to a DataFrame.
|
|
81
|
-
|
|
82
|
-
There is added overhead from duplicating dimensions as columns, which we see as
|
|
83
|
-
worth the convenience of DataFrames.
|
|
116
|
+
amenable to a DataFrame. We take advantage of this light weight metadata change to
|
|
117
|
+
make chunked information scannable by a DB engine (DataFusion).
|
|
84
118
|
|
|
85
119
|
## What are the current limitations?
|
|
86
120
|
|
|
87
|
-
_2025 update_: TBD,
|
|
121
|
+
_2025 update_: TBD, DataFusion provides a whole new world! Currently, we're looking for
|
|
122
|
+
early users – "tire kickers", if you will. We'd love your input to shape the direction of this
|
|
123
|
+
project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
|
|
124
|
+
you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
|
|
88
125
|
|
|
89
126
|
## What would a deeper integration look like?
|
|
90
127
|
|
|
@@ -99,6 +136,13 @@ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
|
|
|
99
136
|
and BigQuery. More thoughts on this
|
|
100
137
|
in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
|
|
101
138
|
|
|
139
|
+
_2025 update_: Something like this is being built across a few projects! The ones I know about are:
|
|
140
|
+
- [CartoDB's Raquet](https://github.com/CartoDB/raquet)
|
|
141
|
+
- The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
|
|
142
|
+
|
|
143
|
+
As of writing, this project is [amid integrating](https://github.com/alxmrs/xarray-sql/pull/69) a
|
|
144
|
+
rust-based DataFusion backend provided by arrow-zarr.
|
|
145
|
+
|
|
102
146
|
## Sponsors & Contributors
|
|
103
147
|
|
|
104
148
|
I want to give a special thanks to the following folks and institutions:
|
|
@@ -108,6 +152,9 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
108
152
|
- Jake Wall and AI2/Ecoscope for compute resources and key use cases.
|
|
109
153
|
- Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
|
|
110
154
|
for the early review and discussion of this project.
|
|
155
|
+
- Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
|
|
156
|
+
Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
|
|
157
|
+
and DataFusion-specific collaboration.
|
|
111
158
|
|
|
112
159
|
## License
|
|
113
160
|
|
|
@@ -150,7 +150,24 @@ def pivot(ds: xr.Dataset) -> pd.DataFrame:
|
|
|
150
150
|
return ds.to_dataframe().reset_index()
|
|
151
151
|
|
|
152
152
|
|
|
153
|
-
def
|
|
153
|
+
def _parse_schema(ds) -> pa.Schema:
|
|
154
|
+
"""Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns."""
|
|
155
|
+
columns = []
|
|
156
|
+
|
|
157
|
+
for coord_name, coord_var in ds.coords.items():
|
|
158
|
+
# Only include dimension coordinates
|
|
159
|
+
if coord_name in ds.dims:
|
|
160
|
+
pa_type = pa.from_numpy_dtype(coord_var.dtype)
|
|
161
|
+
columns.append(pa.field(coord_name, pa_type))
|
|
162
|
+
|
|
163
|
+
for var_name, var in ds.data_vars.items():
|
|
164
|
+
pa_type = pa.from_numpy_dtype(var.dtype)
|
|
165
|
+
columns.append(pa.field(var_name, pa_type))
|
|
166
|
+
|
|
167
|
+
return pa.schema(columns)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
|
|
154
171
|
"""Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
|
|
155
172
|
|
|
156
173
|
Args:
|
|
@@ -162,18 +179,15 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> ArrowStreamExportable:
|
|
|
162
179
|
Returns:
|
|
163
180
|
A PyArrow Table, which is a table representation of the input Dataset.
|
|
164
181
|
"""
|
|
165
|
-
fst = next(iter(ds.values())).dims
|
|
166
|
-
assert all(
|
|
167
|
-
da.dims == fst for da in ds.values()
|
|
168
|
-
), "All dimensions must be equal. Please filter data_vars in the Dataset."
|
|
169
|
-
|
|
170
|
-
blocks = list(block_slices(ds, chunks))
|
|
171
182
|
|
|
172
183
|
def pivot_block(b: Block):
|
|
173
184
|
return pivot(ds.isel(b))
|
|
174
185
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
186
|
+
fst = next(iter(ds.values())).dims
|
|
187
|
+
assert all(
|
|
188
|
+
da.dims == fst for da in ds.values()
|
|
189
|
+
), "All dimensions must be equal. Please filter data_vars in the Dataset."
|
|
178
190
|
|
|
191
|
+
schema = _parse_schema(ds)
|
|
192
|
+
blocks = block_slices(ds, chunks)
|
|
179
193
|
return from_map_batched(pivot_block, blocks, schema=schema)
|