xarray-sql 0.0.2__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/.github/workflows/ci.yml +10 -19
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/.github/workflows/lint.yml +8 -17
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/.github/workflows/publish.yml +7 -7
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/.gitignore +3 -1
- xarray_sql-0.1.0/.python-version +1 -0
- xarray_sql-0.1.0/CONTRIBUTING.md +36 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/PKG-INFO +24 -28
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/README.md +16 -18
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air.py +4 -2
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air_full.py +4 -2
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/pyproject.toml +13 -10
- xarray_sql-0.1.0/uv.lock +2783 -0
- xarray_sql-0.1.0/xarray_sql/__init__.py +2 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql/core.py +4 -3
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql/df.py +59 -40
- xarray_sql-0.1.0/xarray_sql/df_test.py +182 -0
- xarray_sql-0.1.0/xarray_sql/sql.py +17 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql/sql_test.py +22 -18
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql.egg-info/PKG-INFO +24 -28
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql.egg-info/SOURCES.txt +2 -0
- xarray_sql-0.1.0/xarray_sql.egg-info/requires.txt +8 -0
- xarray_sql-0.0.2/CONTRIBUTING.md +0 -22
- xarray_sql-0.0.2/xarray_sql/__init__.py +0 -2
- xarray_sql-0.0.2/xarray_sql/df_test.py +0 -89
- xarray_sql-0.0.2/xarray_sql/sql.py +0 -20
- xarray_sql-0.0.2/xarray_sql.egg-info/requires.txt +0 -12
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/LICENSE +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/README.md +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/compute_air.py +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/compute_air.py-2024-02-26T22:57:59+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/compute_air.py-2024-03-03T07:42:36+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/compute_air.py-2024-03-03T07:50:18+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air.py-2024-02-15T13:01:22+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air.py-2024-02-15T15:03:53+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air.py-2024-02-17T18:42:14+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air.py-2024-02-26T23:04:34+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air.py-2024-03-03T07:43:22+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air.py-2024-03-03T07:50:06+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air_full.py-2024-02-26T23:06:38+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air_full.py-2024-03-03T07:44:17+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/groupby_air_full.py-2024-03-03T07:49:53+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/open_era5.py +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/open_era5.py-2024-02-18T17:21:44+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/open_era5.py-2024-02-18T17:33:02+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/open_era5.py-2024-02-19T17:52:51+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/profile.sh +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-02-15T09:08:28+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-02-15T09:15:07+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-02-15T12:51:37+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-02-15T15:04:48+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-02-17T18:42:08+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-02-26T23:00:00+07:00.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-03-03T07:45:47+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/perf_tests/sanity.py-2024-03-03T07:49:44+05:30.svg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/setup.cfg +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql/df_integrationtest.py +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql.egg-info/dependency_links.txt +0 -0
- {xarray_sql-0.0.2 → xarray_sql-0.1.0}/xarray_sql.egg-info/top_level.txt +0 -0
|
@@ -33,28 +33,19 @@ jobs:
|
|
|
33
33
|
fail-fast: false
|
|
34
34
|
matrix:
|
|
35
35
|
python-version: [
|
|
36
|
-
"3.8",
|
|
37
36
|
"3.9",
|
|
38
37
|
"3.10",
|
|
39
38
|
"3.11",
|
|
40
39
|
"3.12",
|
|
40
|
+
"3.13",
|
|
41
41
|
]
|
|
42
42
|
steps:
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
python-version: ${{ matrix.python-version }}
|
|
53
|
-
cache: 'pip'
|
|
54
|
-
- name: Install xarray_sql
|
|
55
|
-
run: |
|
|
56
|
-
pip install -e .[test]
|
|
57
|
-
- uses: 'actions/checkout@v4'
|
|
58
|
-
- name: Run unit tests
|
|
59
|
-
run: |
|
|
60
|
-
pytest xarray_sql
|
|
43
|
+
- uses: actions/checkout@v4
|
|
44
|
+
- name: Install uv and set the python version
|
|
45
|
+
uses: astral-sh/setup-uv@v5
|
|
46
|
+
with:
|
|
47
|
+
python-version: ${{ matrix.python-version }}
|
|
48
|
+
- name: Install xarray_sql
|
|
49
|
+
run: uv sync --dev
|
|
50
|
+
- name: Run unit tests
|
|
51
|
+
run: uv run pytest
|
|
@@ -32,21 +32,12 @@ jobs:
|
|
|
32
32
|
strategy:
|
|
33
33
|
fail-fast: false
|
|
34
34
|
matrix:
|
|
35
|
-
python-version: ["3.
|
|
35
|
+
python-version: ["3.9", "3.13"]
|
|
36
36
|
steps:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
uses: actions/setup-python@v2
|
|
45
|
-
with:
|
|
46
|
-
python-version: ${{ matrix.python-version }}
|
|
47
|
-
- name: Install linter
|
|
48
|
-
run: |
|
|
49
|
-
pip install pyink
|
|
50
|
-
- name: Lint with pyink
|
|
51
|
-
run: |
|
|
52
|
-
pyink --check .
|
|
37
|
+
- uses: actions/checkout@v4
|
|
38
|
+
- name: Install uv and set the python version
|
|
39
|
+
uses: astral-sh/setup-uv@v5
|
|
40
|
+
with:
|
|
41
|
+
python-version: ${{ matrix.python-version }}
|
|
42
|
+
- name: Lint with pyink
|
|
43
|
+
run: uvx pyink --check .
|
|
@@ -23,11 +23,11 @@ jobs:
|
|
|
23
23
|
build-artifacts:
|
|
24
24
|
runs-on: ubuntu-latest
|
|
25
25
|
steps:
|
|
26
|
-
- uses: actions/checkout@
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
27
|
- name: Set up Python
|
|
28
28
|
uses: actions/setup-python@v2.3.1
|
|
29
29
|
with:
|
|
30
|
-
python-version: 3.
|
|
30
|
+
python-version: 3.11
|
|
31
31
|
|
|
32
32
|
- name: Install dependencies
|
|
33
33
|
run: |
|
|
@@ -43,7 +43,7 @@ jobs:
|
|
|
43
43
|
run: |
|
|
44
44
|
python -m twine check dist/*
|
|
45
45
|
pwd
|
|
46
|
-
- uses: actions/upload-artifact@
|
|
46
|
+
- uses: actions/upload-artifact@v4
|
|
47
47
|
with:
|
|
48
48
|
name: releases
|
|
49
49
|
path: dist
|
|
@@ -55,8 +55,8 @@ jobs:
|
|
|
55
55
|
- uses: actions/setup-python@v2.3.1
|
|
56
56
|
name: Install Python
|
|
57
57
|
with:
|
|
58
|
-
python-version: 3.
|
|
59
|
-
- uses: actions/download-artifact@
|
|
58
|
+
python-version: 3.11
|
|
59
|
+
- uses: actions/download-artifact@v4
|
|
60
60
|
with:
|
|
61
61
|
name: releases
|
|
62
62
|
path: dist
|
|
@@ -79,13 +79,13 @@ jobs:
|
|
|
79
79
|
sleep 3
|
|
80
80
|
python -m pip install --upgrade pip
|
|
81
81
|
python -m pip install --extra-index-url https://test.pypi.org/simple --upgrade de
|
|
82
|
-
python -c "import
|
|
82
|
+
python -c "import xarray_sql; print(xarray_sql.__version__)"
|
|
83
83
|
upload-to-pypi:
|
|
84
84
|
needs: test-built-dist
|
|
85
85
|
if: github.event_name == 'release'
|
|
86
86
|
runs-on: ubuntu-latest
|
|
87
87
|
steps:
|
|
88
|
-
- uses: actions/download-artifact@
|
|
88
|
+
- uses: actions/download-artifact@v4
|
|
89
89
|
with:
|
|
90
90
|
name: releases
|
|
91
91
|
path: dist
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.11
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Contributing Guide
|
|
2
|
+
|
|
3
|
+
## Where to start?
|
|
4
|
+
|
|
5
|
+
Please check out the [issues tab](https://github.com/alxmrs/xarray-sql/issues).
|
|
6
|
+
Let's have a discussion over there before proceeding with any changes. Great
|
|
7
|
+
minds think alike -- someone may have already created an issue related to your
|
|
8
|
+
inquiry. If there's a bug, please let us know.
|
|
9
|
+
|
|
10
|
+
If you're totally new to open source development, I recommend
|
|
11
|
+
reading [Xarray's contributing guide](https://docs.xarray.dev/en/stable/contributing.html).
|
|
12
|
+
|
|
13
|
+
## Developer setup
|
|
14
|
+
|
|
15
|
+
0. Create a project-specific python
|
|
16
|
+
environment with `uv`: https://docs.astral.sh/uv/getting-started/installation/
|
|
17
|
+
1. Clone the repository (bonus: [via SSH](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account))
|
|
18
|
+
and `cd xarray_sql` (the project root).
|
|
19
|
+
2. Install dev dependencies via: `uv sync --dev`
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
## Before submitting a pull request...
|
|
23
|
+
|
|
24
|
+
Thanks so much for your contribution! For a volunteer led project, we so
|
|
25
|
+
appreciate your help. A few things to keep in mind:
|
|
26
|
+
- Please be nice. We assume good intent from you, and we ask you to do the same for us.
|
|
27
|
+
- Development in this project will be slow if not sporadic. Reviews will come
|
|
28
|
+
as time allows.
|
|
29
|
+
- Every contribution, big or small, matters and deserves credit.
|
|
30
|
+
|
|
31
|
+
Here are a few requests for your development process:
|
|
32
|
+
- We require all code to be formatted with `pyink`.
|
|
33
|
+
- Please include unit tests, if possible, and performance tests when you touch the core functionality (see `perf_tests/`).
|
|
34
|
+
- It's polite to do a self review before asking for one from a maintainer. Don't stress if you forget; we all do sometimes.
|
|
35
|
+
- Please add (or update) documentation when adding new code. We use Google Style docstrings.
|
|
36
|
+
- We are thrilled to get documentation-only PRs -- especially spelling and typo fixes (I am a bad speller). If writing tutorials excites you, it would be to everyone's benefit.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: xarray_sql
|
|
3
|
-
Version: 0.0
|
|
3
|
+
Version: 0.1.0
|
|
4
4
|
Summary: Querry Xarray with SQL.
|
|
5
5
|
Author-email: Alexander Merose <al@merose.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -14,26 +14,24 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
14
14
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
15
15
|
Classifier: Operating System :: Microsoft :: Windows
|
|
16
16
|
Classifier: Operating System :: POSIX
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.9
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
23
23
|
Classifier: Topic :: Database :: Front-Ends
|
|
24
|
-
Requires-Python: >=3.
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist:
|
|
27
|
+
Requires-Dist: dask>=2024.8.0
|
|
28
|
+
Requires-Dist: datafusion>=47.0.0
|
|
29
|
+
Requires-Dist: xarray>=2024.7.0
|
|
29
30
|
Provides-Extra: test
|
|
30
31
|
Requires-Dist: pytest; extra == "test"
|
|
31
32
|
Requires-Dist: xarray[io]; extra == "test"
|
|
32
33
|
Requires-Dist: gcsfs; extra == "test"
|
|
33
|
-
|
|
34
|
-
Requires-Dist: xarray_sql[test]; extra == "dev"
|
|
35
|
-
Requires-Dist: pyink; extra == "dev"
|
|
36
|
-
Requires-Dist: py-spy; extra == "dev"
|
|
34
|
+
Dynamic: license-file
|
|
37
35
|
|
|
38
36
|
# xarray-sql
|
|
39
37
|
|
|
@@ -42,6 +40,9 @@ _Query Xarray with SQL_
|
|
|
42
40
|
[](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml)
|
|
43
41
|
[](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml)
|
|
44
42
|
|
|
43
|
+
```shell
|
|
44
|
+
pip install xarray-sql
|
|
45
|
+
```
|
|
45
46
|
|
|
46
47
|
## What is this?
|
|
47
48
|
|
|
@@ -54,8 +55,8 @@ import xarray_sql as qr
|
|
|
54
55
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
55
56
|
|
|
56
57
|
# The same as a dask-sql Context; i.e. an Apache DataFusion Context.
|
|
57
|
-
c = qr.
|
|
58
|
-
c.
|
|
58
|
+
c = qr.XarrayContext()
|
|
59
|
+
c.from_dataset('air', ds, chunks=dict(time=24))
|
|
59
60
|
|
|
60
61
|
df = c.sql('''
|
|
61
62
|
SELECT
|
|
@@ -67,10 +68,10 @@ df = c.sql('''
|
|
|
67
68
|
''')
|
|
68
69
|
|
|
69
70
|
# A table of the average temperature for each location across time.
|
|
70
|
-
df.
|
|
71
|
+
df.to_pandas()
|
|
71
72
|
|
|
72
73
|
# Alternatively, you can just create the DataFrame from the Dataset:
|
|
73
|
-
df = qr.read_xarray(ds)
|
|
74
|
+
df = qr.read_xarray(ds).to_pandas()
|
|
74
75
|
df.head()
|
|
75
76
|
```
|
|
76
77
|
|
|
@@ -82,7 +83,7 @@ SQL queries against them.
|
|
|
82
83
|
A few reasons:
|
|
83
84
|
|
|
84
85
|
* Even though SQL is the lingua franca of data, scientific datasets are often
|
|
85
|
-
inaccessible to non-scientists.
|
|
86
|
+
inaccessible to non-scientists (SQL users).
|
|
86
87
|
* Joining tabular data with raster data is common yet difficult. It could be
|
|
87
88
|
easy.
|
|
88
89
|
* There are many cloud-native, Xarray-openable datasets,
|
|
@@ -102,10 +103,13 @@ All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
|
|
|
102
103
|
`from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
|
|
103
104
|
That's it!
|
|
104
105
|
|
|
106
|
+
_2025 update_: This library now implements a dask-like `from_map` interface in
|
|
107
|
+
pure `datafusion` and `pyarrow`, but works with the same principle!
|
|
108
|
+
|
|
105
109
|
## Why does this work?
|
|
106
110
|
|
|
107
111
|
Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
|
|
108
|
-
|
|
112
|
+
chunks and represented contiguously in memory. It is only a matter of metadata
|
|
109
113
|
that breaks them up into ndarrays. `to_dataframe()`
|
|
110
114
|
just changes this metadata (via a `ravel()`/`reshape()`), back into a column
|
|
111
115
|
amenable to a DataFrame.
|
|
@@ -115,15 +119,7 @@ worth the convenience of DataFrames.
|
|
|
115
119
|
|
|
116
120
|
## What are the current limitations?
|
|
117
121
|
|
|
118
|
-
|
|
119
|
-
`MultiIndex`s ([dask/dask#1493](https://github.com/dask/dask/issues/1493)). If
|
|
120
|
-
it did, I suspect performance for many types of queries would greatly improve.
|
|
121
|
-
|
|
122
|
-
Further, while this does play well with `dask-geopandas` (for geospatial query
|
|
123
|
-
support), certain types of operations don't quite match standard geopandas.
|
|
124
|
-
Spatial joins come to mind as a killer feature, but only inner joins are
|
|
125
|
-
supported ([geopandas/dask-geopandas#72](https://github.com/geopandas/dask-geopandas/issues/72))
|
|
126
|
-
.
|
|
122
|
+
_2025 update_: TBD, `datafusion` provides a whole new world!
|
|
127
123
|
|
|
128
124
|
## What would a deeper integration look like?
|
|
129
125
|
|
|
@@ -134,7 +130,7 @@ Xarray Datasets. This approach is being pursued
|
|
|
134
130
|
Deeper still: I was thinking we could make
|
|
135
131
|
a [virtual](https://fsspec.github.io/kerchunk/)
|
|
136
132
|
filesystem for parquet that would internally map to Zarr. Raster-backed virtual
|
|
137
|
-
parquet would open up integrations to
|
|
133
|
+
parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
|
|
138
134
|
and BigQuery. More thoughts on this
|
|
139
135
|
in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
|
|
140
136
|
|
|
@@ -145,8 +141,8 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
145
141
|
- Pramod Gupta and the Anthromet Team at Google Research for the problem
|
|
146
142
|
formation and design inspiration.
|
|
147
143
|
- Jake Wall and AI2/Ecoscope for compute resources and key use cases.
|
|
148
|
-
- Charles Stern, Stephan Hoyer,
|
|
149
|
-
discussion of this project.
|
|
144
|
+
- Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
|
|
145
|
+
for the early review and discussion of this project.
|
|
150
146
|
|
|
151
147
|
## License
|
|
152
148
|
|
|
@@ -5,6 +5,9 @@ _Query Xarray with SQL_
|
|
|
5
5
|
[](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml)
|
|
6
6
|
[](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml)
|
|
7
7
|
|
|
8
|
+
```shell
|
|
9
|
+
pip install xarray-sql
|
|
10
|
+
```
|
|
8
11
|
|
|
9
12
|
## What is this?
|
|
10
13
|
|
|
@@ -17,8 +20,8 @@ import xarray_sql as qr
|
|
|
17
20
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
18
21
|
|
|
19
22
|
# The same as a dask-sql Context; i.e. an Apache DataFusion Context.
|
|
20
|
-
c = qr.
|
|
21
|
-
c.
|
|
23
|
+
c = qr.XarrayContext()
|
|
24
|
+
c.from_dataset('air', ds, chunks=dict(time=24))
|
|
22
25
|
|
|
23
26
|
df = c.sql('''
|
|
24
27
|
SELECT
|
|
@@ -30,10 +33,10 @@ df = c.sql('''
|
|
|
30
33
|
''')
|
|
31
34
|
|
|
32
35
|
# A table of the average temperature for each location across time.
|
|
33
|
-
df.
|
|
36
|
+
df.to_pandas()
|
|
34
37
|
|
|
35
38
|
# Alternatively, you can just create the DataFrame from the Dataset:
|
|
36
|
-
df = qr.read_xarray(ds)
|
|
39
|
+
df = qr.read_xarray(ds).to_pandas()
|
|
37
40
|
df.head()
|
|
38
41
|
```
|
|
39
42
|
|
|
@@ -45,7 +48,7 @@ SQL queries against them.
|
|
|
45
48
|
A few reasons:
|
|
46
49
|
|
|
47
50
|
* Even though SQL is the lingua franca of data, scientific datasets are often
|
|
48
|
-
inaccessible to non-scientists.
|
|
51
|
+
inaccessible to non-scientists (SQL users).
|
|
49
52
|
* Joining tabular data with raster data is common yet difficult. It could be
|
|
50
53
|
easy.
|
|
51
54
|
* There are many cloud-native, Xarray-openable datasets,
|
|
@@ -65,10 +68,13 @@ All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
|
|
|
65
68
|
`from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
|
|
66
69
|
That's it!
|
|
67
70
|
|
|
71
|
+
_2025 update_: This library now implements a dask-like `from_map` interface in
|
|
72
|
+
pure `datafusion` and `pyarrow`, but works with the same principle!
|
|
73
|
+
|
|
68
74
|
## Why does this work?
|
|
69
75
|
|
|
70
76
|
Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
|
|
71
|
-
|
|
77
|
+
chunks and represented contiguously in memory. It is only a matter of metadata
|
|
72
78
|
that breaks them up into ndarrays. `to_dataframe()`
|
|
73
79
|
just changes this metadata (via a `ravel()`/`reshape()`), back into a column
|
|
74
80
|
amenable to a DataFrame.
|
|
@@ -78,15 +84,7 @@ worth the convenience of DataFrames.
|
|
|
78
84
|
|
|
79
85
|
## What are the current limitations?
|
|
80
86
|
|
|
81
|
-
|
|
82
|
-
`MultiIndex`s ([dask/dask#1493](https://github.com/dask/dask/issues/1493)). If
|
|
83
|
-
it did, I suspect performance for many types of queries would greatly improve.
|
|
84
|
-
|
|
85
|
-
Further, while this does play well with `dask-geopandas` (for geospatial query
|
|
86
|
-
support), certain types of operations don't quite match standard geopandas.
|
|
87
|
-
Spatial joins come to mind as a killer feature, but only inner joins are
|
|
88
|
-
supported ([geopandas/dask-geopandas#72](https://github.com/geopandas/dask-geopandas/issues/72))
|
|
89
|
-
.
|
|
87
|
+
_2025 update_: TBD, `datafusion` provides a whole new world!
|
|
90
88
|
|
|
91
89
|
## What would a deeper integration look like?
|
|
92
90
|
|
|
@@ -97,7 +95,7 @@ Xarray Datasets. This approach is being pursued
|
|
|
97
95
|
Deeper still: I was thinking we could make
|
|
98
96
|
a [virtual](https://fsspec.github.io/kerchunk/)
|
|
99
97
|
filesystem for parquet that would internally map to Zarr. Raster-backed virtual
|
|
100
|
-
parquet would open up integrations to
|
|
98
|
+
parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
|
|
101
99
|
and BigQuery. More thoughts on this
|
|
102
100
|
in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
|
|
103
101
|
|
|
@@ -108,8 +106,8 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
108
106
|
- Pramod Gupta and the Anthromet Team at Google Research for the problem
|
|
109
107
|
formation and design inspiration.
|
|
110
108
|
- Jake Wall and AI2/Ecoscope for compute resources and key use cases.
|
|
111
|
-
- Charles Stern, Stephan Hoyer,
|
|
112
|
-
discussion of this project.
|
|
109
|
+
- Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
|
|
110
|
+
for the early review and discussion of this project.
|
|
113
111
|
|
|
114
112
|
## License
|
|
115
113
|
|
|
@@ -18,14 +18,16 @@ if __name__ == '__main__':
|
|
|
18
18
|
c = Context()
|
|
19
19
|
c.create_table('air', df)
|
|
20
20
|
|
|
21
|
-
query = c.sql(
|
|
21
|
+
query = c.sql(
|
|
22
|
+
"""
|
|
22
23
|
SELECT
|
|
23
24
|
"lat", "lon", SUM("air") as air_total
|
|
24
25
|
FROM
|
|
25
26
|
"air"
|
|
26
27
|
GROUP BY
|
|
27
28
|
"lat", "lon"
|
|
28
|
-
"""
|
|
29
|
+
"""
|
|
30
|
+
)
|
|
29
31
|
|
|
30
32
|
result = query.compute()
|
|
31
33
|
|
|
@@ -15,14 +15,16 @@ if __name__ == '__main__':
|
|
|
15
15
|
c = Context()
|
|
16
16
|
c.create_table('air', df)
|
|
17
17
|
|
|
18
|
-
query = c.sql(
|
|
18
|
+
query = c.sql(
|
|
19
|
+
"""
|
|
19
20
|
SELECT
|
|
20
21
|
"lat", "lon", SUM("air") as air_total
|
|
21
22
|
FROM
|
|
22
23
|
"air"
|
|
23
24
|
GROUP BY
|
|
24
25
|
"lat", "lon"
|
|
25
|
-
"""
|
|
26
|
+
"""
|
|
27
|
+
)
|
|
26
28
|
|
|
27
29
|
result = query.compute()
|
|
28
30
|
|
|
@@ -3,7 +3,7 @@ name = "xarray_sql"
|
|
|
3
3
|
dynamic = ["version"]
|
|
4
4
|
description = "Querry Xarray with SQL."
|
|
5
5
|
readme = "README.md"
|
|
6
|
-
requires-python = ">=3.
|
|
6
|
+
requires-python = ">=3.9"
|
|
7
7
|
license = {text = "Apache-2.0"}
|
|
8
8
|
authors = [
|
|
9
9
|
{name = "Alexander Merose", email = "al@merose.com"},
|
|
@@ -17,17 +17,18 @@ classifiers = [
|
|
|
17
17
|
"Operating System :: MacOS :: MacOS X",
|
|
18
18
|
"Operating System :: Microsoft :: Windows",
|
|
19
19
|
"Operating System :: POSIX",
|
|
20
|
-
"Programming Language :: Python :: 3.8",
|
|
21
20
|
"Programming Language :: Python :: 3.9",
|
|
22
21
|
"Programming Language :: Python :: 3.10",
|
|
23
22
|
"Programming Language :: Python :: 3.11",
|
|
24
23
|
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
25
|
"Topic :: Scientific/Engineering :: Atmospheric Science",
|
|
26
26
|
"Topic :: Database :: Front-Ends",
|
|
27
27
|
]
|
|
28
28
|
dependencies = [
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
"dask>=2024.8.0",
|
|
30
|
+
"datafusion>=47.0.0",
|
|
31
|
+
"xarray>=2024.7.0",
|
|
31
32
|
]
|
|
32
33
|
|
|
33
34
|
[project.optional-dependencies]
|
|
@@ -36,11 +37,6 @@ test = [
|
|
|
36
37
|
"xarray[io]",
|
|
37
38
|
"gcsfs",
|
|
38
39
|
]
|
|
39
|
-
dev = [
|
|
40
|
-
"xarray_sql[test]",
|
|
41
|
-
"pyink",
|
|
42
|
-
"py-spy"
|
|
43
|
-
]
|
|
44
40
|
|
|
45
41
|
[project.urls]
|
|
46
42
|
Homepage = "https://github.com/alxmrs/xarray-sql"
|
|
@@ -59,4 +55,11 @@ preview = true
|
|
|
59
55
|
pyink-indentation = 2
|
|
60
56
|
pyink-use-majority-quotes = true
|
|
61
57
|
|
|
62
|
-
[tool.setuptools_scm]
|
|
58
|
+
[tool.setuptools_scm]
|
|
59
|
+
|
|
60
|
+
[dependency-groups]
|
|
61
|
+
dev = [
|
|
62
|
+
"xarray_sql[test]",
|
|
63
|
+
"py-spy>=0.4.0",
|
|
64
|
+
"pyink>=24.10.1",
|
|
65
|
+
]
|