xarray-sql 0.0.2__py3-none-any.whl → 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xarray_sql/__init__.py +2 -2
- xarray_sql/core.py +4 -3
- xarray_sql/df.py +59 -40
- xarray_sql/df_test.py +114 -21
- xarray_sql/sql.py +7 -10
- xarray_sql/sql_test.py +22 -18
- {xarray_sql-0.0.2.dist-info → xarray_sql-0.1.0.dist-info}/METADATA +27 -31
- xarray_sql-0.1.0.dist-info/RECORD +12 -0
- {xarray_sql-0.0.2.dist-info → xarray_sql-0.1.0.dist-info}/WHEEL +1 -1
- xarray_sql-0.0.2.dist-info/RECORD +0 -12
- {xarray_sql-0.0.2.dist-info → xarray_sql-0.1.0.dist-info/licenses}/LICENSE +0 -0
- {xarray_sql-0.0.2.dist-info → xarray_sql-0.1.0.dist-info}/top_level.txt +0 -0
xarray_sql/__init__.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
from .df import read_xarray
|
|
2
|
-
from .sql import
|
|
1
|
+
from .df import read_xarray, from_map
|
|
2
|
+
from .sql import XarrayContext
|
xarray_sql/core.py
CHANGED
|
@@ -7,13 +7,14 @@ import xarray as xr
|
|
|
7
7
|
Row = t.List[t.Any]
|
|
8
8
|
|
|
9
9
|
|
|
10
|
+
# deprecated
|
|
10
11
|
def get_columns(ds: xr.Dataset) -> t.List[str]:
|
|
11
|
-
return list(ds.
|
|
12
|
+
return list(ds.sizes.keys()) + list(ds.data_vars.keys())
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
# Deprecated
|
|
15
16
|
def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
|
|
16
|
-
dim_keys, dim_vals = zip(*ds.
|
|
17
|
+
dim_keys, dim_vals = zip(*ds.sizes.items())
|
|
17
18
|
|
|
18
19
|
for idx in itertools.product(*(range(d) for d in dim_vals)):
|
|
19
20
|
coord_idx = dict(zip(dim_keys, idx))
|
|
@@ -26,7 +27,7 @@ def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
|
|
|
26
27
|
# Deprecated
|
|
27
28
|
def unbounded_unravel(ds: xr.Dataset) -> np.ndarray:
|
|
28
29
|
"""Unravel with unbounded memory (as a NumPy Array)."""
|
|
29
|
-
dim_keys, dim_vals = zip(*ds.
|
|
30
|
+
dim_keys, dim_vals = zip(*ds.sizes.items())
|
|
30
31
|
columns = get_columns(ds)
|
|
31
32
|
|
|
32
33
|
N = np.prod([d for d in dim_vals])
|
xarray_sql/df.py
CHANGED
|
@@ -1,24 +1,14 @@
|
|
|
1
1
|
import itertools
|
|
2
2
|
import typing as t
|
|
3
3
|
|
|
4
|
-
import dask
|
|
5
|
-
import dask.dataframe as dd
|
|
6
4
|
import numpy as np
|
|
7
5
|
import pandas as pd
|
|
6
|
+
import pyarrow as pa
|
|
8
7
|
import xarray as xr
|
|
9
|
-
from dask.dataframe.io import from_map
|
|
10
|
-
|
|
11
|
-
from . import core
|
|
12
8
|
|
|
13
9
|
Block = t.Dict[str, slice]
|
|
14
10
|
Chunks = t.Optional[t.Dict[str, int]]
|
|
15
11
|
|
|
16
|
-
# Turn on Dask-Expr
|
|
17
|
-
dask.config.set({'dataframe.query-planning-warning': False})
|
|
18
|
-
dask.config.set({'dataframe.query-planning': True})
|
|
19
|
-
# Turn on Copy-On-Write (needs Pandas 2.0).
|
|
20
|
-
pd.options.mode.copy_on_write = True
|
|
21
|
-
|
|
22
12
|
|
|
23
13
|
# Borrowed from Xarray
|
|
24
14
|
def _get_chunk_slicer(
|
|
@@ -42,7 +32,7 @@ def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[Block]:
|
|
|
42
32
|
else:
|
|
43
33
|
chunks = ds.chunks
|
|
44
34
|
|
|
45
|
-
assert chunks,
|
|
35
|
+
assert chunks, "Dataset `ds` must be chunked or `chunks` must be provided."
|
|
46
36
|
|
|
47
37
|
chunk_bounds = {dim: np.cumsum((0,) + c) for dim, c in chunks.items()}
|
|
48
38
|
ichunk = {dim: range(len(c)) for dim, c in chunks.items()}
|
|
@@ -67,8 +57,60 @@ def _block_len(block: Block) -> int:
|
|
|
67
57
|
return np.prod([v.stop - v.start for v in block.values()])
|
|
68
58
|
|
|
69
59
|
|
|
70
|
-
def
|
|
71
|
-
|
|
60
|
+
def from_map(
|
|
61
|
+
func: t.Callable, *iterables, args: t.Optional[t.Tuple] = None, **kwargs
|
|
62
|
+
) -> pa.Table:
|
|
63
|
+
"""Create a PyArrow Table by mapping a function over iterables.
|
|
64
|
+
|
|
65
|
+
This is equivalent to dask's from_map but returns a PyArrow Table
|
|
66
|
+
that can be used with DataFusion instead of a Dask DataFrame.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
func: Function to apply to each element of the iterables.
|
|
70
|
+
*iterables: Iterable objects to map the function over.
|
|
71
|
+
args: Additional positional arguments to pass to func.
|
|
72
|
+
**kwargs: Additional keyword arguments to pass to func.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
A PyArrow Table containing the concatenated results.
|
|
76
|
+
"""
|
|
77
|
+
if args is None:
|
|
78
|
+
args = ()
|
|
79
|
+
|
|
80
|
+
# Apply the function to each combination of iterable elements
|
|
81
|
+
results = []
|
|
82
|
+
for items in zip(*iterables) if len(iterables) > 1 else iterables[0]:
|
|
83
|
+
if isinstance(items, tuple):
|
|
84
|
+
result = func(*items, *args, **kwargs)
|
|
85
|
+
else:
|
|
86
|
+
result = func(items, *args, **kwargs)
|
|
87
|
+
|
|
88
|
+
# Convert result to PyArrow Table
|
|
89
|
+
if isinstance(result, pd.DataFrame):
|
|
90
|
+
pa_table = pa.Table.from_pandas(result)
|
|
91
|
+
elif isinstance(result, pa.Table):
|
|
92
|
+
pa_table = result
|
|
93
|
+
else:
|
|
94
|
+
# Try to convert to pandas first, then to PyArrow
|
|
95
|
+
try:
|
|
96
|
+
df = pd.DataFrame(result)
|
|
97
|
+
pa_table = pa.Table.from_pandas(df)
|
|
98
|
+
except Exception as e:
|
|
99
|
+
raise ValueError(
|
|
100
|
+
f"Cannot convert function result to PyArrow Table: {e}"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
results.append(pa_table)
|
|
104
|
+
|
|
105
|
+
# Concatenate all results
|
|
106
|
+
if not results:
|
|
107
|
+
raise ValueError("No results to concatenate")
|
|
108
|
+
|
|
109
|
+
return pa.concat_tables(results)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
|
|
113
|
+
"""Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
|
|
72
114
|
|
|
73
115
|
Args:
|
|
74
116
|
ds: An Xarray Dataset. All `data_vars` mush share the same dimensions.
|
|
@@ -77,39 +119,16 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> dd.DataFrame:
|
|
|
77
119
|
dataframe partition.
|
|
78
120
|
|
|
79
121
|
Returns:
|
|
80
|
-
A
|
|
122
|
+
A PyArrow Table, which is a table representation of the input Dataset.
|
|
81
123
|
"""
|
|
82
124
|
fst = next(iter(ds.values())).dims
|
|
83
125
|
assert all(
|
|
84
126
|
da.dims == fst for da in ds.values()
|
|
85
|
-
),
|
|
127
|
+
), "All dimensions must be equal. Please filter data_vars in the Dataset."
|
|
86
128
|
|
|
87
129
|
blocks = list(block_slices(ds, chunks))
|
|
88
130
|
|
|
89
|
-
block_lengths = [_block_len(b) for b in blocks]
|
|
90
|
-
divisions = tuple(np.cumsum([0] + block_lengths)) # 0 ==> start partition.
|
|
91
|
-
|
|
92
131
|
def pivot(b: Block) -> pd.DataFrame:
|
|
93
132
|
return ds.isel(b).to_dataframe().reset_index()
|
|
94
133
|
|
|
95
|
-
|
|
96
|
-
# it's own token from the constituent parts.
|
|
97
|
-
token = (
|
|
98
|
-
'xarray-Dataset-'
|
|
99
|
-
f'{"_".join(list(ds.dims.keys()))}'
|
|
100
|
-
'__'
|
|
101
|
-
f'{"_".join(list(ds.data_vars.keys()))}'
|
|
102
|
-
)
|
|
103
|
-
|
|
104
|
-
columns = core.get_columns(ds)
|
|
105
|
-
|
|
106
|
-
# TODO(#18): Is it possible to pass the length (known now) here?
|
|
107
|
-
meta = {c: ds[c].dtype for c in columns}
|
|
108
|
-
|
|
109
|
-
return from_map(
|
|
110
|
-
pivot,
|
|
111
|
-
blocks,
|
|
112
|
-
meta=meta,
|
|
113
|
-
divisions=divisions,
|
|
114
|
-
token=token,
|
|
115
|
-
)
|
|
134
|
+
return from_map(pivot, blocks)
|
xarray_sql/df_test.py
CHANGED
|
@@ -1,11 +1,37 @@
|
|
|
1
1
|
import itertools
|
|
2
2
|
import unittest
|
|
3
3
|
|
|
4
|
-
import dask.dataframe as dd
|
|
5
4
|
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import pyarrow as pa
|
|
6
7
|
import xarray as xr
|
|
7
8
|
|
|
8
|
-
from .df import explode, read_xarray, block_slices
|
|
9
|
+
from .df import explode, read_xarray, block_slices, from_map
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def rand_wx(start: str, end: str) -> xr.Dataset:
|
|
13
|
+
np.random.seed(42)
|
|
14
|
+
lat = np.linspace(-90, 90, num=720)
|
|
15
|
+
lon = np.linspace(-180, 180, num=1440)
|
|
16
|
+
time = pd.date_range(start, end, freq='h')
|
|
17
|
+
level = np.array([1000, 500], dtype=np.int32)
|
|
18
|
+
reference_time = pd.Timestamp(start)
|
|
19
|
+
temperature = 15 + 8 * np.random.randn(720, 1440, len(time), len(level))
|
|
20
|
+
precipitation = 10 * np.random.rand(720, 1440, len(time), len(level))
|
|
21
|
+
return xr.Dataset(
|
|
22
|
+
data_vars=dict(
|
|
23
|
+
temperature=(['lat', 'lon', 'time', 'level'], temperature),
|
|
24
|
+
precipitation=(['lat', 'lon', 'time', 'level'], precipitation),
|
|
25
|
+
),
|
|
26
|
+
coords=dict(
|
|
27
|
+
lat=lat,
|
|
28
|
+
lon=lon,
|
|
29
|
+
time=time,
|
|
30
|
+
level=level,
|
|
31
|
+
reference_time=reference_time,
|
|
32
|
+
),
|
|
33
|
+
attrs=dict(description='Random weather.'),
|
|
34
|
+
)
|
|
9
35
|
|
|
10
36
|
|
|
11
37
|
class DaskTestCase(unittest.TestCase):
|
|
@@ -18,6 +44,7 @@ class DaskTestCase(unittest.TestCase):
|
|
|
18
44
|
self.air_small = self.air.isel(
|
|
19
45
|
time=slice(0, 12), lat=slice(0, 11), lon=slice(0, 10)
|
|
20
46
|
).chunk(self.chunks)
|
|
47
|
+
self.randwx = rand_wx('1995-01-13T00', '1995-01-13T01')
|
|
21
48
|
|
|
22
49
|
|
|
23
50
|
class ExplodeTest(DaskTestCase):
|
|
@@ -32,7 +59,7 @@ class ExplodeTest(DaskTestCase):
|
|
|
32
59
|
ds = next(iter(explode(self.air)))
|
|
33
60
|
for k, v in self.chunks.items():
|
|
34
61
|
self.assertIn(k, ds.dims)
|
|
35
|
-
self.assertEqual(v, ds.
|
|
62
|
+
self.assertEqual(v, ds.sizes[k])
|
|
36
63
|
|
|
37
64
|
def skip_test_dim_sizes__all(self):
|
|
38
65
|
# TODO(alxmrs): Why is this test slow?
|
|
@@ -44,45 +71,111 @@ class ExplodeTest(DaskTestCase):
|
|
|
44
71
|
|
|
45
72
|
def test_data_equal__one__first(self):
|
|
46
73
|
ds = next(iter(explode(self.air)))
|
|
47
|
-
iselection = {dim: slice(0, s) for dim, s in ds.
|
|
74
|
+
iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
|
|
48
75
|
self.assertEqual(self.air.isel(iselection), ds)
|
|
49
76
|
|
|
50
77
|
def test_data_equal__one__last(self):
|
|
51
78
|
dss = list(explode(self.air))
|
|
52
79
|
ds = dss[-1]
|
|
53
|
-
iselection = {dim: slice(0, s) for dim, s in ds.
|
|
80
|
+
iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
|
|
54
81
|
self.assertEqual(self.air.isel(iselection), ds)
|
|
55
82
|
|
|
56
83
|
|
|
57
|
-
class
|
|
84
|
+
class PyArrowTableTest(DaskTestCase):
|
|
58
85
|
|
|
59
86
|
def test_sanity(self):
|
|
60
|
-
|
|
61
|
-
self.assertIsNotNone(
|
|
62
|
-
self.
|
|
87
|
+
table = read_xarray(self.air_small)
|
|
88
|
+
self.assertIsNotNone(table)
|
|
89
|
+
self.assertIsInstance(table, pa.Table)
|
|
90
|
+
self.assertEqual(len(table), np.prod(list(self.air_small.sizes.values())))
|
|
63
91
|
|
|
64
92
|
def test_columns(self):
|
|
65
|
-
|
|
66
|
-
cols =
|
|
93
|
+
table = read_xarray(self.air_small)
|
|
94
|
+
cols = table.column_names
|
|
67
95
|
self.assertEqual(cols, ['lat', 'time', 'lon', 'air'])
|
|
68
96
|
|
|
69
97
|
def test_dtypes(self):
|
|
70
|
-
|
|
98
|
+
table = read_xarray(self.air_small)
|
|
99
|
+
# Convert to pandas to check dtypes
|
|
100
|
+
df = table.to_pandas()
|
|
71
101
|
types = list(df.dtypes)
|
|
72
102
|
self.assertEqual([self.air_small[c].dtype for c in df.columns], types)
|
|
73
103
|
|
|
74
|
-
def
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
chunked: dd.DataFrame = read_xarray(self.air_small, dict(time=5))
|
|
104
|
+
def test_different_chunk_sizes(self):
|
|
105
|
+
default_table = read_xarray(self.air_small)
|
|
106
|
+
chunked_table = read_xarray(self.air_small, dict(time=5))
|
|
78
107
|
|
|
79
|
-
|
|
80
|
-
self.
|
|
108
|
+
# Both should produce valid tables
|
|
109
|
+
self.assertIsInstance(default_table, pa.Table)
|
|
110
|
+
self.assertIsInstance(chunked_table, pa.Table)
|
|
111
|
+
# Should have same number of rows
|
|
112
|
+
self.assertEqual(len(default_table), len(chunked_table))
|
|
81
113
|
|
|
82
114
|
def test_chunk_perf(self):
|
|
83
|
-
|
|
84
|
-
self.assertIsNotNone(
|
|
85
|
-
self.assertEqual(len(
|
|
115
|
+
table = read_xarray(self.air, chunks=dict(time=6))
|
|
116
|
+
self.assertIsNotNone(table)
|
|
117
|
+
self.assertEqual(len(table), np.prod(list(self.air.sizes.values())))
|
|
118
|
+
|
|
119
|
+
def test_column_metadata_preserved(self):
|
|
120
|
+
try:
|
|
121
|
+
table = read_xarray(self.randwx, chunks=dict(time=24))
|
|
122
|
+
self.assertIsInstance(table, pa.Table)
|
|
123
|
+
except Exception as e:
|
|
124
|
+
self.fail(f'Unexpected error: {e}')
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class FromMapTest(unittest.TestCase):
|
|
128
|
+
|
|
129
|
+
def test_basic_from_map(self):
|
|
130
|
+
"""Test basic from_map functionality with pandas DataFrames."""
|
|
131
|
+
|
|
132
|
+
def make_df(x):
|
|
133
|
+
return pd.DataFrame({'value': [x, x * 2], 'index': [0, 1]})
|
|
134
|
+
|
|
135
|
+
result = from_map(make_df, [1, 2, 3])
|
|
136
|
+
self.assertIsInstance(result, pa.Table)
|
|
137
|
+
self.assertEqual(len(result), 6) # 3 inputs * 2 rows each
|
|
138
|
+
self.assertEqual(result.column_names, ['value', 'index'])
|
|
139
|
+
|
|
140
|
+
def test_from_map_with_multiple_iterables(self):
|
|
141
|
+
"""Test from_map with multiple iterables."""
|
|
142
|
+
|
|
143
|
+
def add_values(x, y):
|
|
144
|
+
return pd.DataFrame({'sum': [x + y], 'x': [x], 'y': [y]})
|
|
145
|
+
|
|
146
|
+
result = from_map(add_values, [1, 2], [10, 20])
|
|
147
|
+
self.assertIsInstance(result, pa.Table)
|
|
148
|
+
self.assertEqual(len(result), 2)
|
|
149
|
+
|
|
150
|
+
# Convert to pandas to check values
|
|
151
|
+
df = result.to_pandas()
|
|
152
|
+
self.assertEqual(list(df['sum']), [11, 22])
|
|
153
|
+
|
|
154
|
+
def test_from_map_with_args(self):
|
|
155
|
+
"""Test from_map with additional arguments."""
|
|
156
|
+
|
|
157
|
+
def multiply_and_add(x, multiplier, add_value):
|
|
158
|
+
return pd.DataFrame({'result': [x * multiplier + add_value]})
|
|
159
|
+
|
|
160
|
+
result = from_map(multiply_and_add, [1, 2, 3], args=(2, 10))
|
|
161
|
+
self.assertIsInstance(result, pa.Table)
|
|
162
|
+
self.assertEqual(len(result), 3)
|
|
163
|
+
|
|
164
|
+
df = result.to_pandas()
|
|
165
|
+
self.assertEqual(
|
|
166
|
+
list(df['result']), [12, 14, 16]
|
|
167
|
+
) # (1*2+10, 2*2+10, 3*2+10)
|
|
168
|
+
|
|
169
|
+
def test_from_map_with_pyarrow_tables(self):
|
|
170
|
+
"""Test from_map when function returns PyArrow tables."""
|
|
171
|
+
|
|
172
|
+
def make_arrow_table(x):
|
|
173
|
+
df = pd.DataFrame({'value': [x]})
|
|
174
|
+
return pa.Table.from_pandas(df)
|
|
175
|
+
|
|
176
|
+
result = from_map(make_arrow_table, [1, 2, 3])
|
|
177
|
+
self.assertIsInstance(result, pa.Table)
|
|
178
|
+
self.assertEqual(len(result), 3)
|
|
86
179
|
|
|
87
180
|
|
|
88
181
|
if __name__ == '__main__':
|
xarray_sql/sql.py
CHANGED
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
import xarray as xr
|
|
2
|
-
import
|
|
2
|
+
from datafusion import SessionContext
|
|
3
3
|
|
|
4
4
|
from .df import read_xarray, Chunks
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
class
|
|
8
|
-
"""
|
|
7
|
+
class XarrayContext(SessionContext):
|
|
8
|
+
"""A datafusion `SessionContext` that also supports `xarray.Dataset`s."""
|
|
9
9
|
|
|
10
|
-
def
|
|
10
|
+
def from_dataset(
|
|
11
11
|
self,
|
|
12
12
|
table_name: str,
|
|
13
|
-
input_table:
|
|
13
|
+
input_table: xr.Dataset,
|
|
14
14
|
chunks: Chunks = None,
|
|
15
|
-
*args,
|
|
16
|
-
**kwargs,
|
|
17
15
|
):
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
super().create_table(table_name, input_table, *args, **kwargs)
|
|
16
|
+
arrow_table = read_xarray(input_table, chunks)
|
|
17
|
+
return self.from_arrow(arrow_table, table_name)
|
xarray_sql/sql_test.py
CHANGED
|
@@ -1,58 +1,62 @@
|
|
|
1
1
|
import unittest
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
from . import Context
|
|
3
|
+
from . import XarrayContext
|
|
5
4
|
from .df_test import DaskTestCase
|
|
6
5
|
|
|
7
6
|
|
|
8
7
|
class SqlTestCase(DaskTestCase):
|
|
9
8
|
|
|
10
9
|
def test_sanity(self):
|
|
11
|
-
c =
|
|
12
|
-
c.
|
|
10
|
+
c = XarrayContext()
|
|
11
|
+
c.from_dataset('air', self.air_small)
|
|
13
12
|
|
|
14
13
|
query = c.sql('SELECT "lat", "lon", "time", "air" FROM "air" LIMIT 100')
|
|
15
14
|
|
|
16
|
-
result = query.
|
|
15
|
+
result = query.to_pandas()
|
|
17
16
|
self.assertIsNotNone(result)
|
|
18
|
-
self.
|
|
17
|
+
self.assertLessEqual(len(result), 1320) # Should be all rows or less
|
|
18
|
+
self.assertGreater(len(result), 0) # Should have some rows
|
|
19
19
|
|
|
20
20
|
def test_agg_small(self):
|
|
21
|
-
c =
|
|
22
|
-
c.
|
|
21
|
+
c = XarrayContext()
|
|
22
|
+
c.from_dataset('air', self.air_small)
|
|
23
23
|
|
|
24
|
-
query = c.sql(
|
|
24
|
+
query = c.sql(
|
|
25
|
+
"""
|
|
25
26
|
SELECT
|
|
26
27
|
"lat", "lon", SUM("air") as air_total
|
|
27
28
|
FROM
|
|
28
29
|
"air"
|
|
29
30
|
GROUP BY
|
|
30
31
|
"lat", "lon"
|
|
31
|
-
"""
|
|
32
|
+
"""
|
|
33
|
+
)
|
|
32
34
|
|
|
33
|
-
result = query.
|
|
35
|
+
result = query.to_pandas()
|
|
34
36
|
self.assertIsNotNone(result)
|
|
35
37
|
|
|
36
|
-
expected = self.air_small.
|
|
38
|
+
expected = self.air_small.sizes['lat'] * self.air_small.sizes['lon']
|
|
37
39
|
self.assertEqual(len(result), expected)
|
|
38
40
|
|
|
39
41
|
def test_agg_regular(self):
|
|
40
|
-
c =
|
|
41
|
-
c.
|
|
42
|
+
c = XarrayContext()
|
|
43
|
+
c.from_dataset('air', self.air)
|
|
42
44
|
|
|
43
|
-
query = c.sql(
|
|
45
|
+
query = c.sql(
|
|
46
|
+
"""
|
|
44
47
|
SELECT
|
|
45
48
|
"lat", "lon", AVG("air") as air_total
|
|
46
49
|
FROM
|
|
47
50
|
"air"
|
|
48
51
|
GROUP BY
|
|
49
52
|
"lat", "lon"
|
|
50
|
-
"""
|
|
53
|
+
"""
|
|
54
|
+
)
|
|
51
55
|
|
|
52
|
-
result = query.
|
|
56
|
+
result = query.to_pandas()
|
|
53
57
|
self.assertIsNotNone(result)
|
|
54
58
|
|
|
55
|
-
expected = self.air.
|
|
59
|
+
expected = self.air.sizes['lat'] * self.air.sizes['lon']
|
|
56
60
|
self.assertEqual(len(result), expected)
|
|
57
61
|
|
|
58
62
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: xarray_sql
|
|
3
|
-
Version: 0.0
|
|
3
|
+
Version: 0.1.0
|
|
4
4
|
Summary: Querry Xarray with SQL.
|
|
5
5
|
Author-email: Alexander Merose <al@merose.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -14,26 +14,24 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
14
14
|
Classifier: Operating System :: MacOS :: MacOS X
|
|
15
15
|
Classifier: Operating System :: Microsoft :: Windows
|
|
16
16
|
Classifier: Operating System :: POSIX
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.9
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
22
|
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
23
23
|
Classifier: Topic :: Database :: Front-Ends
|
|
24
|
-
Requires-Python: >=3.
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
|
|
30
|
-
Requires-Dist: xarray-sql[test] ; extra == 'dev'
|
|
31
|
-
Requires-Dist: pyink ; extra == 'dev'
|
|
32
|
-
Requires-Dist: py-spy ; extra == 'dev'
|
|
27
|
+
Requires-Dist: dask>=2024.8.0
|
|
28
|
+
Requires-Dist: datafusion>=47.0.0
|
|
29
|
+
Requires-Dist: xarray>=2024.7.0
|
|
33
30
|
Provides-Extra: test
|
|
34
|
-
Requires-Dist: pytest
|
|
35
|
-
Requires-Dist: xarray[io]
|
|
36
|
-
Requires-Dist: gcsfs
|
|
31
|
+
Requires-Dist: pytest; extra == "test"
|
|
32
|
+
Requires-Dist: xarray[io]; extra == "test"
|
|
33
|
+
Requires-Dist: gcsfs; extra == "test"
|
|
34
|
+
Dynamic: license-file
|
|
37
35
|
|
|
38
36
|
# xarray-sql
|
|
39
37
|
|
|
@@ -42,6 +40,9 @@ _Query Xarray with SQL_
|
|
|
42
40
|
[](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml)
|
|
43
41
|
[](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml)
|
|
44
42
|
|
|
43
|
+
```shell
|
|
44
|
+
pip install xarray-sql
|
|
45
|
+
```
|
|
45
46
|
|
|
46
47
|
## What is this?
|
|
47
48
|
|
|
@@ -54,8 +55,8 @@ import xarray_sql as qr
|
|
|
54
55
|
ds = xr.tutorial.open_dataset('air_temperature')
|
|
55
56
|
|
|
56
57
|
# The same as a dask-sql Context; i.e. an Apache DataFusion Context.
|
|
57
|
-
c = qr.
|
|
58
|
-
c.
|
|
58
|
+
c = qr.XarrayContext()
|
|
59
|
+
c.from_dataset('air', ds, chunks=dict(time=24))
|
|
59
60
|
|
|
60
61
|
df = c.sql('''
|
|
61
62
|
SELECT
|
|
@@ -67,10 +68,10 @@ df = c.sql('''
|
|
|
67
68
|
''')
|
|
68
69
|
|
|
69
70
|
# A table of the average temperature for each location across time.
|
|
70
|
-
df.
|
|
71
|
+
df.to_pandas()
|
|
71
72
|
|
|
72
73
|
# Alternatively, you can just create the DataFrame from the Dataset:
|
|
73
|
-
df = qr.read_xarray(ds)
|
|
74
|
+
df = qr.read_xarray(ds).to_pandas()
|
|
74
75
|
df.head()
|
|
75
76
|
```
|
|
76
77
|
|
|
@@ -82,7 +83,7 @@ SQL queries against them.
|
|
|
82
83
|
A few reasons:
|
|
83
84
|
|
|
84
85
|
* Even though SQL is the lingua franca of data, scientific datasets are often
|
|
85
|
-
inaccessible to non-scientists.
|
|
86
|
+
inaccessible to non-scientists (SQL users).
|
|
86
87
|
* Joining tabular data with raster data is common yet difficult. It could be
|
|
87
88
|
easy.
|
|
88
89
|
* There are many cloud-native, Xarray-openable datasets,
|
|
@@ -102,10 +103,13 @@ All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
|
|
|
102
103
|
`from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
|
|
103
104
|
That's it!
|
|
104
105
|
|
|
106
|
+
_2025 update_: This library now implements a dask-like `from_map` interface in
|
|
107
|
+
pure `datafusion` and `pyarrow`, but works with the same principle!
|
|
108
|
+
|
|
105
109
|
## Why does this work?
|
|
106
110
|
|
|
107
111
|
Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
|
|
108
|
-
|
|
112
|
+
chunks and represented contiguously in memory. It is only a matter of metadata
|
|
109
113
|
that breaks them up into ndarrays. `to_dataframe()`
|
|
110
114
|
just changes this metadata (via a `ravel()`/`reshape()`), back into a column
|
|
111
115
|
amenable to a DataFrame.
|
|
@@ -115,15 +119,7 @@ worth the convenience of DataFrames.
|
|
|
115
119
|
|
|
116
120
|
## What are the current limitations?
|
|
117
121
|
|
|
118
|
-
|
|
119
|
-
`MultiIndex`s ([dask/dask#1493](https://github.com/dask/dask/issues/1493)). If
|
|
120
|
-
it did, I suspect performance for many types of queries would greatly improve.
|
|
121
|
-
|
|
122
|
-
Further, while this does play well with `dask-geopandas` (for geospatial query
|
|
123
|
-
support), certain types of operations don't quite match standard geopandas.
|
|
124
|
-
Spatial joins come to mind as a killer feature, but only inner joins are
|
|
125
|
-
supported ([geopandas/dask-geopandas#72](https://github.com/geopandas/dask-geopandas/issues/72))
|
|
126
|
-
.
|
|
122
|
+
_2025 update_: TBD, `datafusion` provides a whole new world!
|
|
127
123
|
|
|
128
124
|
## What would a deeper integration look like?
|
|
129
125
|
|
|
@@ -134,7 +130,7 @@ Xarray Datasets. This approach is being pursued
|
|
|
134
130
|
Deeper still: I was thinking we could make
|
|
135
131
|
a [virtual](https://fsspec.github.io/kerchunk/)
|
|
136
132
|
filesystem for parquet that would internally map to Zarr. Raster-backed virtual
|
|
137
|
-
parquet would open up integrations to
|
|
133
|
+
parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
|
|
138
134
|
and BigQuery. More thoughts on this
|
|
139
135
|
in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
|
|
140
136
|
|
|
@@ -145,8 +141,8 @@ I want to give a special thanks to the following folks and institutions:
|
|
|
145
141
|
- Pramod Gupta and the Anthromet Team at Google Research for the problem
|
|
146
142
|
formation and design inspiration.
|
|
147
143
|
- Jake Wall and AI2/Ecoscope for compute resources and key use cases.
|
|
148
|
-
- Charles Stern, Stephan Hoyer,
|
|
149
|
-
discussion of this project.
|
|
144
|
+
- Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
|
|
145
|
+
for the early review and discussion of this project.
|
|
150
146
|
|
|
151
147
|
## License
|
|
152
148
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
xarray_sql/__init__.py,sha256=HmuxwFMm3a7EVhCz3z2XUn6MeU7ghwSbO9y7A5G8MY8,69
|
|
2
|
+
xarray_sql/core.py,sha256=EQ1qBjHxnJPpb_HTRQ7B3NKLaI00KOdD4fmt5Qum3BI,1213
|
|
3
|
+
xarray_sql/df.py,sha256=0RPhxPIzyEwUiU_SrHpFPz3zs1BBgJb4GH9vO7X52YY,4101
|
|
4
|
+
xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
|
|
5
|
+
xarray_sql/df_test.py,sha256=LUJJd48RZl6R9Tph8EFN-C1lxuBlpL26SCS2F4hqPZE,5661
|
|
6
|
+
xarray_sql/sql.py,sha256=D6Wz4m79jhgZxDqNs__-n6B8zjqkFGRDQtcq0e72NKI,434
|
|
7
|
+
xarray_sql/sql_test.py,sha256=bScp4cyfuv1AooYRHeg3EPg2nXLsCKIxBPVFH0cFrYo,1384
|
|
8
|
+
xarray_sql-0.1.0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
9
|
+
xarray_sql-0.1.0.dist-info/METADATA,sha256=t3ZZdn03kWbUfjRvGucS6bGTLgst7QYhGciNSKuxvas,6086
|
|
10
|
+
xarray_sql-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
11
|
+
xarray_sql-0.1.0.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
|
|
12
|
+
xarray_sql-0.1.0.dist-info/RECORD,,
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
xarray_sql/__init__.py,sha256=7ij_x7HZfcLn1GdZkluH8Lgx3l-cflE7Qmg1ZBHJCiA,53
|
|
2
|
-
xarray_sql/core.py,sha256=BbKJO9GnjO8hXPUPdQrmttgDeHe_8crdh5he-c9rJsE,1197
|
|
3
|
-
xarray_sql/df.py,sha256=hsGEwPTTiO5fimB1u3-ywYH50MlTF20dq6Hpcb_A9XU,3432
|
|
4
|
-
xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
|
|
5
|
-
xarray_sql/df_test.py,sha256=souhCNstQHML-idSYiPuQ5BEjHH_D9XE-YpUCJhnNSk,2694
|
|
6
|
-
xarray_sql/sql.py,sha256=68ySKbvPKL_inHJ7FI6mF7tNrZB2BCeWKFO4FdysNFA,495
|
|
7
|
-
xarray_sql/sql_test.py,sha256=wFp4dC0_FQiQUM66isdliQSTqPeLB35h2LKHbpicYms,1224
|
|
8
|
-
xarray_sql-0.0.2.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
9
|
-
xarray_sql-0.0.2.dist-info/METADATA,sha256=tyZ8NQqG95gx74re302VHzKj8jJuSPK-zGzMIPLXZ10,6367
|
|
10
|
-
xarray_sql-0.0.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
11
|
-
xarray_sql-0.0.2.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
|
|
12
|
-
xarray_sql-0.0.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|