xarray-sql 0.0.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/__init__.py CHANGED
@@ -1,2 +1,2 @@
1
- from .df import read_xarray
2
- from .sql import Context
1
+ from .df import read_xarray, from_map
2
+ from .sql import XarrayContext
xarray_sql/core.py CHANGED
@@ -7,13 +7,14 @@ import xarray as xr
7
7
  Row = t.List[t.Any]
8
8
 
9
9
 
10
+ # deprecated
10
11
  def get_columns(ds: xr.Dataset) -> t.List[str]:
11
- return list(ds.dims.keys()) + list(ds.data_vars.keys())
12
+ return list(ds.sizes.keys()) + list(ds.data_vars.keys())
12
13
 
13
14
 
14
15
  # Deprecated
15
16
  def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
16
- dim_keys, dim_vals = zip(*ds.dims.items())
17
+ dim_keys, dim_vals = zip(*ds.sizes.items())
17
18
 
18
19
  for idx in itertools.product(*(range(d) for d in dim_vals)):
19
20
  coord_idx = dict(zip(dim_keys, idx))
@@ -26,7 +27,7 @@ def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
26
27
  # Deprecated
27
28
  def unbounded_unravel(ds: xr.Dataset) -> np.ndarray:
28
29
  """Unravel with unbounded memory (as a NumPy Array)."""
29
- dim_keys, dim_vals = zip(*ds.dims.items())
30
+ dim_keys, dim_vals = zip(*ds.sizes.items())
30
31
  columns = get_columns(ds)
31
32
 
32
33
  N = np.prod([d for d in dim_vals])
xarray_sql/df.py CHANGED
@@ -1,24 +1,14 @@
1
1
  import itertools
2
2
  import typing as t
3
3
 
4
- import dask
5
- import dask.dataframe as dd
6
4
  import numpy as np
7
5
  import pandas as pd
6
+ import pyarrow as pa
8
7
  import xarray as xr
9
- from dask.dataframe.io import from_map
10
-
11
- from . import core
12
8
 
13
9
  Block = t.Dict[str, slice]
14
10
  Chunks = t.Optional[t.Dict[str, int]]
15
11
 
16
- # Turn on Dask-Expr
17
- dask.config.set({'dataframe.query-planning-warning': False})
18
- dask.config.set({'dataframe.query-planning': True})
19
- # Turn on Copy-On-Write (needs Pandas 2.0).
20
- pd.options.mode.copy_on_write = True
21
-
22
12
 
23
13
  # Borrowed from Xarray
24
14
  def _get_chunk_slicer(
@@ -42,7 +32,7 @@ def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[Block]:
42
32
  else:
43
33
  chunks = ds.chunks
44
34
 
45
- assert chunks, 'Dataset `ds` must be chunked or `chunks` must be provided.'
35
+ assert chunks, "Dataset `ds` must be chunked or `chunks` must be provided."
46
36
 
47
37
  chunk_bounds = {dim: np.cumsum((0,) + c) for dim, c in chunks.items()}
48
38
  ichunk = {dim: range(len(c)) for dim, c in chunks.items()}
@@ -67,8 +57,60 @@ def _block_len(block: Block) -> int:
67
57
  return np.prod([v.stop - v.start for v in block.values()])
68
58
 
69
59
 
70
- def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> dd.DataFrame:
71
- """Pivots an Xarray Dataset into a Dask Dataframe, partitioned by chunks.
60
+ def from_map(
61
+ func: t.Callable, *iterables, args: t.Optional[t.Tuple] = None, **kwargs
62
+ ) -> pa.Table:
63
+ """Create a PyArrow Table by mapping a function over iterables.
64
+
65
+ This is equivalent to dask's from_map but returns a PyArrow Table
66
+ that can be used with DataFusion instead of a Dask DataFrame.
67
+
68
+ Args:
69
+ func: Function to apply to each element of the iterables.
70
+ *iterables: Iterable objects to map the function over.
71
+ args: Additional positional arguments to pass to func.
72
+ **kwargs: Additional keyword arguments to pass to func.
73
+
74
+ Returns:
75
+ A PyArrow Table containing the concatenated results.
76
+ """
77
+ if args is None:
78
+ args = ()
79
+
80
+ # Apply the function to each combination of iterable elements
81
+ results = []
82
+ for items in zip(*iterables) if len(iterables) > 1 else iterables[0]:
83
+ if isinstance(items, tuple):
84
+ result = func(*items, *args, **kwargs)
85
+ else:
86
+ result = func(items, *args, **kwargs)
87
+
88
+ # Convert result to PyArrow Table
89
+ if isinstance(result, pd.DataFrame):
90
+ pa_table = pa.Table.from_pandas(result)
91
+ elif isinstance(result, pa.Table):
92
+ pa_table = result
93
+ else:
94
+ # Try to convert to pandas first, then to PyArrow
95
+ try:
96
+ df = pd.DataFrame(result)
97
+ pa_table = pa.Table.from_pandas(df)
98
+ except Exception as e:
99
+ raise ValueError(
100
+ f"Cannot convert function result to PyArrow Table: {e}"
101
+ )
102
+
103
+ results.append(pa_table)
104
+
105
+ # Concatenate all results
106
+ if not results:
107
+ raise ValueError("No results to concatenate")
108
+
109
+ return pa.concat_tables(results)
110
+
111
+
112
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
113
+ """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
72
114
 
73
115
  Args:
74
116
  ds: An Xarray Dataset. All `data_vars` mush share the same dimensions.
@@ -77,39 +119,16 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> dd.DataFrame:
77
119
  dataframe partition.
78
120
 
79
121
  Returns:
80
- A Dask Dataframe, which is a table representation of the input Dataset.
122
+ A PyArrow Table, which is a table representation of the input Dataset.
81
123
  """
82
124
  fst = next(iter(ds.values())).dims
83
125
  assert all(
84
126
  da.dims == fst for da in ds.values()
85
- ), 'All dimensions must be equal. Please filter data_vars in the Dataset.'
127
+ ), "All dimensions must be equal. Please filter data_vars in the Dataset."
86
128
 
87
129
  blocks = list(block_slices(ds, chunks))
88
130
 
89
- block_lengths = [_block_len(b) for b in blocks]
90
- divisions = tuple(np.cumsum([0] + block_lengths)) # 0 ==> start partition.
91
-
92
131
  def pivot(b: Block) -> pd.DataFrame:
93
132
  return ds.isel(b).to_dataframe().reset_index()
94
133
 
95
- # Token is needed to prevent Dask from spending too many cycles calculating
96
- # it's own token from the constituent parts.
97
- token = (
98
- 'xarray-Dataset-'
99
- f'{"_".join(list(ds.dims.keys()))}'
100
- '__'
101
- f'{"_".join(list(ds.data_vars.keys()))}'
102
- )
103
-
104
- columns = core.get_columns(ds)
105
-
106
- # TODO(#18): Is it possible to pass the length (known now) here?
107
- meta = {c: ds[c].dtype for c in columns}
108
-
109
- return from_map(
110
- pivot,
111
- blocks,
112
- meta=meta,
113
- divisions=divisions,
114
- token=token,
115
- )
134
+ return from_map(pivot, blocks)
xarray_sql/df_test.py CHANGED
@@ -1,11 +1,37 @@
1
1
  import itertools
2
2
  import unittest
3
3
 
4
- import dask.dataframe as dd
5
4
  import numpy as np
5
+ import pandas as pd
6
+ import pyarrow as pa
6
7
  import xarray as xr
7
8
 
8
- from .df import explode, read_xarray, block_slices
9
+ from .df import explode, read_xarray, block_slices, from_map
10
+
11
+
12
+ def rand_wx(start: str, end: str) -> xr.Dataset:
13
+ np.random.seed(42)
14
+ lat = np.linspace(-90, 90, num=720)
15
+ lon = np.linspace(-180, 180, num=1440)
16
+ time = pd.date_range(start, end, freq='h')
17
+ level = np.array([1000, 500], dtype=np.int32)
18
+ reference_time = pd.Timestamp(start)
19
+ temperature = 15 + 8 * np.random.randn(720, 1440, len(time), len(level))
20
+ precipitation = 10 * np.random.rand(720, 1440, len(time), len(level))
21
+ return xr.Dataset(
22
+ data_vars=dict(
23
+ temperature=(['lat', 'lon', 'time', 'level'], temperature),
24
+ precipitation=(['lat', 'lon', 'time', 'level'], precipitation),
25
+ ),
26
+ coords=dict(
27
+ lat=lat,
28
+ lon=lon,
29
+ time=time,
30
+ level=level,
31
+ reference_time=reference_time,
32
+ ),
33
+ attrs=dict(description='Random weather.'),
34
+ )
9
35
 
10
36
 
11
37
  class DaskTestCase(unittest.TestCase):
@@ -18,6 +44,7 @@ class DaskTestCase(unittest.TestCase):
18
44
  self.air_small = self.air.isel(
19
45
  time=slice(0, 12), lat=slice(0, 11), lon=slice(0, 10)
20
46
  ).chunk(self.chunks)
47
+ self.randwx = rand_wx('1995-01-13T00', '1995-01-13T01')
21
48
 
22
49
 
23
50
  class ExplodeTest(DaskTestCase):
@@ -32,7 +59,7 @@ class ExplodeTest(DaskTestCase):
32
59
  ds = next(iter(explode(self.air)))
33
60
  for k, v in self.chunks.items():
34
61
  self.assertIn(k, ds.dims)
35
- self.assertEqual(v, ds.dims[k])
62
+ self.assertEqual(v, ds.sizes[k])
36
63
 
37
64
  def skip_test_dim_sizes__all(self):
38
65
  # TODO(alxmrs): Why is this test slow?
@@ -44,45 +71,111 @@ class ExplodeTest(DaskTestCase):
44
71
 
45
72
  def test_data_equal__one__first(self):
46
73
  ds = next(iter(explode(self.air)))
47
- iselection = {dim: slice(0, s) for dim, s in ds.dims.items()}
74
+ iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
48
75
  self.assertEqual(self.air.isel(iselection), ds)
49
76
 
50
77
  def test_data_equal__one__last(self):
51
78
  dss = list(explode(self.air))
52
79
  ds = dss[-1]
53
- iselection = {dim: slice(0, s) for dim, s in ds.dims.items()}
80
+ iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
54
81
  self.assertEqual(self.air.isel(iselection), ds)
55
82
 
56
83
 
57
- class DaskDataframeTest(DaskTestCase):
84
+ class PyArrowTableTest(DaskTestCase):
58
85
 
59
86
  def test_sanity(self):
60
- df = read_xarray(self.air_small).compute()
61
- self.assertIsNotNone(df)
62
- self.assertEqual(len(df), np.prod(list(self.air_small.dims.values())))
87
+ table = read_xarray(self.air_small)
88
+ self.assertIsNotNone(table)
89
+ self.assertIsInstance(table, pa.Table)
90
+ self.assertEqual(len(table), np.prod(list(self.air_small.sizes.values())))
63
91
 
64
92
  def test_columns(self):
65
- df = read_xarray(self.air_small).compute()
66
- cols = list(df.columns)
93
+ table = read_xarray(self.air_small)
94
+ cols = table.column_names
67
95
  self.assertEqual(cols, ['lat', 'time', 'lon', 'air'])
68
96
 
69
97
  def test_dtypes(self):
70
- df: dd.DataFrame = read_xarray(self.air_small).compute()
98
+ table = read_xarray(self.air_small)
99
+ # Convert to pandas to check dtypes
100
+ df = table.to_pandas()
71
101
  types = list(df.dtypes)
72
102
  self.assertEqual([self.air_small[c].dtype for c in df.columns], types)
73
103
 
74
- def test_partitions_dont_match_dataset_chunks(self):
75
- standard_blocks = list(block_slices(self.air_small))
76
- default: dd.DataFrame = read_xarray(self.air_small)
77
- chunked: dd.DataFrame = read_xarray(self.air_small, dict(time=5))
104
+ def test_different_chunk_sizes(self):
105
+ default_table = read_xarray(self.air_small)
106
+ chunked_table = read_xarray(self.air_small, dict(time=5))
78
107
 
79
- self.assertEqual(default.npartitions, len(standard_blocks))
80
- self.assertNotEqual(chunked.npartitions, len(standard_blocks))
108
+ # Both should produce valid tables
109
+ self.assertIsInstance(default_table, pa.Table)
110
+ self.assertIsInstance(chunked_table, pa.Table)
111
+ # Should have same number of rows
112
+ self.assertEqual(len(default_table), len(chunked_table))
81
113
 
82
114
  def test_chunk_perf(self):
83
- df = read_xarray(self.air, chunks=dict(time=6)).compute()
84
- self.assertIsNotNone(df)
85
- self.assertEqual(len(df), np.prod(list(self.air.dims.values())))
115
+ table = read_xarray(self.air, chunks=dict(time=6))
116
+ self.assertIsNotNone(table)
117
+ self.assertEqual(len(table), np.prod(list(self.air.sizes.values())))
118
+
119
+ def test_column_metadata_preserved(self):
120
+ try:
121
+ table = read_xarray(self.randwx, chunks=dict(time=24))
122
+ self.assertIsInstance(table, pa.Table)
123
+ except Exception as e:
124
+ self.fail(f'Unexpected error: {e}')
125
+
126
+
127
+ class FromMapTest(unittest.TestCase):
128
+
129
+ def test_basic_from_map(self):
130
+ """Test basic from_map functionality with pandas DataFrames."""
131
+
132
+ def make_df(x):
133
+ return pd.DataFrame({'value': [x, x * 2], 'index': [0, 1]})
134
+
135
+ result = from_map(make_df, [1, 2, 3])
136
+ self.assertIsInstance(result, pa.Table)
137
+ self.assertEqual(len(result), 6) # 3 inputs * 2 rows each
138
+ self.assertEqual(result.column_names, ['value', 'index'])
139
+
140
+ def test_from_map_with_multiple_iterables(self):
141
+ """Test from_map with multiple iterables."""
142
+
143
+ def add_values(x, y):
144
+ return pd.DataFrame({'sum': [x + y], 'x': [x], 'y': [y]})
145
+
146
+ result = from_map(add_values, [1, 2], [10, 20])
147
+ self.assertIsInstance(result, pa.Table)
148
+ self.assertEqual(len(result), 2)
149
+
150
+ # Convert to pandas to check values
151
+ df = result.to_pandas()
152
+ self.assertEqual(list(df['sum']), [11, 22])
153
+
154
+ def test_from_map_with_args(self):
155
+ """Test from_map with additional arguments."""
156
+
157
+ def multiply_and_add(x, multiplier, add_value):
158
+ return pd.DataFrame({'result': [x * multiplier + add_value]})
159
+
160
+ result = from_map(multiply_and_add, [1, 2, 3], args=(2, 10))
161
+ self.assertIsInstance(result, pa.Table)
162
+ self.assertEqual(len(result), 3)
163
+
164
+ df = result.to_pandas()
165
+ self.assertEqual(
166
+ list(df['result']), [12, 14, 16]
167
+ ) # (1*2+10, 2*2+10, 3*2+10)
168
+
169
+ def test_from_map_with_pyarrow_tables(self):
170
+ """Test from_map when function returns PyArrow tables."""
171
+
172
+ def make_arrow_table(x):
173
+ df = pd.DataFrame({'value': [x]})
174
+ return pa.Table.from_pandas(df)
175
+
176
+ result = from_map(make_arrow_table, [1, 2, 3])
177
+ self.assertIsInstance(result, pa.Table)
178
+ self.assertEqual(len(result), 3)
86
179
 
87
180
 
88
181
  if __name__ == '__main__':
xarray_sql/sql.py CHANGED
@@ -1,20 +1,17 @@
1
1
  import xarray as xr
2
- import dask_sql.input_utils
2
+ from datafusion import SessionContext
3
3
 
4
4
  from .df import read_xarray, Chunks
5
5
 
6
6
 
7
- class Context(dask_sql.Context):
8
- """See the `dask_sql.Context` docs."""
7
+ class XarrayContext(SessionContext):
8
+ """A datafusion `SessionContext` that also supports `xarray.Dataset`s."""
9
9
 
10
- def create_table(
10
+ def from_dataset(
11
11
  self,
12
12
  table_name: str,
13
- input_table: dask_sql.input_utils.InputType,
13
+ input_table: xr.Dataset,
14
14
  chunks: Chunks = None,
15
- *args,
16
- **kwargs,
17
15
  ):
18
- if isinstance(input_table, xr.Dataset):
19
- input_table = read_xarray(input_table, chunks)
20
- super().create_table(table_name, input_table, *args, **kwargs)
16
+ arrow_table = read_xarray(input_table, chunks)
17
+ return self.from_arrow(arrow_table, table_name)
xarray_sql/sql_test.py CHANGED
@@ -1,58 +1,62 @@
1
1
  import unittest
2
2
 
3
-
4
- from . import Context
3
+ from . import XarrayContext
5
4
  from .df_test import DaskTestCase
6
5
 
7
6
 
8
7
  class SqlTestCase(DaskTestCase):
9
8
 
10
9
  def test_sanity(self):
11
- c = Context()
12
- c.create_table('air', self.air_small)
10
+ c = XarrayContext()
11
+ c.from_dataset('air', self.air_small)
13
12
 
14
13
  query = c.sql('SELECT "lat", "lon", "time", "air" FROM "air" LIMIT 100')
15
14
 
16
- result = query.compute()
15
+ result = query.to_pandas()
17
16
  self.assertIsNotNone(result)
18
- self.assertEqual(len(result), 100)
17
+ self.assertLessEqual(len(result), 1320) # Should be all rows or less
18
+ self.assertGreater(len(result), 0) # Should have some rows
19
19
 
20
20
  def test_agg_small(self):
21
- c = Context()
22
- c.create_table('air', self.air_small)
21
+ c = XarrayContext()
22
+ c.from_dataset('air', self.air_small)
23
23
 
24
- query = c.sql("""
24
+ query = c.sql(
25
+ """
25
26
  SELECT
26
27
  "lat", "lon", SUM("air") as air_total
27
28
  FROM
28
29
  "air"
29
30
  GROUP BY
30
31
  "lat", "lon"
31
- """)
32
+ """
33
+ )
32
34
 
33
- result = query.compute()
35
+ result = query.to_pandas()
34
36
  self.assertIsNotNone(result)
35
37
 
36
- expected = self.air_small.dims['lat'] * self.air_small.dims['lon']
38
+ expected = self.air_small.sizes['lat'] * self.air_small.sizes['lon']
37
39
  self.assertEqual(len(result), expected)
38
40
 
39
41
  def test_agg_regular(self):
40
- c = Context()
41
- c.create_table('air', self.air)
42
+ c = XarrayContext()
43
+ c.from_dataset('air', self.air)
42
44
 
43
- query = c.sql("""
45
+ query = c.sql(
46
+ """
44
47
  SELECT
45
48
  "lat", "lon", AVG("air") as air_total
46
49
  FROM
47
50
  "air"
48
51
  GROUP BY
49
52
  "lat", "lon"
50
- """)
53
+ """
54
+ )
51
55
 
52
- result = query.compute()
56
+ result = query.to_pandas()
53
57
  self.assertIsNotNone(result)
54
58
 
55
- expected = self.air.dims['lat'] * self.air.dims['lon']
59
+ expected = self.air.sizes['lat'] * self.air.sizes['lon']
56
60
  self.assertEqual(len(result), expected)
57
61
 
58
62
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.0.2
3
+ Version: 0.1.0
4
4
  Summary: Querry Xarray with SQL.
5
5
  Author-email: Alexander Merose <al@merose.com>
6
6
  License: Apache-2.0
@@ -14,26 +14,24 @@ Classifier: License :: OSI Approved :: Apache Software License
14
14
  Classifier: Operating System :: MacOS :: MacOS X
15
15
  Classifier: Operating System :: Microsoft :: Windows
16
16
  Classifier: Operating System :: POSIX
17
- Classifier: Programming Language :: Python :: 3.8
18
17
  Classifier: Programming Language :: Python :: 3.9
19
18
  Classifier: Programming Language :: Python :: 3.10
20
19
  Classifier: Programming Language :: Python :: 3.11
21
20
  Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
22
  Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
23
23
  Classifier: Topic :: Database :: Front-Ends
24
- Requires-Python: >=3.8
24
+ Requires-Python: >=3.9
25
25
  Description-Content-Type: text/markdown
26
26
  License-File: LICENSE
27
- Requires-Dist: xarray
28
- Requires-Dist: dask-sql
29
- Provides-Extra: dev
30
- Requires-Dist: xarray-sql[test] ; extra == 'dev'
31
- Requires-Dist: pyink ; extra == 'dev'
32
- Requires-Dist: py-spy ; extra == 'dev'
27
+ Requires-Dist: dask>=2024.8.0
28
+ Requires-Dist: datafusion>=47.0.0
29
+ Requires-Dist: xarray>=2024.7.0
33
30
  Provides-Extra: test
34
- Requires-Dist: pytest ; extra == 'test'
35
- Requires-Dist: xarray[io] ; extra == 'test'
36
- Requires-Dist: gcsfs ; extra == 'test'
31
+ Requires-Dist: pytest; extra == "test"
32
+ Requires-Dist: xarray[io]; extra == "test"
33
+ Requires-Dist: gcsfs; extra == "test"
34
+ Dynamic: license-file
37
35
 
38
36
  # xarray-sql
39
37
 
@@ -42,6 +40,9 @@ _Query Xarray with SQL_
42
40
  [![ci](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml)
43
41
  [![lint](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml)
44
42
 
43
+ ```shell
44
+ pip install xarray-sql
45
+ ```
45
46
 
46
47
  ## What is this?
47
48
 
@@ -54,8 +55,8 @@ import xarray_sql as qr
54
55
  ds = xr.tutorial.open_dataset('air_temperature')
55
56
 
56
57
  # The same as a dask-sql Context; i.e. an Apache DataFusion Context.
57
- c = qr.Context(ds)
58
- c.create_table('air', ds, chunks=dict(time=24))
58
+ c = qr.XarrayContext()
59
+ c.from_dataset('air', ds, chunks=dict(time=24))
59
60
 
60
61
  df = c.sql('''
61
62
  SELECT
@@ -67,10 +68,10 @@ df = c.sql('''
67
68
  ''')
68
69
 
69
70
  # A table of the average temperature for each location across time.
70
- df.compute()
71
+ df.to_pandas()
71
72
 
72
73
  # Alternatively, you can just create the DataFrame from the Dataset:
73
- df = qr.read_xarray(ds)
74
+ df = qr.read_xarray(ds).to_pandas()
74
75
  df.head()
75
76
  ```
76
77
 
@@ -82,7 +83,7 @@ SQL queries against them.
82
83
  A few reasons:
83
84
 
84
85
  * Even though SQL is the lingua franca of data, scientific datasets are often
85
- inaccessible to non-scientists.
86
+ inaccessible to non-scientists (SQL users).
86
87
  * Joining tabular data with raster data is common yet difficult. It could be
87
88
  easy.
88
89
  * There are many cloud-native, Xarray-openable datasets,
@@ -102,10 +103,13 @@ All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
102
103
  `from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
103
104
  That's it!
104
105
 
106
+ _2025 update_: This library now implements a dask-like `from_map` interface in
107
+ pure `datafusion` and `pyarrow`, but works with the same principle!
108
+
105
109
  ## Why does this work?
106
110
 
107
111
  Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
108
- chucks and represented contiguously in memory. It is only a matter of metadata
112
+ chunks and represented contiguously in memory. It is only a matter of metadata
109
113
  that breaks them up into ndarrays. `to_dataframe()`
110
114
  just changes this metadata (via a `ravel()`/`reshape()`), back into a column
111
115
  amenable to a DataFrame.
@@ -115,15 +119,7 @@ worth the convenience of DataFrames.
115
119
 
116
120
  ## What are the current limitations?
117
121
 
118
- Dask doesn't support
119
- `MultiIndex`s ([dask/dask#1493](https://github.com/dask/dask/issues/1493)). If
120
- it did, I suspect performance for many types of queries would greatly improve.
121
-
122
- Further, while this does play well with `dask-geopandas` (for geospatial query
123
- support), certain types of operations don't quite match standard geopandas.
124
- Spatial joins come to mind as a killer feature, but only inner joins are
125
- supported ([geopandas/dask-geopandas#72](https://github.com/geopandas/dask-geopandas/issues/72))
126
- .
122
+ _2025 update_: TBD, `datafusion` provides a whole new world!
127
123
 
128
124
  ## What would a deeper integration look like?
129
125
 
@@ -134,7 +130,7 @@ Xarray Datasets. This approach is being pursued
134
130
  Deeper still: I was thinking we could make
135
131
  a [virtual](https://fsspec.github.io/kerchunk/)
136
132
  filesystem for parquet that would internally map to Zarr. Raster-backed virtual
137
- parquet would open up integrations to numeroustools like dask, pyarrow, duckdb,
133
+ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
138
134
  and BigQuery. More thoughts on this
139
135
  in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
140
136
 
@@ -145,8 +141,8 @@ I want to give a special thanks to the following folks and institutions:
145
141
  - Pramod Gupta and the Anthromet Team at Google Research for the problem
146
142
  formation and design inspiration.
147
143
  - Jake Wall and AI2/Ecoscope for compute resources and key use cases.
148
- - Charles Stern, Stephan Hoyer, and Qiusheng Wu for the early review and
149
- discussion of this project.
144
+ - Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
145
+ for the early review and discussion of this project.
150
146
 
151
147
  ## License
152
148
 
@@ -0,0 +1,12 @@
1
+ xarray_sql/__init__.py,sha256=HmuxwFMm3a7EVhCz3z2XUn6MeU7ghwSbO9y7A5G8MY8,69
2
+ xarray_sql/core.py,sha256=EQ1qBjHxnJPpb_HTRQ7B3NKLaI00KOdD4fmt5Qum3BI,1213
3
+ xarray_sql/df.py,sha256=0RPhxPIzyEwUiU_SrHpFPz3zs1BBgJb4GH9vO7X52YY,4101
4
+ xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
5
+ xarray_sql/df_test.py,sha256=LUJJd48RZl6R9Tph8EFN-C1lxuBlpL26SCS2F4hqPZE,5661
6
+ xarray_sql/sql.py,sha256=D6Wz4m79jhgZxDqNs__-n6B8zjqkFGRDQtcq0e72NKI,434
7
+ xarray_sql/sql_test.py,sha256=bScp4cyfuv1AooYRHeg3EPg2nXLsCKIxBPVFH0cFrYo,1384
8
+ xarray_sql-0.1.0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
+ xarray_sql-0.1.0.dist-info/METADATA,sha256=t3ZZdn03kWbUfjRvGucS6bGTLgst7QYhGciNSKuxvas,6086
10
+ xarray_sql-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
11
+ xarray_sql-0.1.0.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
+ xarray_sql-0.1.0.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.42.0)
2
+ Generator: setuptools (80.9.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,12 +0,0 @@
1
- xarray_sql/__init__.py,sha256=7ij_x7HZfcLn1GdZkluH8Lgx3l-cflE7Qmg1ZBHJCiA,53
2
- xarray_sql/core.py,sha256=BbKJO9GnjO8hXPUPdQrmttgDeHe_8crdh5he-c9rJsE,1197
3
- xarray_sql/df.py,sha256=hsGEwPTTiO5fimB1u3-ywYH50MlTF20dq6Hpcb_A9XU,3432
4
- xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
5
- xarray_sql/df_test.py,sha256=souhCNstQHML-idSYiPuQ5BEjHH_D9XE-YpUCJhnNSk,2694
6
- xarray_sql/sql.py,sha256=68ySKbvPKL_inHJ7FI6mF7tNrZB2BCeWKFO4FdysNFA,495
7
- xarray_sql/sql_test.py,sha256=wFp4dC0_FQiQUM66isdliQSTqPeLB35h2LKHbpicYms,1224
8
- xarray_sql-0.0.2.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
- xarray_sql-0.0.2.dist-info/METADATA,sha256=tyZ8NQqG95gx74re302VHzKj8jJuSPK-zGzMIPLXZ10,6367
10
- xarray_sql-0.0.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
11
- xarray_sql-0.0.2.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
- xarray_sql-0.0.2.dist-info/RECORD,,