xarray-sql 0.0.2__py3-none-any.whl → 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/__init__.py CHANGED
@@ -1,2 +1,2 @@
1
- from .df import read_xarray
2
- from .sql import Context
1
+ from .df import read_xarray, from_map
2
+ from .sql import XarrayContext
xarray_sql/core.py CHANGED
@@ -7,13 +7,14 @@ import xarray as xr
7
7
  Row = t.List[t.Any]
8
8
 
9
9
 
10
+ # deprecated
10
11
  def get_columns(ds: xr.Dataset) -> t.List[str]:
11
- return list(ds.dims.keys()) + list(ds.data_vars.keys())
12
+ return list(ds.sizes.keys()) + list(ds.data_vars.keys())
12
13
 
13
14
 
14
15
  # Deprecated
15
16
  def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
16
- dim_keys, dim_vals = zip(*ds.dims.items())
17
+ dim_keys, dim_vals = zip(*ds.sizes.items())
17
18
 
18
19
  for idx in itertools.product(*(range(d) for d in dim_vals)):
19
20
  coord_idx = dict(zip(dim_keys, idx))
@@ -26,7 +27,7 @@ def unravel(ds: xr.Dataset) -> t.Iterator[Row]:
26
27
  # Deprecated
27
28
  def unbounded_unravel(ds: xr.Dataset) -> np.ndarray:
28
29
  """Unravel with unbounded memory (as a NumPy Array)."""
29
- dim_keys, dim_vals = zip(*ds.dims.items())
30
+ dim_keys, dim_vals = zip(*ds.sizes.items())
30
31
  columns = get_columns(ds)
31
32
 
32
33
  N = np.prod([d for d in dim_vals])
xarray_sql/df.py CHANGED
@@ -1,24 +1,15 @@
1
1
  import itertools
2
2
  import typing as t
3
3
 
4
- import dask
5
- import dask.dataframe as dd
6
4
  import numpy as np
7
5
  import pandas as pd
6
+ import pyarrow as pa
8
7
  import xarray as xr
9
- from dask.dataframe.io import from_map
8
+ from datafusion.context import ArrowStreamExportable
10
9
 
11
- from . import core
12
-
13
- Block = t.Dict[str, slice]
10
+ Block = t.Dict[t.Hashable, slice]
14
11
  Chunks = t.Optional[t.Dict[str, int]]
15
12
 
16
- # Turn on Dask-Expr
17
- dask.config.set({'dataframe.query-planning-warning': False})
18
- dask.config.set({'dataframe.query-planning': True})
19
- # Turn on Copy-On-Write (needs Pandas 2.0).
20
- pd.options.mode.copy_on_write = True
21
-
22
13
 
23
14
  # Borrowed from Xarray
24
15
  def _get_chunk_slicer(
@@ -42,7 +33,7 @@ def block_slices(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[Block]:
42
33
  else:
43
34
  chunks = ds.chunks
44
35
 
45
- assert chunks, 'Dataset `ds` must be chunked or `chunks` must be provided.'
36
+ assert chunks, "Dataset `ds` must be chunked or `chunks` must be provided."
46
37
 
47
38
  chunk_bounds = {dim: np.cumsum((0,) + c) for dim, c in chunks.items()}
48
39
  ichunk = {dim: range(len(c)) for dim, c in chunks.items()}
@@ -64,11 +55,103 @@ def explode(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[xr.Dataset]:
64
55
 
65
56
 
66
57
  def _block_len(block: Block) -> int:
67
- return np.prod([v.stop - v.start for v in block.values()])
58
+ return int(np.prod([v.stop - v.start for v in block.values()]))
59
+
60
+
61
+ def from_map_batched(
62
+ func: t.Callable[[...], pd.DataFrame],
63
+ *iterables,
64
+ args: t.Optional[t.Tuple] = None,
65
+ schema: pa.Schema = None,
66
+ **kwargs,
67
+ ) -> pa.RecordBatchReader:
68
+ """Create a PyArrow RecordBatchReader by mapping a function over iterables.
69
+
70
+ This is equivalent to dask's from_map but returns a PyArrow
71
+ RecordBatchReader that can be used with DataFusion. It iterates over
72
+ RecordBatches which are created via the `func` one-at-a-time.
73
+
74
+ Args:
75
+ func: Function to apply to each element of the iterables. Currently, the function
76
+ must return a Pandas DataFrame.
77
+ *iterables: Iterable objects to map the function over.
78
+ schema: Optional schema needed for the RecordBatchReader.
79
+ args: Additional positional arguments to pass to func.
80
+ **kwargs: Additional keyword arguments to pass to func.
81
+
82
+ Returns:
83
+ A PyArrow RecordBatchReader containing the stream of RecordBatches.
84
+ """
85
+ if args is None:
86
+ args = ()
87
+
88
+ def map_batches():
89
+ for items in zip(*iterables):
90
+ df = func(*items, *args, **kwargs)
91
+ yield pa.RecordBatch.from_pandas(df, schema=schema)
92
+
93
+ return pa.RecordBatchReader.from_batches(schema, map_batches())
94
+
95
+
96
+ def from_map(
97
+ func: t.Callable, *iterables, args: t.Optional[t.Tuple] = None, **kwargs
98
+ ) -> pa.Table:
99
+ """Create a PyArrow Table by mapping a function over iterables.
100
+
101
+ This is equivalent to dask's from_map but returns a PyArrow Table
102
+ that can be used with DataFusion instead of a Dask DataFrame.
103
+
104
+ Args:
105
+ func: Function to apply to each element of the iterables.
106
+ *iterables: Iterable objects to map the function over.
107
+ args: Additional positional arguments to pass to func.
108
+ **kwargs: Additional keyword arguments to pass to func.
109
+
110
+ Returns:
111
+ A PyArrow Table containing the concatenated results.
112
+ """
113
+ if args is None:
114
+ args = ()
115
+
116
+ # Apply the function to each combination of iterable elements
117
+ results = []
118
+ for items in zip(*iterables) if len(iterables) > 1 else iterables[0]:
119
+ if isinstance(items, tuple):
120
+ result = func(*items, *args, **kwargs)
121
+ else:
122
+ result = func(items, *args, **kwargs)
123
+
124
+ # Convert result to PyArrow Table
125
+ if isinstance(result, pd.DataFrame):
126
+ pa_table = pa.Table.from_pandas(result)
127
+ elif isinstance(result, pa.Table):
128
+ pa_table = result
129
+ else:
130
+ # Try to convert to pandas first, then to PyArrow
131
+ try:
132
+ df = pd.DataFrame(result)
133
+ pa_table = pa.Table.from_pandas(df)
134
+ except Exception as e:
135
+ raise ValueError(
136
+ f"Cannot convert function result to PyArrow Table: {e}"
137
+ )
138
+
139
+ results.append(pa_table)
140
+
141
+ # Concatenate all results
142
+ if not results:
143
+ raise ValueError("No results to concatenate")
68
144
 
145
+ return pa.concat_tables(results)
69
146
 
70
- def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> dd.DataFrame:
71
- """Pivots an Xarray Dataset into a Dask Dataframe, partitioned by chunks.
147
+
148
+ def pivot(ds: xr.Dataset) -> pd.DataFrame:
149
+ """Converts an xarray Dataset to a pandas DataFrame."""
150
+ return ds.to_dataframe().reset_index()
151
+
152
+
153
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> ArrowStreamExportable:
154
+ """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
72
155
 
73
156
  Args:
74
157
  ds: An Xarray Dataset. All `data_vars` mush share the same dimensions.
@@ -77,39 +160,20 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> dd.DataFrame:
77
160
  dataframe partition.
78
161
 
79
162
  Returns:
80
- A Dask Dataframe, which is a table representation of the input Dataset.
163
+ A PyArrow Table, which is a table representation of the input Dataset.
81
164
  """
82
165
  fst = next(iter(ds.values())).dims
83
166
  assert all(
84
167
  da.dims == fst for da in ds.values()
85
- ), 'All dimensions must be equal. Please filter data_vars in the Dataset.'
168
+ ), "All dimensions must be equal. Please filter data_vars in the Dataset."
86
169
 
87
170
  blocks = list(block_slices(ds, chunks))
88
171
 
89
- block_lengths = [_block_len(b) for b in blocks]
90
- divisions = tuple(np.cumsum([0] + block_lengths)) # 0 ==> start partition.
91
-
92
- def pivot(b: Block) -> pd.DataFrame:
93
- return ds.isel(b).to_dataframe().reset_index()
172
+ def pivot_block(b: Block):
173
+ return pivot(ds.isel(b))
94
174
 
95
- # Token is needed to prevent Dask from spending too many cycles calculating
96
- # it's own token from the constituent parts.
97
- token = (
98
- 'xarray-Dataset-'
99
- f'{"_".join(list(ds.dims.keys()))}'
100
- '__'
101
- f'{"_".join(list(ds.data_vars.keys()))}'
102
- )
103
-
104
- columns = core.get_columns(ds)
175
+ schema = pa.Schema.from_pandas(pivot_block(blocks[0]))
176
+ last_schema = pa.Schema.from_pandas(pivot_block(blocks[-1]))
177
+ assert schema == last_schema, "Schemas must be consistent across blocks!"
105
178
 
106
- # TODO(#18): Is it possible to pass the length (known now) here?
107
- meta = {c: ds[c].dtype for c in columns}
108
-
109
- return from_map(
110
- pivot,
111
- blocks,
112
- meta=meta,
113
- divisions=divisions,
114
- token=token,
115
- )
179
+ return from_map_batched(pivot_block, blocks, schema=schema)
xarray_sql/df_test.py CHANGED
@@ -1,11 +1,70 @@
1
1
  import itertools
2
+ import tracemalloc
2
3
  import unittest
3
4
 
4
- import dask.dataframe as dd
5
5
  import numpy as np
6
+ import pandas as pd
7
+ import pyarrow as pa
6
8
  import xarray as xr
7
9
 
8
- from .df import explode, read_xarray, block_slices
10
+ from .df import explode, read_xarray, block_slices, from_map, pivot, from_map_batched
11
+
12
+
13
+ def rand_wx(start: str, end: str) -> xr.Dataset:
14
+ np.random.seed(42)
15
+ lat = np.linspace(-90, 90, num=720)
16
+ lon = np.linspace(-180, 180, num=1440)
17
+ time = pd.date_range(start, end, freq='h')
18
+ level = np.array([1000, 500], dtype=np.int32)
19
+ reference_time = pd.Timestamp(start)
20
+ temperature = 15 + 8 * np.random.randn(720, 1440, len(time), len(level))
21
+ precipitation = 10 * np.random.rand(720, 1440, len(time), len(level))
22
+ return xr.Dataset(
23
+ data_vars=dict(
24
+ temperature=(['lat', 'lon', 'time', 'level'], temperature),
25
+ precipitation=(['lat', 'lon', 'time', 'level'], precipitation),
26
+ ),
27
+ coords=dict(
28
+ lat=lat,
29
+ lon=lon,
30
+ time=time,
31
+ level=level,
32
+ reference_time=reference_time,
33
+ ),
34
+ attrs=dict(description='Random weather.'),
35
+ )
36
+
37
+
38
+ def create_large_dataset(time_steps=1000, lat_points=100, lon_points=100):
39
+ """Create a large xarray dataset for memory testing."""
40
+ np.random.seed(42)
41
+
42
+ # Create coordinates
43
+ time = pd.date_range('2020-01-01', periods=time_steps, freq='h')
44
+ lat = np.linspace(-90, 90, lat_points)
45
+ lon = np.linspace(-180, 180, lon_points)
46
+
47
+ # Create large data arrays
48
+ temp_data = np.random.rand(time_steps, lat_points, lon_points) * 40 - 10
49
+ precip_data = np.random.rand(time_steps, lat_points, lon_points) * 100
50
+
51
+ return xr.Dataset(
52
+ {
53
+ 'temperature': (['time', 'lat', 'lon'], temp_data),
54
+ 'precipitation': (['time', 'lat', 'lon'], precip_data),
55
+ },
56
+ coords={
57
+ 'time': time,
58
+ 'lat': lat,
59
+ 'lon': lon,
60
+ },
61
+ )
62
+
63
+
64
+ def adding_function(x, y):
65
+ """Simple function that adds two values and returns a DataFrame."""
66
+ result = pd.DataFrame({'x': [x], 'y': [y], 'sum': [x + y]})
67
+ return result
9
68
 
10
69
 
11
70
  class DaskTestCase(unittest.TestCase):
@@ -18,6 +77,7 @@ class DaskTestCase(unittest.TestCase):
18
77
  self.air_small = self.air.isel(
19
78
  time=slice(0, 12), lat=slice(0, 11), lon=slice(0, 10)
20
79
  ).chunk(self.chunks)
80
+ self.randwx = rand_wx('1995-01-13T00', '1995-01-13T01')
21
81
 
22
82
 
23
83
  class ExplodeTest(DaskTestCase):
@@ -32,7 +92,7 @@ class ExplodeTest(DaskTestCase):
32
92
  ds = next(iter(explode(self.air)))
33
93
  for k, v in self.chunks.items():
34
94
  self.assertIn(k, ds.dims)
35
- self.assertEqual(v, ds.dims[k])
95
+ self.assertEqual(v, ds.sizes[k])
36
96
 
37
97
  def skip_test_dim_sizes__all(self):
38
98
  # TODO(alxmrs): Why is this test slow?
@@ -44,45 +104,283 @@ class ExplodeTest(DaskTestCase):
44
104
 
45
105
  def test_data_equal__one__first(self):
46
106
  ds = next(iter(explode(self.air)))
47
- iselection = {dim: slice(0, s) for dim, s in ds.dims.items()}
107
+ iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
48
108
  self.assertEqual(self.air.isel(iselection), ds)
49
109
 
50
110
  def test_data_equal__one__last(self):
51
111
  dss = list(explode(self.air))
52
112
  ds = dss[-1]
53
- iselection = {dim: slice(0, s) for dim, s in ds.dims.items()}
113
+ iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
54
114
  self.assertEqual(self.air.isel(iselection), ds)
55
115
 
56
116
 
57
- class DaskDataframeTest(DaskTestCase):
117
+ class FromMapTest(unittest.TestCase):
118
+
119
+ def test_basic_from_map(self):
120
+ """Test basic from_map functionality with pandas DataFrames."""
121
+
122
+ def make_df(x):
123
+ return pd.DataFrame({'value': [x, x * 2], 'index': [0, 1]})
124
+
125
+ result = from_map(make_df, [1, 2, 3])
126
+ self.assertIsInstance(result, pa.Table)
127
+ self.assertEqual(len(result), 6) # 3 inputs * 2 rows each
128
+ self.assertEqual(result.column_names, ['value', 'index'])
129
+
130
+ def test_from_map_with_multiple_iterables(self):
131
+ """Test from_map with multiple iterables."""
132
+
133
+ def add_values(x, y):
134
+ return pd.DataFrame({'sum': [x + y], 'x': [x], 'y': [y]})
135
+
136
+ result = from_map(add_values, [1, 2], [10, 20])
137
+ self.assertIsInstance(result, pa.Table)
138
+ self.assertEqual(len(result), 2)
139
+
140
+ # Convert to pandas to check values
141
+ df = result.to_pandas()
142
+ self.assertEqual(list(df['sum']), [11, 22])
143
+
144
+ def test_from_map_with_args(self):
145
+ """Test from_map with additional arguments."""
146
+
147
+ def multiply_and_add(x, multiplier, add_value):
148
+ return pd.DataFrame({'result': [x * multiplier + add_value]})
149
+
150
+ result = from_map(multiply_and_add, [1, 2, 3], args=(2, 10))
151
+ self.assertIsInstance(result, pa.Table)
152
+ self.assertEqual(len(result), 3)
153
+
154
+ df = result.to_pandas()
155
+ self.assertEqual(
156
+ list(df['result']), [12, 14, 16]
157
+ ) # (1*2+10, 2*2+10, 3*2+10)
158
+
159
+ def test_from_map_with_pyarrow_tables(self):
160
+ """Test from_map when function returns PyArrow tables."""
161
+
162
+ def make_arrow_table(x):
163
+ df = pd.DataFrame({'value': [x]})
164
+ return pa.Table.from_pandas(df)
165
+
166
+ result = from_map(make_arrow_table, [1, 2, 3])
167
+ self.assertIsInstance(result, pa.Table)
168
+ self.assertEqual(len(result), 3)
169
+
170
+
171
+ class TestFromMapBatchedCorrectness(DaskTestCase):
172
+ """Test correctness of from_map_batched function."""
173
+
174
+ def test_basic_functionality(self):
175
+ """Test that from_map_batched produces correct RecordBatchReader."""
176
+ blocks = list(
177
+ block_slices(self.air_small, chunks={'time': 4, 'lat': 3, 'lon': 4})
178
+ )
179
+
180
+ # Get expected schema
181
+ first_block_df = pivot(self.air_small.isel(blocks[0]))
182
+ expected_schema = pa.Schema.from_pandas(first_block_df)
183
+
184
+ # Create RecordBatchReader
185
+ reader = from_map_batched(
186
+ pivot,
187
+ [self.air_small.isel(block) for block in blocks],
188
+ schema=expected_schema,
189
+ )
190
+
191
+ # Verify it's a RecordBatchReader
192
+ self.assertIsInstance(reader, pa.RecordBatchReader)
193
+
194
+ # Verify schema
195
+ self.assertEqual(reader.schema, expected_schema)
196
+
197
+ # Read all batches and verify content
198
+ batches = list(reader)
199
+ self.assertGreater(len(batches), 0)
200
+
201
+ # Verify each batch has the correct schema
202
+ for batch in batches:
203
+ self.assertEqual(batch.schema, expected_schema)
204
+ self.assertGreater(len(batch), 0)
205
+
206
+ def test_multiple_iterables(self):
207
+ """Test from_map_batched with multiple iterables."""
208
+ x_values = [1, 2, 3, 4, 5]
209
+ y_values = [10, 20, 30, 40, 50]
210
+
211
+ expected_schema = pa.schema(
212
+ [('x', pa.int64()), ('y', pa.int64()), ('sum', pa.int64())]
213
+ )
214
+
215
+ reader = from_map_batched(
216
+ adding_function, x_values, y_values, schema=expected_schema
217
+ )
218
+
219
+ # Read all data
220
+ table = reader.read_all()
221
+ df = table.to_pandas()
222
+
223
+ # Verify results
224
+ expected_df = pd.DataFrame(
225
+ {
226
+ 'x': x_values,
227
+ 'y': y_values,
228
+ 'sum': [x + y for x, y in zip(x_values, y_values)],
229
+ }
230
+ )
231
+
232
+ pd.testing.assert_frame_equal(df, expected_df)
233
+
234
+ def test_with_args_and_kwargs(self):
235
+ """Test from_map_batched with additional args and kwargs."""
236
+
237
+ def multiply_and_add(x, multiplier, offset=0):
238
+ result = pd.DataFrame({'x': [x], 'result': [x * multiplier + offset]})
239
+ return result
240
+
241
+ values = [1, 2, 3]
242
+ expected_schema = pa.schema([('x', pa.int64()), ('result', pa.int64())])
243
+
244
+ reader = from_map_batched(
245
+ multiply_and_add,
246
+ values,
247
+ args=(2,), # multiplier = 2
248
+ offset=5, # offset = 5
249
+ schema=expected_schema,
250
+ )
251
+
252
+ table = reader.read_all()
253
+ df = table.to_pandas()
254
+
255
+ # Verify results: x * 2 + 5
256
+ expected_df = pd.DataFrame(
257
+ {'x': [1, 2, 3], 'result': [7, 9, 11]} # (1*2+5, 2*2+5, 3*2+5)
258
+ )
259
+
260
+ pd.testing.assert_frame_equal(df, expected_df)
261
+
262
+ def test_empty_iterables(self):
263
+ """Test from_map_batched with empty iterables."""
264
+ empty_schema = pa.schema([('value', pa.int64())])
265
+
266
+ reader = from_map_batched(
267
+ lambda x: pd.DataFrame({'value': [x]}), [], schema=empty_schema
268
+ )
269
+
270
+ batches = list(reader)
271
+ self.assertEqual(len(batches), 0)
272
+
273
+ def test_consistency_with_regular_map(self):
274
+ """Test that results are consistent with regular mapping."""
275
+ blocks = list(block_slices(self.air_small, chunks={'time': 4, 'lat': 3}))
276
+ datasets = [self.air_small.isel(block) for block in blocks]
277
+
278
+ # Get schema from first block
279
+ first_df = pivot(datasets[0])
280
+ schema = pa.Schema.from_pandas(first_df)
281
+
282
+ # Use from_map_batched
283
+ reader = from_map_batched(pivot, datasets, schema=schema)
284
+ batched_table = reader.read_all()
285
+
286
+ # Regular map approach
287
+ regular_dfs = [pivot(ds) for ds in datasets]
288
+ regular_table = pa.Table.from_pandas(
289
+ pd.concat(regular_dfs, ignore_index=True)
290
+ )
291
+
292
+ # Results should be identical
293
+ self.assertEqual(batched_table.schema, regular_table.schema)
294
+ self.assertEqual(len(batched_table), len(regular_table))
295
+
296
+ # Compare data (allowing for potential column order differences)
297
+ batched_df = (
298
+ batched_table.to_pandas()
299
+ .sort_values(['time', 'lat', 'lon'])
300
+ .reset_index(drop=True)
301
+ )
302
+ regular_df = (
303
+ regular_table.to_pandas()
304
+ .sort_values(['time', 'lat', 'lon'])
305
+ .reset_index(drop=True)
306
+ )
307
+
308
+ pd.testing.assert_frame_equal(batched_df, regular_df)
309
+
310
+ def test_integration_with_datafusion_via_read_xarray(self):
311
+ """Test integration with the read_xarray function that uses from_map_batched."""
312
+
313
+ # Use a small dataset
314
+ air = xr.tutorial.open_dataset('air_temperature')
315
+ air_small = air.isel(time=slice(0, 50), lat=slice(0, 10), lon=slice(0, 15))
316
+ air_chunked = air_small.chunk({'time': 25, 'lat': 5, 'lon': 8})
317
+
318
+ # read_xarray uses from_map_batched internally
319
+ arrow_stream = read_xarray(
320
+ air_chunked, chunks={'time': 25, 'lat': 5, 'lon': 8}
321
+ )
322
+
323
+ # Verify it returns a proper ArrowStreamExportable (RecordBatchReader)
324
+ self.assertTrue(hasattr(arrow_stream, 'schema'))
325
+ self.assertTrue(hasattr(arrow_stream, '__iter__'))
326
+
327
+ # Should be able to read all data
328
+ table = arrow_stream.read_all()
329
+ self.assertGreater(len(table), 0)
330
+
331
+ # Should have the expected columns
332
+ expected_columns = {'time', 'lat', 'lon', 'air'}
333
+ actual_columns = set(table.column_names)
334
+ self.assertTrue(expected_columns.issubset(actual_columns))
335
+
336
+
337
+ class ReadXarrayStreamingTest(unittest.TestCase):
338
+
339
+ def setUp(self):
340
+ self.large_ds = create_large_dataset().chunk({'time': 25})
341
+
342
+ def test_read_xarray__loads_one_chunk_at_a_time(self):
343
+ tracemalloc.start()
344
+ iterable = read_xarray(self.large_ds)
345
+ first_size, first_peak = tracemalloc.get_traced_memory()
346
+ tracemalloc.reset_peak()
347
+
348
+ sizes, peaks = [], []
349
+
350
+ first_chunk = self.large_ds.isel(next(block_slices(self.large_ds)))
351
+ chunk_size = first_chunk.nbytes
352
+
353
+ # Creating the iterator should be inexpensive -- less than one chunk.
354
+ # We multiply by constant factors because chunks have additional overhead
355
+ self.assertLess(first_size, chunk_size * 3)
356
+ self.assertLess(first_peak, chunk_size * 6)
58
357
 
59
- def test_sanity(self):
60
- df = read_xarray(self.air_small).compute()
61
- self.assertIsNotNone(df)
62
- self.assertEqual(len(df), np.prod(list(self.air_small.dims.values())))
358
+ for it in iterable:
359
+ _ = it
360
+ cur_size, cur_peak = tracemalloc.get_traced_memory()
361
+ tracemalloc.reset_peak()
362
+ sizes.append(cur_size), peaks.append(cur_peak)
63
363
 
64
- def test_columns(self):
65
- df = read_xarray(self.air_small).compute()
66
- cols = list(df.columns)
67
- self.assertEqual(cols, ['lat', 'time', 'lon', 'air'])
364
+ mean_size = np.mean(sizes)
365
+ mean_peak = np.mean(peaks)
68
366
 
69
- def test_dtypes(self):
70
- df: dd.DataFrame = read_xarray(self.air_small).compute()
71
- types = list(df.dtypes)
72
- self.assertEqual([self.air_small[c].dtype for c in df.columns], types)
367
+ # Provide a bound for each size and peak
368
+ for size in sizes:
369
+ self.assertGreater(mean_size * 1.1, size)
370
+ self.assertGreater(chunk_size * 3, size)
371
+ # malloc size is about 2.66x chunk_size on average
372
+ self.assertLess(chunk_size * 2, size)
73
373
 
74
- def test_partitions_dont_match_dataset_chunks(self):
75
- standard_blocks = list(block_slices(self.air_small))
76
- default: dd.DataFrame = read_xarray(self.air_small)
77
- chunked: dd.DataFrame = read_xarray(self.air_small, dict(time=5))
374
+ for peak in peaks:
375
+ self.assertGreater(mean_peak * 1.1, peak)
376
+ self.assertGreater(chunk_size * 7, peak)
377
+ # malloc peak is about 6.89x chunk_size on average
378
+ self.assertLess(chunk_size * 4, peak)
78
379
 
79
- self.assertEqual(default.npartitions, len(standard_blocks))
80
- self.assertNotEqual(chunked.npartitions, len(standard_blocks))
380
+ # The peak malloc should never be more than the original dataset!
381
+ self.assertLess(max(peaks), self.large_ds.nbytes)
81
382
 
82
- def test_chunk_perf(self):
83
- df = read_xarray(self.air, chunks=dict(time=6)).compute()
84
- self.assertIsNotNone(df)
85
- self.assertEqual(len(df), np.prod(list(self.air.dims.values())))
383
+ tracemalloc.stop()
86
384
 
87
385
 
88
386
  if __name__ == '__main__':
xarray_sql/sql.py CHANGED
@@ -1,20 +1,17 @@
1
1
  import xarray as xr
2
- import dask_sql.input_utils
2
+ from datafusion import SessionContext
3
3
 
4
4
  from .df import read_xarray, Chunks
5
5
 
6
6
 
7
- class Context(dask_sql.Context):
8
- """See the `dask_sql.Context` docs."""
7
+ class XarrayContext(SessionContext):
8
+ """A datafusion `SessionContext` that also supports `xarray.Dataset`s."""
9
9
 
10
- def create_table(
10
+ def from_dataset(
11
11
  self,
12
12
  table_name: str,
13
- input_table: dask_sql.input_utils.InputType,
13
+ input_table: xr.Dataset,
14
14
  chunks: Chunks = None,
15
- *args,
16
- **kwargs,
17
15
  ):
18
- if isinstance(input_table, xr.Dataset):
19
- input_table = read_xarray(input_table, chunks)
20
- super().create_table(table_name, input_table, *args, **kwargs)
16
+ arrow_table = read_xarray(input_table, chunks)
17
+ return self.from_arrow(arrow_table, table_name)
xarray_sql/sql_test.py CHANGED
@@ -1,58 +1,62 @@
1
1
  import unittest
2
2
 
3
-
4
- from . import Context
3
+ from . import XarrayContext
5
4
  from .df_test import DaskTestCase
6
5
 
7
6
 
8
7
  class SqlTestCase(DaskTestCase):
9
8
 
10
9
  def test_sanity(self):
11
- c = Context()
12
- c.create_table('air', self.air_small)
10
+ c = XarrayContext()
11
+ c.from_dataset('air', self.air_small)
13
12
 
14
13
  query = c.sql('SELECT "lat", "lon", "time", "air" FROM "air" LIMIT 100')
15
14
 
16
- result = query.compute()
15
+ result = query.to_pandas()
17
16
  self.assertIsNotNone(result)
18
- self.assertEqual(len(result), 100)
17
+ self.assertLessEqual(len(result), 1320) # Should be all rows or less
18
+ self.assertGreater(len(result), 0) # Should have some rows
19
19
 
20
20
  def test_agg_small(self):
21
- c = Context()
22
- c.create_table('air', self.air_small)
23
-
24
- query = c.sql("""
25
- SELECT
26
- "lat", "lon", SUM("air") as air_total
27
- FROM
28
- "air"
29
- GROUP BY
30
- "lat", "lon"
31
- """)
32
-
33
- result = query.compute()
21
+ c = XarrayContext()
22
+ c.from_dataset('air', self.air_small)
23
+
24
+ query = c.sql(
25
+ """
26
+ SELECT
27
+ "lat", "lon", SUM("air") as air_total
28
+ FROM
29
+ "air"
30
+ GROUP BY
31
+ "lat", "lon"
32
+ """
33
+ )
34
+
35
+ result = query.to_pandas()
34
36
  self.assertIsNotNone(result)
35
37
 
36
- expected = self.air_small.dims['lat'] * self.air_small.dims['lon']
38
+ expected = self.air_small.sizes['lat'] * self.air_small.sizes['lon']
37
39
  self.assertEqual(len(result), expected)
38
40
 
39
41
  def test_agg_regular(self):
40
- c = Context()
41
- c.create_table('air', self.air)
42
+ c = XarrayContext()
43
+ c.from_dataset('air', self.air)
42
44
 
43
- query = c.sql("""
45
+ query = c.sql(
46
+ """
44
47
  SELECT
45
48
  "lat", "lon", AVG("air") as air_total
46
49
  FROM
47
50
  "air"
48
51
  GROUP BY
49
52
  "lat", "lon"
50
- """)
53
+ """
54
+ )
51
55
 
52
- result = query.compute()
56
+ result = query.to_pandas()
53
57
  self.assertIsNotNone(result)
54
58
 
55
- expected = self.air.dims['lat'] * self.air.dims['lon']
59
+ expected = self.air.sizes['lat'] * self.air.sizes['lon']
56
60
  self.assertEqual(len(result), expected)
57
61
 
58
62
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.0.2
3
+ Version: 0.1.1
4
4
  Summary: Querry Xarray with SQL.
5
5
  Author-email: Alexander Merose <al@merose.com>
6
6
  License: Apache-2.0
@@ -14,26 +14,24 @@ Classifier: License :: OSI Approved :: Apache Software License
14
14
  Classifier: Operating System :: MacOS :: MacOS X
15
15
  Classifier: Operating System :: Microsoft :: Windows
16
16
  Classifier: Operating System :: POSIX
17
- Classifier: Programming Language :: Python :: 3.8
18
17
  Classifier: Programming Language :: Python :: 3.9
19
18
  Classifier: Programming Language :: Python :: 3.10
20
19
  Classifier: Programming Language :: Python :: 3.11
21
20
  Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
22
  Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
23
23
  Classifier: Topic :: Database :: Front-Ends
24
- Requires-Python: >=3.8
24
+ Requires-Python: >=3.9
25
25
  Description-Content-Type: text/markdown
26
26
  License-File: LICENSE
27
- Requires-Dist: xarray
28
- Requires-Dist: dask-sql
29
- Provides-Extra: dev
30
- Requires-Dist: xarray-sql[test] ; extra == 'dev'
31
- Requires-Dist: pyink ; extra == 'dev'
32
- Requires-Dist: py-spy ; extra == 'dev'
27
+ Requires-Dist: dask>=2024.8.0
28
+ Requires-Dist: datafusion>=47.0.0
29
+ Requires-Dist: xarray>=2024.7.0
33
30
  Provides-Extra: test
34
- Requires-Dist: pytest ; extra == 'test'
35
- Requires-Dist: xarray[io] ; extra == 'test'
36
- Requires-Dist: gcsfs ; extra == 'test'
31
+ Requires-Dist: pytest; extra == "test"
32
+ Requires-Dist: xarray[io]; extra == "test"
33
+ Requires-Dist: gcsfs; extra == "test"
34
+ Dynamic: license-file
37
35
 
38
36
  # xarray-sql
39
37
 
@@ -42,6 +40,9 @@ _Query Xarray with SQL_
42
40
  [![ci](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/ci.yml)
43
41
  [![lint](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml/badge.svg)](https://github.com/alxmrs/xarray-sql/actions/workflows/lint.yml)
44
42
 
43
+ ```shell
44
+ pip install xarray-sql
45
+ ```
45
46
 
46
47
  ## What is this?
47
48
 
@@ -54,8 +55,8 @@ import xarray_sql as qr
54
55
  ds = xr.tutorial.open_dataset('air_temperature')
55
56
 
56
57
  # The same as a dask-sql Context; i.e. an Apache DataFusion Context.
57
- c = qr.Context(ds)
58
- c.create_table('air', ds, chunks=dict(time=24))
58
+ c = qr.XarrayContext()
59
+ c.from_dataset('air', ds, chunks=dict(time=24))
59
60
 
60
61
  df = c.sql('''
61
62
  SELECT
@@ -67,10 +68,10 @@ df = c.sql('''
67
68
  ''')
68
69
 
69
70
  # A table of the average temperature for each location across time.
70
- df.compute()
71
+ df.to_pandas()
71
72
 
72
73
  # Alternatively, you can just create the DataFrame from the Dataset:
73
- df = qr.read_xarray(ds)
74
+ df = qr.read_xarray(ds).to_pandas()
74
75
  df.head()
75
76
  ```
76
77
 
@@ -82,7 +83,7 @@ SQL queries against them.
82
83
  A few reasons:
83
84
 
84
85
  * Even though SQL is the lingua franca of data, scientific datasets are often
85
- inaccessible to non-scientists.
86
+ inaccessible to non-scientists (SQL users).
86
87
  * Joining tabular data with raster data is common yet difficult. It could be
87
88
  easy.
88
89
  * There are many cloud-native, Xarray-openable datasets,
@@ -102,10 +103,13 @@ All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
102
103
  `from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
103
104
  That's it!
104
105
 
106
+ _2025 update_: This library now implements a dask-like `from_map` interface in
107
+ pure `datafusion` and `pyarrow`, but works with the same principle!
108
+
105
109
  ## Why does this work?
106
110
 
107
111
  Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
108
- chucks and represented contiguously in memory. It is only a matter of metadata
112
+ chunks and represented contiguously in memory. It is only a matter of metadata
109
113
  that breaks them up into ndarrays. `to_dataframe()`
110
114
  just changes this metadata (via a `ravel()`/`reshape()`), back into a column
111
115
  amenable to a DataFrame.
@@ -115,15 +119,7 @@ worth the convenience of DataFrames.
115
119
 
116
120
  ## What are the current limitations?
117
121
 
118
- Dask doesn't support
119
- `MultiIndex`s ([dask/dask#1493](https://github.com/dask/dask/issues/1493)). If
120
- it did, I suspect performance for many types of queries would greatly improve.
121
-
122
- Further, while this does play well with `dask-geopandas` (for geospatial query
123
- support), certain types of operations don't quite match standard geopandas.
124
- Spatial joins come to mind as a killer feature, but only inner joins are
125
- supported ([geopandas/dask-geopandas#72](https://github.com/geopandas/dask-geopandas/issues/72))
126
- .
122
+ _2025 update_: TBD, `datafusion` provides a whole new world!
127
123
 
128
124
  ## What would a deeper integration look like?
129
125
 
@@ -134,7 +130,7 @@ Xarray Datasets. This approach is being pursued
134
130
  Deeper still: I was thinking we could make
135
131
  a [virtual](https://fsspec.github.io/kerchunk/)
136
132
  filesystem for parquet that would internally map to Zarr. Raster-backed virtual
137
- parquet would open up integrations to numeroustools like dask, pyarrow, duckdb,
133
+ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
138
134
  and BigQuery. More thoughts on this
139
135
  in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
140
136
 
@@ -145,8 +141,8 @@ I want to give a special thanks to the following folks and institutions:
145
141
  - Pramod Gupta and the Anthromet Team at Google Research for the problem
146
142
  formation and design inspiration.
147
143
  - Jake Wall and AI2/Ecoscope for compute resources and key use cases.
148
- - Charles Stern, Stephan Hoyer, and Qiusheng Wu for the early review and
149
- discussion of this project.
144
+ - Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
145
+ for the early review and discussion of this project.
150
146
 
151
147
  ## License
152
148
 
@@ -0,0 +1,12 @@
1
+ xarray_sql/__init__.py,sha256=HmuxwFMm3a7EVhCz3z2XUn6MeU7ghwSbO9y7A5G8MY8,69
2
+ xarray_sql/core.py,sha256=EQ1qBjHxnJPpb_HTRQ7B3NKLaI00KOdD4fmt5Qum3BI,1213
3
+ xarray_sql/df.py,sha256=3HoHmWCZQkkeOkJfP1lEaFjaqAykimY0ILQpkZ3ZaZY,5703
4
+ xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
5
+ xarray_sql/df_test.py,sha256=w2GqqaD2iUlIPA9jkyzUlHJJsFhQoHZB5pgtfz8xFgs,11965
6
+ xarray_sql/sql.py,sha256=D6Wz4m79jhgZxDqNs__-n6B8zjqkFGRDQtcq0e72NKI,434
7
+ xarray_sql/sql_test.py,sha256=eQ2w9wliBIKE32x_tBhjezMbykwQvOpgm0ectObPJb4,1370
8
+ xarray_sql-0.1.1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
+ xarray_sql-0.1.1.dist-info/METADATA,sha256=KJS24M8Vvcrxd_CVusz6DtucLAbnlahjpwRDZi-osEI,6086
10
+ xarray_sql-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
11
+ xarray_sql-0.1.1.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
+ xarray_sql-0.1.1.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.42.0)
2
+ Generator: setuptools (80.9.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,12 +0,0 @@
1
- xarray_sql/__init__.py,sha256=7ij_x7HZfcLn1GdZkluH8Lgx3l-cflE7Qmg1ZBHJCiA,53
2
- xarray_sql/core.py,sha256=BbKJO9GnjO8hXPUPdQrmttgDeHe_8crdh5he-c9rJsE,1197
3
- xarray_sql/df.py,sha256=hsGEwPTTiO5fimB1u3-ywYH50MlTF20dq6Hpcb_A9XU,3432
4
- xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
5
- xarray_sql/df_test.py,sha256=souhCNstQHML-idSYiPuQ5BEjHH_D9XE-YpUCJhnNSk,2694
6
- xarray_sql/sql.py,sha256=68ySKbvPKL_inHJ7FI6mF7tNrZB2BCeWKFO4FdysNFA,495
7
- xarray_sql/sql_test.py,sha256=wFp4dC0_FQiQUM66isdliQSTqPeLB35h2LKHbpicYms,1224
8
- xarray_sql-0.0.2.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
- xarray_sql-0.0.2.dist-info/METADATA,sha256=tyZ8NQqG95gx74re302VHzKj8jJuSPK-zGzMIPLXZ10,6367
10
- xarray_sql-0.0.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
11
- xarray_sql-0.0.2.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
- xarray_sql-0.0.2.dist-info/RECORD,,