xarray-sql 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/df.py CHANGED
@@ -5,8 +5,9 @@ import numpy as np
5
5
  import pandas as pd
6
6
  import pyarrow as pa
7
7
  import xarray as xr
8
+ from datafusion.context import ArrowStreamExportable
8
9
 
9
- Block = t.Dict[str, slice]
10
+ Block = t.Dict[t.Hashable, slice]
10
11
  Chunks = t.Optional[t.Dict[str, int]]
11
12
 
12
13
 
@@ -54,7 +55,42 @@ def explode(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[xr.Dataset]:
54
55
 
55
56
 
56
57
  def _block_len(block: Block) -> int:
57
- return np.prod([v.stop - v.start for v in block.values()])
58
+ return int(np.prod([v.stop - v.start for v in block.values()]))
59
+
60
+
61
+ def from_map_batched(
62
+ func: t.Callable[[...], pd.DataFrame],
63
+ *iterables,
64
+ args: t.Optional[t.Tuple] = None,
65
+ schema: pa.Schema = None,
66
+ **kwargs,
67
+ ) -> pa.RecordBatchReader:
68
+ """Create a PyArrow RecordBatchReader by mapping a function over iterables.
69
+
70
+ This is equivalent to dask's from_map but returns a PyArrow
71
+ RecordBatchReader that can be used with DataFusion. It iterates over
72
+ RecordBatches which are created via the `func` one-at-a-time.
73
+
74
+ Args:
75
+ func: Function to apply to each element of the iterables. Currently, the function
76
+ must return a Pandas DataFrame.
77
+ *iterables: Iterable objects to map the function over.
78
+ schema: Optional schema needed for the RecordBatchReader.
79
+ args: Additional positional arguments to pass to func.
80
+ **kwargs: Additional keyword arguments to pass to func.
81
+
82
+ Returns:
83
+ A PyArrow RecordBatchReader containing the stream of RecordBatches.
84
+ """
85
+ if args is None:
86
+ args = ()
87
+
88
+ def map_batches():
89
+ for items in zip(*iterables):
90
+ df = func(*items, *args, **kwargs)
91
+ yield pa.RecordBatch.from_pandas(df, schema=schema)
92
+
93
+ return pa.RecordBatchReader.from_batches(schema, map_batches())
58
94
 
59
95
 
60
96
  def from_map(
@@ -109,7 +145,12 @@ def from_map(
109
145
  return pa.concat_tables(results)
110
146
 
111
147
 
112
- def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
148
+ def pivot(ds: xr.Dataset) -> pd.DataFrame:
149
+ """Converts an xarray Dataset to a pandas DataFrame."""
150
+ return ds.to_dataframe().reset_index()
151
+
152
+
153
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> ArrowStreamExportable:
113
154
  """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
114
155
 
115
156
  Args:
@@ -128,7 +169,11 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
128
169
 
129
170
  blocks = list(block_slices(ds, chunks))
130
171
 
131
- def pivot(b: Block) -> pd.DataFrame:
132
- return ds.isel(b).to_dataframe().reset_index()
172
+ def pivot_block(b: Block):
173
+ return pivot(ds.isel(b))
174
+
175
+ schema = pa.Schema.from_pandas(pivot_block(blocks[0]))
176
+ last_schema = pa.Schema.from_pandas(pivot_block(blocks[-1]))
177
+ assert schema == last_schema, "Schemas must be consistent across blocks!"
133
178
 
134
- return from_map(pivot, blocks)
179
+ return from_map_batched(pivot_block, blocks, schema=schema)
xarray_sql/df_test.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import itertools
2
+ import tracemalloc
2
3
  import unittest
3
4
 
4
5
  import numpy as np
@@ -6,7 +7,7 @@ import pandas as pd
6
7
  import pyarrow as pa
7
8
  import xarray as xr
8
9
 
9
- from .df import explode, read_xarray, block_slices, from_map
10
+ from .df import explode, read_xarray, block_slices, from_map, pivot, from_map_batched
10
11
 
11
12
 
12
13
  def rand_wx(start: str, end: str) -> xr.Dataset:
@@ -34,6 +35,38 @@ def rand_wx(start: str, end: str) -> xr.Dataset:
34
35
  )
35
36
 
36
37
 
38
+ def create_large_dataset(time_steps=1000, lat_points=100, lon_points=100):
39
+ """Create a large xarray dataset for memory testing."""
40
+ np.random.seed(42)
41
+
42
+ # Create coordinates
43
+ time = pd.date_range('2020-01-01', periods=time_steps, freq='h')
44
+ lat = np.linspace(-90, 90, lat_points)
45
+ lon = np.linspace(-180, 180, lon_points)
46
+
47
+ # Create large data arrays
48
+ temp_data = np.random.rand(time_steps, lat_points, lon_points) * 40 - 10
49
+ precip_data = np.random.rand(time_steps, lat_points, lon_points) * 100
50
+
51
+ return xr.Dataset(
52
+ {
53
+ 'temperature': (['time', 'lat', 'lon'], temp_data),
54
+ 'precipitation': (['time', 'lat', 'lon'], precip_data),
55
+ },
56
+ coords={
57
+ 'time': time,
58
+ 'lat': lat,
59
+ 'lon': lon,
60
+ },
61
+ )
62
+
63
+
64
+ def adding_function(x, y):
65
+ """Simple function that adds two values and returns a DataFrame."""
66
+ result = pd.DataFrame({'x': [x], 'y': [y], 'sum': [x + y]})
67
+ return result
68
+
69
+
37
70
  class DaskTestCase(unittest.TestCase):
38
71
 
39
72
  def setUp(self) -> None:
@@ -81,49 +114,6 @@ class ExplodeTest(DaskTestCase):
81
114
  self.assertEqual(self.air.isel(iselection), ds)
82
115
 
83
116
 
84
- class PyArrowTableTest(DaskTestCase):
85
-
86
- def test_sanity(self):
87
- table = read_xarray(self.air_small)
88
- self.assertIsNotNone(table)
89
- self.assertIsInstance(table, pa.Table)
90
- self.assertEqual(len(table), np.prod(list(self.air_small.sizes.values())))
91
-
92
- def test_columns(self):
93
- table = read_xarray(self.air_small)
94
- cols = table.column_names
95
- self.assertEqual(cols, ['lat', 'time', 'lon', 'air'])
96
-
97
- def test_dtypes(self):
98
- table = read_xarray(self.air_small)
99
- # Convert to pandas to check dtypes
100
- df = table.to_pandas()
101
- types = list(df.dtypes)
102
- self.assertEqual([self.air_small[c].dtype for c in df.columns], types)
103
-
104
- def test_different_chunk_sizes(self):
105
- default_table = read_xarray(self.air_small)
106
- chunked_table = read_xarray(self.air_small, dict(time=5))
107
-
108
- # Both should produce valid tables
109
- self.assertIsInstance(default_table, pa.Table)
110
- self.assertIsInstance(chunked_table, pa.Table)
111
- # Should have same number of rows
112
- self.assertEqual(len(default_table), len(chunked_table))
113
-
114
- def test_chunk_perf(self):
115
- table = read_xarray(self.air, chunks=dict(time=6))
116
- self.assertIsNotNone(table)
117
- self.assertEqual(len(table), np.prod(list(self.air.sizes.values())))
118
-
119
- def test_column_metadata_preserved(self):
120
- try:
121
- table = read_xarray(self.randwx, chunks=dict(time=24))
122
- self.assertIsInstance(table, pa.Table)
123
- except Exception as e:
124
- self.fail(f'Unexpected error: {e}')
125
-
126
-
127
117
  class FromMapTest(unittest.TestCase):
128
118
 
129
119
  def test_basic_from_map(self):
@@ -178,5 +168,220 @@ class FromMapTest(unittest.TestCase):
178
168
  self.assertEqual(len(result), 3)
179
169
 
180
170
 
171
+ class TestFromMapBatchedCorrectness(DaskTestCase):
172
+ """Test correctness of from_map_batched function."""
173
+
174
+ def test_basic_functionality(self):
175
+ """Test that from_map_batched produces correct RecordBatchReader."""
176
+ blocks = list(
177
+ block_slices(self.air_small, chunks={'time': 4, 'lat': 3, 'lon': 4})
178
+ )
179
+
180
+ # Get expected schema
181
+ first_block_df = pivot(self.air_small.isel(blocks[0]))
182
+ expected_schema = pa.Schema.from_pandas(first_block_df)
183
+
184
+ # Create RecordBatchReader
185
+ reader = from_map_batched(
186
+ pivot,
187
+ [self.air_small.isel(block) for block in blocks],
188
+ schema=expected_schema,
189
+ )
190
+
191
+ # Verify it's a RecordBatchReader
192
+ self.assertIsInstance(reader, pa.RecordBatchReader)
193
+
194
+ # Verify schema
195
+ self.assertEqual(reader.schema, expected_schema)
196
+
197
+ # Read all batches and verify content
198
+ batches = list(reader)
199
+ self.assertGreater(len(batches), 0)
200
+
201
+ # Verify each batch has the correct schema
202
+ for batch in batches:
203
+ self.assertEqual(batch.schema, expected_schema)
204
+ self.assertGreater(len(batch), 0)
205
+
206
+ def test_multiple_iterables(self):
207
+ """Test from_map_batched with multiple iterables."""
208
+ x_values = [1, 2, 3, 4, 5]
209
+ y_values = [10, 20, 30, 40, 50]
210
+
211
+ expected_schema = pa.schema(
212
+ [('x', pa.int64()), ('y', pa.int64()), ('sum', pa.int64())]
213
+ )
214
+
215
+ reader = from_map_batched(
216
+ adding_function, x_values, y_values, schema=expected_schema
217
+ )
218
+
219
+ # Read all data
220
+ table = reader.read_all()
221
+ df = table.to_pandas()
222
+
223
+ # Verify results
224
+ expected_df = pd.DataFrame(
225
+ {
226
+ 'x': x_values,
227
+ 'y': y_values,
228
+ 'sum': [x + y for x, y in zip(x_values, y_values)],
229
+ }
230
+ )
231
+
232
+ pd.testing.assert_frame_equal(df, expected_df)
233
+
234
+ def test_with_args_and_kwargs(self):
235
+ """Test from_map_batched with additional args and kwargs."""
236
+
237
+ def multiply_and_add(x, multiplier, offset=0):
238
+ result = pd.DataFrame({'x': [x], 'result': [x * multiplier + offset]})
239
+ return result
240
+
241
+ values = [1, 2, 3]
242
+ expected_schema = pa.schema([('x', pa.int64()), ('result', pa.int64())])
243
+
244
+ reader = from_map_batched(
245
+ multiply_and_add,
246
+ values,
247
+ args=(2,), # multiplier = 2
248
+ offset=5, # offset = 5
249
+ schema=expected_schema,
250
+ )
251
+
252
+ table = reader.read_all()
253
+ df = table.to_pandas()
254
+
255
+ # Verify results: x * 2 + 5
256
+ expected_df = pd.DataFrame(
257
+ {'x': [1, 2, 3], 'result': [7, 9, 11]} # (1*2+5, 2*2+5, 3*2+5)
258
+ )
259
+
260
+ pd.testing.assert_frame_equal(df, expected_df)
261
+
262
+ def test_empty_iterables(self):
263
+ """Test from_map_batched with empty iterables."""
264
+ empty_schema = pa.schema([('value', pa.int64())])
265
+
266
+ reader = from_map_batched(
267
+ lambda x: pd.DataFrame({'value': [x]}), [], schema=empty_schema
268
+ )
269
+
270
+ batches = list(reader)
271
+ self.assertEqual(len(batches), 0)
272
+
273
+ def test_consistency_with_regular_map(self):
274
+ """Test that results are consistent with regular mapping."""
275
+ blocks = list(block_slices(self.air_small, chunks={'time': 4, 'lat': 3}))
276
+ datasets = [self.air_small.isel(block) for block in blocks]
277
+
278
+ # Get schema from first block
279
+ first_df = pivot(datasets[0])
280
+ schema = pa.Schema.from_pandas(first_df)
281
+
282
+ # Use from_map_batched
283
+ reader = from_map_batched(pivot, datasets, schema=schema)
284
+ batched_table = reader.read_all()
285
+
286
+ # Regular map approach
287
+ regular_dfs = [pivot(ds) for ds in datasets]
288
+ regular_table = pa.Table.from_pandas(
289
+ pd.concat(regular_dfs, ignore_index=True)
290
+ )
291
+
292
+ # Results should be identical
293
+ self.assertEqual(batched_table.schema, regular_table.schema)
294
+ self.assertEqual(len(batched_table), len(regular_table))
295
+
296
+ # Compare data (allowing for potential column order differences)
297
+ batched_df = (
298
+ batched_table.to_pandas()
299
+ .sort_values(['time', 'lat', 'lon'])
300
+ .reset_index(drop=True)
301
+ )
302
+ regular_df = (
303
+ regular_table.to_pandas()
304
+ .sort_values(['time', 'lat', 'lon'])
305
+ .reset_index(drop=True)
306
+ )
307
+
308
+ pd.testing.assert_frame_equal(batched_df, regular_df)
309
+
310
+ def test_integration_with_datafusion_via_read_xarray(self):
311
+ """Test integration with the read_xarray function that uses from_map_batched."""
312
+
313
+ # Use a small dataset
314
+ air = xr.tutorial.open_dataset('air_temperature')
315
+ air_small = air.isel(time=slice(0, 50), lat=slice(0, 10), lon=slice(0, 15))
316
+ air_chunked = air_small.chunk({'time': 25, 'lat': 5, 'lon': 8})
317
+
318
+ # read_xarray uses from_map_batched internally
319
+ arrow_stream = read_xarray(
320
+ air_chunked, chunks={'time': 25, 'lat': 5, 'lon': 8}
321
+ )
322
+
323
+ # Verify it returns a proper ArrowStreamExportable (RecordBatchReader)
324
+ self.assertTrue(hasattr(arrow_stream, 'schema'))
325
+ self.assertTrue(hasattr(arrow_stream, '__iter__'))
326
+
327
+ # Should be able to read all data
328
+ table = arrow_stream.read_all()
329
+ self.assertGreater(len(table), 0)
330
+
331
+ # Should have the expected columns
332
+ expected_columns = {'time', 'lat', 'lon', 'air'}
333
+ actual_columns = set(table.column_names)
334
+ self.assertTrue(expected_columns.issubset(actual_columns))
335
+
336
+
337
+ class ReadXarrayStreamingTest(unittest.TestCase):
338
+
339
+ def setUp(self):
340
+ self.large_ds = create_large_dataset().chunk({'time': 25})
341
+
342
+ def test_read_xarray__loads_one_chunk_at_a_time(self):
343
+ tracemalloc.start()
344
+ iterable = read_xarray(self.large_ds)
345
+ first_size, first_peak = tracemalloc.get_traced_memory()
346
+ tracemalloc.reset_peak()
347
+
348
+ sizes, peaks = [], []
349
+
350
+ first_chunk = self.large_ds.isel(next(block_slices(self.large_ds)))
351
+ chunk_size = first_chunk.nbytes
352
+
353
+ # Creating the iterator should be inexpensive -- less than one chunk.
354
+ # We multiply by constant factors because chunks have additional overhead
355
+ self.assertLess(first_size, chunk_size * 3)
356
+ self.assertLess(first_peak, chunk_size * 6)
357
+
358
+ for it in iterable:
359
+ _ = it
360
+ cur_size, cur_peak = tracemalloc.get_traced_memory()
361
+ tracemalloc.reset_peak()
362
+ sizes.append(cur_size), peaks.append(cur_peak)
363
+
364
+ mean_size = np.mean(sizes)
365
+ mean_peak = np.mean(peaks)
366
+
367
+ # Provide a bound for each size and peak
368
+ for size in sizes:
369
+ self.assertGreater(mean_size * 1.1, size)
370
+ self.assertGreater(chunk_size * 3, size)
371
+ # malloc size is about 2.66x chunk_size on average
372
+ self.assertLess(chunk_size * 2, size)
373
+
374
+ for peak in peaks:
375
+ self.assertGreater(mean_peak * 1.1, peak)
376
+ self.assertGreater(chunk_size * 7, peak)
377
+ # malloc peak is about 6.89x chunk_size on average
378
+ self.assertLess(chunk_size * 4, peak)
379
+
380
+ # The peak malloc should never be more than the original dataset!
381
+ self.assertLess(max(peaks), self.large_ds.nbytes)
382
+
383
+ tracemalloc.stop()
384
+
385
+
181
386
  if __name__ == '__main__':
182
387
  unittest.main()
xarray_sql/sql_test.py CHANGED
@@ -23,13 +23,13 @@ class SqlTestCase(DaskTestCase):
23
23
 
24
24
  query = c.sql(
25
25
  """
26
- SELECT
27
- "lat", "lon", SUM("air") as air_total
28
- FROM
29
- "air"
30
- GROUP BY
31
- "lat", "lon"
32
- """
26
+ SELECT
27
+ "lat", "lon", SUM("air") as air_total
28
+ FROM
29
+ "air"
30
+ GROUP BY
31
+ "lat", "lon"
32
+ """
33
33
  )
34
34
 
35
35
  result = query.to_pandas()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.1.0
3
+ Version: 0.1.1
4
4
  Summary: Querry Xarray with SQL.
5
5
  Author-email: Alexander Merose <al@merose.com>
6
6
  License: Apache-2.0
@@ -0,0 +1,12 @@
1
+ xarray_sql/__init__.py,sha256=HmuxwFMm3a7EVhCz3z2XUn6MeU7ghwSbO9y7A5G8MY8,69
2
+ xarray_sql/core.py,sha256=EQ1qBjHxnJPpb_HTRQ7B3NKLaI00KOdD4fmt5Qum3BI,1213
3
+ xarray_sql/df.py,sha256=3HoHmWCZQkkeOkJfP1lEaFjaqAykimY0ILQpkZ3ZaZY,5703
4
+ xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
5
+ xarray_sql/df_test.py,sha256=w2GqqaD2iUlIPA9jkyzUlHJJsFhQoHZB5pgtfz8xFgs,11965
6
+ xarray_sql/sql.py,sha256=D6Wz4m79jhgZxDqNs__-n6B8zjqkFGRDQtcq0e72NKI,434
7
+ xarray_sql/sql_test.py,sha256=eQ2w9wliBIKE32x_tBhjezMbykwQvOpgm0ectObPJb4,1370
8
+ xarray_sql-0.1.1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
+ xarray_sql-0.1.1.dist-info/METADATA,sha256=KJS24M8Vvcrxd_CVusz6DtucLAbnlahjpwRDZi-osEI,6086
10
+ xarray_sql-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
11
+ xarray_sql-0.1.1.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
+ xarray_sql-0.1.1.dist-info/RECORD,,
@@ -1,12 +0,0 @@
1
- xarray_sql/__init__.py,sha256=HmuxwFMm3a7EVhCz3z2XUn6MeU7ghwSbO9y7A5G8MY8,69
2
- xarray_sql/core.py,sha256=EQ1qBjHxnJPpb_HTRQ7B3NKLaI00KOdD4fmt5Qum3BI,1213
3
- xarray_sql/df.py,sha256=0RPhxPIzyEwUiU_SrHpFPz3zs1BBgJb4GH9vO7X52YY,4101
4
- xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
5
- xarray_sql/df_test.py,sha256=LUJJd48RZl6R9Tph8EFN-C1lxuBlpL26SCS2F4hqPZE,5661
6
- xarray_sql/sql.py,sha256=D6Wz4m79jhgZxDqNs__-n6B8zjqkFGRDQtcq0e72NKI,434
7
- xarray_sql/sql_test.py,sha256=bScp4cyfuv1AooYRHeg3EPg2nXLsCKIxBPVFH0cFrYo,1384
8
- xarray_sql-0.1.0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
- xarray_sql-0.1.0.dist-info/METADATA,sha256=t3ZZdn03kWbUfjRvGucS6bGTLgst7QYhGciNSKuxvas,6086
10
- xarray_sql-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
11
- xarray_sql-0.1.0.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
- xarray_sql-0.1.0.dist-info/RECORD,,