xarray-sql 0.1.0__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/PKG-INFO +1 -1
  2. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql/df.py +51 -6
  3. xarray_sql-0.1.1/xarray_sql/df_test.py +387 -0
  4. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql/sql_test.py +7 -7
  5. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql.egg-info/PKG-INFO +1 -1
  6. xarray_sql-0.1.0/xarray_sql/df_test.py +0 -182
  7. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/.github/workflows/ci.yml +0 -0
  8. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/.github/workflows/lint.yml +0 -0
  9. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/.github/workflows/publish.yml +0 -0
  10. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/.gitignore +0 -0
  11. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/.python-version +0 -0
  12. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/CONTRIBUTING.md +0 -0
  13. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/LICENSE +0 -0
  14. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/README.md +0 -0
  15. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/README.md +0 -0
  16. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/compute_air.py +0 -0
  17. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/compute_air.py-2024-02-26T22:57:59+07:00.svg +0 -0
  18. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/compute_air.py-2024-03-03T07:42:36+05:30.svg +0 -0
  19. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/compute_air.py-2024-03-03T07:50:18+05:30.svg +0 -0
  20. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air.py +0 -0
  21. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air.py-2024-02-15T13:01:22+07:00.svg +0 -0
  22. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air.py-2024-02-15T15:03:53+07:00.svg +0 -0
  23. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air.py-2024-02-17T18:42:14+07:00.svg +0 -0
  24. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air.py-2024-02-26T23:04:34+07:00.svg +0 -0
  25. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air.py-2024-03-03T07:43:22+05:30.svg +0 -0
  26. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air.py-2024-03-03T07:50:06+05:30.svg +0 -0
  27. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air_full.py +0 -0
  28. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air_full.py-2024-02-26T23:06:38+07:00.svg +0 -0
  29. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air_full.py-2024-03-03T07:44:17+05:30.svg +0 -0
  30. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/groupby_air_full.py-2024-03-03T07:49:53+05:30.svg +0 -0
  31. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/open_era5.py +0 -0
  32. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/open_era5.py-2024-02-18T17:21:44+07:00.svg +0 -0
  33. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/open_era5.py-2024-02-18T17:33:02+07:00.svg +0 -0
  34. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/open_era5.py-2024-02-19T17:52:51+07:00.svg +0 -0
  35. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/profile.sh +0 -0
  36. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py +0 -0
  37. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-02-15T09:08:28+07:00.svg +0 -0
  38. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-02-15T09:15:07+07:00.svg +0 -0
  39. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-02-15T12:51:37+07:00.svg +0 -0
  40. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-02-15T15:04:48+07:00.svg +0 -0
  41. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-02-17T18:42:08+07:00.svg +0 -0
  42. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-02-26T23:00:00+07:00.svg +0 -0
  43. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-03-03T07:45:47+05:30.svg +0 -0
  44. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/perf_tests/sanity.py-2024-03-03T07:49:44+05:30.svg +0 -0
  45. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/pyproject.toml +0 -0
  46. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/setup.cfg +0 -0
  47. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/uv.lock +0 -0
  48. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql/__init__.py +0 -0
  49. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql/core.py +0 -0
  50. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql/df_integrationtest.py +0 -0
  51. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql/sql.py +0 -0
  52. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql.egg-info/SOURCES.txt +0 -0
  53. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql.egg-info/dependency_links.txt +0 -0
  54. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql.egg-info/requires.txt +0 -0
  55. {xarray_sql-0.1.0 → xarray_sql-0.1.1}/xarray_sql.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.1.0
3
+ Version: 0.1.1
4
4
  Summary: Querry Xarray with SQL.
5
5
  Author-email: Alexander Merose <al@merose.com>
6
6
  License: Apache-2.0
@@ -5,8 +5,9 @@ import numpy as np
5
5
  import pandas as pd
6
6
  import pyarrow as pa
7
7
  import xarray as xr
8
+ from datafusion.context import ArrowStreamExportable
8
9
 
9
- Block = t.Dict[str, slice]
10
+ Block = t.Dict[t.Hashable, slice]
10
11
  Chunks = t.Optional[t.Dict[str, int]]
11
12
 
12
13
 
@@ -54,7 +55,42 @@ def explode(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[xr.Dataset]:
54
55
 
55
56
 
56
57
  def _block_len(block: Block) -> int:
57
- return np.prod([v.stop - v.start for v in block.values()])
58
+ return int(np.prod([v.stop - v.start for v in block.values()]))
59
+
60
+
61
+ def from_map_batched(
62
+ func: t.Callable[[...], pd.DataFrame],
63
+ *iterables,
64
+ args: t.Optional[t.Tuple] = None,
65
+ schema: pa.Schema = None,
66
+ **kwargs,
67
+ ) -> pa.RecordBatchReader:
68
+ """Create a PyArrow RecordBatchReader by mapping a function over iterables.
69
+
70
+ This is equivalent to dask's from_map but returns a PyArrow
71
+ RecordBatchReader that can be used with DataFusion. It iterates over
72
+ RecordBatches which are created via the `func` one-at-a-time.
73
+
74
+ Args:
75
+ func: Function to apply to each element of the iterables. Currently, the function
76
+ must return a Pandas DataFrame.
77
+ *iterables: Iterable objects to map the function over.
78
+ schema: Optional schema needed for the RecordBatchReader.
79
+ args: Additional positional arguments to pass to func.
80
+ **kwargs: Additional keyword arguments to pass to func.
81
+
82
+ Returns:
83
+ A PyArrow RecordBatchReader containing the stream of RecordBatches.
84
+ """
85
+ if args is None:
86
+ args = ()
87
+
88
+ def map_batches():
89
+ for items in zip(*iterables):
90
+ df = func(*items, *args, **kwargs)
91
+ yield pa.RecordBatch.from_pandas(df, schema=schema)
92
+
93
+ return pa.RecordBatchReader.from_batches(schema, map_batches())
58
94
 
59
95
 
60
96
  def from_map(
@@ -109,7 +145,12 @@ def from_map(
109
145
  return pa.concat_tables(results)
110
146
 
111
147
 
112
- def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
148
+ def pivot(ds: xr.Dataset) -> pd.DataFrame:
149
+ """Converts an xarray Dataset to a pandas DataFrame."""
150
+ return ds.to_dataframe().reset_index()
151
+
152
+
153
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> ArrowStreamExportable:
113
154
  """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
114
155
 
115
156
  Args:
@@ -128,7 +169,11 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
128
169
 
129
170
  blocks = list(block_slices(ds, chunks))
130
171
 
131
- def pivot(b: Block) -> pd.DataFrame:
132
- return ds.isel(b).to_dataframe().reset_index()
172
+ def pivot_block(b: Block):
173
+ return pivot(ds.isel(b))
174
+
175
+ schema = pa.Schema.from_pandas(pivot_block(blocks[0]))
176
+ last_schema = pa.Schema.from_pandas(pivot_block(blocks[-1]))
177
+ assert schema == last_schema, "Schemas must be consistent across blocks!"
133
178
 
134
- return from_map(pivot, blocks)
179
+ return from_map_batched(pivot_block, blocks, schema=schema)
@@ -0,0 +1,387 @@
1
+ import itertools
2
+ import tracemalloc
3
+ import unittest
4
+
5
+ import numpy as np
6
+ import pandas as pd
7
+ import pyarrow as pa
8
+ import xarray as xr
9
+
10
+ from .df import explode, read_xarray, block_slices, from_map, pivot, from_map_batched
11
+
12
+
13
+ def rand_wx(start: str, end: str) -> xr.Dataset:
14
+ np.random.seed(42)
15
+ lat = np.linspace(-90, 90, num=720)
16
+ lon = np.linspace(-180, 180, num=1440)
17
+ time = pd.date_range(start, end, freq='h')
18
+ level = np.array([1000, 500], dtype=np.int32)
19
+ reference_time = pd.Timestamp(start)
20
+ temperature = 15 + 8 * np.random.randn(720, 1440, len(time), len(level))
21
+ precipitation = 10 * np.random.rand(720, 1440, len(time), len(level))
22
+ return xr.Dataset(
23
+ data_vars=dict(
24
+ temperature=(['lat', 'lon', 'time', 'level'], temperature),
25
+ precipitation=(['lat', 'lon', 'time', 'level'], precipitation),
26
+ ),
27
+ coords=dict(
28
+ lat=lat,
29
+ lon=lon,
30
+ time=time,
31
+ level=level,
32
+ reference_time=reference_time,
33
+ ),
34
+ attrs=dict(description='Random weather.'),
35
+ )
36
+
37
+
38
+ def create_large_dataset(time_steps=1000, lat_points=100, lon_points=100):
39
+ """Create a large xarray dataset for memory testing."""
40
+ np.random.seed(42)
41
+
42
+ # Create coordinates
43
+ time = pd.date_range('2020-01-01', periods=time_steps, freq='h')
44
+ lat = np.linspace(-90, 90, lat_points)
45
+ lon = np.linspace(-180, 180, lon_points)
46
+
47
+ # Create large data arrays
48
+ temp_data = np.random.rand(time_steps, lat_points, lon_points) * 40 - 10
49
+ precip_data = np.random.rand(time_steps, lat_points, lon_points) * 100
50
+
51
+ return xr.Dataset(
52
+ {
53
+ 'temperature': (['time', 'lat', 'lon'], temp_data),
54
+ 'precipitation': (['time', 'lat', 'lon'], precip_data),
55
+ },
56
+ coords={
57
+ 'time': time,
58
+ 'lat': lat,
59
+ 'lon': lon,
60
+ },
61
+ )
62
+
63
+
64
+ def adding_function(x, y):
65
+ """Simple function that adds two values and returns a DataFrame."""
66
+ result = pd.DataFrame({'x': [x], 'y': [y], 'sum': [x + y]})
67
+ return result
68
+
69
+
70
+ class DaskTestCase(unittest.TestCase):
71
+
72
+ def setUp(self) -> None:
73
+ self.air = xr.tutorial.open_dataset('air_temperature')
74
+ self.chunks = {'time': 240}
75
+ self.air = self.air.chunk(self.chunks)
76
+
77
+ self.air_small = self.air.isel(
78
+ time=slice(0, 12), lat=slice(0, 11), lon=slice(0, 10)
79
+ ).chunk(self.chunks)
80
+ self.randwx = rand_wx('1995-01-13T00', '1995-01-13T01')
81
+
82
+
83
+ class ExplodeTest(DaskTestCase):
84
+
85
+ def test_cardinality(self):
86
+ dss = explode(self.air)
87
+ self.assertEqual(
88
+ len(list(dss)), np.prod([len(c) for c in self.air.chunks.values()])
89
+ )
90
+
91
+ def test_dim_sizes__one(self):
92
+ ds = next(iter(explode(self.air)))
93
+ for k, v in self.chunks.items():
94
+ self.assertIn(k, ds.dims)
95
+ self.assertEqual(v, ds.sizes[k])
96
+
97
+ def skip_test_dim_sizes__all(self):
98
+ # TODO(alxmrs): Why is this test slow?
99
+ dss = explode(self.air)
100
+ self.assertEqual(
101
+ [tuple(ds.dims.values()) for ds in dss],
102
+ list(itertools.product(*self.air.chunksizes.values())),
103
+ )
104
+
105
+ def test_data_equal__one__first(self):
106
+ ds = next(iter(explode(self.air)))
107
+ iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
108
+ self.assertEqual(self.air.isel(iselection), ds)
109
+
110
+ def test_data_equal__one__last(self):
111
+ dss = list(explode(self.air))
112
+ ds = dss[-1]
113
+ iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
114
+ self.assertEqual(self.air.isel(iselection), ds)
115
+
116
+
117
+ class FromMapTest(unittest.TestCase):
118
+
119
+ def test_basic_from_map(self):
120
+ """Test basic from_map functionality with pandas DataFrames."""
121
+
122
+ def make_df(x):
123
+ return pd.DataFrame({'value': [x, x * 2], 'index': [0, 1]})
124
+
125
+ result = from_map(make_df, [1, 2, 3])
126
+ self.assertIsInstance(result, pa.Table)
127
+ self.assertEqual(len(result), 6) # 3 inputs * 2 rows each
128
+ self.assertEqual(result.column_names, ['value', 'index'])
129
+
130
+ def test_from_map_with_multiple_iterables(self):
131
+ """Test from_map with multiple iterables."""
132
+
133
+ def add_values(x, y):
134
+ return pd.DataFrame({'sum': [x + y], 'x': [x], 'y': [y]})
135
+
136
+ result = from_map(add_values, [1, 2], [10, 20])
137
+ self.assertIsInstance(result, pa.Table)
138
+ self.assertEqual(len(result), 2)
139
+
140
+ # Convert to pandas to check values
141
+ df = result.to_pandas()
142
+ self.assertEqual(list(df['sum']), [11, 22])
143
+
144
+ def test_from_map_with_args(self):
145
+ """Test from_map with additional arguments."""
146
+
147
+ def multiply_and_add(x, multiplier, add_value):
148
+ return pd.DataFrame({'result': [x * multiplier + add_value]})
149
+
150
+ result = from_map(multiply_and_add, [1, 2, 3], args=(2, 10))
151
+ self.assertIsInstance(result, pa.Table)
152
+ self.assertEqual(len(result), 3)
153
+
154
+ df = result.to_pandas()
155
+ self.assertEqual(
156
+ list(df['result']), [12, 14, 16]
157
+ ) # (1*2+10, 2*2+10, 3*2+10)
158
+
159
+ def test_from_map_with_pyarrow_tables(self):
160
+ """Test from_map when function returns PyArrow tables."""
161
+
162
+ def make_arrow_table(x):
163
+ df = pd.DataFrame({'value': [x]})
164
+ return pa.Table.from_pandas(df)
165
+
166
+ result = from_map(make_arrow_table, [1, 2, 3])
167
+ self.assertIsInstance(result, pa.Table)
168
+ self.assertEqual(len(result), 3)
169
+
170
+
171
+ class TestFromMapBatchedCorrectness(DaskTestCase):
172
+ """Test correctness of from_map_batched function."""
173
+
174
+ def test_basic_functionality(self):
175
+ """Test that from_map_batched produces correct RecordBatchReader."""
176
+ blocks = list(
177
+ block_slices(self.air_small, chunks={'time': 4, 'lat': 3, 'lon': 4})
178
+ )
179
+
180
+ # Get expected schema
181
+ first_block_df = pivot(self.air_small.isel(blocks[0]))
182
+ expected_schema = pa.Schema.from_pandas(first_block_df)
183
+
184
+ # Create RecordBatchReader
185
+ reader = from_map_batched(
186
+ pivot,
187
+ [self.air_small.isel(block) for block in blocks],
188
+ schema=expected_schema,
189
+ )
190
+
191
+ # Verify it's a RecordBatchReader
192
+ self.assertIsInstance(reader, pa.RecordBatchReader)
193
+
194
+ # Verify schema
195
+ self.assertEqual(reader.schema, expected_schema)
196
+
197
+ # Read all batches and verify content
198
+ batches = list(reader)
199
+ self.assertGreater(len(batches), 0)
200
+
201
+ # Verify each batch has the correct schema
202
+ for batch in batches:
203
+ self.assertEqual(batch.schema, expected_schema)
204
+ self.assertGreater(len(batch), 0)
205
+
206
+ def test_multiple_iterables(self):
207
+ """Test from_map_batched with multiple iterables."""
208
+ x_values = [1, 2, 3, 4, 5]
209
+ y_values = [10, 20, 30, 40, 50]
210
+
211
+ expected_schema = pa.schema(
212
+ [('x', pa.int64()), ('y', pa.int64()), ('sum', pa.int64())]
213
+ )
214
+
215
+ reader = from_map_batched(
216
+ adding_function, x_values, y_values, schema=expected_schema
217
+ )
218
+
219
+ # Read all data
220
+ table = reader.read_all()
221
+ df = table.to_pandas()
222
+
223
+ # Verify results
224
+ expected_df = pd.DataFrame(
225
+ {
226
+ 'x': x_values,
227
+ 'y': y_values,
228
+ 'sum': [x + y for x, y in zip(x_values, y_values)],
229
+ }
230
+ )
231
+
232
+ pd.testing.assert_frame_equal(df, expected_df)
233
+
234
+ def test_with_args_and_kwargs(self):
235
+ """Test from_map_batched with additional args and kwargs."""
236
+
237
+ def multiply_and_add(x, multiplier, offset=0):
238
+ result = pd.DataFrame({'x': [x], 'result': [x * multiplier + offset]})
239
+ return result
240
+
241
+ values = [1, 2, 3]
242
+ expected_schema = pa.schema([('x', pa.int64()), ('result', pa.int64())])
243
+
244
+ reader = from_map_batched(
245
+ multiply_and_add,
246
+ values,
247
+ args=(2,), # multiplier = 2
248
+ offset=5, # offset = 5
249
+ schema=expected_schema,
250
+ )
251
+
252
+ table = reader.read_all()
253
+ df = table.to_pandas()
254
+
255
+ # Verify results: x * 2 + 5
256
+ expected_df = pd.DataFrame(
257
+ {'x': [1, 2, 3], 'result': [7, 9, 11]} # (1*2+5, 2*2+5, 3*2+5)
258
+ )
259
+
260
+ pd.testing.assert_frame_equal(df, expected_df)
261
+
262
+ def test_empty_iterables(self):
263
+ """Test from_map_batched with empty iterables."""
264
+ empty_schema = pa.schema([('value', pa.int64())])
265
+
266
+ reader = from_map_batched(
267
+ lambda x: pd.DataFrame({'value': [x]}), [], schema=empty_schema
268
+ )
269
+
270
+ batches = list(reader)
271
+ self.assertEqual(len(batches), 0)
272
+
273
+ def test_consistency_with_regular_map(self):
274
+ """Test that results are consistent with regular mapping."""
275
+ blocks = list(block_slices(self.air_small, chunks={'time': 4, 'lat': 3}))
276
+ datasets = [self.air_small.isel(block) for block in blocks]
277
+
278
+ # Get schema from first block
279
+ first_df = pivot(datasets[0])
280
+ schema = pa.Schema.from_pandas(first_df)
281
+
282
+ # Use from_map_batched
283
+ reader = from_map_batched(pivot, datasets, schema=schema)
284
+ batched_table = reader.read_all()
285
+
286
+ # Regular map approach
287
+ regular_dfs = [pivot(ds) for ds in datasets]
288
+ regular_table = pa.Table.from_pandas(
289
+ pd.concat(regular_dfs, ignore_index=True)
290
+ )
291
+
292
+ # Results should be identical
293
+ self.assertEqual(batched_table.schema, regular_table.schema)
294
+ self.assertEqual(len(batched_table), len(regular_table))
295
+
296
+ # Compare data (allowing for potential column order differences)
297
+ batched_df = (
298
+ batched_table.to_pandas()
299
+ .sort_values(['time', 'lat', 'lon'])
300
+ .reset_index(drop=True)
301
+ )
302
+ regular_df = (
303
+ regular_table.to_pandas()
304
+ .sort_values(['time', 'lat', 'lon'])
305
+ .reset_index(drop=True)
306
+ )
307
+
308
+ pd.testing.assert_frame_equal(batched_df, regular_df)
309
+
310
+ def test_integration_with_datafusion_via_read_xarray(self):
311
+ """Test integration with the read_xarray function that uses from_map_batched."""
312
+
313
+ # Use a small dataset
314
+ air = xr.tutorial.open_dataset('air_temperature')
315
+ air_small = air.isel(time=slice(0, 50), lat=slice(0, 10), lon=slice(0, 15))
316
+ air_chunked = air_small.chunk({'time': 25, 'lat': 5, 'lon': 8})
317
+
318
+ # read_xarray uses from_map_batched internally
319
+ arrow_stream = read_xarray(
320
+ air_chunked, chunks={'time': 25, 'lat': 5, 'lon': 8}
321
+ )
322
+
323
+ # Verify it returns a proper ArrowStreamExportable (RecordBatchReader)
324
+ self.assertTrue(hasattr(arrow_stream, 'schema'))
325
+ self.assertTrue(hasattr(arrow_stream, '__iter__'))
326
+
327
+ # Should be able to read all data
328
+ table = arrow_stream.read_all()
329
+ self.assertGreater(len(table), 0)
330
+
331
+ # Should have the expected columns
332
+ expected_columns = {'time', 'lat', 'lon', 'air'}
333
+ actual_columns = set(table.column_names)
334
+ self.assertTrue(expected_columns.issubset(actual_columns))
335
+
336
+
337
+ class ReadXarrayStreamingTest(unittest.TestCase):
338
+
339
+ def setUp(self):
340
+ self.large_ds = create_large_dataset().chunk({'time': 25})
341
+
342
+ def test_read_xarray__loads_one_chunk_at_a_time(self):
343
+ tracemalloc.start()
344
+ iterable = read_xarray(self.large_ds)
345
+ first_size, first_peak = tracemalloc.get_traced_memory()
346
+ tracemalloc.reset_peak()
347
+
348
+ sizes, peaks = [], []
349
+
350
+ first_chunk = self.large_ds.isel(next(block_slices(self.large_ds)))
351
+ chunk_size = first_chunk.nbytes
352
+
353
+ # Creating the iterator should be inexpensive -- less than one chunk.
354
+ # We multiply by constant factors because chunks have additional overhead
355
+ self.assertLess(first_size, chunk_size * 3)
356
+ self.assertLess(first_peak, chunk_size * 6)
357
+
358
+ for it in iterable:
359
+ _ = it
360
+ cur_size, cur_peak = tracemalloc.get_traced_memory()
361
+ tracemalloc.reset_peak()
362
+ sizes.append(cur_size), peaks.append(cur_peak)
363
+
364
+ mean_size = np.mean(sizes)
365
+ mean_peak = np.mean(peaks)
366
+
367
+ # Provide a bound for each size and peak
368
+ for size in sizes:
369
+ self.assertGreater(mean_size * 1.1, size)
370
+ self.assertGreater(chunk_size * 3, size)
371
+ # malloc size is about 2.66x chunk_size on average
372
+ self.assertLess(chunk_size * 2, size)
373
+
374
+ for peak in peaks:
375
+ self.assertGreater(mean_peak * 1.1, peak)
376
+ self.assertGreater(chunk_size * 7, peak)
377
+ # malloc peak is about 6.89x chunk_size on average
378
+ self.assertLess(chunk_size * 4, peak)
379
+
380
+ # The peak malloc should never be more than the original dataset!
381
+ self.assertLess(max(peaks), self.large_ds.nbytes)
382
+
383
+ tracemalloc.stop()
384
+
385
+
386
+ if __name__ == '__main__':
387
+ unittest.main()
@@ -23,13 +23,13 @@ class SqlTestCase(DaskTestCase):
23
23
 
24
24
  query = c.sql(
25
25
  """
26
- SELECT
27
- "lat", "lon", SUM("air") as air_total
28
- FROM
29
- "air"
30
- GROUP BY
31
- "lat", "lon"
32
- """
26
+ SELECT
27
+ "lat", "lon", SUM("air") as air_total
28
+ FROM
29
+ "air"
30
+ GROUP BY
31
+ "lat", "lon"
32
+ """
33
33
  )
34
34
 
35
35
  result = query.to_pandas()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.1.0
3
+ Version: 0.1.1
4
4
  Summary: Querry Xarray with SQL.
5
5
  Author-email: Alexander Merose <al@merose.com>
6
6
  License: Apache-2.0
@@ -1,182 +0,0 @@
1
- import itertools
2
- import unittest
3
-
4
- import numpy as np
5
- import pandas as pd
6
- import pyarrow as pa
7
- import xarray as xr
8
-
9
- from .df import explode, read_xarray, block_slices, from_map
10
-
11
-
12
- def rand_wx(start: str, end: str) -> xr.Dataset:
13
- np.random.seed(42)
14
- lat = np.linspace(-90, 90, num=720)
15
- lon = np.linspace(-180, 180, num=1440)
16
- time = pd.date_range(start, end, freq='h')
17
- level = np.array([1000, 500], dtype=np.int32)
18
- reference_time = pd.Timestamp(start)
19
- temperature = 15 + 8 * np.random.randn(720, 1440, len(time), len(level))
20
- precipitation = 10 * np.random.rand(720, 1440, len(time), len(level))
21
- return xr.Dataset(
22
- data_vars=dict(
23
- temperature=(['lat', 'lon', 'time', 'level'], temperature),
24
- precipitation=(['lat', 'lon', 'time', 'level'], precipitation),
25
- ),
26
- coords=dict(
27
- lat=lat,
28
- lon=lon,
29
- time=time,
30
- level=level,
31
- reference_time=reference_time,
32
- ),
33
- attrs=dict(description='Random weather.'),
34
- )
35
-
36
-
37
- class DaskTestCase(unittest.TestCase):
38
-
39
- def setUp(self) -> None:
40
- self.air = xr.tutorial.open_dataset('air_temperature')
41
- self.chunks = {'time': 240}
42
- self.air = self.air.chunk(self.chunks)
43
-
44
- self.air_small = self.air.isel(
45
- time=slice(0, 12), lat=slice(0, 11), lon=slice(0, 10)
46
- ).chunk(self.chunks)
47
- self.randwx = rand_wx('1995-01-13T00', '1995-01-13T01')
48
-
49
-
50
- class ExplodeTest(DaskTestCase):
51
-
52
- def test_cardinality(self):
53
- dss = explode(self.air)
54
- self.assertEqual(
55
- len(list(dss)), np.prod([len(c) for c in self.air.chunks.values()])
56
- )
57
-
58
- def test_dim_sizes__one(self):
59
- ds = next(iter(explode(self.air)))
60
- for k, v in self.chunks.items():
61
- self.assertIn(k, ds.dims)
62
- self.assertEqual(v, ds.sizes[k])
63
-
64
- def skip_test_dim_sizes__all(self):
65
- # TODO(alxmrs): Why is this test slow?
66
- dss = explode(self.air)
67
- self.assertEqual(
68
- [tuple(ds.dims.values()) for ds in dss],
69
- list(itertools.product(*self.air.chunksizes.values())),
70
- )
71
-
72
- def test_data_equal__one__first(self):
73
- ds = next(iter(explode(self.air)))
74
- iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
75
- self.assertEqual(self.air.isel(iselection), ds)
76
-
77
- def test_data_equal__one__last(self):
78
- dss = list(explode(self.air))
79
- ds = dss[-1]
80
- iselection = {dim: slice(0, s) for dim, s in ds.sizes.items()}
81
- self.assertEqual(self.air.isel(iselection), ds)
82
-
83
-
84
- class PyArrowTableTest(DaskTestCase):
85
-
86
- def test_sanity(self):
87
- table = read_xarray(self.air_small)
88
- self.assertIsNotNone(table)
89
- self.assertIsInstance(table, pa.Table)
90
- self.assertEqual(len(table), np.prod(list(self.air_small.sizes.values())))
91
-
92
- def test_columns(self):
93
- table = read_xarray(self.air_small)
94
- cols = table.column_names
95
- self.assertEqual(cols, ['lat', 'time', 'lon', 'air'])
96
-
97
- def test_dtypes(self):
98
- table = read_xarray(self.air_small)
99
- # Convert to pandas to check dtypes
100
- df = table.to_pandas()
101
- types = list(df.dtypes)
102
- self.assertEqual([self.air_small[c].dtype for c in df.columns], types)
103
-
104
- def test_different_chunk_sizes(self):
105
- default_table = read_xarray(self.air_small)
106
- chunked_table = read_xarray(self.air_small, dict(time=5))
107
-
108
- # Both should produce valid tables
109
- self.assertIsInstance(default_table, pa.Table)
110
- self.assertIsInstance(chunked_table, pa.Table)
111
- # Should have same number of rows
112
- self.assertEqual(len(default_table), len(chunked_table))
113
-
114
- def test_chunk_perf(self):
115
- table = read_xarray(self.air, chunks=dict(time=6))
116
- self.assertIsNotNone(table)
117
- self.assertEqual(len(table), np.prod(list(self.air.sizes.values())))
118
-
119
- def test_column_metadata_preserved(self):
120
- try:
121
- table = read_xarray(self.randwx, chunks=dict(time=24))
122
- self.assertIsInstance(table, pa.Table)
123
- except Exception as e:
124
- self.fail(f'Unexpected error: {e}')
125
-
126
-
127
- class FromMapTest(unittest.TestCase):
128
-
129
- def test_basic_from_map(self):
130
- """Test basic from_map functionality with pandas DataFrames."""
131
-
132
- def make_df(x):
133
- return pd.DataFrame({'value': [x, x * 2], 'index': [0, 1]})
134
-
135
- result = from_map(make_df, [1, 2, 3])
136
- self.assertIsInstance(result, pa.Table)
137
- self.assertEqual(len(result), 6) # 3 inputs * 2 rows each
138
- self.assertEqual(result.column_names, ['value', 'index'])
139
-
140
- def test_from_map_with_multiple_iterables(self):
141
- """Test from_map with multiple iterables."""
142
-
143
- def add_values(x, y):
144
- return pd.DataFrame({'sum': [x + y], 'x': [x], 'y': [y]})
145
-
146
- result = from_map(add_values, [1, 2], [10, 20])
147
- self.assertIsInstance(result, pa.Table)
148
- self.assertEqual(len(result), 2)
149
-
150
- # Convert to pandas to check values
151
- df = result.to_pandas()
152
- self.assertEqual(list(df['sum']), [11, 22])
153
-
154
- def test_from_map_with_args(self):
155
- """Test from_map with additional arguments."""
156
-
157
- def multiply_and_add(x, multiplier, add_value):
158
- return pd.DataFrame({'result': [x * multiplier + add_value]})
159
-
160
- result = from_map(multiply_and_add, [1, 2, 3], args=(2, 10))
161
- self.assertIsInstance(result, pa.Table)
162
- self.assertEqual(len(result), 3)
163
-
164
- df = result.to_pandas()
165
- self.assertEqual(
166
- list(df['result']), [12, 14, 16]
167
- ) # (1*2+10, 2*2+10, 3*2+10)
168
-
169
- def test_from_map_with_pyarrow_tables(self):
170
- """Test from_map when function returns PyArrow tables."""
171
-
172
- def make_arrow_table(x):
173
- df = pd.DataFrame({'value': [x]})
174
- return pa.Table.from_pandas(df)
175
-
176
- result = from_map(make_arrow_table, [1, 2, 3])
177
- self.assertIsInstance(result, pa.Table)
178
- self.assertEqual(len(result), 3)
179
-
180
-
181
- if __name__ == '__main__':
182
- unittest.main()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes