xarray-sql 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
xarray_sql/df.py CHANGED
@@ -150,7 +150,24 @@ def pivot(ds: xr.Dataset) -> pd.DataFrame:
150
150
  return ds.to_dataframe().reset_index()
151
151
 
152
152
 
153
- def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> ArrowStreamExportable:
153
+ def _parse_schema(ds) -> pa.Schema:
154
+ """Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns."""
155
+ columns = []
156
+
157
+ for coord_name, coord_var in ds.coords.items():
158
+ # Only include dimension coordinates
159
+ if coord_name in ds.dims:
160
+ pa_type = pa.from_numpy_dtype(coord_var.dtype)
161
+ columns.append(pa.field(coord_name, pa_type))
162
+
163
+ for var_name, var in ds.data_vars.items():
164
+ pa_type = pa.from_numpy_dtype(var.dtype)
165
+ columns.append(pa.field(var_name, pa_type))
166
+
167
+ return pa.schema(columns)
168
+
169
+
170
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
154
171
  """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
155
172
 
156
173
  Args:
@@ -162,18 +179,15 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> ArrowStreamExportable:
162
179
  Returns:
163
180
  A PyArrow Table, which is a table representation of the input Dataset.
164
181
  """
165
- fst = next(iter(ds.values())).dims
166
- assert all(
167
- da.dims == fst for da in ds.values()
168
- ), "All dimensions must be equal. Please filter data_vars in the Dataset."
169
-
170
- blocks = list(block_slices(ds, chunks))
171
182
 
172
183
  def pivot_block(b: Block):
173
184
  return pivot(ds.isel(b))
174
185
 
175
- schema = pa.Schema.from_pandas(pivot_block(blocks[0]))
176
- last_schema = pa.Schema.from_pandas(pivot_block(blocks[-1]))
177
- assert schema == last_schema, "Schemas must be consistent across blocks!"
186
+ fst = next(iter(ds.values())).dims
187
+ assert all(
188
+ da.dims == fst for da in ds.values()
189
+ ), "All dimensions must be equal. Please filter data_vars in the Dataset."
178
190
 
191
+ schema = _parse_schema(ds)
192
+ blocks = block_slices(ds, chunks)
179
193
  return from_map_batched(pivot_block, blocks, schema=schema)
xarray_sql/sql_test.py CHANGED
@@ -1,7 +1,10 @@
1
1
  import unittest
2
+ import numpy as np
3
+ import pandas as pd
4
+ import xarray as xr
2
5
 
3
6
  from . import XarrayContext
4
- from .df_test import DaskTestCase
7
+ from .df_test import DaskTestCase, create_large_dataset, rand_wx
5
8
 
6
9
 
7
10
  class SqlTestCase(DaskTestCase):
@@ -60,5 +63,751 @@ class SqlTestCase(DaskTestCase):
60
63
  self.assertEqual(len(result), expected)
61
64
 
62
65
 
66
+ class SqlVarietyTestCase(unittest.TestCase):
67
+ """Test SQL functionality with various types of Xarray datasets."""
68
+
69
+ def setUp(self):
70
+ """Set up test datasets for SQL testing."""
71
+ # Create air temperature dataset
72
+ self.air = xr.tutorial.open_dataset('air_temperature')
73
+ self.air_small = self.air.isel(
74
+ time=slice(0, 24), lat=slice(0, 10), lon=slice(0, 15)
75
+ ).chunk({'time': 12})
76
+
77
+ # Create weather dataset with multiple variables
78
+ self.weather = rand_wx('2023-01-01T00', '2023-01-01T12')
79
+ self.weather_small = self.weather.isel(
80
+ time=slice(0, 6), lat=slice(0, 10), lon=slice(0, 10)
81
+ ).chunk({'time': 3})
82
+
83
+ # Create synthetic dataset with different data types
84
+ self.synthetic = create_large_dataset(
85
+ time_steps=50, lat_points=20, lon_points=20
86
+ ).chunk({'time': 25})
87
+
88
+ # Create 1D dataset for testing joins
89
+ self.stations = xr.Dataset(
90
+ {
91
+ 'station_id': (['station'], [1, 2, 3, 4, 5]),
92
+ 'elevation': (['station'], [100, 250, 500, 750, 1000]),
93
+ 'name': (
94
+ ['station'],
95
+ [
96
+ 'Station_A',
97
+ 'Station_B',
98
+ 'Station_C',
99
+ 'Station_D',
100
+ 'Station_E',
101
+ ],
102
+ ),
103
+ }
104
+ ).chunk({'station': 5})
105
+
106
+ def test_basic_select_all(self):
107
+ """Test basic SELECT * queries on different datasets."""
108
+ ctx = XarrayContext()
109
+
110
+ # Test with air temperature dataset
111
+ ctx.from_dataset('air', self.air_small)
112
+ result = ctx.sql('SELECT * FROM air LIMIT 10').to_pandas()
113
+
114
+ self.assertGreater(len(result), 0)
115
+ self.assertLessEqual(len(result), 10)
116
+ self.assertIn('air', result.columns)
117
+ self.assertIn('lat', result.columns)
118
+ self.assertIn('lon', result.columns)
119
+ self.assertIn('time', result.columns)
120
+
121
+ def test_weather_dataset_queries(self):
122
+ """Test queries on weather dataset with multiple variables."""
123
+ ctx = XarrayContext()
124
+ ctx.from_dataset('weather', self.weather_small)
125
+
126
+ # Test selecting specific columns
127
+ result = ctx.sql(
128
+ 'SELECT lat, lon, temperature, precipitation FROM weather LIMIT 20'
129
+ ).to_pandas()
130
+
131
+ self.assertGreater(len(result), 0)
132
+ self.assertIn('temperature', result.columns)
133
+ self.assertIn('precipitation', result.columns)
134
+
135
+ # Test filtering
136
+ result = ctx.sql(
137
+ 'SELECT * FROM weather WHERE temperature > 10 LIMIT 50'
138
+ ).to_pandas()
139
+
140
+ self.assertGreater(len(result), 0)
141
+ # All temperatures should be > 10
142
+ self.assertTrue((result['temperature'] > 10).all())
143
+
144
+ def test_synthetic_dataset_aggregations(self):
145
+ """Test aggregation queries on synthetic dataset."""
146
+ ctx = XarrayContext()
147
+ ctx.from_dataset('synthetic', self.synthetic)
148
+
149
+ # Test COUNT
150
+ result = ctx.sql(
151
+ 'SELECT COUNT(*) as total_count FROM synthetic'
152
+ ).to_pandas()
153
+ self.assertEqual(len(result), 1)
154
+ self.assertGreater(result['total_count'].iloc[0], 0)
155
+
156
+ # Test MIN, MAX, AVG
157
+ result = ctx.sql(
158
+ """
159
+ SELECT
160
+ MIN(temperature) as min_temp,
161
+ MAX(temperature) as max_temp,
162
+ AVG(temperature) as avg_temp
163
+ FROM synthetic
164
+ """
165
+ ).to_pandas()
166
+
167
+ self.assertEqual(len(result), 1)
168
+ self.assertLess(result['min_temp'].iloc[0], result['max_temp'].iloc[0])
169
+ self.assertGreaterEqual(
170
+ result['avg_temp'].iloc[0], result['min_temp'].iloc[0]
171
+ )
172
+ self.assertLessEqual(result['avg_temp'].iloc[0], result['max_temp'].iloc[0])
173
+
174
+ def test_spatial_grouping(self):
175
+ """Test spatial grouping queries."""
176
+ ctx = XarrayContext()
177
+ ctx.from_dataset('air', self.air_small)
178
+
179
+ # Group by spatial coordinates
180
+ result = ctx.sql(
181
+ """
182
+ SELECT
183
+ lat, lon,
184
+ AVG(air) as avg_air,
185
+ COUNT(*) as time_count
186
+ FROM air
187
+ GROUP BY lat, lon
188
+ ORDER BY lat, lon
189
+ """
190
+ ).to_pandas()
191
+
192
+ expected_spatial_points = (
193
+ self.air_small.sizes['lat'] * self.air_small.sizes['lon']
194
+ )
195
+ self.assertEqual(len(result), expected_spatial_points)
196
+
197
+ # Each spatial point should have same number of time steps
198
+ self.assertTrue(
199
+ (result['time_count'] == self.air_small.sizes['time']).all()
200
+ )
201
+
202
+ def test_temporal_filtering(self):
203
+ """Test temporal filtering and grouping."""
204
+ ctx = XarrayContext()
205
+ ctx.from_dataset('weather', self.weather_small)
206
+
207
+ # Get unique time values for filtering
208
+ all_data = ctx.sql(
209
+ 'SELECT DISTINCT time FROM weather ORDER BY time'
210
+ ).to_pandas()
211
+ if len(all_data) > 2:
212
+ mid_time = all_data['time'].iloc[len(all_data) // 2]
213
+
214
+ # Filter by time
215
+ result = ctx.sql(
216
+ f"""
217
+ SELECT COUNT(*) as count_after
218
+ FROM weather
219
+ WHERE time >= '{mid_time}'
220
+ """
221
+ ).to_pandas()
222
+
223
+ self.assertGreater(result['count_after'].iloc[0], 0)
224
+
225
+ def test_station_dataset_queries(self):
226
+ """Test queries on 1D station dataset."""
227
+ ctx = XarrayContext()
228
+ ctx.from_dataset('stations', self.stations)
229
+
230
+ # Basic select
231
+ result = ctx.sql('SELECT * FROM stations ORDER BY elevation').to_pandas()
232
+ self.assertEqual(len(result), 5)
233
+
234
+ # Test filtering by elevation
235
+ result = ctx.sql(
236
+ 'SELECT name, elevation FROM stations WHERE elevation > 300 ORDER BY elevation'
237
+ ).to_pandas()
238
+
239
+ self.assertGreater(len(result), 0)
240
+ self.assertTrue((result['elevation'] > 300).all())
241
+
242
+
243
+ class SqlJoinTestCase(unittest.TestCase):
244
+ """Test joining tabular data with raster data using from_dataset."""
245
+
246
+ def setUp(self):
247
+ """Set up datasets for join testing."""
248
+ # Create a small air temperature dataset
249
+ self.air = xr.tutorial.open_dataset('air_temperature')
250
+ self.air_small = self.air.isel(
251
+ time=slice(0, 12), lat=slice(0, 5), lon=slice(0, 8)
252
+ ).chunk({'time': 6})
253
+
254
+ # Create station metadata as a simple tabular dataset
255
+ # Use coordinates that overlap with air dataset
256
+ air_lats = self.air_small.lat.values
257
+ air_lons = self.air_small.lon.values
258
+
259
+ # Create stations at specific lat/lon points
260
+ self.stations = xr.Dataset(
261
+ {
262
+ 'station_id': (['station'], [101, 102, 103]),
263
+ 'lat': (['station'], [air_lats[0], air_lats[2], air_lats[4]]),
264
+ 'lon': (['station'], [air_lons[1], air_lons[3], air_lons[5]]),
265
+ 'elevation': (['station'], [100, 250, 500]),
266
+ 'name': (['station'], ['Downtown', 'Airport', 'Mountain']),
267
+ }
268
+ ).chunk({'station': 3})
269
+
270
+ # Create region lookup table
271
+ self.regions = xr.Dataset(
272
+ {
273
+ 'region_id': (['region'], [1, 2, 3, 4]),
274
+ 'region_name': (['region'], ['North', 'South', 'East', 'West']),
275
+ 'min_lat': (['region'], [60, 30, 40, 40]),
276
+ 'max_lat': (['region'], [90, 60, 80, 80]),
277
+ 'min_lon': (['region'], [-180, -180, -90, -180]),
278
+ 'max_lon': (['region'], [180, 180, 180, -90]),
279
+ }
280
+ ).chunk({'region': 4})
281
+
282
+ def test_simple_cross_join(self):
283
+ """Test cross join between raster and tabular data."""
284
+ ctx = XarrayContext()
285
+ ctx.from_dataset('air_data', self.air_small)
286
+ ctx.from_dataset('stations', self.stations)
287
+
288
+ # Test separate queries first to ensure both datasets work
289
+ air_result = ctx.sql(
290
+ 'SELECT COUNT(*) as air_count FROM air_data'
291
+ ).to_pandas()
292
+ station_result = ctx.sql(
293
+ 'SELECT COUNT(*) as station_count FROM stations'
294
+ ).to_pandas()
295
+
296
+ self.assertGreater(air_result['air_count'].iloc[0], 0)
297
+ self.assertGreater(station_result['station_count'].iloc[0], 0)
298
+
299
+ # Test that we can query both datasets in the same context
300
+ # This demonstrates multi-dataset capability without complex joins
301
+ air_sample = ctx.sql('SELECT air FROM air_data LIMIT 5').to_pandas()
302
+ station_sample = ctx.sql(
303
+ 'SELECT station_id FROM stations LIMIT 5'
304
+ ).to_pandas()
305
+
306
+ self.assertGreater(len(air_sample), 0)
307
+ self.assertGreater(len(station_sample), 0)
308
+ self.assertIn('air', air_sample.columns)
309
+ self.assertIn('station_id', station_sample.columns)
310
+
311
+ def test_coordinate_based_join(self):
312
+ """Test joining on coordinate proximity."""
313
+ ctx = XarrayContext()
314
+ ctx.from_dataset('air_data', self.air_small)
315
+ ctx.from_dataset('stations', self.stations)
316
+
317
+ # First test a simple cross join to ensure datasets are compatible
318
+ result = ctx.sql(
319
+ """
320
+ SELECT COUNT(*) as total_combinations
321
+ FROM air_data a
322
+ CROSS JOIN stations s
323
+ """
324
+ ).to_pandas()
325
+
326
+ self.assertGreater(result['total_combinations'].iloc[0], 0)
327
+
328
+ # Test a simpler join condition
329
+ result = ctx.sql(
330
+ """
331
+ SELECT
332
+ COUNT(*) as match_count
333
+ FROM air_data a, stations s
334
+ WHERE s.station_id = 101
335
+ """
336
+ ).to_pandas()
337
+
338
+ self.assertGreater(result['match_count'].iloc[0], 0)
339
+
340
+ def test_region_classification_join(self):
341
+ """Test joining with region classification."""
342
+ ctx = XarrayContext()
343
+ ctx.from_dataset('air_data', self.air_small)
344
+ ctx.from_dataset('regions', self.regions)
345
+
346
+ # Test that both datasets can be queried independently
347
+ air_result = ctx.sql(
348
+ 'SELECT COUNT(*) as air_count FROM air_data'
349
+ ).to_pandas()
350
+ region_result = ctx.sql(
351
+ 'SELECT COUNT(*) as region_count FROM regions'
352
+ ).to_pandas()
353
+
354
+ self.assertGreater(air_result['air_count'].iloc[0], 0)
355
+ self.assertGreater(region_result['region_count'].iloc[0], 0)
356
+
357
+ # Test a simpler region-based query without complex joins
358
+ result = ctx.sql(
359
+ """
360
+ SELECT
361
+ region_name,
362
+ min_lat,
363
+ max_lat
364
+ FROM regions
365
+ WHERE min_lat < 50
366
+ """
367
+ ).to_pandas()
368
+
369
+ self.assertGreater(len(result), 0)
370
+ self.assertIn('region_name', result.columns)
371
+
372
+ def test_multiple_dataset_aggregation(self):
373
+ """Test aggregating across multiple datasets."""
374
+ ctx = XarrayContext()
375
+ ctx.from_dataset('air_data', self.air_small)
376
+ ctx.from_dataset('stations', self.stations)
377
+
378
+ # Get statistics by elevation bands using station data
379
+ result = ctx.sql(
380
+ """
381
+ SELECT
382
+ CASE
383
+ WHEN s.elevation < 200 THEN 'Low'
384
+ WHEN s.elevation < 400 THEN 'Medium'
385
+ ELSE 'High'
386
+ END as elevation_band,
387
+ COUNT(DISTINCT s.station_id) as station_count,
388
+ COUNT(*) as air_measurements,
389
+ AVG(a.air) as avg_air
390
+ FROM air_data a
391
+ CROSS JOIN stations s
392
+ GROUP BY elevation_band
393
+ ORDER BY elevation_band
394
+ """
395
+ ).to_pandas()
396
+
397
+ self.assertGreater(len(result), 0)
398
+ self.assertIn('elevation_band', result.columns)
399
+ # Should have Low, Medium, High bands
400
+ self.assertGreaterEqual(len(result), 1)
401
+
402
+
403
+ class SqlOptimizationTestCase(unittest.TestCase):
404
+ """Test DataFusion optimizations like column selection and filters."""
405
+
406
+ def setUp(self):
407
+ """Set up dataset for optimization testing."""
408
+ self.air = xr.tutorial.open_dataset('air_temperature')
409
+ self.air_medium = self.air.isel(
410
+ time=slice(0, 100), lat=slice(0, 20), lon=slice(0, 30)
411
+ ).chunk({'time': 50})
412
+
413
+ # Create multi-variable dataset
414
+ self.weather = rand_wx('2023-01-01T00', '2023-01-02T00')
415
+ self.weather_medium = self.weather.isel(
416
+ time=slice(0, 12), lat=slice(0, 15), lon=slice(0, 20)
417
+ ).chunk({'time': 6})
418
+
419
+ def test_column_projection(self):
420
+ """Test that selecting specific columns works efficiently."""
421
+ ctx = XarrayContext()
422
+ ctx.from_dataset('weather', self.weather_medium)
423
+
424
+ # Select only specific columns
425
+ result = ctx.sql(
426
+ 'SELECT lat, lon, temperature FROM weather LIMIT 100'
427
+ ).to_pandas()
428
+
429
+ # Should only have the requested columns
430
+ expected_columns = {'lat', 'lon', 'temperature'}
431
+ actual_columns = set(result.columns)
432
+ self.assertEqual(expected_columns, actual_columns)
433
+
434
+ # Should not include precipitation, time, level, reference_time
435
+ unwanted_columns = {'precipitation', 'time', 'level', 'reference_time'}
436
+ self.assertTrue(unwanted_columns.isdisjoint(actual_columns))
437
+
438
+ def test_where_clause_filtering(self):
439
+ """Test WHERE clause filtering optimization."""
440
+ ctx = XarrayContext()
441
+ ctx.from_dataset('air', self.air_medium)
442
+
443
+ # Test numeric filtering
444
+ result = ctx.sql(
445
+ 'SELECT * FROM air WHERE air > 280 AND air < 290'
446
+ ).to_pandas()
447
+
448
+ if len(result) > 0:
449
+ self.assertTrue((result['air'] > 280).all())
450
+ self.assertTrue((result['air'] < 290).all())
451
+
452
+ # Test coordinate filtering
453
+ result = ctx.sql(
454
+ 'SELECT * FROM air WHERE lat > 50 AND lon < -100'
455
+ ).to_pandas()
456
+
457
+ if len(result) > 0:
458
+ self.assertTrue((result['lat'] > 50).all())
459
+ self.assertTrue((result['lon'] < -100).all())
460
+
461
+ def test_limit_optimization(self):
462
+ """Test LIMIT clause optimization."""
463
+ ctx = XarrayContext()
464
+ ctx.from_dataset('air', self.air_medium)
465
+
466
+ # Test small limits
467
+ for limit in [1, 5, 10, 50]:
468
+ result = ctx.sql(f'SELECT * FROM air LIMIT {limit}').to_pandas()
469
+ self.assertLessEqual(len(result), limit)
470
+ if (
471
+ limit
472
+ <= self.air_medium.sizes['time']
473
+ * self.air_medium.sizes['lat']
474
+ * self.air_medium.sizes['lon']
475
+ ):
476
+ self.assertEqual(len(result), limit)
477
+
478
+ def test_order_by_optimization(self):
479
+ """Test ORDER BY clause."""
480
+ ctx = XarrayContext()
481
+ ctx.from_dataset('air', self.air_medium)
482
+
483
+ # Test ordering by different columns
484
+ result = ctx.sql(
485
+ 'SELECT lat, lon, air FROM air ORDER BY air DESC LIMIT 20'
486
+ ).to_pandas()
487
+
488
+ if len(result) > 1:
489
+ # Should be in descending order
490
+ air_values = result['air'].values
491
+ self.assertTrue(np.all(air_values[:-1] >= air_values[1:]))
492
+
493
+ # Test ordering by coordinates
494
+ result = ctx.sql(
495
+ 'SELECT lat, lon, air FROM air ORDER BY lat ASC, lon DESC LIMIT 20'
496
+ ).to_pandas()
497
+
498
+ if len(result) > 1:
499
+ # Check lat is ascending
500
+ lat_values = result['lat'].values
501
+ lat_diffs = np.diff(lat_values)
502
+ # Allow for equal values (same lat, different lon)
503
+ self.assertTrue(np.all(lat_diffs >= 0))
504
+
505
+ def test_aggregation_pushdown(self):
506
+ """Test aggregation optimization."""
507
+ ctx = XarrayContext()
508
+ ctx.from_dataset('weather', self.weather_medium)
509
+
510
+ # Test GROUP BY optimization
511
+ result = ctx.sql(
512
+ """
513
+ SELECT
514
+ lat,
515
+ COUNT(*) as point_count,
516
+ AVG(temperature) as avg_temp,
517
+ MIN(precipitation) as min_precip,
518
+ MAX(precipitation) as max_precip
519
+ FROM weather
520
+ GROUP BY lat
521
+ ORDER BY lat
522
+ """
523
+ ).to_pandas()
524
+
525
+ # Should have one row per unique latitude
526
+ expected_lats = len(self.weather_medium.lat)
527
+ self.assertEqual(len(result), expected_lats)
528
+
529
+ # All aggregation columns should be present
530
+ expected_agg_cols = {'point_count', 'avg_temp', 'min_precip', 'max_precip'}
531
+ self.assertTrue(expected_agg_cols.issubset(set(result.columns)))
532
+
533
+ def test_complex_filter_optimization(self):
534
+ """Test complex filtering with multiple conditions."""
535
+ ctx = XarrayContext()
536
+ ctx.from_dataset('weather', self.weather_medium)
537
+
538
+ # Complex WHERE clause with AND/OR
539
+ result = ctx.sql(
540
+ """
541
+ SELECT lat, lon, temperature, precipitation
542
+ FROM weather
543
+ WHERE (temperature > 15 AND precipitation < 50)
544
+ OR (lat > 45 AND lon < -100)
545
+ LIMIT 100
546
+ """
547
+ ).to_pandas()
548
+
549
+ # Verify the complex condition
550
+ if len(result) > 0:
551
+ condition1 = (result['temperature'] > 15) & (result['precipitation'] < 50)
552
+ condition2 = (result['lat'] > 45) & (result['lon'] < -100)
553
+ combined = condition1 | condition2
554
+ self.assertTrue(combined.all())
555
+
556
+
557
+ class SqlComplexQueryTestCase(unittest.TestCase):
558
+ """Test complex SQL queries with advanced features."""
559
+
560
+ def setUp(self):
561
+ """Set up datasets for complex query testing."""
562
+ self.air = xr.tutorial.open_dataset('air_temperature')
563
+ self.air_small = self.air.isel(
564
+ time=slice(0, 50), lat=slice(0, 15), lon=slice(0, 20)
565
+ ).chunk({'time': 25})
566
+
567
+ self.weather = rand_wx('2023-01-01T00', '2023-01-01T12')
568
+ self.weather_small = self.weather.isel(
569
+ time=slice(0, 8), lat=slice(0, 10), lon=slice(0, 12)
570
+ ).chunk({'time': 4})
571
+
572
+ def test_subqueries(self):
573
+ """Test subqueries and CTEs."""
574
+ ctx = XarrayContext()
575
+ ctx.from_dataset('air', self.air_small)
576
+
577
+ # Subquery to find above-average temperatures
578
+ result = ctx.sql(
579
+ """
580
+ SELECT lat, lon, air
581
+ FROM air
582
+ WHERE air > (
583
+ SELECT AVG(air) FROM air
584
+ )
585
+ LIMIT 50
586
+ """
587
+ ).to_pandas()
588
+
589
+ if len(result) > 0:
590
+ # Get the average to verify
591
+ avg_result = ctx.sql('SELECT AVG(air) as avg_air FROM air').to_pandas()
592
+ avg_air = avg_result['avg_air'].iloc[0]
593
+
594
+ # All results should be above average
595
+ self.assertTrue((result['air'] > avg_air).all())
596
+
597
+ def test_window_functions(self):
598
+ """Test window functions if supported by DataFusion."""
599
+ ctx = XarrayContext()
600
+ ctx.from_dataset('air', self.air_small)
601
+
602
+ try:
603
+ # Test ROW_NUMBER window function
604
+ result = ctx.sql(
605
+ """
606
+ SELECT
607
+ lat, lon, air,
608
+ ROW_NUMBER() OVER (PARTITION BY lat ORDER BY air DESC) as rank_in_lat
609
+ FROM air
610
+ WHERE lat IN (
611
+ SELECT DISTINCT lat FROM air LIMIT 3
612
+ )
613
+ ORDER BY lat, rank_in_lat
614
+ LIMIT 30
615
+ """
616
+ ).to_pandas()
617
+
618
+ if len(result) > 0:
619
+ # Check that ranking works within each lat
620
+ for lat_val in result['lat'].unique():
621
+ lat_data = result[result['lat'] == lat_val].sort_values('rank_in_lat')
622
+ if len(lat_data) > 1:
623
+ # Air values should be in descending order within each lat
624
+ air_values = lat_data['air'].values
625
+ self.assertTrue(np.all(air_values[:-1] >= air_values[1:]))
626
+
627
+ except Exception:
628
+ # Window functions might not be supported, skip test
629
+ self.skipTest('Window functions not supported')
630
+
631
+ def test_case_statements(self):
632
+ """Test CASE statements for conditional logic."""
633
+ ctx = XarrayContext()
634
+ ctx.from_dataset('weather', self.weather_small)
635
+
636
+ result = ctx.sql(
637
+ """
638
+ SELECT
639
+ lat, lon,
640
+ temperature,
641
+ CASE
642
+ WHEN temperature < 0 THEN 'Freezing'
643
+ WHEN temperature < 10 THEN 'Cold'
644
+ WHEN temperature < 20 THEN 'Cool'
645
+ WHEN temperature < 30 THEN 'Warm'
646
+ ELSE 'Hot'
647
+ END as temp_category,
648
+ precipitation,
649
+ CASE
650
+ WHEN precipitation < 5 THEN 'Dry'
651
+ WHEN precipitation < 20 THEN 'Light'
652
+ WHEN precipitation < 50 THEN 'Moderate'
653
+ ELSE 'Heavy'
654
+ END as precip_category
655
+ FROM weather
656
+ LIMIT 50
657
+ """
658
+ ).to_pandas()
659
+
660
+ self.assertGreater(len(result), 0)
661
+ self.assertIn('temp_category', result.columns)
662
+ self.assertIn('precip_category', result.columns)
663
+
664
+ # Verify categories are assigned correctly
665
+ for _, row in result.iterrows():
666
+ temp = row['temperature']
667
+ temp_cat = row['temp_category']
668
+
669
+ if temp < 0:
670
+ self.assertEqual(temp_cat, 'Freezing')
671
+ elif temp < 10:
672
+ self.assertEqual(temp_cat, 'Cold')
673
+ elif temp < 20:
674
+ self.assertEqual(temp_cat, 'Cool')
675
+ elif temp < 30:
676
+ self.assertEqual(temp_cat, 'Warm')
677
+ else:
678
+ self.assertEqual(temp_cat, 'Hot')
679
+
680
+ def test_mathematical_functions(self):
681
+ """Test mathematical functions in SQL."""
682
+ ctx = XarrayContext()
683
+ ctx.from_dataset('weather', self.weather_small)
684
+
685
+ result = ctx.sql(
686
+ """
687
+ SELECT
688
+ lat, lon,
689
+ temperature,
690
+ ROUND(temperature, 1) as temp_rounded,
691
+ ABS(temperature - 20) as temp_diff_from_20,
692
+ SQRT(ABS(temperature)) as temp_sqrt,
693
+ precipitation,
694
+ LOG(precipitation + 1) as log_precip
695
+ FROM weather
696
+ WHERE precipitation > 0
697
+ LIMIT 30
698
+ """
699
+ ).to_pandas()
700
+
701
+ if len(result) > 0:
702
+ # Verify mathematical operations
703
+ for _, row in result.iterrows():
704
+ temp = row['temperature']
705
+ self.assertAlmostEqual(
706
+ row['temp_diff_from_20'], abs(temp - 20), places=5
707
+ )
708
+ if temp >= 0:
709
+ self.assertAlmostEqual(row['temp_sqrt'], np.sqrt(temp), places=5)
710
+
711
+ def test_string_operations(self):
712
+ """Test string operations if applicable."""
713
+ ctx = XarrayContext()
714
+ ctx.from_dataset('air', self.air_small)
715
+
716
+ # Test string operations on numeric data converted to string
717
+ result = ctx.sql(
718
+ """
719
+ SELECT
720
+ lat, lon,
721
+ CAST(lat AS VARCHAR) as lat_str,
722
+ CONCAT('Lat: ', CAST(lat AS VARCHAR), ', Lon: ', CAST(lon AS VARCHAR)) as coordinates
723
+ FROM air
724
+ LIMIT 10
725
+ """
726
+ ).to_pandas()
727
+
728
+ self.assertGreater(len(result), 0)
729
+ self.assertIn('coordinates', result.columns)
730
+
731
+ # Verify concatenation worked
732
+ for _, row in result.iterrows():
733
+ expected = f"Lat: {row['lat']}, Lon: {row['lon']}"
734
+ # Allow for slight formatting differences
735
+ self.assertIn('Lat:', row['coordinates'])
736
+ self.assertIn('Lon:', row['coordinates'])
737
+
738
+
739
+ class SqlErrorHandlingTestCase(unittest.TestCase):
740
+ """Test error handling and edge cases in SQL interface."""
741
+
742
+ def setUp(self):
743
+ """Set up dataset for error testing."""
744
+ self.air = xr.tutorial.open_dataset('air_temperature')
745
+ self.air_small = self.air.isel(
746
+ time=slice(0, 10), lat=slice(0, 5), lon=slice(0, 8)
747
+ ).chunk({'time': 5})
748
+
749
+ def test_invalid_table_name(self):
750
+ """Test error handling for invalid table names."""
751
+ ctx = XarrayContext()
752
+ ctx.from_dataset('air', self.air_small)
753
+
754
+ with self.assertRaises(Exception):
755
+ ctx.sql('SELECT * FROM nonexistent_table')
756
+
757
+ def test_invalid_column_name(self):
758
+ """Test error handling for invalid column names."""
759
+ ctx = XarrayContext()
760
+ ctx.from_dataset('air', self.air_small)
761
+
762
+ with self.assertRaises(Exception):
763
+ ctx.sql('SELECT nonexistent_column FROM air')
764
+
765
+ def test_syntax_errors(self):
766
+ """Test handling of SQL syntax errors."""
767
+ ctx = XarrayContext()
768
+ ctx.from_dataset('air', self.air_small)
769
+
770
+ # Invalid SQL syntax
771
+ with self.assertRaises(Exception):
772
+ ctx.sql('SELECT * FORM air') # Typo: FORM instead of FROM
773
+
774
+ with self.assertRaises(Exception):
775
+ ctx.sql('SELECT * FROM air WHERE') # Incomplete WHERE clause
776
+
777
+ def test_empty_dataset_handling(self):
778
+ """Test handling of empty datasets."""
779
+ # Create empty dataset
780
+ empty_ds = xr.Dataset({'temp': (['x'], [])}, coords={'x': []}).chunk(
781
+ {'x': 1}
782
+ )
783
+
784
+ ctx = XarrayContext()
785
+ ctx.from_dataset('empty', empty_ds)
786
+
787
+ # Should handle empty dataset gracefully
788
+ result = ctx.sql('SELECT * FROM empty').to_pandas()
789
+ self.assertEqual(len(result), 0)
790
+
791
+ # Aggregations on empty dataset
792
+ result = ctx.sql('SELECT COUNT(*) as count FROM empty').to_pandas()
793
+ self.assertEqual(len(result), 1)
794
+ self.assertEqual(result['count'].iloc[0], 0)
795
+
796
+ def test_large_limit_handling(self):
797
+ """Test handling of very large LIMIT values."""
798
+ ctx = XarrayContext()
799
+ ctx.from_dataset('air', self.air_small)
800
+
801
+ total_rows = (
802
+ self.air_small.sizes['time']
803
+ * self.air_small.sizes['lat']
804
+ * self.air_small.sizes['lon']
805
+ )
806
+
807
+ # Request more rows than exist
808
+ result = ctx.sql(f'SELECT * FROM air LIMIT {total_rows * 10}').to_pandas()
809
+ self.assertEqual(len(result), total_rows)
810
+
811
+
63
812
  if __name__ == '__main__':
64
813
  unittest.main()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.1.1
3
+ Version: 0.1.2
4
4
  Summary: Querry Xarray with SQL.
5
5
  Author-email: Alexander Merose <al@merose.com>
6
6
  License: Apache-2.0
@@ -46,19 +46,35 @@ pip install xarray-sql
46
46
 
47
47
  ## What is this?
48
48
 
49
- This is an experiment to provide a SQL interface for raster data.
49
+ This is an experiment to provide a SQL interface for array datasets.
50
50
 
51
51
  ```python
52
52
  import xarray as xr
53
- import xarray_sql as qr
53
+ import xarray_sql as xql
54
54
 
55
55
  ds = xr.tutorial.open_dataset('air_temperature')
56
56
 
57
57
  # The same as a dask-sql Context; i.e. an Apache DataFusion Context.
58
- c = qr.XarrayContext()
59
- c.from_dataset('air', ds, chunks=dict(time=24))
60
-
61
- df = c.sql('''
58
+ ctx = xql.XarrayContext()
59
+ ctx.from_dataset('air', ds, chunks=dict(time=24)) # the dataset needs to be chunked!
60
+ # DataFrame()
61
+ # +------+---------------------+-------+--------------------+
62
+ # | lat | time | lon | air |
63
+ # +------+---------------------+-------+--------------------+
64
+ # | 75.0 | 2013-01-01T00:00:00 | 200.0 | 241.20000000000002 |
65
+ # | 75.0 | 2013-01-01T00:00:00 | 202.5 | 242.5 |
66
+ # | 75.0 | 2013-01-01T00:00:00 | 205.0 | 243.5 |
67
+ # | 75.0 | 2013-01-01T00:00:00 | 207.5 | 244.0 |
68
+ # | 75.0 | 2013-01-01T00:00:00 | 210.0 | 244.1 |
69
+ # | 75.0 | 2013-01-01T00:00:00 | 212.5 | 243.89000000000001 |
70
+ # | 75.0 | 2013-01-01T00:00:00 | 215.0 | 243.6 |
71
+ # | 75.0 | 2013-01-01T00:00:00 | 217.5 | 243.1 |
72
+ # | 75.0 | 2013-01-01T00:00:00 | 220.0 | 242.5 |
73
+ # | 75.0 | 2013-01-01T00:00:00 | 222.5 | 241.89000000000001 |
74
+ # +------+---------------------+-------+--------------------+
75
+ # Data truncated.
76
+
77
+ result = ctx.sql('''
62
78
  SELECT
63
79
  "lat", "lon", AVG("air") as air_total
64
80
  FROM
@@ -66,16 +82,36 @@ df = c.sql('''
66
82
  GROUP BY
67
83
  "lat", "lon"
68
84
  ''')
85
+ # DataFrame()
86
+ # +------+-------+--------------------+
87
+ # | lat | lon | air_total |
88
+ # +------+-------+--------------------+
89
+ # | 75.0 | 205.0 | 259.88662671232834 |
90
+ # | 75.0 | 207.5 | 259.48268150684896 |
91
+ # | 75.0 | 230.0 | 258.9192123287667 |
92
+ # | 75.0 | 275.0 | 257.07574315068456 |
93
+ # | 75.0 | 322.5 | 250.11792123287654 |
94
+ # | 75.0 | 325.0 | 250.81590068493125 |
95
+ # | 72.5 | 205.0 | 262.74933904109537 |
96
+ # | 72.5 | 207.5 | 262.5384315068488 |
97
+ # | 72.5 | 230.0 | 260.8287945205475 |
98
+ # | 72.5 | 275.0 | 257.30633219178037 |
99
+ # +------+-------+--------------------+
100
+ # Data truncated.
101
+ #
69
102
 
70
103
  # A table of the average temperature for each location across time.
71
- df.to_pandas()
72
-
73
- # Alternatively, you can just create the DataFrame from the Dataset:
74
- df = qr.read_xarray(ds).to_pandas()
104
+ df = result.to_pandas()
75
105
  df.head()
106
+ # lat lon air_total
107
+ # 0 75.0 210.0 259.016562
108
+ # 1 75.0 222.5 258.362212
109
+ # 2 75.0 237.5 258.318240
110
+ # 3 75.0 267.5 256.928497
111
+ # 4 75.0 285.0 261.614103
76
112
  ```
77
113
 
78
- Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
114
+ Succinctly, we "pivot" Xarray Datasets (with consistent dimensions) to treat them like tables so we can run
79
115
  SQL queries against them.
80
116
 
81
117
  ## Why build this?
@@ -93,18 +129,18 @@ A few reasons:
93
129
 
94
130
  This is a light-weight way to prove the value of the interface.
95
131
 
96
- The larger goal is to explore the hypothesis that [Pangeo](https://pangeo.io/)
97
- is a scientific database. Here, xarray-sql can be thought of as a missing DB
98
- front end.
132
+ The larger goal is to explore the hypothesis that the [Pangeo](https://pangeo.io/)
133
+ ecosystem is a scientific database. Here, xarray-sql can be thought of as a missing
134
+ DB front end.
99
135
 
100
136
  ## How does it work?
101
137
 
102
- All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
138
+ All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
103
139
  `from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
104
140
  That's it!
105
141
 
106
- _2025 update_: This library now implements a dask-like `from_map` interface in
107
- pure `datafusion` and `pyarrow`, but works with the same principle!
142
+ _2025 update_: This library now implements a Dask-like `from_map` interface in
143
+ pure DataFusion and PyArrow, but works with the same principle!
108
144
 
109
145
  ## Why does this work?
110
146
 
@@ -112,14 +148,15 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
112
148
  chunks and represented contiguously in memory. It is only a matter of metadata
113
149
  that breaks them up into ndarrays. `to_dataframe()`
114
150
  just changes this metadata (via a `ravel()`/`reshape()`), back into a column
115
- amenable to a DataFrame.
116
-
117
- There is added overhead from duplicating dimensions as columns, which we see as
118
- worth the convenience of DataFrames.
151
+ amenable to a DataFrame. We take advantage of this light weight metadata change to
152
+ make chunked information scannable by a DB engine (DataFusion).
119
153
 
120
154
  ## What are the current limitations?
121
155
 
122
- _2025 update_: TBD, `datafusion` provides a whole new world!
156
+ _2025 update_: TBD, DataFusion provides a whole new world! Currently, we're looking for
157
+ early users – "tire kickers", if you will. We'd love your input to shape the direction of this
158
+ project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
159
+ you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
123
160
 
124
161
  ## What would a deeper integration look like?
125
162
 
@@ -134,6 +171,13 @@ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
134
171
  and BigQuery. More thoughts on this
135
172
  in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
136
173
 
174
+ _2025 update_: Something like this is being built across a few projects! The ones I know about are:
175
+ - [CartoDB's Raquet](https://github.com/CartoDB/raquet)
176
+ - The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
177
+
178
+ As of writing, this project is [amid integrating](https://github.com/alxmrs/xarray-sql/pull/69) a
179
+ rust-based DataFusion backend provided by arrow-zarr.
180
+
137
181
  ## Sponsors & Contributors
138
182
 
139
183
  I want to give a special thanks to the following folks and institutions:
@@ -143,6 +187,9 @@ I want to give a special thanks to the following folks and institutions:
143
187
  - Jake Wall and AI2/Ecoscope for compute resources and key use cases.
144
188
  - Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
145
189
  for the early review and discussion of this project.
190
+ - Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
191
+ Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
192
+ and DataFusion-specific collaboration.
146
193
 
147
194
  ## License
148
195
 
@@ -1,12 +1,12 @@
1
1
  xarray_sql/__init__.py,sha256=HmuxwFMm3a7EVhCz3z2XUn6MeU7ghwSbO9y7A5G8MY8,69
2
2
  xarray_sql/core.py,sha256=EQ1qBjHxnJPpb_HTRQ7B3NKLaI00KOdD4fmt5Qum3BI,1213
3
- xarray_sql/df.py,sha256=3HoHmWCZQkkeOkJfP1lEaFjaqAykimY0ILQpkZ3ZaZY,5703
3
+ xarray_sql/df.py,sha256=1BmcA8mq59er_6bxsxxZnHQ3zyMEeXoBblQtj-eVTLU,6065
4
4
  xarray_sql/df_integrationtest.py,sha256=ziAq6t3W23mehRDFUlFC4j-3TWAdrN7BaWp_p_Rsjs0,648
5
5
  xarray_sql/df_test.py,sha256=w2GqqaD2iUlIPA9jkyzUlHJJsFhQoHZB5pgtfz8xFgs,11965
6
6
  xarray_sql/sql.py,sha256=D6Wz4m79jhgZxDqNs__-n6B8zjqkFGRDQtcq0e72NKI,434
7
- xarray_sql/sql_test.py,sha256=eQ2w9wliBIKE32x_tBhjezMbykwQvOpgm0ectObPJb4,1370
8
- xarray_sql-0.1.1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
- xarray_sql-0.1.1.dist-info/METADATA,sha256=KJS24M8Vvcrxd_CVusz6DtucLAbnlahjpwRDZi-osEI,6086
10
- xarray_sql-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
11
- xarray_sql-0.1.1.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
- xarray_sql-0.1.1.dist-info/RECORD,,
7
+ xarray_sql/sql_test.py,sha256=Iy_w7EWSCTqQJuXa04WXLg-e05cZ6XL8DQVCsaBuXhA,24944
8
+ xarray_sql-0.1.2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
9
+ xarray_sql-0.1.2.dist-info/METADATA,sha256=q7EdLLIRJH70cjG3vKNMiUCAGuj0afSQrKvj_fQIqss,8667
10
+ xarray_sql-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
11
+ xarray_sql-0.1.2.dist-info/top_level.txt,sha256=GooBATfgVuXZoJQ7FresTpPl7G2KHbiBmrMw5r6FmXk,16
12
+ xarray_sql-0.1.2.dist-info/RECORD,,